From b2cd5c541b021b46837fb4a8c157ea9d0dcb2daa Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 18 Feb 2025 12:42:55 -0600 Subject: [PATCH 01/99] Move uploaders into @ditto/uploaders --- deno.json | 3 ++- packages/ditto/middleware/uploaderMiddleware.ts | 4 +--- packages/{ditto => }/uploaders/DenoUploader.ts | 3 ++- packages/{ditto => }/uploaders/IPFSUploader.ts | 3 ++- packages/{ditto => }/uploaders/S3Uploader.ts | 3 ++- packages/uploaders/deno.json | 7 +++++++ packages/uploaders/mod.ts | 3 +++ 7 files changed, 19 insertions(+), 7 deletions(-) rename packages/{ditto => }/uploaders/DenoUploader.ts (95%) rename packages/{ditto => }/uploaders/IPFSUploader.ts (96%) rename packages/{ditto => }/uploaders/S3Uploader.ts (96%) create mode 100644 packages/uploaders/deno.json create mode 100644 packages/uploaders/mod.ts diff --git a/deno.json b/deno.json index 412f32a3..70b8cd74 100644 --- a/deno.json +++ b/deno.json @@ -5,7 +5,8 @@ "./packages/conf", "./packages/db", "./packages/ditto", - "./packages/metrics" + "./packages/metrics", + "./packages/uploaders" ], "tasks": { "start": "deno run -A --env-file --deny-read=.env packages/ditto/server.ts", diff --git a/packages/ditto/middleware/uploaderMiddleware.ts b/packages/ditto/middleware/uploaderMiddleware.ts index 056106c1..10cd3d2b 100644 --- a/packages/ditto/middleware/uploaderMiddleware.ts +++ b/packages/ditto/middleware/uploaderMiddleware.ts @@ -1,10 +1,8 @@ +import { DenoUploader, IPFSUploader, S3Uploader } from '@ditto/uploaders'; import { BlossomUploader, NostrBuildUploader } from '@nostrify/nostrify/uploaders'; import { safeFetch } from '@soapbox/safe-fetch'; import { AppMiddleware } from '@/app.ts'; -import { DenoUploader } from '@/uploaders/DenoUploader.ts'; -import { IPFSUploader } from '@/uploaders/IPFSUploader.ts'; -import { S3Uploader } from '@/uploaders/S3Uploader.ts'; /** Set an uploader for the user. */ export const uploaderMiddleware: AppMiddleware = async (c, next) => { diff --git a/packages/ditto/uploaders/DenoUploader.ts b/packages/uploaders/DenoUploader.ts similarity index 95% rename from packages/ditto/uploaders/DenoUploader.ts rename to packages/uploaders/DenoUploader.ts index fd30d8c6..a97bdb52 100644 --- a/packages/ditto/uploaders/DenoUploader.ts +++ b/packages/uploaders/DenoUploader.ts @@ -1,10 +1,11 @@ import { join } from 'node:path'; -import { NUploader } from '@nostrify/nostrify'; import { crypto } from '@std/crypto'; import { encodeHex } from '@std/encoding/hex'; import { extensionsByType } from '@std/media-types'; +import type { NUploader } from '@nostrify/nostrify'; + export interface DenoUploaderOpts { baseUrl: string; dir: string; diff --git a/packages/ditto/uploaders/IPFSUploader.ts b/packages/uploaders/IPFSUploader.ts similarity index 96% rename from packages/ditto/uploaders/IPFSUploader.ts rename to packages/uploaders/IPFSUploader.ts index 7bf5165b..cf9c1516 100644 --- a/packages/ditto/uploaders/IPFSUploader.ts +++ b/packages/uploaders/IPFSUploader.ts @@ -1,6 +1,7 @@ -import { NUploader } from '@nostrify/nostrify'; import { z } from 'zod'; +import type { NUploader } from '@nostrify/nostrify'; + export interface IPFSUploaderOpts { baseUrl: string; apiUrl?: string; diff --git a/packages/ditto/uploaders/S3Uploader.ts b/packages/uploaders/S3Uploader.ts similarity index 96% rename from packages/ditto/uploaders/S3Uploader.ts rename to packages/uploaders/S3Uploader.ts index c0d776f8..551a554d 100644 --- a/packages/ditto/uploaders/S3Uploader.ts +++ b/packages/uploaders/S3Uploader.ts @@ -1,11 +1,12 @@ import { join } from 'node:path'; import { S3Client } from '@bradenmacdonald/s3-lite-client'; -import { NUploader } from '@nostrify/nostrify'; import { crypto } from '@std/crypto'; import { encodeHex } from '@std/encoding/hex'; import { extensionsByType } from '@std/media-types'; +import type { NUploader } from '@nostrify/nostrify'; + export interface S3UploaderOpts { endPoint: string; region: string; diff --git a/packages/uploaders/deno.json b/packages/uploaders/deno.json new file mode 100644 index 00000000..b37b8aa7 --- /dev/null +++ b/packages/uploaders/deno.json @@ -0,0 +1,7 @@ +{ + "name": "@ditto/uploaders", + "version": "1.1.0", + "exports": { + ".": "./mod.ts" + } +} diff --git a/packages/uploaders/mod.ts b/packages/uploaders/mod.ts new file mode 100644 index 00000000..c5405344 --- /dev/null +++ b/packages/uploaders/mod.ts @@ -0,0 +1,3 @@ +export { DenoUploader } from './DenoUploader.ts'; +export { IPFSUploader } from './IPFSUploader.ts'; +export { S3Uploader } from './S3Uploader.ts'; From d901a722e56baa54fde14c5f5b9a917c08b78b85 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 18 Feb 2025 12:48:27 -0600 Subject: [PATCH 02/99] Make @ditto/lang its own package --- deno.json | 1 + packages/ditto/storages/EventsDB.ts | 2 +- packages/lang/deno.json | 7 +++++++ packages/{ditto/utils => lang}/language.test.ts | 3 ++- packages/{ditto/utils => lang}/language.ts | 0 5 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 packages/lang/deno.json rename packages/{ditto/utils => lang}/language.test.ts (98%) rename packages/{ditto/utils => lang}/language.ts (100%) diff --git a/deno.json b/deno.json index 70b8cd74..33c8e1d6 100644 --- a/deno.json +++ b/deno.json @@ -5,6 +5,7 @@ "./packages/conf", "./packages/db", "./packages/ditto", + "./packages/lang", "./packages/metrics", "./packages/uploaders" ], diff --git a/packages/ditto/storages/EventsDB.ts b/packages/ditto/storages/EventsDB.ts index 622f5811..e7669861 100644 --- a/packages/ditto/storages/EventsDB.ts +++ b/packages/ditto/storages/EventsDB.ts @@ -1,6 +1,7 @@ // deno-lint-ignore-file require-await import { DittoTables } from '@ditto/db'; +import { detectLanguage } from '@ditto/lang'; import { NPostgres, NPostgresSchema } from '@nostrify/db'; import { dbEventsCounter } from '@ditto/metrics'; import { NIP50, NKinds, NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; @@ -18,7 +19,6 @@ import { isNostrId } from '@/utils.ts'; import { abortError } from '@/utils/abort.ts'; import { purifyEvent } from '@/utils/purify.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { detectLanguage } from '@/utils/language.ts'; import { getMediaLinks } from '@/utils/note.ts'; /** Function to decide whether or not to index a tag. */ diff --git a/packages/lang/deno.json b/packages/lang/deno.json new file mode 100644 index 00000000..f192fb0f --- /dev/null +++ b/packages/lang/deno.json @@ -0,0 +1,7 @@ +{ + "name": "@ditto/lang", + "version": "1.1.0", + "exports": { + ".": "./language.ts" + } +} diff --git a/packages/ditto/utils/language.test.ts b/packages/lang/language.test.ts similarity index 98% rename from packages/ditto/utils/language.test.ts rename to packages/lang/language.test.ts index 66a26edd..09dbb66a 100644 --- a/packages/ditto/utils/language.test.ts +++ b/packages/lang/language.test.ts @@ -1,6 +1,7 @@ -import { detectLanguage } from '@/utils/language.ts'; import { assertEquals } from '@std/assert'; +import { detectLanguage } from './language.ts'; + Deno.test('Detect English language', () => { assertEquals(detectLanguage(``, 0.90), undefined); assertEquals(detectLanguage(`Good morning my fellow friends`, 0.90), 'en'); diff --git a/packages/ditto/utils/language.ts b/packages/lang/language.ts similarity index 100% rename from packages/ditto/utils/language.ts rename to packages/lang/language.ts From 990646da26cbd266a1afe1463b66a3ca696bf9b9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 18 Feb 2025 13:00:49 -0600 Subject: [PATCH 03/99] Make @ditto/translators its own package --- deno.json | 1 + packages/ditto/app.ts | 2 +- .../ditto/middleware/translatorMiddleware.ts | 3 +-- packages/ditto/test.ts | 14 ------------- .../translators/DeepLTranslator.test.ts | 20 +++++++++---------- .../translators/DeepLTranslator.ts | 7 ++++--- .../DittoTranslator.ts | 0 .../LibreTranslateTranslator.test.ts | 20 +++++++++---------- .../translators/LibreTranslateTranslator.ts | 7 ++++--- packages/translators/deno.json | 7 +++++++ packages/translators/mod.ts | 4 ++++ packages/translators/schema.ts | 8 ++++++++ 12 files changed, 50 insertions(+), 43 deletions(-) rename packages/{ditto => }/translators/DeepLTranslator.test.ts (79%) rename packages/{ditto => }/translators/DeepLTranslator.ts (93%) rename packages/{ditto/interfaces => translators}/DittoTranslator.ts (100%) rename packages/{ditto => }/translators/LibreTranslateTranslator.test.ts (70%) rename packages/{ditto => }/translators/LibreTranslateTranslator.ts (94%) create mode 100644 packages/translators/deno.json create mode 100644 packages/translators/mod.ts create mode 100644 packages/translators/schema.ts diff --git a/deno.json b/deno.json index 33c8e1d6..888db8cf 100644 --- a/deno.json +++ b/deno.json @@ -7,6 +7,7 @@ "./packages/ditto", "./packages/lang", "./packages/metrics", + "./packages/translators", "./packages/uploaders" ], "tasks": { diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index 6a54f66f..88bfa7f9 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -1,6 +1,7 @@ import { confMw } from '@ditto/api/middleware'; import { type DittoConf } from '@ditto/conf'; import { DittoTables } from '@ditto/db'; +import { type DittoTranslator } from '@ditto/translators'; import { type Context, Env as HonoEnv, Handler, Hono, Input as HonoInput, MiddlewareHandler } from '@hono/hono'; import { every } from '@hono/hono/combine'; import { cors } from '@hono/hono/cors'; @@ -134,7 +135,6 @@ import { metricsController } from '@/controllers/metrics.ts'; import { manifestController } from '@/controllers/manifest.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; import { nostrController } from '@/controllers/well-known/nostr.ts'; -import { DittoTranslator } from '@/interfaces/DittoTranslator.ts'; import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts'; import { cacheControlMiddleware } from '@/middleware/cacheControlMiddleware.ts'; import { cspMiddleware } from '@/middleware/cspMiddleware.ts'; diff --git a/packages/ditto/middleware/translatorMiddleware.ts b/packages/ditto/middleware/translatorMiddleware.ts index eb97ae44..478c2fb9 100644 --- a/packages/ditto/middleware/translatorMiddleware.ts +++ b/packages/ditto/middleware/translatorMiddleware.ts @@ -1,8 +1,7 @@ +import { DeepLTranslator, LibreTranslateTranslator } from '@ditto/translators'; import { safeFetch } from '@soapbox/safe-fetch'; import { AppMiddleware } from '@/app.ts'; -import { DeepLTranslator } from '@/translators/DeepLTranslator.ts'; -import { LibreTranslateTranslator } from '@/translators/LibreTranslateTranslator.ts'; /** Set the translator used for translating posts. */ export const translatorMiddleware: AppMiddleware = async (c, next) => { diff --git a/packages/ditto/test.ts b/packages/ditto/test.ts index bc9a6787..47052b8d 100644 --- a/packages/ditto/test.ts +++ b/packages/ditto/test.ts @@ -1,6 +1,4 @@ import { DittoDB } from '@ditto/db'; -import ISO6391, { LanguageCode } from 'iso-639-1'; -import lande from 'lande'; import { NostrEvent } from '@nostrify/nostrify'; import { finalizeEvent, generateSecretKey } from 'nostr-tools'; @@ -68,15 +66,3 @@ export async function createTestDB(opts?: { pure?: boolean }) { export function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } - -export function getLanguage(text: string): LanguageCode | undefined { - const [topResult] = lande(text); - if (topResult) { - const [iso6393] = topResult; - const locale = new Intl.Locale(iso6393); - if (ISO6391.validate(locale.language)) { - return locale.language; - } - } - return; -} diff --git a/packages/ditto/translators/DeepLTranslator.test.ts b/packages/translators/DeepLTranslator.test.ts similarity index 79% rename from packages/ditto/translators/DeepLTranslator.test.ts rename to packages/translators/DeepLTranslator.test.ts index 08f16a66..ae1565c9 100644 --- a/packages/ditto/translators/DeepLTranslator.test.ts +++ b/packages/translators/DeepLTranslator.test.ts @@ -1,14 +1,14 @@ +import { DittoConf } from '@ditto/conf'; +import { detectLanguage } from '@ditto/lang'; import { assert, assertEquals } from '@std/assert'; -import { Conf } from '@/config.ts'; -import { DeepLTranslator } from '@/translators/DeepLTranslator.ts'; -import { getLanguage } from '@/test.ts'; +import { DeepLTranslator } from './DeepLTranslator.ts'; const { deeplBaseUrl: baseUrl, deeplApiKey: apiKey, translationProvider, -} = Conf; +} = new DittoConf(Deno.env); const deepl = 'deepl'; @@ -28,9 +28,9 @@ Deno.test('DeepL translation with source language omitted', { ); assertEquals(data.source_lang, 'pt'); - assertEquals(getLanguage(data.results[0]), 'en'); - assertEquals(getLanguage(data.results[1]), 'en'); - assertEquals(getLanguage(data.results[2]), 'en'); + assertEquals(detectLanguage(data.results[0], 0), 'en'); + assertEquals(detectLanguage(data.results[1], 0), 'en'); + assertEquals(detectLanguage(data.results[2], 0), 'en'); }); Deno.test('DeepL translation with source language set', { @@ -49,9 +49,9 @@ Deno.test('DeepL translation with source language set', { ); assertEquals(data.source_lang, 'pt'); - assertEquals(getLanguage(data.results[0]), 'en'); - assertEquals(getLanguage(data.results[1]), 'en'); - assertEquals(getLanguage(data.results[2]), 'en'); + assertEquals(detectLanguage(data.results[0], 0), 'en'); + assertEquals(detectLanguage(data.results[1], 0), 'en'); + assertEquals(detectLanguage(data.results[2], 0), 'en'); }); Deno.test("DeepL translation doesn't alter Nostr URIs", { diff --git a/packages/ditto/translators/DeepLTranslator.ts b/packages/translators/DeepLTranslator.ts similarity index 93% rename from packages/ditto/translators/DeepLTranslator.ts rename to packages/translators/DeepLTranslator.ts index d1cefaaa..4c077a87 100644 --- a/packages/ditto/translators/DeepLTranslator.ts +++ b/packages/translators/DeepLTranslator.ts @@ -1,8 +1,9 @@ -import { LanguageCode } from 'iso-639-1'; import { z } from 'zod'; -import { DittoTranslator } from '@/interfaces/DittoTranslator.ts'; -import { languageSchema } from '@/schema.ts'; +import { languageSchema } from './schema.ts'; + +import type { LanguageCode } from 'iso-639-1'; +import type { DittoTranslator } from './DittoTranslator.ts'; interface DeepLTranslatorOpts { /** DeepL base URL to use. Default: 'https://api.deepl.com' */ diff --git a/packages/ditto/interfaces/DittoTranslator.ts b/packages/translators/DittoTranslator.ts similarity index 100% rename from packages/ditto/interfaces/DittoTranslator.ts rename to packages/translators/DittoTranslator.ts diff --git a/packages/ditto/translators/LibreTranslateTranslator.test.ts b/packages/translators/LibreTranslateTranslator.test.ts similarity index 70% rename from packages/ditto/translators/LibreTranslateTranslator.test.ts rename to packages/translators/LibreTranslateTranslator.test.ts index edda3039..fc6c0a55 100644 --- a/packages/ditto/translators/LibreTranslateTranslator.test.ts +++ b/packages/translators/LibreTranslateTranslator.test.ts @@ -1,14 +1,14 @@ +import { DittoConf } from '@ditto/conf'; +import { detectLanguage } from '@ditto/lang'; import { assertEquals } from '@std/assert'; -import { Conf } from '@/config.ts'; -import { LibreTranslateTranslator } from '@/translators/LibreTranslateTranslator.ts'; -import { getLanguage } from '@/test.ts'; +import { LibreTranslateTranslator } from './LibreTranslateTranslator.ts'; const { libretranslateBaseUrl: baseUrl, libretranslateApiKey: apiKey, translationProvider, -} = Conf; +} = new DittoConf(Deno.env); const libretranslate = 'libretranslate'; @@ -28,9 +28,9 @@ Deno.test('LibreTranslate translation with source language omitted', { ); assertEquals(data.source_lang, 'pt'); - assertEquals(getLanguage(data.results[0]), 'ca'); - assertEquals(getLanguage(data.results[1]), 'ca'); - assertEquals(getLanguage(data.results[2]), 'ca'); + assertEquals(detectLanguage(data.results[0], 0), 'ca'); + assertEquals(detectLanguage(data.results[1], 0), 'ca'); + assertEquals(detectLanguage(data.results[2], 0), 'ca'); }); Deno.test('LibreTranslate translation with source language set', { @@ -49,7 +49,7 @@ Deno.test('LibreTranslate translation with source language set', { ); assertEquals(data.source_lang, 'pt'); - assertEquals(getLanguage(data.results[0]), 'ca'); - assertEquals(getLanguage(data.results[1]), 'ca'); - assertEquals(getLanguage(data.results[2]), 'ca'); + assertEquals(detectLanguage(data.results[0], 0), 'ca'); + assertEquals(detectLanguage(data.results[1], 0), 'ca'); + assertEquals(detectLanguage(data.results[2], 0), 'ca'); }); diff --git a/packages/ditto/translators/LibreTranslateTranslator.ts b/packages/translators/LibreTranslateTranslator.ts similarity index 94% rename from packages/ditto/translators/LibreTranslateTranslator.ts rename to packages/translators/LibreTranslateTranslator.ts index ef7fb1f8..041a0ee7 100644 --- a/packages/ditto/translators/LibreTranslateTranslator.ts +++ b/packages/translators/LibreTranslateTranslator.ts @@ -1,8 +1,9 @@ -import { LanguageCode } from 'iso-639-1'; import { z } from 'zod'; -import { DittoTranslator } from '@/interfaces/DittoTranslator.ts'; -import { languageSchema } from '@/schema.ts'; +import { languageSchema } from './schema.ts'; + +import type { LanguageCode } from 'iso-639-1'; +import type { DittoTranslator } from './DittoTranslator.ts'; interface LibreTranslateTranslatorOpts { /** Libretranslate endpoint to use. Default: 'https://libretranslate.com' */ diff --git a/packages/translators/deno.json b/packages/translators/deno.json new file mode 100644 index 00000000..5d603f3a --- /dev/null +++ b/packages/translators/deno.json @@ -0,0 +1,7 @@ +{ + "name": "@ditto/translators", + "version": "1.1.0", + "exports": { + ".": "./mod.ts" + } +} diff --git a/packages/translators/mod.ts b/packages/translators/mod.ts new file mode 100644 index 00000000..e60f19c7 --- /dev/null +++ b/packages/translators/mod.ts @@ -0,0 +1,4 @@ +export { DeepLTranslator } from './DeepLTranslator.ts'; +export { LibreTranslateTranslator } from './LibreTranslateTranslator.ts'; + +export type { DittoTranslator } from './DittoTranslator.ts'; diff --git a/packages/translators/schema.ts b/packages/translators/schema.ts new file mode 100644 index 00000000..803ef1b0 --- /dev/null +++ b/packages/translators/schema.ts @@ -0,0 +1,8 @@ +import ISO6391 from 'iso-639-1'; +import z from 'zod'; + +/** Value is a ISO-639-1 language code. */ +export const languageSchema = z.string().refine( + (val) => ISO6391.validate(val), + { message: 'Not a valid language in ISO-639-1 format' }, +); From 025a86fda2c22c77744fe4405fbdc8eb1e91f894 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 18 Feb 2025 13:05:47 -0600 Subject: [PATCH 04/99] translators: add missing return types --- packages/translators/DeepLTranslator.ts | 2 +- packages/translators/LibreTranslateTranslator.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/translators/DeepLTranslator.ts b/packages/translators/DeepLTranslator.ts index 4c077a87..f4b6f918 100644 --- a/packages/translators/DeepLTranslator.ts +++ b/packages/translators/DeepLTranslator.ts @@ -32,7 +32,7 @@ export class DeepLTranslator implements DittoTranslator { source: LanguageCode | undefined, dest: LanguageCode, opts?: { signal?: AbortSignal }, - ) { + ): Promise<{ results: string[]; source_lang: LanguageCode }> { const { translations } = await this.translateMany(texts, source, dest, opts); return { diff --git a/packages/translators/LibreTranslateTranslator.ts b/packages/translators/LibreTranslateTranslator.ts index 041a0ee7..b75f9b54 100644 --- a/packages/translators/LibreTranslateTranslator.ts +++ b/packages/translators/LibreTranslateTranslator.ts @@ -32,7 +32,7 @@ export class LibreTranslateTranslator implements DittoTranslator { source: LanguageCode | undefined, dest: LanguageCode, opts?: { signal?: AbortSignal }, - ) { + ): Promise<{ results: string[]; source_lang: LanguageCode }> { const translations = await Promise.all( texts.map((text) => this.translateOne(text, source, dest, 'html', { signal: opts?.signal })), ); From 6f9081bbafe7532800bcfa761d2dc1d9f00847ac Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 18 Feb 2025 13:13:05 -0600 Subject: [PATCH 05/99] Make @ditto/policies its own package --- deno.json | 1 + packages/ditto/controllers/api/streaming.ts | 2 +- .../policies/MuteListPolicy.test.ts | 41 ++++++++----------- .../{ditto => }/policies/MuteListPolicy.ts | 13 ++++-- packages/policies/deno.json | 7 ++++ packages/policies/mod.ts | 1 + 6 files changed, 37 insertions(+), 28 deletions(-) rename packages/{ditto => }/policies/MuteListPolicy.test.ts (67%) rename packages/{ditto => }/policies/MuteListPolicy.ts (64%) create mode 100644 packages/policies/deno.json create mode 100644 packages/policies/mod.ts diff --git a/deno.json b/deno.json index 888db8cf..0dd5034c 100644 --- a/deno.json +++ b/deno.json @@ -7,6 +7,7 @@ "./packages/ditto", "./packages/lang", "./packages/metrics", + "./packages/policies", "./packages/translators", "./packages/uploaders" ], diff --git a/packages/ditto/controllers/api/streaming.ts b/packages/ditto/controllers/api/streaming.ts index 7f2f8b64..01eaaed8 100644 --- a/packages/ditto/controllers/api/streaming.ts +++ b/packages/ditto/controllers/api/streaming.ts @@ -1,3 +1,4 @@ +import { MuteListPolicy } from '@ditto/policies'; import { streamingClientMessagesCounter, streamingConnectionsGauge, @@ -9,7 +10,6 @@ import { logi } from '@soapbox/logi'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; -import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; import { getFeedPubkeys } from '@/queries.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; diff --git a/packages/ditto/policies/MuteListPolicy.test.ts b/packages/policies/MuteListPolicy.test.ts similarity index 67% rename from packages/ditto/policies/MuteListPolicy.test.ts rename to packages/policies/MuteListPolicy.test.ts index 89d7d993..d07c4472 100644 --- a/packages/ditto/policies/MuteListPolicy.test.ts +++ b/packages/policies/MuteListPolicy.test.ts @@ -1,8 +1,8 @@ import { MockRelay } from '@nostrify/nostrify/test'; import { assertEquals } from '@std/assert'; -import { UserStore } from '@/storages/UserStore.ts'; -import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; + +import { MuteListPolicy } from './MuteListPolicy.ts'; import userBlack from '~/fixtures/events/kind-0-black.json' with { type: 'json' }; import userMe from '~/fixtures/events/event-0-makes-repost-with-quote-repost.json' with { type: 'json' }; @@ -16,14 +16,12 @@ Deno.test('block event: muted user cannot post', async () => { const blockEventCopy = structuredClone(blockEvent); const event1authorUserMeCopy = structuredClone(event1authorUserMe); - const db = new MockRelay(); + const relay = new MockRelay(); + const policy = new MuteListPolicy(userBlack.pubkey, relay); - const store = new UserStore(userBlackCopy.pubkey, db); - const policy = new MuteListPolicy(userBlack.pubkey, db); - - await store.event(blockEventCopy); - await store.event(userBlackCopy); - await store.event(userMeCopy); + await relay.event(blockEventCopy); + await relay.event(userBlackCopy); + await relay.event(userMeCopy); const ok = await policy.call(event1authorUserMeCopy); @@ -35,13 +33,11 @@ Deno.test('allow event: user is NOT muted because there is no muted event', asyn const userMeCopy = structuredClone(userMe); const event1authorUserMeCopy = structuredClone(event1authorUserMe); - const db = new MockRelay(); + const relay = new MockRelay(); + const policy = new MuteListPolicy(userBlack.pubkey, relay); - const store = new UserStore(userBlackCopy.pubkey, db); - const policy = new MuteListPolicy(userBlack.pubkey, db); - - await store.event(userBlackCopy); - await store.event(userMeCopy); + await relay.event(userBlackCopy); + await relay.event(userMeCopy); const ok = await policy.call(event1authorUserMeCopy); @@ -55,16 +51,15 @@ Deno.test('allow event: user is NOT muted because he is not in mute event', asyn const blockEventCopy = structuredClone(blockEvent); const event1copy = structuredClone(event1); - const db = new MockRelay(); + const relay = new MockRelay(); - const store = new UserStore(userBlackCopy.pubkey, db); - const policy = new MuteListPolicy(userBlack.pubkey, db); + const policy = new MuteListPolicy(userBlack.pubkey, relay); - await store.event(userBlackCopy); - await store.event(blockEventCopy); - await store.event(userMeCopy); - await store.event(event1copy); - await store.event(event1authorUserMeCopy); + await relay.event(userBlackCopy); + await relay.event(blockEventCopy); + await relay.event(userMeCopy); + await relay.event(event1copy); + await relay.event(event1authorUserMeCopy); const ok = await policy.call(event1copy); diff --git a/packages/ditto/policies/MuteListPolicy.ts b/packages/policies/MuteListPolicy.ts similarity index 64% rename from packages/ditto/policies/MuteListPolicy.ts rename to packages/policies/MuteListPolicy.ts index 130d10df..d880c57d 100644 --- a/packages/ditto/policies/MuteListPolicy.ts +++ b/packages/policies/MuteListPolicy.ts @@ -1,13 +1,18 @@ -import { NostrEvent, NostrRelayOK, NPolicy, NStore } from '@nostrify/nostrify'; - -import { getTagSet } from '@/utils/tags.ts'; +import type { NostrEvent, NostrRelayOK, NPolicy, NStore } from '@nostrify/nostrify'; export class MuteListPolicy implements NPolicy { constructor(private pubkey: string, private store: NStore) {} async call(event: NostrEvent): Promise { + const pubkeys = new Set(); + const [muteList] = await this.store.query([{ authors: [this.pubkey], kinds: [10000], limit: 1 }]); - const pubkeys = getTagSet(muteList?.tags ?? [], 'p'); + + for (const [name, value] of muteList?.tags ?? []) { + if (name === 'p') { + pubkeys.add(value); + } + } if (pubkeys.has(event.pubkey)) { return ['OK', event.id, false, 'blocked: Your account has been deactivated.']; diff --git a/packages/policies/deno.json b/packages/policies/deno.json new file mode 100644 index 00000000..ca190883 --- /dev/null +++ b/packages/policies/deno.json @@ -0,0 +1,7 @@ +{ + "name": "@ditto/policies", + "version": "1.1.0", + "exports": { + ".": "./mod.ts" + } +} diff --git a/packages/policies/mod.ts b/packages/policies/mod.ts new file mode 100644 index 00000000..9748a4cf --- /dev/null +++ b/packages/policies/mod.ts @@ -0,0 +1 @@ +export { MuteListPolicy } from './MuteListPolicy.ts'; From ac3a9fdf5aa3ebd585f9d50aa1082fbb5750e776 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 18 Feb 2025 13:16:42 -0600 Subject: [PATCH 06/99] Make @ditto/ratelimiter its own package --- deno.json | 1 + packages/ditto/controllers/nostr/relay.ts | 4 +--- .../utils => }/ratelimiter/MemoryRateLimiter.test.ts | 0 .../{ditto/utils => }/ratelimiter/MemoryRateLimiter.ts | 3 ++- .../{ditto/utils => }/ratelimiter/MultiRateLimiter.test.ts | 0 packages/{ditto/utils => }/ratelimiter/MultiRateLimiter.ts | 2 +- packages/{ditto/utils => }/ratelimiter/RateLimitError.ts | 2 +- packages/ratelimiter/deno.json | 7 +++++++ packages/ratelimiter/mod.ts | 5 +++++ packages/{ditto/utils => }/ratelimiter/types.ts | 0 10 files changed, 18 insertions(+), 6 deletions(-) rename packages/{ditto/utils => }/ratelimiter/MemoryRateLimiter.test.ts (100%) rename packages/{ditto/utils => }/ratelimiter/MemoryRateLimiter.ts (95%) rename packages/{ditto/utils => }/ratelimiter/MultiRateLimiter.test.ts (100%) rename packages/{ditto/utils => }/ratelimiter/MultiRateLimiter.ts (94%) rename packages/{ditto/utils => }/ratelimiter/RateLimitError.ts (73%) create mode 100644 packages/ratelimiter/deno.json create mode 100644 packages/ratelimiter/mod.ts rename packages/{ditto/utils => }/ratelimiter/types.ts (100%) diff --git a/deno.json b/deno.json index 0dd5034c..2601e28d 100644 --- a/deno.json +++ b/deno.json @@ -8,6 +8,7 @@ "./packages/lang", "./packages/metrics", "./packages/policies", + "./packages/ratelimiter", "./packages/translators", "./packages/uploaders" ], diff --git a/packages/ditto/controllers/nostr/relay.ts b/packages/ditto/controllers/nostr/relay.ts index 92906d04..b4924f22 100644 --- a/packages/ditto/controllers/nostr/relay.ts +++ b/packages/ditto/controllers/nostr/relay.ts @@ -1,5 +1,6 @@ import { type DittoConf } from '@ditto/conf'; import { relayConnectionsGauge, relayEventsCounter, relayMessagesCounter } from '@ditto/metrics'; +import { MemoryRateLimiter, MultiRateLimiter, type RateLimiter } from '@ditto/ratelimiter'; import { logi } from '@soapbox/logi'; import { JsonValue } from '@std/json'; import { @@ -20,9 +21,6 @@ import { RelayError } from '@/RelayError.ts'; import { Storages } from '@/storages.ts'; import { errorJson } from '@/utils/log.ts'; import { purifyEvent } from '@/utils/purify.ts'; -import { MemoryRateLimiter } from '@/utils/ratelimiter/MemoryRateLimiter.ts'; -import { MultiRateLimiter } from '@/utils/ratelimiter/MultiRateLimiter.ts'; -import { RateLimiter } from '@/utils/ratelimiter/types.ts'; import { Time } from '@/utils/time.ts'; /** Limit of initial events returned for a subscription. */ diff --git a/packages/ditto/utils/ratelimiter/MemoryRateLimiter.test.ts b/packages/ratelimiter/MemoryRateLimiter.test.ts similarity index 100% rename from packages/ditto/utils/ratelimiter/MemoryRateLimiter.test.ts rename to packages/ratelimiter/MemoryRateLimiter.test.ts diff --git a/packages/ditto/utils/ratelimiter/MemoryRateLimiter.ts b/packages/ratelimiter/MemoryRateLimiter.ts similarity index 95% rename from packages/ditto/utils/ratelimiter/MemoryRateLimiter.ts rename to packages/ratelimiter/MemoryRateLimiter.ts index 0eaa5540..15546fd0 100644 --- a/packages/ditto/utils/ratelimiter/MemoryRateLimiter.ts +++ b/packages/ratelimiter/MemoryRateLimiter.ts @@ -1,5 +1,6 @@ import { RateLimitError } from './RateLimitError.ts'; -import { RateLimiter, RateLimiterClient } from './types.ts'; + +import type { RateLimiter, RateLimiterClient } from './types.ts'; interface MemoryRateLimiterOpts { limit: number; diff --git a/packages/ditto/utils/ratelimiter/MultiRateLimiter.test.ts b/packages/ratelimiter/MultiRateLimiter.test.ts similarity index 100% rename from packages/ditto/utils/ratelimiter/MultiRateLimiter.test.ts rename to packages/ratelimiter/MultiRateLimiter.test.ts diff --git a/packages/ditto/utils/ratelimiter/MultiRateLimiter.ts b/packages/ratelimiter/MultiRateLimiter.ts similarity index 94% rename from packages/ditto/utils/ratelimiter/MultiRateLimiter.ts rename to packages/ratelimiter/MultiRateLimiter.ts index 14b23142..189ca177 100644 --- a/packages/ditto/utils/ratelimiter/MultiRateLimiter.ts +++ b/packages/ratelimiter/MultiRateLimiter.ts @@ -1,4 +1,4 @@ -import { RateLimiter, RateLimiterClient } from './types.ts'; +import type { RateLimiter, RateLimiterClient } from './types.ts'; export class MultiRateLimiter { constructor(private limiters: RateLimiter[]) {} diff --git a/packages/ditto/utils/ratelimiter/RateLimitError.ts b/packages/ratelimiter/RateLimitError.ts similarity index 73% rename from packages/ditto/utils/ratelimiter/RateLimitError.ts rename to packages/ratelimiter/RateLimitError.ts index ce21af72..da3a4fd8 100644 --- a/packages/ditto/utils/ratelimiter/RateLimitError.ts +++ b/packages/ratelimiter/RateLimitError.ts @@ -1,4 +1,4 @@ -import { RateLimiter, RateLimiterClient } from './types.ts'; +import type { RateLimiter, RateLimiterClient } from './types.ts'; export class RateLimitError extends Error { constructor( diff --git a/packages/ratelimiter/deno.json b/packages/ratelimiter/deno.json new file mode 100644 index 00000000..66e97171 --- /dev/null +++ b/packages/ratelimiter/deno.json @@ -0,0 +1,7 @@ +{ + "name": "@ditto/ratelimiter", + "version": "1.1.0", + "exports": { + ".": "./mod.ts" + } +} diff --git a/packages/ratelimiter/mod.ts b/packages/ratelimiter/mod.ts new file mode 100644 index 00000000..58bbbeaa --- /dev/null +++ b/packages/ratelimiter/mod.ts @@ -0,0 +1,5 @@ +export { MemoryRateLimiter } from './MemoryRateLimiter.ts'; +export { MultiRateLimiter } from './MultiRateLimiter.ts'; +export { RateLimitError } from './RateLimitError.ts'; + +export type { RateLimiter, RateLimiterClient } from './types.ts'; diff --git a/packages/ditto/utils/ratelimiter/types.ts b/packages/ratelimiter/types.ts similarity index 100% rename from packages/ditto/utils/ratelimiter/types.ts rename to packages/ratelimiter/types.ts From 37f418899bceb99c27962236550f180159371445 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 18 Feb 2025 13:32:32 -0600 Subject: [PATCH 07/99] EventsDB -> DittoPgStore --- packages/ditto/storages.ts | 8 +++--- ...{EventsDB.test.ts => DittoPgStore.test.ts} | 6 ++--- .../storages/{EventsDB.ts => DittoPgStore.ts} | 26 +++++++++---------- packages/ditto/test.ts | 4 +-- packages/ditto/workers/policy.worker.ts | 6 ++--- scripts/db-populate-extensions.ts | 4 +-- 6 files changed, 27 insertions(+), 27 deletions(-) rename packages/ditto/storages/{EventsDB.test.ts => DittoPgStore.test.ts} (97%) rename packages/ditto/storages/{EventsDB.ts => DittoPgStore.ts} (95%) diff --git a/packages/ditto/storages.ts b/packages/ditto/storages.ts index be61beb6..4bd7fa30 100644 --- a/packages/ditto/storages.ts +++ b/packages/ditto/storages.ts @@ -6,7 +6,7 @@ import { logi } from '@soapbox/logi'; import { Conf } from '@/config.ts'; import { wsUrlSchema } from '@/schema.ts'; import { AdminStore } from '@/storages/AdminStore.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; +import { DittoPgStore } from '@/storages/DittoPgStore.ts'; import { SearchStore } from '@/storages/search-store.ts'; import { InternalRelay } from '@/storages/InternalRelay.ts'; import { NPool, NRelay1 } from '@nostrify/nostrify'; @@ -14,7 +14,7 @@ import { getRelays } from '@/utils/outbox.ts'; import { seedZapSplits } from '@/utils/zap-split.ts'; export class Storages { - private static _db: Promise | undefined; + private static _db: Promise | undefined; private static _database: Promise | undefined; private static _admin: Promise | undefined; private static _client: Promise> | undefined; @@ -41,11 +41,11 @@ export class Storages { } /** SQL database to store events this Ditto server cares about. */ - public static async db(): Promise { + public static async db(): Promise { if (!this._db) { this._db = (async () => { const kysely = await this.kysely(); - const store = new EventsDB({ kysely, pubkey: Conf.pubkey, timeout: Conf.db.timeouts.default }); + const store = new DittoPgStore({ kysely, pubkey: Conf.pubkey, timeout: Conf.db.timeouts.default }); await seedZapSplits(store); return store; })(); diff --git a/packages/ditto/storages/EventsDB.test.ts b/packages/ditto/storages/DittoPgStore.test.ts similarity index 97% rename from packages/ditto/storages/EventsDB.test.ts rename to packages/ditto/storages/DittoPgStore.test.ts index d0947075..b74e91b9 100644 --- a/packages/ditto/storages/EventsDB.test.ts +++ b/packages/ditto/storages/DittoPgStore.test.ts @@ -4,7 +4,7 @@ import { generateSecretKey } from 'nostr-tools'; import { RelayError } from '@/RelayError.ts'; import { eventFixture, genEvent } from '@/test.ts'; import { Conf } from '@/config.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; +import { DittoPgStore } from '@/storages/DittoPgStore.ts'; import { createTestDB } from '@/test.ts'; Deno.test('count filters', async () => { @@ -254,7 +254,7 @@ Deno.test('NPostgres.query with search', async (t) => { }); }); -Deno.test('EventsDB.indexTags indexes only the final `e` and `p` tag of kind 7 events', () => { +Deno.test('DittoPgStore.indexTags indexes only the final `e` and `p` tag of kind 7 events', () => { const event = { kind: 7, id: 'a92549a442d306b32273aa9456ba48e3851a4e6203af3f567543298ab964b35b', @@ -285,7 +285,7 @@ Deno.test('EventsDB.indexTags indexes only the final `e` and `p` tag of kind 7 e '44639d039a7f7fb8772fcfa13d134d3cda684ec34b6a777ead589676f9e8d81b08a24234066dcde1aacfbe193224940fba7586e7197c159757d3caf8f2b57e1b', }; - const tags = EventsDB.indexTags(event); + const tags = DittoPgStore.indexTags(event); assertEquals(tags, [ ['e', 'e3653ae41ffb510e5fc071555ecfbc94d2fc31e355d61d941e39a97ac6acb15b'], diff --git a/packages/ditto/storages/EventsDB.ts b/packages/ditto/storages/DittoPgStore.ts similarity index 95% rename from packages/ditto/storages/EventsDB.ts rename to packages/ditto/storages/DittoPgStore.ts index e7669861..36040f4c 100644 --- a/packages/ditto/storages/EventsDB.ts +++ b/packages/ditto/storages/DittoPgStore.ts @@ -36,8 +36,8 @@ interface TagConditionOpts { value: string; } -/** Options for the EventsDB store. */ -interface EventsDBOpts { +/** Options for the DittoPgStore store. */ +interface DittoPgStoreOpts { /** Kysely instance to use. */ kysely: Kysely; /** Pubkey of the admin account. */ @@ -49,18 +49,18 @@ interface EventsDBOpts { } /** SQL database storage adapter for Nostr events. */ -class EventsDB extends NPostgres { +class DittoPgStore extends NPostgres { /** Conditions for when to index certain tags. */ static tagConditions: Record = { 'a': ({ count }) => count < 15, 'd': ({ event, count }) => count === 0 && NKinds.parameterizedReplaceable(event.kind), - 'e': EventsDB.eTagCondition, + 'e': DittoPgStore.eTagCondition, 'k': ({ count, value }) => count === 0 && Number.isInteger(Number(value)), 'L': ({ event, count }) => event.kind === 1985 || count === 0, 'l': ({ event, count }) => event.kind === 1985 || count === 0, 'n': ({ count, value }) => count < 50 && value.length < 50, 'P': ({ count, value }) => count === 0 && isNostrId(value), - 'p': EventsDB.pTagCondition, + 'p': DittoPgStore.pTagCondition, 'proxy': ({ count, value }) => count === 0 && value.length < 256, 'q': ({ event, count, value }) => count === 0 && event.kind === 1 && isNostrId(value), 'r': ({ event, count }) => (event.kind === 1985 ? count < 20 : count < 3), @@ -119,11 +119,11 @@ class EventsDB extends NPostgres { return ext; } - constructor(private opts: EventsDBOpts) { + constructor(private opts: DittoPgStoreOpts) { super(opts.kysely, { - indexTags: EventsDB.indexTags, - indexSearch: EventsDB.searchText, - indexExtensions: EventsDB.indexExtensions, + indexTags: DittoPgStore.indexTags, + indexSearch: DittoPgStore.searchText, + indexExtensions: DittoPgStore.indexExtensions, }); } @@ -323,7 +323,7 @@ class EventsDB extends NPostgres { return event.tags.reduce((results, tag, index) => { const [name, value] = tag; - const condition = EventsDB.tagConditions[name] as TagCondition | undefined; + const condition = DittoPgStore.tagConditions[name] as TagCondition | undefined; if (value && condition && value.length < 200 && checkCondition(name, value, condition, index)) { results.push(tag); @@ -338,12 +338,12 @@ class EventsDB extends NPostgres { static searchText(event: NostrEvent): string { switch (event.kind) { case 0: - return EventsDB.buildUserSearchContent(event); + return DittoPgStore.buildUserSearchContent(event); case 1: case 20: return nip27.replaceAll(event.content, () => ''); case 30009: - return EventsDB.buildTagsSearchContent(event.tags.filter(([t]) => t !== 'alt')); + return DittoPgStore.buildTagsSearchContent(event.tags.filter(([t]) => t !== 'alt')); case 30360: return event.tags.find(([name]) => name === 'd')?.[1] || ''; default: @@ -434,4 +434,4 @@ class EventsDB extends NPostgres { } } -export { EventsDB }; +export { DittoPgStore }; diff --git a/packages/ditto/test.ts b/packages/ditto/test.ts index 47052b8d..3c6a555b 100644 --- a/packages/ditto/test.ts +++ b/packages/ditto/test.ts @@ -3,7 +3,7 @@ import { NostrEvent } from '@nostrify/nostrify'; import { finalizeEvent, generateSecretKey } from 'nostr-tools'; import { Conf } from '@/config.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; +import { DittoPgStore } from '@/storages/DittoPgStore.ts'; import { purifyEvent } from '@/utils/purify.ts'; import { sql } from 'kysely'; @@ -38,7 +38,7 @@ export async function createTestDB(opts?: { pure?: boolean }) { await DittoDB.migrate(kysely); - const store = new EventsDB({ + const store = new DittoPgStore({ kysely, timeout: Conf.db.timeouts.default, pubkey: Conf.pubkey, diff --git a/packages/ditto/workers/policy.worker.ts b/packages/ditto/workers/policy.worker.ts index 85a98240..acf7b2f1 100644 --- a/packages/ditto/workers/policy.worker.ts +++ b/packages/ditto/workers/policy.worker.ts @@ -4,7 +4,7 @@ import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify'; import { ReadOnlyPolicy } from '@nostrify/policies'; import * as Comlink from 'comlink'; -import { EventsDB } from '@/storages/EventsDB.ts'; +import { DittoPgStore } from '@/storages/DittoPgStore.ts'; // @ts-ignore Don't try to access the env from this worker. Deno.env = new Map(); @@ -15,7 +15,7 @@ interface PolicyInit { path: string; /** Database URL to connect to. */ databaseUrl: string; - /** Admin pubkey to use for EventsDB checks. */ + /** Admin pubkey to use for DittoPgStore checks. */ pubkey: string; } @@ -32,7 +32,7 @@ export class CustomPolicy implements NPolicy { const { kysely } = DittoDB.create(databaseUrl, { poolSize: 1 }); - const store = new EventsDB({ + const store = new DittoPgStore({ kysely, pubkey, timeout: 5_000, diff --git a/scripts/db-populate-extensions.ts b/scripts/db-populate-extensions.ts index 2b40bd3d..0cb3a49b 100644 --- a/scripts/db-populate-extensions.ts +++ b/scripts/db-populate-extensions.ts @@ -1,7 +1,7 @@ import { NostrEvent } from '@nostrify/nostrify'; import { Storages } from '../packages/ditto/storages.ts'; -import { EventsDB } from '../packages/ditto/storages/EventsDB.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; const kysely = await Storages.kysely(); @@ -11,7 +11,7 @@ const query = kysely for await (const row of query.stream()) { const event: NostrEvent = { ...row, created_at: Number(row.created_at) }; - const ext = EventsDB.indexExtensions(event); + const ext = DittoPgStore.indexExtensions(event); try { await kysely From 6fb873e72f3e59f7f54813f864105e0c0d9957cf Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 18 Feb 2025 13:34:32 -0600 Subject: [PATCH 08/99] Make DittoPgStore pubsub capable --- packages/ditto/storages.ts | 4 +- packages/ditto/storages/DittoPgStore.test.ts | 20 ++ packages/ditto/storages/DittoPgStore.ts | 206 +++++++++++++------ packages/ditto/test.ts | 14 +- packages/ditto/workers/policy.worker.ts | 4 +- 5 files changed, 178 insertions(+), 70 deletions(-) diff --git a/packages/ditto/storages.ts b/packages/ditto/storages.ts index 4bd7fa30..c99e4252 100644 --- a/packages/ditto/storages.ts +++ b/packages/ditto/storages.ts @@ -44,8 +44,8 @@ export class Storages { public static async db(): Promise { if (!this._db) { this._db = (async () => { - const kysely = await this.kysely(); - const store = new DittoPgStore({ kysely, pubkey: Conf.pubkey, timeout: Conf.db.timeouts.default }); + const db = await this.database(); + const store = new DittoPgStore({ db, pubkey: Conf.pubkey, timeout: Conf.db.timeouts.default }); await seedZapSplits(store); return store; })(); diff --git a/packages/ditto/storages/DittoPgStore.test.ts b/packages/ditto/storages/DittoPgStore.test.ts index b74e91b9..3d2ee611 100644 --- a/packages/ditto/storages/DittoPgStore.test.ts +++ b/packages/ditto/storages/DittoPgStore.test.ts @@ -7,6 +7,26 @@ import { Conf } from '@/config.ts'; import { DittoPgStore } from '@/storages/DittoPgStore.ts'; import { createTestDB } from '@/test.ts'; +Deno.test('req streaming', async () => { + await using db = await createTestDB({ pure: true }); + const { store: relay } = db; + + const event1 = await eventFixture('event-1'); + + const promise = new Promise((resolve) => setTimeout(() => resolve(relay.event(event1)), 0)); + + for await (const msg of relay.req([{ since: 0 }])) { + if (msg[0] === 'EVENT') { + assertEquals(relay.subs.size, 1); + assertEquals(msg[2], event1); + break; + } + } + + await promise; + assertEquals(relay.subs.size, 0); // cleanup +}); + Deno.test('count filters', async () => { await using db = await createTestDB({ pure: true }); const { store } = db; diff --git a/packages/ditto/storages/DittoPgStore.ts b/packages/ditto/storages/DittoPgStore.ts index 36040f4c..4c966c8a 100644 --- a/packages/ditto/storages/DittoPgStore.ts +++ b/packages/ditto/storages/DittoPgStore.ts @@ -1,16 +1,27 @@ // deno-lint-ignore-file require-await -import { DittoTables } from '@ditto/db'; +import { DittoDatabase, DittoTables } from '@ditto/db'; import { detectLanguage } from '@ditto/lang'; import { NPostgres, NPostgresSchema } from '@nostrify/db'; -import { dbEventsCounter } from '@ditto/metrics'; -import { NIP50, NKinds, NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; +import { dbEventsCounter, internalSubscriptionsSizeGauge } from '@ditto/metrics'; +import { + NIP50, + NKinds, + NostrEvent, + NostrFilter, + NostrRelayCLOSED, + NostrRelayEOSE, + NostrRelayEVENT, + NSchema as n, +} from '@nostrify/nostrify'; +import { Machina } from '@nostrify/nostrify/utils'; import { logi } from '@soapbox/logi'; import { JsonValue } from '@std/json'; import { LanguageCode } from 'iso-639-1'; import { Kysely } from 'kysely'; import linkify from 'linkifyjs'; -import { nip27 } from 'nostr-tools'; +import { LRUCache } from 'lru-cache'; +import { matchFilter, nip27 } from 'nostr-tools'; import tldts from 'tldts'; import { z } from 'zod'; @@ -36,20 +47,25 @@ interface TagConditionOpts { value: string; } -/** Options for the DittoPgStore store. */ +/** Options for the EventsDB store. */ interface DittoPgStoreOpts { /** Kysely instance to use. */ - kysely: Kysely; + db: DittoDatabase; /** Pubkey of the admin account. */ pubkey: string; /** Timeout in milliseconds for database queries. */ timeout: number; /** Whether the event returned should be a Nostr event or a Ditto event. Defaults to false. */ pure?: boolean; + /** Chunk size for streaming events. Defaults to 100. */ + chunkSize?: number; } /** SQL database storage adapter for Nostr events. */ -class DittoPgStore extends NPostgres { +export class DittoPgStore extends NPostgres { + readonly subs = new Map }>(); + readonly encounters = new LRUCache({ max: 100 }); + /** Conditions for when to index certain tags. */ static tagConditions: Record = { 'a': ({ count }) => count < 15, @@ -72,65 +88,33 @@ class DittoPgStore extends NPostgres { }, }; - static indexExtensions(event: NostrEvent): Record { - const ext: Record = {}; - - if (event.kind === 1) { - ext.reply = event.tags.some(([name]) => name === 'e').toString(); - } else if (event.kind === 1111) { - ext.reply = event.tags.some(([name]) => ['e', 'E'].includes(name)).toString(); - } else if (event.kind === 6) { - ext.reply = 'false'; - } - - if ([1, 20, 30023].includes(event.kind)) { - const language = detectLanguage(event.content, 0.90); - - if (language) { - ext.language = language; - } - } - - const imeta: string[][][] = event.tags - .filter(([name]) => name === 'imeta') - .map(([_, ...entries]) => - entries.map((entry) => { - const split = entry.split(' '); - return [split[0], split.splice(1).join(' ')]; - }) - ); - - // quirks mode - if (!imeta.length && event.kind === 1) { - const links = linkify.find(event.content).filter(({ type }) => type === 'url'); - imeta.push(...getMediaLinks(links)); - } - - if (imeta.length) { - ext.media = 'true'; - - if (imeta.every((tags) => tags.some(([name, value]) => name === 'm' && value.startsWith('video/')))) { - ext.video = 'true'; - } - } - - ext.protocol = event.tags.find(([name]) => name === 'proxy')?.[2] ?? 'nostr'; - - return ext; - } - constructor(private opts: DittoPgStoreOpts) { - super(opts.kysely, { + super(opts.db.kysely, { indexTags: DittoPgStore.indexTags, indexSearch: DittoPgStore.searchText, indexExtensions: DittoPgStore.indexExtensions, + chunkSize: opts.chunkSize, + }); + + opts.db.listen('nostr_event', async (id) => { + if (this.encounters.has(id)) return; + this.encounters.set(id, true); + + const [event] = await this.query([{ ids: [id] }]); + + if (event) { + this.streamOut(event); + } }); } /** Insert an event (and its tags) into the database. */ override async event(event: NostrEvent, opts: { signal?: AbortSignal; timeout?: number } = {}): Promise { event = purifyEvent(event); + logi({ level: 'debug', ns: 'ditto.event', source: 'db', id: event.id, kind: event.kind }); + + this.encounters.set(event.id, true); dbEventsCounter.inc({ kind: event.kind }); if (await this.isDeletedAdmin(event)) { @@ -141,6 +125,7 @@ class DittoPgStore extends NPostgres { try { await super.event(event, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); + this.streamOut(event); } catch (e) { if (e instanceof Error && e.message === 'Cannot add a deleted event') { throw new RelayError('blocked', 'event deleted by user'); @@ -152,6 +137,21 @@ class DittoPgStore extends NPostgres { } } + protected matchesFilter(event: NostrEvent, filter: NostrFilter): boolean { + // TODO: support streaming by search. + return matchFilter(filter, event) && filter.search === undefined; + } + + protected streamOut(event: NostrEvent): void { + for (const { filters, machina } of this.subs.values()) { + for (const filter of filters) { + if (this.matchesFilter(event, filter)) { + machina.push(event); + } + } + } + } + /** Check if an event has been deleted by the admin. */ private async isDeletedAdmin(event: NostrEvent): Promise { const filters: NostrFilter[] = [ @@ -213,10 +213,53 @@ class DittoPgStore extends NPostgres { } } + override async *req( + filters: NostrFilter[], + opts?: { signal?: AbortSignal }, + ): AsyncIterable { + const subId = crypto.randomUUID(); + const normalFilters = this.normalizeFilters(filters); + + if (normalFilters.length) { + const { db, chunkSize = 100 } = this.opts; + const rows = this.getEventsQuery(db.kysely as unknown as Kysely, normalFilters).stream( + chunkSize, + ); + + for await (const row of rows) { + const event = this.parseEventRow(row); + yield ['EVENT', subId, event]; + + if (opts?.signal?.aborted) { + yield ['CLOSED', subId, 'aborted']; + return; + } + } + } + + yield ['EOSE', subId]; + + const machina = new Machina(opts?.signal); + + this.subs.set(subId, { filters, machina }); + internalSubscriptionsSizeGauge.set(this.subs.size); + + try { + for await (const event of machina) { + yield ['EVENT', subId, event]; + } + } catch { + yield ['CLOSED', subId, 'error: something went wrong']; + } finally { + this.subs.delete(subId); + internalSubscriptionsSizeGauge.set(this.subs.size); + } + } + /** Get events for filters from the database. */ override async query( filters: NostrFilter[], - opts: { signal?: AbortSignal; timeout?: number; limit?: number } = {}, + opts: { signal?: AbortSignal; pure?: boolean; timeout?: number; limit?: number } = {}, ): Promise { filters = await this.expandFilters(filters); @@ -334,6 +377,53 @@ class DittoPgStore extends NPostgres { }, []); } + static indexExtensions(event: NostrEvent): Record { + const ext: Record = {}; + + if (event.kind === 1) { + ext.reply = event.tags.some(([name]) => name === 'e').toString(); + } else if (event.kind === 1111) { + ext.reply = event.tags.some(([name]) => ['e', 'E'].includes(name)).toString(); + } else if (event.kind === 6) { + ext.reply = 'false'; + } + + if ([1, 20, 30023].includes(event.kind)) { + const language = detectLanguage(event.content, 0.90); + + if (language) { + ext.language = language; + } + } + + const imeta: string[][][] = event.tags + .filter(([name]) => name === 'imeta') + .map(([_, ...entries]) => + entries.map((entry) => { + const split = entry.split(' '); + return [split[0], split.splice(1).join(' ')]; + }) + ); + + // quirks mode + if (!imeta.length && event.kind === 1) { + const links = linkify.find(event.content).filter(({ type }) => type === 'url'); + imeta.push(...getMediaLinks(links)); + } + + if (imeta.length) { + ext.media = 'true'; + + if (imeta.every((tags) => tags.some(([name, value]) => name === 'm' && value.startsWith('video/')))) { + ext.video = 'true'; + } + } + + ext.protocol = event.tags.find(([name]) => name === 'proxy')?.[2] ?? 'nostr'; + + return ext; + } + /** Build a search index from the event. */ static searchText(event: NostrEvent): string { switch (event.kind) { @@ -385,7 +475,7 @@ class DittoPgStore extends NPostgres { } if (domains.size || hostnames.size) { - let query = this.opts.kysely + let query = this.opts.db.kysely .selectFrom('author_stats') .select('pubkey') .where((eb) => { @@ -433,5 +523,3 @@ class DittoPgStore extends NPostgres { return super.transaction((store, kysely) => callback(store, kysely as unknown as Kysely)); } } - -export { DittoPgStore }; diff --git a/packages/ditto/test.ts b/packages/ditto/test.ts index 3c6a555b..dd1ae6cb 100644 --- a/packages/ditto/test.ts +++ b/packages/ditto/test.ts @@ -34,31 +34,31 @@ export function genEvent(t: Partial = {}, sk: Uint8Array = generateS /** Create a database for testing. It uses `DATABASE_URL`, or creates an in-memory database by default. */ export async function createTestDB(opts?: { pure?: boolean }) { - const { kysely } = DittoDB.create(Conf.databaseUrl, { poolSize: 1 }); + const db = DittoDB.create(Conf.databaseUrl, { poolSize: 1 }); - await DittoDB.migrate(kysely); + await DittoDB.migrate(db.kysely); const store = new DittoPgStore({ - kysely, + db, timeout: Conf.db.timeouts.default, pubkey: Conf.pubkey, pure: opts?.pure ?? false, }); return { + ...db, store, - kysely, [Symbol.asyncDispose]: async () => { const { rows } = await sql< { tablename: string } - >`select tablename from pg_tables where schemaname = current_schema()`.execute(kysely); + >`select tablename from pg_tables where schemaname = current_schema()`.execute(db.kysely); for (const { tablename } of rows) { if (tablename.startsWith('kysely_')) continue; - await sql`truncate table ${sql.ref(tablename)} cascade`.execute(kysely); + await sql`truncate table ${sql.ref(tablename)} cascade`.execute(db.kysely); } - await kysely.destroy(); + await db.kysely.destroy(); }, }; } diff --git a/packages/ditto/workers/policy.worker.ts b/packages/ditto/workers/policy.worker.ts index acf7b2f1..852c24b5 100644 --- a/packages/ditto/workers/policy.worker.ts +++ b/packages/ditto/workers/policy.worker.ts @@ -30,10 +30,10 @@ export class CustomPolicy implements NPolicy { async init({ path, databaseUrl, pubkey }: PolicyInit): Promise { const Policy = (await import(path)).default; - const { kysely } = DittoDB.create(databaseUrl, { poolSize: 1 }); + const db = DittoDB.create(databaseUrl, { poolSize: 1 }); const store = new DittoPgStore({ - kysely, + db, pubkey, timeout: 5_000, }); From aabe6350a765d4988bdc04d661f49d391405f045 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 18 Feb 2025 15:08:00 -0600 Subject: [PATCH 09/99] Remove SearchStore --- packages/ditto/controllers/api/accounts.ts | 2 +- packages/ditto/controllers/api/search.ts | 4 +- packages/ditto/filter.test.ts | 46 ---------- packages/ditto/filter.ts | 97 ---------------------- packages/ditto/storages.ts | 15 ---- packages/ditto/storages/search-store.ts | 60 ------------- 6 files changed, 3 insertions(+), 221 deletions(-) delete mode 100644 packages/ditto/filter.test.ts delete mode 100644 packages/ditto/filter.ts delete mode 100644 packages/ditto/storages/search-store.ts diff --git a/packages/ditto/controllers/api/accounts.ts b/packages/ditto/controllers/api/accounts.ts index 252ddad6..8a1b9e3d 100644 --- a/packages/ditto/controllers/api/accounts.ts +++ b/packages/ditto/controllers/api/accounts.ts @@ -115,6 +115,7 @@ const accountSearchQuerySchema = z.object({ }); const accountSearchController: AppController = async (c) => { + const { store } = c.var; const { signal } = c.req.raw; const { limit } = c.get('pagination'); @@ -128,7 +129,6 @@ const accountSearchController: AppController = async (c) => { } const query = decodeURIComponent(result.data.q); - const store = await Storages.search(); const lookup = extractIdentifier(query); const event = await lookupAccount(lookup ?? query); diff --git a/packages/ditto/controllers/api/search.ts b/packages/ditto/controllers/api/search.ts index e5761f32..e890f166 100644 --- a/packages/ditto/controllers/api/search.ts +++ b/packages/ditto/controllers/api/search.ts @@ -94,7 +94,7 @@ async function searchEvents( return Promise.resolve([]); } - const store = await Storages.search(); + const store = await Storages.db(); const filter: NostrFilter = { kinds: typeToKinds(type), @@ -150,7 +150,7 @@ function typeToKinds(type: SearchQuery['type']): number[] { /** Resolve a searched value into an event, if applicable. */ async function lookupEvent(query: SearchQuery, signal: AbortSignal): Promise { const filters = await getLookupFilters(query, signal); - const store = await Storages.search(); + const store = await Storages.db(); return store.query(filters, { limit: 1, signal }) .then((events) => hydrateEvents({ events, store, signal })) diff --git a/packages/ditto/filter.test.ts b/packages/ditto/filter.test.ts deleted file mode 100644 index 9379208e..00000000 --- a/packages/ditto/filter.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { assertEquals } from '@std/assert'; - -import event0 from '~/fixtures/events/event-0.json' with { type: 'json' }; -import event1 from '~/fixtures/events/event-1.json' with { type: 'json' }; - -import { eventToMicroFilter, getFilterId, getFilterLimit, getMicroFilters, isMicrofilter } from './filter.ts'; - -Deno.test('getMicroFilters', () => { - const event = event0; - const microfilters = getMicroFilters(event); - assertEquals(microfilters.length, 2); - assertEquals(microfilters[0], { authors: [event.pubkey], kinds: [0] }); - assertEquals(microfilters[1], { ids: [event.id] }); -}); - -Deno.test('eventToMicroFilter', () => { - assertEquals(eventToMicroFilter(event0), { authors: [event0.pubkey], kinds: [0] }); - assertEquals(eventToMicroFilter(event1), { ids: [event1.id] }); -}); - -Deno.test('isMicrofilter', () => { - assertEquals(isMicrofilter({ ids: [event0.id] }), true); - assertEquals(isMicrofilter({ authors: [event0.pubkey], kinds: [0] }), true); - assertEquals(isMicrofilter({ ids: [event0.id], authors: [event0.pubkey], kinds: [0] }), false); -}); - -Deno.test('getFilterId', () => { - assertEquals( - getFilterId({ ids: [event0.id] }), - '{"ids":["63d38c9b483d2d98a46382eadefd272e0e4bdb106a5b6eddb400c4e76f693d35"]}', - ); - assertEquals( - getFilterId({ authors: [event0.pubkey], kinds: [0] }), - '{"authors":["79c2cae114ea28a981e7559b4fe7854a473521a8d22a66bbab9fa248eb820ff6"],"kinds":[0]}', - ); -}); - -Deno.test('getFilterLimit', () => { - assertEquals(getFilterLimit({ ids: [event0.id] }), 1); - assertEquals(getFilterLimit({ ids: [event0.id], limit: 2 }), 1); - assertEquals(getFilterLimit({ ids: [event0.id], limit: 0 }), 0); - assertEquals(getFilterLimit({ ids: [event0.id], limit: -1 }), 0); - assertEquals(getFilterLimit({ kinds: [0], authors: [event0.pubkey] }), 1); - assertEquals(getFilterLimit({ kinds: [1], authors: [event0.pubkey] }), Infinity); - assertEquals(getFilterLimit({}), Infinity); -}); diff --git a/packages/ditto/filter.ts b/packages/ditto/filter.ts deleted file mode 100644 index f9288c8a..00000000 --- a/packages/ditto/filter.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { NKinds, NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; -import stringifyStable from 'fast-stable-stringify'; -import { z } from 'zod'; - -/** Microfilter to get one specific event by ID. */ -type IdMicrofilter = { ids: [NostrEvent['id']] }; -/** Microfilter to get an author. */ -type AuthorMicrofilter = { kinds: [0]; authors: [NostrEvent['pubkey']] }; -/** Filter to get one specific event. */ -type MicroFilter = IdMicrofilter | AuthorMicrofilter; - -/** Get deterministic ID for a microfilter. */ -function getFilterId(filter: MicroFilter): string { - if ('ids' in filter) { - return stringifyStable({ ids: [filter.ids[0]] }); - } else { - return stringifyStable({ - kinds: [filter.kinds[0]], - authors: [filter.authors[0]], - }); - } -} - -/** Get a microfilter from a Nostr event. */ -function eventToMicroFilter(event: NostrEvent): MicroFilter { - const [microfilter] = getMicroFilters(event); - return microfilter; -} - -/** Get all the microfilters for an event, in order of priority. */ -function getMicroFilters(event: NostrEvent): MicroFilter[] { - const microfilters: MicroFilter[] = []; - if (event.kind === 0) { - microfilters.push({ kinds: [0], authors: [event.pubkey] }); - } - microfilters.push({ ids: [event.id] }); - return microfilters; -} - -/** Microfilter schema. */ -const microFilterSchema = z.union([ - z.object({ ids: z.tuple([n.id()]) }).strict(), - z.object({ kinds: z.tuple([z.literal(0)]), authors: z.tuple([n.id()]) }).strict(), -]); - -/** Checks whether the filter is a microfilter. */ -function isMicrofilter(filter: NostrFilter): filter is MicroFilter { - return microFilterSchema.safeParse(filter).success; -} - -/** Returns true if the filter could potentially return any stored events at all. */ -function canFilter(filter: NostrFilter): boolean { - return getFilterLimit(filter) > 0; -} - -/** Normalize the `limit` of each filter, and remove filters that can't produce any events. */ -function normalizeFilters(filters: F[]): F[] { - return filters.reduce((acc, filter) => { - const limit = getFilterLimit(filter); - if (limit > 0) { - acc.push(limit === Infinity ? filter : { ...filter, limit }); - } - return acc; - }, []); -} - -/** Calculate the intrinsic limit of a filter. This function may return `Infinity`. */ -function getFilterLimit(filter: NostrFilter): number { - if (filter.ids && !filter.ids.length) return 0; - if (filter.kinds && !filter.kinds.length) return 0; - if (filter.authors && !filter.authors.length) return 0; - - for (const [key, value] of Object.entries(filter)) { - if (key[0] === '#' && Array.isArray(value) && !value.length) return 0; - } - - return Math.min( - Math.max(0, filter.limit ?? Infinity), - filter.ids?.length ?? Infinity, - filter.authors?.length && filter.kinds?.every((kind) => NKinds.replaceable(kind)) - ? filter.authors.length * filter.kinds.length - : Infinity, - ); -} - -export { - type AuthorMicrofilter, - canFilter, - eventToMicroFilter, - getFilterId, - getFilterLimit, - getMicroFilters, - type IdMicrofilter, - isMicrofilter, - type MicroFilter, - normalizeFilters, -}; diff --git a/packages/ditto/storages.ts b/packages/ditto/storages.ts index be61beb6..1494dc8c 100644 --- a/packages/ditto/storages.ts +++ b/packages/ditto/storages.ts @@ -7,7 +7,6 @@ import { Conf } from '@/config.ts'; import { wsUrlSchema } from '@/schema.ts'; import { AdminStore } from '@/storages/AdminStore.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; -import { SearchStore } from '@/storages/search-store.ts'; import { InternalRelay } from '@/storages/InternalRelay.ts'; import { NPool, NRelay1 } from '@nostrify/nostrify'; import { getRelays } from '@/utils/outbox.ts'; @@ -19,7 +18,6 @@ export class Storages { private static _admin: Promise | undefined; private static _client: Promise> | undefined; private static _pubsub: Promise | undefined; - private static _search: Promise | undefined; public static async database(): Promise { if (!this._database) { @@ -124,17 +122,4 @@ export class Storages { } return this._client; } - - /** Storage to use for remote search. */ - public static async search(): Promise { - if (!this._search) { - this._search = Promise.resolve( - new SearchStore({ - relay: Conf.searchRelay, - fallback: await this.db(), - }), - ); - } - return this._search; - } } diff --git a/packages/ditto/storages/search-store.ts b/packages/ditto/storages/search-store.ts deleted file mode 100644 index 44dc1519..00000000 --- a/packages/ditto/storages/search-store.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { NostrEvent, NostrFilter, NRelay1, NStore } from '@nostrify/nostrify'; -import { logi } from '@soapbox/logi'; -import { JsonValue } from '@std/json'; - -import { normalizeFilters } from '@/filter.ts'; -import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { hydrateEvents } from '@/storages/hydrate.ts'; -import { abortError } from '@/utils/abort.ts'; - -interface SearchStoreOpts { - relay: string | undefined; - fallback: NStore; - hydrator?: NStore; -} - -class SearchStore implements NStore { - #fallback: NStore; - #hydrator: NStore; - #relay: NRelay1 | undefined; - - constructor(opts: SearchStoreOpts) { - this.#fallback = opts.fallback; - this.#hydrator = opts.hydrator ?? this; - - if (opts.relay) { - this.#relay = new NRelay1(opts.relay); - } - } - - event(_event: NostrEvent, _opts?: { signal?: AbortSignal }): Promise { - return Promise.reject(new Error('EVENT not implemented.')); - } - - async query(filters: NostrFilter[], opts?: { signal?: AbortSignal; limit?: number }): Promise { - filters = normalizeFilters(filters); - - if (opts?.signal?.aborted) return Promise.reject(abortError()); - if (!filters.length) return Promise.resolve([]); - - logi({ level: 'debug', ns: 'ditto.req', source: 'search', filters: filters as JsonValue }); - const query = filters[0]?.search; - - if (this.#relay && this.#relay.socket.readyState === WebSocket.OPEN) { - logi({ level: 'debug', ns: 'ditto.search', query, source: 'relay', relay: this.#relay.socket.url }); - - const events = await this.#relay.query(filters, opts); - - return hydrateEvents({ - events, - store: this.#hydrator, - signal: opts?.signal, - }); - } else { - logi({ level: 'debug', ns: 'ditto.search', query, source: 'db' }); - return this.#fallback.query(filters, opts); - } - } -} - -export { SearchStore }; From c29fc57a8cc5bfa76315b3c9c11d12843347e5fa Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 18 Feb 2025 16:35:45 -0600 Subject: [PATCH 10/99] Switch to genEvent from Nostrify --- packages/ditto/controllers/api/cashu.test.ts | 3 ++- packages/ditto/storages/EventsDB.test.ts | 3 ++- packages/ditto/storages/hydrate.bench.ts | 3 ++- packages/ditto/test.ts | 21 -------------------- packages/ditto/trends.test.ts | 3 ++- packages/ditto/utils/stats.test.ts | 3 ++- 6 files changed, 10 insertions(+), 26 deletions(-) diff --git a/packages/ditto/controllers/api/cashu.test.ts b/packages/ditto/controllers/api/cashu.test.ts index 773e9800..ee73661b 100644 --- a/packages/ditto/controllers/api/cashu.test.ts +++ b/packages/ditto/controllers/api/cashu.test.ts @@ -1,12 +1,13 @@ import { confMw } from '@ditto/api/middleware'; import { Env as HonoEnv, Hono } from '@hono/hono'; import { NostrSigner, NSecSigner, NStore } from '@nostrify/nostrify'; +import { genEvent } from '@nostrify/nostrify/test'; import { generateSecretKey, getPublicKey } from 'nostr-tools'; import { bytesToString, stringToBytes } from '@scure/base'; import { stub } from '@std/testing/mock'; import { assertEquals, assertExists, assertObjectMatch } from '@std/assert'; -import { createTestDB, genEvent } from '@/test.ts'; +import { createTestDB } from '@/test.ts'; import cashuApp from '@/controllers/api/cashu.ts'; import { walletSchema } from '@/schema.ts'; diff --git a/packages/ditto/storages/EventsDB.test.ts b/packages/ditto/storages/EventsDB.test.ts index d0947075..03f31d35 100644 --- a/packages/ditto/storages/EventsDB.test.ts +++ b/packages/ditto/storages/EventsDB.test.ts @@ -1,8 +1,9 @@ import { assertEquals, assertRejects } from '@std/assert'; +import { genEvent } from '@nostrify/nostrify/test'; import { generateSecretKey } from 'nostr-tools'; import { RelayError } from '@/RelayError.ts'; -import { eventFixture, genEvent } from '@/test.ts'; +import { eventFixture } from '@/test.ts'; import { Conf } from '@/config.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; import { createTestDB } from '@/test.ts'; diff --git a/packages/ditto/storages/hydrate.bench.ts b/packages/ditto/storages/hydrate.bench.ts index eeacec50..026b1f81 100644 --- a/packages/ditto/storages/hydrate.bench.ts +++ b/packages/ditto/storages/hydrate.bench.ts @@ -1,5 +1,6 @@ +import { jsonlEvents } from '@nostrify/nostrify/test'; + import { assembleEvents } from '@/storages/hydrate.ts'; -import { jsonlEvents } from '@/test.ts'; const testEvents = await jsonlEvents('fixtures/hydrated.jsonl'); const testStats = JSON.parse(await Deno.readTextFile('fixtures/stats.json')); diff --git a/packages/ditto/test.ts b/packages/ditto/test.ts index 47052b8d..dcf428a6 100644 --- a/packages/ditto/test.ts +++ b/packages/ditto/test.ts @@ -1,10 +1,8 @@ import { DittoDB } from '@ditto/db'; import { NostrEvent } from '@nostrify/nostrify'; -import { finalizeEvent, generateSecretKey } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; -import { purifyEvent } from '@/utils/purify.ts'; import { sql } from 'kysely'; /** Import an event fixture by name in tests. */ @@ -13,25 +11,6 @@ export async function eventFixture(name: string): Promise { return structuredClone(result.default); } -/** Import a JSONL fixture by name in tests. */ -export async function jsonlEvents(path: string): Promise { - const data = await Deno.readTextFile(path); - return data.split('\n').map((line) => JSON.parse(line)); -} - -/** Generate an event for use in tests. */ -export function genEvent(t: Partial = {}, sk: Uint8Array = generateSecretKey()): NostrEvent { - const event = finalizeEvent({ - kind: 255, - created_at: 0, - content: '', - tags: [], - ...t, - }, sk); - - return purifyEvent(event); -} - /** Create a database for testing. It uses `DATABASE_URL`, or creates an in-memory database by default. */ export async function createTestDB(opts?: { pure?: boolean }) { const { kysely } = DittoDB.create(Conf.databaseUrl, { poolSize: 1 }); diff --git a/packages/ditto/trends.test.ts b/packages/ditto/trends.test.ts index 79eaf8e0..a99b4eb4 100644 --- a/packages/ditto/trends.test.ts +++ b/packages/ditto/trends.test.ts @@ -1,8 +1,9 @@ import { assertEquals } from '@std/assert'; +import { genEvent } from '@nostrify/nostrify/test'; import { generateSecretKey, NostrEvent } from 'nostr-tools'; import { getTrendingTagValues } from '@/trends.ts'; -import { createTestDB, genEvent } from '@/test.ts'; +import { createTestDB } from '@/test.ts'; Deno.test("getTrendingTagValues(): 'e' tag and WITHOUT language parameter", async () => { await using db = await createTestDB(); diff --git a/packages/ditto/utils/stats.test.ts b/packages/ditto/utils/stats.test.ts index 797f78da..762db37c 100644 --- a/packages/ditto/utils/stats.test.ts +++ b/packages/ditto/utils/stats.test.ts @@ -1,7 +1,8 @@ +import { genEvent } from '@nostrify/nostrify/test'; import { assertEquals } from '@std/assert'; import { generateSecretKey, getPublicKey } from 'nostr-tools'; -import { createTestDB, genEvent } from '@/test.ts'; +import { createTestDB } from '@/test.ts'; import { countAuthorStats, getAuthorStats, getEventStats, getFollowDiff, updateStats } from '@/utils/stats.ts'; Deno.test('updateStats with kind 1 increments notes count', async () => { From 7deec54a2ed4284634250d820bb0cc19c2506109 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 18 Feb 2025 20:03:03 -0600 Subject: [PATCH 11/99] Upgrade Deno to v2.2.0 --- .gitlab-ci.yml | 2 +- .tool-versions | 2 +- Dockerfile | 2 +- packages/conf/DittoConf.ts | 12 +----------- 4 files changed, 4 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 766a144d..b754ff1e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:2.1.10 +image: denoland/deno:2.2.0 default: interruptible: true diff --git a/.tool-versions b/.tool-versions index a3cfae3c..f9adf79b 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -deno 2.1.10 \ No newline at end of file +deno 2.2.0 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 78ae7fad..0b8724a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM denoland/deno:2.1.10 +FROM denoland/deno:2.2.0 ENV PORT 5000 WORKDIR /app diff --git a/packages/conf/DittoConf.ts b/packages/conf/DittoConf.ts index 6d4b45d7..456e9cd2 100644 --- a/packages/conf/DittoConf.ts +++ b/packages/conf/DittoConf.ts @@ -1,4 +1,3 @@ -import Module from 'node:module'; import os from 'node:os'; import path from 'node:path'; @@ -354,7 +353,7 @@ export class DittoConf { /** Absolute path to the data directory used by Ditto. */ get dataDir(): string { - return this.env.get('DITTO_DATA_DIR') || path.join(cwd(), 'data'); + return this.env.get('DITTO_DATA_DIR') || path.join(Deno.cwd(), 'data'); } /** Absolute path of the Deno directory. */ @@ -465,12 +464,3 @@ export class DittoConf { return Number(this.env.get('STREAK_WINDOW') || 129600); } } - -/** - * HACK: get cwd without read permissions. - * https://github.com/denoland/deno/issues/27080#issuecomment-2504150155 - */ -function cwd() { - // @ts-ignore Internal method, but it does exist. - return Module._nodeModulePaths('a')[0].slice(0, -15); -} From 6568dca19109148d09d2444f78f8bdef6f1db0bc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 19 Feb 2025 15:55:09 -0600 Subject: [PATCH 12/99] DittoPgStore: support timeout in req, add special treatment for ephemeral events, yield event loop when processing many subscriptions --- deno.json | 2 +- deno.lock | 8 +-- packages/ditto/storages/DittoPgStore.ts | 83 +++++++++++++++++++------ 3 files changed, 70 insertions(+), 23 deletions(-) diff --git a/deno.json b/deno.json index 2601e28d..50f814fe 100644 --- a/deno.json +++ b/deno.json @@ -61,7 +61,7 @@ "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@negrel/webpush": "jsr:@negrel/webpush@^0.3.0", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/db": "jsr:@nostrify/db@^0.39.0", + "@nostrify/db": "jsr:@nostrify/db@^0.39.2", "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.38.1", "@nostrify/policies": "jsr:@nostrify/policies@^0.36.1", "@nostrify/types": "jsr:@nostrify/types@^0.36.0", diff --git a/deno.lock b/deno.lock index b46ce6da..b7475e3a 100644 --- a/deno.lock +++ b/deno.lock @@ -31,7 +31,7 @@ "jsr:@hono/hono@^4.4.6": "4.6.15", "jsr:@negrel/http-ece@0.6.0": "0.6.0", "jsr:@negrel/webpush@0.3": "0.3.0", - "jsr:@nostrify/db@0.39": "0.39.0", + "jsr:@nostrify/db@~0.39.2": "0.39.2", "jsr:@nostrify/nostrify@0.31": "0.31.0", "jsr:@nostrify/nostrify@0.32": "0.32.0", "jsr:@nostrify/nostrify@0.36": "0.36.2", @@ -363,8 +363,8 @@ "jsr:@std/path@0.224.0" ] }, - "@nostrify/db@0.39.0": { - "integrity": "13a88c610eb15a5dd13848d5beec9170406376c9d05299ce5e5298452a5431ac", + "@nostrify/db@0.39.2": { + "integrity": "65df8e636d172a62319060f77398f992541a674bcc0298d19608fdba639e0b13", "dependencies": [ "jsr:@nostrify/nostrify@~0.38.1", "jsr:@nostrify/types@0.36", @@ -2460,7 +2460,7 @@ "jsr:@gfx/canvas-wasm@~0.4.2", "jsr:@hono/hono@^4.4.6", "jsr:@negrel/webpush@0.3", - "jsr:@nostrify/db@0.39", + "jsr:@nostrify/db@~0.39.2", "jsr:@nostrify/nostrify@~0.38.1", "jsr:@nostrify/policies@~0.36.1", "jsr:@nostrify/types@0.36", diff --git a/packages/ditto/storages/DittoPgStore.ts b/packages/ditto/storages/DittoPgStore.ts index 4c966c8a..22671185 100644 --- a/packages/ditto/storages/DittoPgStore.ts +++ b/packages/ditto/storages/DittoPgStore.ts @@ -57,14 +57,18 @@ interface DittoPgStoreOpts { timeout: number; /** Whether the event returned should be a Nostr event or a Ditto event. Defaults to false. */ pure?: boolean; - /** Chunk size for streaming events. Defaults to 100. */ + /** Chunk size for streaming events. Defaults to 20. */ chunkSize?: number; + /** Batch size for fulfilling subscriptions. Defaults to 500. */ + batchSize?: number; + /** Max age (in **seconds**) an event can be to be fulfilled to realtime subscribers. */ + maxAge?: number; } /** SQL database storage adapter for Nostr events. */ export class DittoPgStore extends NPostgres { readonly subs = new Map }>(); - readonly encounters = new LRUCache({ max: 100 }); + readonly encounters = new LRUCache({ max: 1000 }); /** Conditions for when to index certain tags. */ static tagConditions: Record = { @@ -103,7 +107,7 @@ export class DittoPgStore extends NPostgres { const [event] = await this.query([{ ids: [id] }]); if (event) { - this.streamOut(event); + await this.fulfill(event); } }); } @@ -117,6 +121,10 @@ export class DittoPgStore extends NPostgres { this.encounters.set(event.id, true); dbEventsCounter.inc({ kind: event.kind }); + if (NKinds.ephemeral(event.kind)) { + return await this.fulfill(event); + } + if (await this.isDeletedAdmin(event)) { throw new RelayError('blocked', 'event deleted by admin'); } @@ -125,7 +133,7 @@ export class DittoPgStore extends NPostgres { try { await super.event(event, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); - this.streamOut(event); + this.fulfill(event); // don't await or catch (should never reject) } catch (e) { if (e instanceof Error && e.message === 'Cannot add a deleted event') { throw new RelayError('blocked', 'event deleted by user'); @@ -137,21 +145,48 @@ export class DittoPgStore extends NPostgres { } } - protected matchesFilter(event: NostrEvent, filter: NostrFilter): boolean { - // TODO: support streaming by search. - return matchFilter(filter, event) && filter.search === undefined; - } + /** Fulfill active subscriptions with this event. */ + protected async fulfill(event: NostrEvent): Promise { + const { maxAge = 60, batchSize = 500 } = this.opts; + + const now = Math.floor(Date.now() / 1000); + const age = now - event.created_at; + + if (age > maxAge) { + // Ephemeral events must be fulfilled, or else return an error to the client. + if (NKinds.ephemeral(event.kind)) { + throw new RelayError('invalid', 'event too old'); + } else { + // Silently ignore old events. + return; + } + } + + let count = 0; - protected streamOut(event: NostrEvent): void { for (const { filters, machina } of this.subs.values()) { for (const filter of filters) { + count++; + if (this.matchesFilter(event, filter)) { machina.push(event); + break; + } + + // Yield to event loop. + if (count % batchSize === 0) { + await new Promise((resolve) => setTimeout(resolve, 0)); } } } } + /** Check if the event fulfills the filter, according to Ditto criteria. */ + protected matchesFilter(event: NostrEvent, filter: NostrFilter): boolean { + // TODO: support streaming by search. + return typeof filter.search !== 'string' && matchFilter(filter, event); + } + /** Check if an event has been deleted by the admin. */ private async isDeletedAdmin(event: NostrEvent): Promise { const filters: NostrFilter[] = [ @@ -215,23 +250,26 @@ export class DittoPgStore extends NPostgres { override async *req( filters: NostrFilter[], - opts?: { signal?: AbortSignal }, + opts: { timeout?: number; signal?: AbortSignal } = {}, ): AsyncIterable { const subId = crypto.randomUUID(); const normalFilters = this.normalizeFilters(filters); if (normalFilters.length) { - const { db, chunkSize = 100 } = this.opts; - const rows = this.getEventsQuery(db.kysely as unknown as Kysely, normalFilters).stream( - chunkSize, + const { db, timeout, chunkSize = 20 } = this.opts; + + const rows = await this.withTimeout( + db.kysely as unknown as Kysely, + (trx) => this.getEventsQuery(trx, normalFilters).stream(chunkSize), + opts.timeout ?? timeout, ); for await (const row of rows) { const event = this.parseEventRow(row); yield ['EVENT', subId, event]; - if (opts?.signal?.aborted) { - yield ['CLOSED', subId, 'aborted']; + if (opts.signal?.aborted) { + yield ['CLOSED', subId, 'error: the relay could not respond fast enough']; return; } } @@ -239,7 +277,12 @@ export class DittoPgStore extends NPostgres { yield ['EOSE', subId]; - const machina = new Machina(opts?.signal); + if (opts.signal?.aborted) { + yield ['CLOSED', subId, 'error: the relay could not respond fast enough']; + return; + } + + const machina = new Machina(opts.signal); this.subs.set(subId, { filters, machina }); internalSubscriptionsSizeGauge.set(this.subs.size); @@ -248,8 +291,12 @@ export class DittoPgStore extends NPostgres { for await (const event of machina) { yield ['EVENT', subId, event]; } - } catch { - yield ['CLOSED', subId, 'error: something went wrong']; + } catch (e) { + if (e instanceof Error && e.message.includes('timeout')) { + yield ['CLOSED', subId, 'error: the relay could not respond fast enough']; + } else { + yield ['CLOSED', subId, 'error: something went wrong']; + } } finally { this.subs.delete(subId); internalSubscriptionsSizeGauge.set(this.subs.size); From d9a466c0ee14ff30fac498a3260f2b8f9213a68e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 19 Feb 2025 16:13:22 -0600 Subject: [PATCH 13/99] Remove InternalRelay (pubsub) store --- packages/ditto/controllers/api/oauth.ts | 2 +- packages/ditto/controllers/api/streaming.ts | 4 +- packages/ditto/controllers/nostr/relay.ts | 20 +---- packages/ditto/pipeline.ts | 69 ++++----------- packages/ditto/signers/ConnectSigner.ts | 2 +- packages/ditto/storages.ts | 13 +-- packages/ditto/storages/InternalRelay.test.ts | 23 ----- packages/ditto/storages/InternalRelay.ts | 86 ------------------- 8 files changed, 23 insertions(+), 196 deletions(-) delete mode 100644 packages/ditto/storages/InternalRelay.test.ts delete mode 100644 packages/ditto/storages/InternalRelay.ts diff --git a/packages/ditto/controllers/api/oauth.ts b/packages/ditto/controllers/api/oauth.ts index 7ac2c2b2..c48963a9 100644 --- a/packages/ditto/controllers/api/oauth.ts +++ b/packages/ditto/controllers/api/oauth.ts @@ -123,7 +123,7 @@ async function getToken( encryption: 'nip44', pubkey: bunkerPubkey, signer: new NSecSigner(nip46Seckey), - relay: await Storages.pubsub(), // TODO: Use the relays from the request. + relay: await Storages.db(), // TODO: Use the relays from the request. timeout: 60_000, }); diff --git a/packages/ditto/controllers/api/streaming.ts b/packages/ditto/controllers/api/streaming.ts index 01eaaed8..25fe877d 100644 --- a/packages/ditto/controllers/api/streaming.ts +++ b/packages/ditto/controllers/api/streaming.ts @@ -94,8 +94,6 @@ const streamingController: AppController = async (c) => { const { socket, response } = Deno.upgradeWebSocket(c.req.raw, { protocol: token, idleTimeout: 30 }); const store = await Storages.db(); - const pubsub = await Storages.pubsub(); - const policy = pubkey ? new MuteListPolicy(pubkey, await Storages.admin()) : undefined; function send(e: StreamingEvent) { @@ -107,7 +105,7 @@ const streamingController: AppController = async (c) => { async function sub(filters: NostrFilter[], render: (event: NostrEvent) => Promise) { try { - for await (const msg of pubsub.req(filters, { signal: controller.signal })) { + for await (const msg of store.req(filters, { signal: controller.signal })) { if (msg[0] === 'EVENT') { const event = msg[2]; diff --git a/packages/ditto/controllers/nostr/relay.ts b/packages/ditto/controllers/nostr/relay.ts index b4924f22..c549c594 100644 --- a/packages/ditto/controllers/nostr/relay.ts +++ b/packages/ditto/controllers/nostr/relay.ts @@ -23,9 +23,6 @@ import { errorJson } from '@/utils/log.ts'; import { purifyEvent } from '@/utils/purify.ts'; import { Time } from '@/utils/time.ts'; -/** Limit of initial events returned for a subscription. */ -const FILTER_LIMIT = 100; - const limiters = { msg: new MemoryRateLimiter({ limit: 300, window: Time.minutes(1) }), req: new MultiRateLimiter([ @@ -126,11 +123,10 @@ function connectStream(socket: WebSocket, ip: string | undefined, conf: DittoCon controllers.set(subId, controller); const store = await Storages.db(); - const pubsub = await Storages.pubsub(); try { - for (const event of await store.query(filters, { limit: FILTER_LIMIT, timeout: conf.db.timeouts.relay })) { - send(['EVENT', subId, purifyEvent(event)]); + for await (const [verb, , ...rest] of store.req(filters, { timeout: conf.db.timeouts.relay })) { + send([verb, subId, ...rest] as NostrRelayMsg); } } catch (e) { if (e instanceof RelayError) { @@ -143,18 +139,6 @@ function connectStream(socket: WebSocket, ip: string | undefined, conf: DittoCon controllers.delete(subId); return; } - - send(['EOSE', subId]); - - try { - for await (const msg of pubsub.req(filters, { signal: controller.signal })) { - if (msg[0] === 'EVENT') { - send(['EVENT', subId, msg[2]]); - } - } - } catch { - controllers.delete(subId); - } } /** Handle EVENT. Store the event. */ diff --git a/packages/ditto/pipeline.ts b/packages/ditto/pipeline.ts index d3168c0e..1c49e930 100644 --- a/packages/ditto/pipeline.ts +++ b/packages/ditto/pipeline.ts @@ -77,42 +77,21 @@ async function handleEvent(event: DittoEvent, opts: PipelineOpts): Promise // NIP-46 events get special treatment. // They are exempt from policies and other side-effects, and should be streamed out immediately. // If streaming fails, an error should be returned. - if (event.kind === 24133) { - await streamOut(event); - return; - } + if (event.kind !== 24133) { + // Ensure the event doesn't violate the policy. + if (event.pubkey !== Conf.pubkey) { + await policyFilter(event, opts.signal); + } - // Ensure the event doesn't violate the policy. - if (event.pubkey !== Conf.pubkey) { - await policyFilter(event, opts.signal); - } + // Prepare the event for additional checks. + // FIXME: This is kind of hacky. Should be reorganized to fetch only what's needed for each stage. + await hydrateEvent(event, opts.signal); - // Prepare the event for additional checks. - // FIXME: This is kind of hacky. Should be reorganized to fetch only what's needed for each stage. - await hydrateEvent(event, opts.signal); - - // Ensure that the author is not banned. - const n = getTagSet(event.user?.tags ?? [], 'n'); - if (n.has('disabled')) { - throw new RelayError('blocked', 'author is blocked'); - } - - // Ephemeral events must throw if they are not streamed out. - if (NKinds.ephemeral(event.kind)) { - await Promise.all([ - streamOut(event), - webPush(event), - ]); - return; - } - - // Events received through notify are thought to already be in the database, so they only need to be streamed. - if (opts.source === 'notify') { - await Promise.all([ - streamOut(event), - webPush(event), - ]); - return; + // Ensure that the author is not banned. + const n = getTagSet(event.user?.tags ?? [], 'n'); + if (n.has('disabled')) { + throw new RelayError('blocked', 'author is blocked'); + } } const kysely = await Storages.kysely(); @@ -127,12 +106,7 @@ async function handleEvent(event: DittoEvent, opts: PipelineOpts): Promise prewarmLinkPreview(event, opts.signal), generateSetEvents(event), ]) - .then(() => - Promise.allSettled([ - streamOut(event), - webPush(event), - ]) - ); + .then(() => webPush(event)); } } @@ -165,12 +139,13 @@ async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { - if (NKinds.ephemeral(event.kind)) return; const store = await Storages.db(); try { await store.transaction(async (store, kysely) => { - await updateStats({ event, store, kysely }); + if (!NKinds.ephemeral(event.kind)) { + await updateStats({ event, store, kysely }); + } await store.event(event, { signal }); }); } catch (e) { @@ -274,16 +249,6 @@ function isFresh(event: NostrEvent): boolean { return eventAge(event) < Time.minutes(1); } -/** Distribute the event through active subscriptions. */ -async function streamOut(event: NostrEvent): Promise { - if (!isFresh(event)) { - throw new RelayError('invalid', 'event too old'); - } - - const pubsub = await Storages.pubsub(); - await pubsub.event(event); -} - async function webPush(event: NostrEvent): Promise { if (!isFresh(event)) { throw new RelayError('invalid', 'event too old'); diff --git a/packages/ditto/signers/ConnectSigner.ts b/packages/ditto/signers/ConnectSigner.ts index 89c62679..c6d23d37 100644 --- a/packages/ditto/signers/ConnectSigner.ts +++ b/packages/ditto/signers/ConnectSigner.ts @@ -28,7 +28,7 @@ export class ConnectSigner implements NostrSigner { encryption: 'nip44', pubkey: this.opts.bunkerPubkey, // TODO: use a remote relay for `nprofile` signing (if present and `Conf.relay` isn't already in the list) - relay: await Storages.pubsub(), + relay: await Storages.db(), signer, timeout: 60_000, }); diff --git a/packages/ditto/storages.ts b/packages/ditto/storages.ts index f7bde886..1fe46e83 100644 --- a/packages/ditto/storages.ts +++ b/packages/ditto/storages.ts @@ -1,14 +1,12 @@ // deno-lint-ignore-file require-await import { type DittoDatabase, DittoDB } from '@ditto/db'; -import { internalSubscriptionsSizeGauge } from '@ditto/metrics'; +import { NPool, NRelay1 } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import { Conf } from '@/config.ts'; import { wsUrlSchema } from '@/schema.ts'; import { AdminStore } from '@/storages/AdminStore.ts'; import { DittoPgStore } from '@/storages/DittoPgStore.ts'; -import { InternalRelay } from '@/storages/InternalRelay.ts'; -import { NPool, NRelay1 } from '@nostrify/nostrify'; import { getRelays } from '@/utils/outbox.ts'; import { seedZapSplits } from '@/utils/zap-split.ts'; @@ -17,7 +15,6 @@ export class Storages { private static _database: Promise | undefined; private static _admin: Promise | undefined; private static _client: Promise> | undefined; - private static _pubsub: Promise | undefined; public static async database(): Promise { if (!this._database) { @@ -59,14 +56,6 @@ export class Storages { return this._admin; } - /** Internal pubsub relay between controllers and the pipeline. */ - public static async pubsub(): Promise { - if (!this._pubsub) { - this._pubsub = Promise.resolve(new InternalRelay({ gauge: internalSubscriptionsSizeGauge })); - } - return this._pubsub; - } - /** Relay pool storage. */ public static async client(): Promise> { if (!this._client) { diff --git a/packages/ditto/storages/InternalRelay.test.ts b/packages/ditto/storages/InternalRelay.test.ts deleted file mode 100644 index c97dcd39..00000000 --- a/packages/ditto/storages/InternalRelay.test.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { assertEquals } from '@std/assert'; - -import { eventFixture } from '@/test.ts'; - -import { InternalRelay } from './InternalRelay.ts'; - -Deno.test('InternalRelay', async () => { - const relay = new InternalRelay(); - const event1 = await eventFixture('event-1'); - - const promise = new Promise((resolve) => setTimeout(() => resolve(relay.event(event1)), 0)); - - for await (const msg of relay.req([{}])) { - if (msg[0] === 'EVENT') { - assertEquals(relay.subs.size, 1); - assertEquals(msg[2], event1); - break; - } - } - - await promise; - assertEquals(relay.subs.size, 0); // cleanup -}); diff --git a/packages/ditto/storages/InternalRelay.ts b/packages/ditto/storages/InternalRelay.ts deleted file mode 100644 index 9ab942fb..00000000 --- a/packages/ditto/storages/InternalRelay.ts +++ /dev/null @@ -1,86 +0,0 @@ -// deno-lint-ignore-file require-await -import { - NIP50, - NostrEvent, - NostrFilter, - NostrRelayCLOSED, - NostrRelayEOSE, - NostrRelayEVENT, - NRelay, -} from '@nostrify/nostrify'; -import { Machina } from '@nostrify/nostrify/utils'; -import { matchFilter } from 'nostr-tools'; -import { Gauge } from 'prom-client'; - -import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { purifyEvent } from '@/utils/purify.ts'; - -interface InternalRelayOpts { - gauge?: Gauge; -} - -/** - * PubSub event store for streaming events within the application. - * The pipeline should push events to it, then anything in the application can subscribe to it. - */ -export class InternalRelay implements NRelay { - readonly subs = new Map }>(); - - constructor(private opts: InternalRelayOpts = {}) {} - - async *req( - filters: NostrFilter[], - opts?: { signal?: AbortSignal }, - ): AsyncGenerator { - const id = crypto.randomUUID(); - const machina = new Machina(opts?.signal); - - yield ['EOSE', id]; - - this.subs.set(id, { filters, machina }); - this.opts.gauge?.set(this.subs.size); - - try { - for await (const event of machina) { - yield ['EVENT', id, event]; - } - } finally { - this.subs.delete(id); - this.opts.gauge?.set(this.subs.size); - } - } - - async event(event: DittoEvent): Promise { - for (const { filters, machina } of this.subs.values()) { - for (const filter of filters) { - if (matchFilter(filter, event)) { - if (filter.search) { - const tokens = NIP50.parseInput(filter.search); - - const domain = (tokens.find((t) => - typeof t === 'object' && t.key === 'domain' - ) as { key: 'domain'; value: string } | undefined)?.value; - - if (domain === event.author_stats?.nip05_hostname) { - machina.push(purifyEvent(event)); - break; - } - } else { - machina.push(purifyEvent(event)); - break; - } - } - } - } - - return Promise.resolve(); - } - - async query(): Promise { - return []; - } - - async close(): Promise { - return Promise.resolve(); - } -} From bc0830785a24e743d9c3cd1c3bff082c1e813c85 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 19 Feb 2025 16:19:16 -0600 Subject: [PATCH 14/99] Remove old startNotify code --- packages/ditto/notify.ts | 38 ------------------------- packages/ditto/startup.ts | 5 ---- packages/ditto/storages.ts | 7 ++++- packages/ditto/storages/DittoPgStore.ts | 26 ++++++++++------- 4 files changed, 22 insertions(+), 54 deletions(-) delete mode 100644 packages/ditto/notify.ts diff --git a/packages/ditto/notify.ts b/packages/ditto/notify.ts deleted file mode 100644 index 44ed5619..00000000 --- a/packages/ditto/notify.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { Semaphore } from '@core/asyncutil'; - -import { pipelineEncounters } from '@/caches/pipelineEncounters.ts'; -import { Conf } from '@/config.ts'; -import * as pipeline from '@/pipeline.ts'; -import { Storages } from '@/storages.ts'; -import { logi } from '@soapbox/logi'; - -const sem = new Semaphore(1); - -export async function startNotify(): Promise { - const { listen } = await Storages.database(); - const store = await Storages.db(); - - listen('nostr_event', (id) => { - if (pipelineEncounters.has(id)) { - logi({ level: 'debug', ns: 'ditto.notify', id, skipped: true }); - return; - } - - logi({ level: 'debug', ns: 'ditto.notify', id, skipped: false }); - - sem.lock(async () => { - try { - const signal = AbortSignal.timeout(Conf.db.timeouts.default); - - const [event] = await store.query([{ ids: [id], limit: 1 }], { signal }); - - if (event) { - logi({ level: 'debug', ns: 'ditto.event', source: 'notify', id: event.id, kind: event.kind }); - await pipeline.handleEvent(event, { source: 'notify', signal }); - } - } catch { - // Ignore - } - }); - }); -} diff --git a/packages/ditto/startup.ts b/packages/ditto/startup.ts index 0cc2f26a..0372a1d1 100644 --- a/packages/ditto/startup.ts +++ b/packages/ditto/startup.ts @@ -2,16 +2,11 @@ import { Conf } from '@/config.ts'; import { cron } from '@/cron.ts'; import { startFirehose } from '@/firehose.ts'; -import { startNotify } from '@/notify.ts'; if (Conf.firehoseEnabled) { startFirehose(); } -if (Conf.notifyEnabled) { - startNotify(); -} - if (Conf.cronEnabled) { cron(); } diff --git a/packages/ditto/storages.ts b/packages/ditto/storages.ts index 1fe46e83..ff7b2954 100644 --- a/packages/ditto/storages.ts +++ b/packages/ditto/storages.ts @@ -40,7 +40,12 @@ export class Storages { if (!this._db) { this._db = (async () => { const db = await this.database(); - const store = new DittoPgStore({ db, pubkey: Conf.pubkey, timeout: Conf.db.timeouts.default }); + const store = new DittoPgStore({ + db, + pubkey: Conf.pubkey, + timeout: Conf.db.timeouts.default, + notify: Conf.notifyEnabled, + }); await seedZapSplits(store); return store; })(); diff --git a/packages/ditto/storages/DittoPgStore.ts b/packages/ditto/storages/DittoPgStore.ts index 22671185..7bd22d00 100644 --- a/packages/ditto/storages/DittoPgStore.ts +++ b/packages/ditto/storages/DittoPgStore.ts @@ -63,6 +63,8 @@ interface DittoPgStoreOpts { batchSize?: number; /** Max age (in **seconds**) an event can be to be fulfilled to realtime subscribers. */ maxAge?: number; + /** Whether to listen for events from the database with NOTIFY. */ + notify?: boolean; } /** SQL database storage adapter for Nostr events. */ @@ -100,25 +102,29 @@ export class DittoPgStore extends NPostgres { chunkSize: opts.chunkSize, }); - opts.db.listen('nostr_event', async (id) => { - if (this.encounters.has(id)) return; - this.encounters.set(id, true); + if (opts.notify) { + opts.db.listen('nostr_event', async (id) => { + if (this.encounters.has(id)) return; + this.encounters.set(id, true); - const [event] = await this.query([{ ids: [id] }]); + const [event] = await this.query([{ ids: [id] }]); - if (event) { - await this.fulfill(event); - } - }); + if (event) { + await this.fulfill(event); + } + }); + } } /** Insert an event (and its tags) into the database. */ override async event(event: NostrEvent, opts: { signal?: AbortSignal; timeout?: number } = {}): Promise { event = purifyEvent(event); - logi({ level: 'debug', ns: 'ditto.event', source: 'db', id: event.id, kind: event.kind }); + if (this.opts.notify) { + this.encounters.set(event.id, true); + } - this.encounters.set(event.id, true); + logi({ level: 'debug', ns: 'ditto.event', source: 'db', id: event.id, kind: event.kind }); dbEventsCounter.inc({ kind: event.kind }); if (NKinds.ephemeral(event.kind)) { From f87f19d06cfca7a2b62ab27d8b92fa92036ea4c5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 19 Feb 2025 20:17:53 -0600 Subject: [PATCH 15/99] DittoPgStore: rework realtime streaming so it actually works --- deno.json | 2 +- deno.lock | 8 +- packages/ditto/storages/DittoPgStore.test.ts | 26 ++++--- packages/ditto/storages/DittoPgStore.ts | 77 ++++++++++++-------- packages/ditto/test.ts | 1 + 5 files changed, 71 insertions(+), 43 deletions(-) diff --git a/deno.json b/deno.json index 50f814fe..a3f06bd5 100644 --- a/deno.json +++ b/deno.json @@ -61,7 +61,7 @@ "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@negrel/webpush": "jsr:@negrel/webpush@^0.3.0", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/db": "jsr:@nostrify/db@^0.39.2", + "@nostrify/db": "jsr:@nostrify/db@^0.39.3", "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.38.1", "@nostrify/policies": "jsr:@nostrify/policies@^0.36.1", "@nostrify/types": "jsr:@nostrify/types@^0.36.0", diff --git a/deno.lock b/deno.lock index b7475e3a..19c7aba4 100644 --- a/deno.lock +++ b/deno.lock @@ -31,7 +31,7 @@ "jsr:@hono/hono@^4.4.6": "4.6.15", "jsr:@negrel/http-ece@0.6.0": "0.6.0", "jsr:@negrel/webpush@0.3": "0.3.0", - "jsr:@nostrify/db@~0.39.2": "0.39.2", + "jsr:@nostrify/db@~0.39.3": "0.39.3", "jsr:@nostrify/nostrify@0.31": "0.31.0", "jsr:@nostrify/nostrify@0.32": "0.32.0", "jsr:@nostrify/nostrify@0.36": "0.36.2", @@ -363,8 +363,8 @@ "jsr:@std/path@0.224.0" ] }, - "@nostrify/db@0.39.2": { - "integrity": "65df8e636d172a62319060f77398f992541a674bcc0298d19608fdba639e0b13", + "@nostrify/db@0.39.3": { + "integrity": "d1f1104316b33e0fd3c263086b325ee49f86859abc1a966b43bb9f9a21c15429", "dependencies": [ "jsr:@nostrify/nostrify@~0.38.1", "jsr:@nostrify/types@0.36", @@ -2460,7 +2460,7 @@ "jsr:@gfx/canvas-wasm@~0.4.2", "jsr:@hono/hono@^4.4.6", "jsr:@negrel/webpush@0.3", - "jsr:@nostrify/db@~0.39.2", + "jsr:@nostrify/db@~0.39.3", "jsr:@nostrify/nostrify@~0.38.1", "jsr:@nostrify/policies@~0.36.1", "jsr:@nostrify/types@0.36", diff --git a/packages/ditto/storages/DittoPgStore.test.ts b/packages/ditto/storages/DittoPgStore.test.ts index e119a85f..756cd98b 100644 --- a/packages/ditto/storages/DittoPgStore.test.ts +++ b/packages/ditto/storages/DittoPgStore.test.ts @@ -1,4 +1,5 @@ import { assertEquals, assertRejects } from '@std/assert'; +import { NostrRelayMsg } from '@nostrify/nostrify'; import { genEvent } from '@nostrify/nostrify/test'; import { generateSecretKey } from 'nostr-tools'; @@ -12,19 +13,26 @@ Deno.test('req streaming', async () => { await using db = await createTestDB({ pure: true }); const { store: relay } = db; - const event1 = await eventFixture('event-1'); + const msgs: NostrRelayMsg[] = []; + const controller = new AbortController(); - const promise = new Promise((resolve) => setTimeout(() => resolve(relay.event(event1)), 0)); - - for await (const msg of relay.req([{ since: 0 }])) { - if (msg[0] === 'EVENT') { - assertEquals(relay.subs.size, 1); - assertEquals(msg[2], event1); - break; + const promise = (async () => { + for await (const msg of relay.req([{ since: 0 }], { signal: controller.signal })) { + msgs.push(msg); } - } + })(); + + const event = genEvent({ created_at: Math.floor(Date.now() / 1000) }); + await relay.event(event); + + controller.abort(); await promise; + + const verbs = msgs.map(([verb]) => verb); + + assertEquals(verbs, ['EOSE', 'EVENT', 'CLOSED']); + assertEquals(msgs[1][2], event); assertEquals(relay.subs.size, 0); // cleanup }); diff --git a/packages/ditto/storages/DittoPgStore.ts b/packages/ditto/storages/DittoPgStore.ts index 7bd22d00..000ef536 100644 --- a/packages/ditto/storages/DittoPgStore.ts +++ b/packages/ditto/storages/DittoPgStore.ts @@ -67,9 +67,15 @@ interface DittoPgStoreOpts { notify?: boolean; } +/** Realtime subscription. */ +interface Subscription { + filters: NostrFilter[]; + machina: Machina; +} + /** SQL database storage adapter for Nostr events. */ export class DittoPgStore extends NPostgres { - readonly subs = new Map }>(); + readonly subs = new Map(); readonly encounters = new LRUCache({ max: 1000 }); /** Conditions for when to index certain tags. */ @@ -170,12 +176,12 @@ export class DittoPgStore extends NPostgres { let count = 0; - for (const { filters, machina } of this.subs.values()) { + for (const [subId, { filters, machina }] of this.subs.entries()) { for (const filter of filters) { count++; if (this.matchesFilter(event, filter)) { - machina.push(event); + machina.push(['EVENT', subId, event]); break; } @@ -258,47 +264,60 @@ export class DittoPgStore extends NPostgres { filters: NostrFilter[], opts: { timeout?: number; signal?: AbortSignal } = {}, ): AsyncIterable { + const { db, chunkSize = 20 } = this.opts; + const { timeout = this.opts.timeout, signal } = opts; + const subId = crypto.randomUUID(); const normalFilters = this.normalizeFilters(filters); + const machina = new Machina(signal); if (normalFilters.length) { - const { db, timeout, chunkSize = 20 } = this.opts; + this.withTimeout(db.kysely as unknown as Kysely, timeout, async (trx) => { + const rows = this.getEventsQuery(trx, normalFilters).stream(chunkSize); - const rows = await this.withTimeout( - db.kysely as unknown as Kysely, - (trx) => this.getEventsQuery(trx, normalFilters).stream(chunkSize), - opts.timeout ?? timeout, - ); - - for await (const row of rows) { - const event = this.parseEventRow(row); - yield ['EVENT', subId, event]; - - if (opts.signal?.aborted) { - yield ['CLOSED', subId, 'error: the relay could not respond fast enough']; - return; + for await (const row of rows) { + const event = this.parseEventRow(row); + machina.push(['EVENT', subId, event]); } + + machina.push(['EOSE', subId]); + }).catch((error) => { + if (error instanceof Error && error.message.includes('timeout')) { + machina.push(['CLOSED', subId, 'error: the relay could not respond fast enough']); + } else { + machina.push(['CLOSED', subId, 'error: something went wrong']); + } + }); + + try { + for await (const msg of machina) { + const [verb] = msg; + + yield msg; + + if (verb === 'EOSE') { + break; + } + + if (verb === 'CLOSED') { + return; + } + } + } catch { + yield ['CLOSED', subId, 'error: the relay could not respond fast enough']; + return; } } - yield ['EOSE', subId]; - - if (opts.signal?.aborted) { - yield ['CLOSED', subId, 'error: the relay could not respond fast enough']; - return; - } - - const machina = new Machina(opts.signal); - this.subs.set(subId, { filters, machina }); internalSubscriptionsSizeGauge.set(this.subs.size); try { - for await (const event of machina) { - yield ['EVENT', subId, event]; + for await (const msg of machina) { + yield msg; } } catch (e) { - if (e instanceof Error && e.message.includes('timeout')) { + if (e instanceof Error && e.name === 'AbortError') { yield ['CLOSED', subId, 'error: the relay could not respond fast enough']; } else { yield ['CLOSED', subId, 'error: something went wrong']; diff --git a/packages/ditto/test.ts b/packages/ditto/test.ts index 38801093..84303d76 100644 --- a/packages/ditto/test.ts +++ b/packages/ditto/test.ts @@ -22,6 +22,7 @@ export async function createTestDB(opts?: { pure?: boolean }) { timeout: Conf.db.timeouts.default, pubkey: Conf.pubkey, pure: opts?.pure ?? false, + notify: true, }); return { From f0c7ec0a99931c29f3eed26f8a5d738012b4e9c3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 19 Feb 2025 20:25:12 -0600 Subject: [PATCH 16/99] Prevent the streaming API from paginating the whole database --- packages/ditto/controllers/api/streaming.ts | 23 ++++++++++++--------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/packages/ditto/controllers/api/streaming.ts b/packages/ditto/controllers/api/streaming.ts index 25fe877d..4171e1be 100644 --- a/packages/ditto/controllers/api/streaming.ts +++ b/packages/ditto/controllers/api/streaming.ts @@ -103,9 +103,12 @@ const streamingController: AppController = async (c) => { } } - async function sub(filters: NostrFilter[], render: (event: NostrEvent) => Promise) { + async function sub( + filter: NostrFilter & { limit: 0 }, + render: (event: NostrEvent) => Promise, + ) { try { - for await (const msg of store.req(filters, { signal: controller.signal })) { + for await (const msg of store.req([filter], { signal: controller.signal })) { if (msg[0] === 'EVENT') { const event = msg[2]; @@ -138,7 +141,7 @@ const streamingController: AppController = async (c) => { const topicFilter = await topicToFilter(stream, c.req.query(), pubkey, conf.url.host); if (topicFilter) { - sub([topicFilter], async (event) => { + sub(topicFilter, async (event) => { let payload: object | undefined; if (event.kind === 1) { @@ -159,7 +162,7 @@ const streamingController: AppController = async (c) => { } if (['user', 'user:notification'].includes(stream) && pubkey) { - sub([{ '#p': [pubkey] }], async (event) => { + sub({ '#p': [pubkey], limit: 0 }, async (event) => { if (event.pubkey === pubkey) return; // skip own events const payload = await renderNotification(event, { viewerPubkey: pubkey }); if (payload) { @@ -207,23 +210,23 @@ async function topicToFilter( query: Record, pubkey: string | undefined, host: string, -): Promise { +): Promise<(NostrFilter & { limit: 0 }) | undefined> { switch (topic) { case 'public': - return { kinds: [1, 6, 20] }; + return { kinds: [1, 6, 20], limit: 0 }; case 'public:local': - return { kinds: [1, 6, 20], search: `domain:${host}` }; + return { kinds: [1, 6, 20], search: `domain:${host}`, limit: 0 }; case 'hashtag': - if (query.tag) return { kinds: [1, 6, 20], '#t': [query.tag] }; + if (query.tag) return { kinds: [1, 6, 20], '#t': [query.tag], limit: 0 }; break; case 'hashtag:local': - if (query.tag) return { kinds: [1, 6, 20], '#t': [query.tag], search: `domain:${host}` }; + if (query.tag) return { kinds: [1, 6, 20], '#t': [query.tag], search: `domain:${host}`, limit: 0 }; break; case 'user': // HACK: this puts the user's entire contacts list into RAM, // and then calls `matchFilters` over it. Refreshing the page // is required after following a new user. - return pubkey ? { kinds: [1, 6, 20], authors: [...await getFeedPubkeys(pubkey)] } : undefined; + return pubkey ? { kinds: [1, 6, 20], authors: [...await getFeedPubkeys(pubkey)], limit: 0 } : undefined; } } From d05dd1650707cd708f35131ae5233d60183212ed Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 19 Feb 2025 20:29:24 -0600 Subject: [PATCH 17/99] EOSE after empty initial filters --- packages/ditto/storages/DittoPgStore.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/ditto/storages/DittoPgStore.ts b/packages/ditto/storages/DittoPgStore.ts index 000ef536..4ec7a6ca 100644 --- a/packages/ditto/storages/DittoPgStore.ts +++ b/packages/ditto/storages/DittoPgStore.ts @@ -307,6 +307,8 @@ export class DittoPgStore extends NPostgres { yield ['CLOSED', subId, 'error: the relay could not respond fast enough']; return; } + } else { + yield ['EOSE', subId]; } this.subs.set(subId, { filters, machina }); From 9401c0e0131498110b3e16fc822337b8128a4c47 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 19 Feb 2025 20:40:30 -0600 Subject: [PATCH 18/99] DittoPgStore: call expandFilters in .req --- packages/ditto/storages/DittoPgStore.ts | 34 ++++++++++--------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/packages/ditto/storages/DittoPgStore.ts b/packages/ditto/storages/DittoPgStore.ts index 4ec7a6ca..bcbb9197 100644 --- a/packages/ditto/storages/DittoPgStore.ts +++ b/packages/ditto/storages/DittoPgStore.ts @@ -267,6 +267,8 @@ export class DittoPgStore extends NPostgres { const { db, chunkSize = 20 } = this.opts; const { timeout = this.opts.timeout, signal } = opts; + filters = await this.expandFilters(filters); + const subId = crypto.randomUUID(); const normalFilters = this.normalizeFilters(filters); const machina = new Machina(signal); @@ -337,20 +339,6 @@ export class DittoPgStore extends NPostgres { ): Promise { filters = await this.expandFilters(filters); - for (const filter of filters) { - if (filter.since && filter.since >= 2_147_483_647) { - throw new RelayError('invalid', 'since filter too far into the future'); - } - if (filter.until && filter.until >= 2_147_483_647) { - throw new RelayError('invalid', 'until filter too far into the future'); - } - for (const kind of filter.kinds ?? []) { - if (kind >= 2_147_483_647) { - throw new RelayError('invalid', 'kind filter too far into the future'); - } - } - } - if (opts.signal?.aborted) return Promise.resolve([]); logi({ level: 'debug', ns: 'ditto.req', source: 'db', filters: filters as JsonValue }); @@ -531,6 +519,18 @@ export class DittoPgStore extends NPostgres { filters = structuredClone(filters); for (const filter of filters) { + if (filter.since && filter.since >= 2_147_483_647) { + throw new RelayError('invalid', 'since filter too far into the future'); + } + if (filter.until && filter.until >= 2_147_483_647) { + throw new RelayError('invalid', 'until filter too far into the future'); + } + for (const kind of filter.kinds ?? []) { + if (kind >= 2_147_483_647) { + throw new RelayError('invalid', 'kind filter too far into the future'); + } + } + if (filter.search) { const tokens = NIP50.parseInput(filter.search); @@ -581,12 +581,6 @@ export class DittoPgStore extends NPostgres { .map((t) => typeof t === 'object' ? `${t.key}:${t.value}` : t) .join(' '); } - - if (filter.kinds) { - // Ephemeral events are not stored, so don't bother querying for them. - // If this results in an empty kinds array, NDatabase will remove the filter before querying and return no results. - filter.kinds = filter.kinds.filter((kind) => !NKinds.ephemeral(kind)); - } } return filters; From aefa6bed6ee93f307b6202b27625c03e190e0381 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 19 Feb 2025 20:51:07 -0600 Subject: [PATCH 19/99] Add an initial limit back to the relay --- packages/ditto/controllers/nostr/relay.ts | 2 +- packages/ditto/storages/DittoPgStore.ts | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/ditto/controllers/nostr/relay.ts b/packages/ditto/controllers/nostr/relay.ts index c549c594..9c29f89d 100644 --- a/packages/ditto/controllers/nostr/relay.ts +++ b/packages/ditto/controllers/nostr/relay.ts @@ -125,7 +125,7 @@ function connectStream(socket: WebSocket, ip: string | undefined, conf: DittoCon const store = await Storages.db(); try { - for await (const [verb, , ...rest] of store.req(filters, { timeout: conf.db.timeouts.relay })) { + for await (const [verb, , ...rest] of store.req(filters, { limit: 100, timeout: conf.db.timeouts.relay })) { send([verb, subId, ...rest] as NostrRelayMsg); } } catch (e) { diff --git a/packages/ditto/storages/DittoPgStore.ts b/packages/ditto/storages/DittoPgStore.ts index bcbb9197..e7b88fd4 100644 --- a/packages/ditto/storages/DittoPgStore.ts +++ b/packages/ditto/storages/DittoPgStore.ts @@ -262,10 +262,10 @@ export class DittoPgStore extends NPostgres { override async *req( filters: NostrFilter[], - opts: { timeout?: number; signal?: AbortSignal } = {}, + opts: { timeout?: number; signal?: AbortSignal; limit?: number } = {}, ): AsyncIterable { const { db, chunkSize = 20 } = this.opts; - const { timeout = this.opts.timeout, signal } = opts; + const { limit, timeout = this.opts.timeout, signal } = opts; filters = await this.expandFilters(filters); @@ -273,11 +273,15 @@ export class DittoPgStore extends NPostgres { const normalFilters = this.normalizeFilters(filters); const machina = new Machina(signal); - if (normalFilters.length) { + if (normalFilters.length && limit !== 0) { this.withTimeout(db.kysely as unknown as Kysely, timeout, async (trx) => { - const rows = this.getEventsQuery(trx, normalFilters).stream(chunkSize); + let query = this.getEventsQuery(trx, normalFilters); - for await (const row of rows) { + if (typeof opts.limit === 'number') { + query = query.limit(opts.limit); + } + + for await (const row of query.stream(chunkSize)) { const event = this.parseEventRow(row); machina.push(['EVENT', subId, event]); } From c6605ece77c4c55e5879f04089ee766ad6b2c79c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 19 Feb 2025 21:50:19 -0600 Subject: [PATCH 20/99] Fix not being able to log in for chrissakes --- packages/ditto/pipeline.ts | 29 +++++++++++---------- packages/ditto/storages/DittoPgStore.ts | 34 ++++++++++++++++++++----- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/packages/ditto/pipeline.ts b/packages/ditto/pipeline.ts index 1c49e930..d7536c91 100644 --- a/packages/ditto/pipeline.ts +++ b/packages/ditto/pipeline.ts @@ -77,21 +77,24 @@ async function handleEvent(event: DittoEvent, opts: PipelineOpts): Promise // NIP-46 events get special treatment. // They are exempt from policies and other side-effects, and should be streamed out immediately. // If streaming fails, an error should be returned. - if (event.kind !== 24133) { - // Ensure the event doesn't violate the policy. - if (event.pubkey !== Conf.pubkey) { - await policyFilter(event, opts.signal); - } + if (event.kind === 24133) { + const store = await Storages.db(); + await store.event(event, { signal: opts.signal }); + } - // Prepare the event for additional checks. - // FIXME: This is kind of hacky. Should be reorganized to fetch only what's needed for each stage. - await hydrateEvent(event, opts.signal); + // Ensure the event doesn't violate the policy. + if (event.pubkey !== Conf.pubkey) { + await policyFilter(event, opts.signal); + } - // Ensure that the author is not banned. - const n = getTagSet(event.user?.tags ?? [], 'n'); - if (n.has('disabled')) { - throw new RelayError('blocked', 'author is blocked'); - } + // Prepare the event for additional checks. + // FIXME: This is kind of hacky. Should be reorganized to fetch only what's needed for each stage. + await hydrateEvent(event, opts.signal); + + // Ensure that the author is not banned. + const n = getTagSet(event.user?.tags ?? [], 'n'); + if (n.has('disabled')) { + throw new RelayError('blocked', 'author is blocked'); } const kysely = await Storages.kysely(); diff --git a/packages/ditto/storages/DittoPgStore.ts b/packages/ditto/storages/DittoPgStore.ts index e7b88fd4..a921a309 100644 --- a/packages/ditto/storages/DittoPgStore.ts +++ b/packages/ditto/storages/DittoPgStore.ts @@ -126,10 +126,6 @@ export class DittoPgStore extends NPostgres { override async event(event: NostrEvent, opts: { signal?: AbortSignal; timeout?: number } = {}): Promise { event = purifyEvent(event); - if (this.opts.notify) { - this.encounters.set(event.id, true); - } - logi({ level: 'debug', ns: 'ditto.event', source: 'db', id: event.id, kind: event.kind }); dbEventsCounter.inc({ kind: event.kind }); @@ -137,6 +133,10 @@ export class DittoPgStore extends NPostgres { return await this.fulfill(event); } + if (this.opts.notify) { + this.encounters.set(event.id, true); + } + if (await this.isDeletedAdmin(event)) { throw new RelayError('blocked', 'event deleted by admin'); } @@ -590,8 +590,28 @@ export class DittoPgStore extends NPostgres { return filters; } - // deno-lint-ignore no-explicit-any - override async transaction(callback: (store: NPostgres, kysely: Kysely) => Promise): Promise { - return super.transaction((store, kysely) => callback(store, kysely as unknown as Kysely)); + /** Execute the callback in a new transaction, unless the Kysely instance is already a transaction. */ + private static override async trx( + db: Kysely, + callback: (trx: Kysely) => Promise, + ): Promise { + if (db.isTransaction) { + return await callback(db); + } else { + return await db.transaction().execute((trx) => callback(trx)); + } + } + + /** Execute NPostgres functions in a transaction. */ + // @ts-ignore gg + override async transaction( + callback: (store: DittoPgStore, kysely: Kysely) => Promise, + ): Promise { + const { db } = this.opts; + + await DittoPgStore.trx(db.kysely, async (trx) => { + const store = new DittoPgStore({ ...this.opts, db: { ...db, kysely: trx }, notify: false }); + await callback(store, trx); + }); } } From 6f7fc116356bfb4058eb2bca8baef7c598a25cb6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 19 Feb 2025 22:00:09 -0600 Subject: [PATCH 21/99] Super duper extra close the database --- packages/db/DittoDatabase.ts | 2 +- packages/db/adapters/DittoPglite.ts | 4 ++++ packages/db/adapters/DittoPostgres.ts | 4 ++++ packages/ditto/test.ts | 2 +- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/db/DittoDatabase.ts b/packages/db/DittoDatabase.ts index e43356a0..ebe97cec 100644 --- a/packages/db/DittoDatabase.ts +++ b/packages/db/DittoDatabase.ts @@ -2,7 +2,7 @@ import type { Kysely } from 'kysely'; import type { DittoTables } from './DittoTables.ts'; -export interface DittoDatabase { +export interface DittoDatabase extends AsyncDisposable { readonly kysely: Kysely; readonly poolSize: number; readonly availableConnections: number; diff --git a/packages/db/adapters/DittoPglite.ts b/packages/db/adapters/DittoPglite.ts index 2e7ca3fc..5e7e6ca4 100644 --- a/packages/db/adapters/DittoPglite.ts +++ b/packages/db/adapters/DittoPglite.ts @@ -36,6 +36,10 @@ export class DittoPglite { poolSize: 1, availableConnections: 1, listen, + [Symbol.asyncDispose]: async () => { + await pglite.close(); + await kysely.destroy(); + }, }; } } diff --git a/packages/db/adapters/DittoPostgres.ts b/packages/db/adapters/DittoPostgres.ts index 9ab8156f..b62a878b 100644 --- a/packages/db/adapters/DittoPostgres.ts +++ b/packages/db/adapters/DittoPostgres.ts @@ -54,6 +54,10 @@ export class DittoPostgres { return pg.connections.idle; }, listen, + [Symbol.asyncDispose]: async () => { + await pg.end(); + await kysely.destroy(); + }, }; } } diff --git a/packages/ditto/test.ts b/packages/ditto/test.ts index 84303d76..c363963f 100644 --- a/packages/ditto/test.ts +++ b/packages/ditto/test.ts @@ -38,7 +38,7 @@ export async function createTestDB(opts?: { pure?: boolean }) { await sql`truncate table ${sql.ref(tablename)} cascade`.execute(db.kysely); } - await db.kysely.destroy(); + await db[Symbol.asyncDispose](); }, }; } From 841b83f573834da64f7c904631e1b1fdaced5f68 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 19 Feb 2025 22:01:16 -0600 Subject: [PATCH 22/99] Hopeless Cashu tests leak even more --- packages/ditto/controllers/api/cashu.test.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/ditto/controllers/api/cashu.test.ts b/packages/ditto/controllers/api/cashu.test.ts index ee73661b..57be895d 100644 --- a/packages/ditto/controllers/api/cashu.test.ts +++ b/packages/ditto/controllers/api/cashu.test.ts @@ -140,7 +140,10 @@ Deno.test('PUT /wallet must NOT be successful: wrong request body/schema', async assertObjectMatch(body, { error: 'Bad schema' }); }); -Deno.test('PUT /wallet must NOT be successful: wallet already exists', async () => { +Deno.test('PUT /wallet must NOT be successful: wallet already exists', { + sanitizeOps: false, + sanitizeResources: false, +}, async () => { using _mock = mockFetch(); await using db = await createTestDB(); const store = db.store; @@ -178,7 +181,10 @@ Deno.test('PUT /wallet must NOT be successful: wallet already exists', async () assertEquals(body2, { error: 'You already have a wallet 😏' }); }); -Deno.test('GET /wallet must be successful', async () => { +Deno.test('GET /wallet must be successful', { + sanitizeOps: false, + sanitizeResources: false, +}, async () => { using _mock = mockFetch(); await using db = await createTestDB(); const store = db.store; From 351d03bde742473389144ee4528859b959a3b799 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 19 Feb 2025 22:02:49 -0600 Subject: [PATCH 23/99] Remove accidentally added DittoAPIStore --- packages/ditto/storages/DittoAPIStore.ts | 38 ------------------------ 1 file changed, 38 deletions(-) delete mode 100644 packages/ditto/storages/DittoAPIStore.ts diff --git a/packages/ditto/storages/DittoAPIStore.ts b/packages/ditto/storages/DittoAPIStore.ts deleted file mode 100644 index 46c56df2..00000000 --- a/packages/ditto/storages/DittoAPIStore.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { - NostrEvent, - NostrFilter, - NostrRelayCLOSED, - NostrRelayCOUNT, - NostrRelayEOSE, - NostrRelayEVENT, - NRelay, -} from '@nostrify/nostrify'; - -export class DittoAPIStore implements NRelay { - req( - filters: NostrFilter[], - opts?: { signal?: AbortSignal }, - ): AsyncIterable { - throw new Error('Method not implemented.'); - } - - close(): Promise { - throw new Error('Method not implemented.'); - } - - event(event: NostrEvent, opts?: { signal?: AbortSignal }): Promise { - throw new Error('Method not implemented.'); - } - - query(filters: NostrFilter[], opts?: { signal?: AbortSignal }): Promise { - throw new Error('Method not implemented.'); - } - - count(filters: NostrFilter[], opts?: { signal?: AbortSignal }): Promise { - throw new Error('Method not implemented.'); - } - - remove(filters: NostrFilter[], opts?: { signal?: AbortSignal }): Promise { - throw new Error('Method not implemented.'); - } -} From 521b63185b4edf12fa6464cc335707676f958b3b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 19 Feb 2025 22:14:38 -0600 Subject: [PATCH 24/99] Catch webPush --- packages/ditto/pipeline.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/ditto/pipeline.ts b/packages/ditto/pipeline.ts index d7536c91..07be1bd9 100644 --- a/packages/ditto/pipeline.ts +++ b/packages/ditto/pipeline.ts @@ -109,7 +109,8 @@ async function handleEvent(event: DittoEvent, opts: PipelineOpts): Promise prewarmLinkPreview(event, opts.signal), generateSetEvents(event), ]) - .then(() => webPush(event)); + .then(() => webPush(event)) + .catch(() => {}); } } From f72fcdbd653f44a8bf39a0c24eb0e9e7220bfa59 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 00:23:06 -0600 Subject: [PATCH 25/99] Upgrade socket before closing with 1008 (ratelimit) --- packages/ditto/controllers/nostr/relay.ts | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/ditto/controllers/nostr/relay.ts b/packages/ditto/controllers/nostr/relay.ts index 9c29f89d..0284ce64 100644 --- a/packages/ditto/controllers/nostr/relay.ts +++ b/packages/ditto/controllers/nostr/relay.ts @@ -45,6 +45,17 @@ const connections = new Set(); function connectStream(socket: WebSocket, ip: string | undefined, conf: DittoConf) { const controllers = new Map(); + if (ip) { + const remaining = Object + .values(limiters) + .reduce((acc, limiter) => Math.min(acc, limiter.client(ip).remaining), Infinity); + + if (remaining < 0) { + socket.close(1008, 'Rate limit exceeded'); + return; + } + } + socket.onopen = () => { connections.add(socket); relayConnectionsGauge.set(connections.size); @@ -206,16 +217,6 @@ const relayController: AppController = (c, next) => { ip = undefined; } - if (ip) { - const remaining = Object - .values(limiters) - .reduce((acc, limiter) => Math.min(acc, limiter.client(ip).remaining), Infinity); - - if (remaining < 0) { - return c.json({ error: 'Rate limit exceeded' }, 429); - } - } - const { socket, response } = Deno.upgradeWebSocket(c.req.raw, { idleTimeout: 30 }); connectStream(socket, ip, conf); From 2ce283e9a55ceb9181b23f29a4bc14cf9364dace Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 00:57:47 -0600 Subject: [PATCH 26/99] return new Response() -> return c.newResponse() --- packages/ditto/controllers/api/captcha.ts | 2 +- packages/ditto/controllers/api/ditto.ts | 8 ++++---- packages/ditto/controllers/api/pleroma.ts | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/ditto/controllers/api/captcha.ts b/packages/ditto/controllers/api/captcha.ts index 6bbcc49f..7b310e53 100644 --- a/packages/ditto/controllers/api/captcha.ts +++ b/packages/ditto/controllers/api/captcha.ts @@ -171,7 +171,7 @@ export const captchaVerifyController: AppController = async (c) => { if (solved) { captchas.delete(id); await updateUser(pubkey, { captcha_solved: true }, c); - return new Response(null, { status: 204 }); + return c.newResponse(null, { status: 204 }); } return c.json({ error: 'Incorrect solution' }, { status: 400 }); diff --git a/packages/ditto/controllers/api/ditto.ts b/packages/ditto/controllers/api/ditto.ts index 9465517c..b9fef08a 100644 --- a/packages/ditto/controllers/api/ditto.ts +++ b/packages/ditto/controllers/api/ditto.ts @@ -188,7 +188,7 @@ export const updateZapSplitsController: AppController = async (c) => { const pubkeys = Object.keys(data); if (pubkeys.length < 1) { - return c.json(200); + return c.newResponse(null, { status: 204 }); } await updateListAdminEvent( @@ -200,7 +200,7 @@ export const updateZapSplitsController: AppController = async (c) => { c, ); - return c.json(200); + return c.newResponse(null, { status: 204 }); }; const deleteZapSplitSchema = z.array(n.id()).min(1); @@ -231,7 +231,7 @@ export const deleteZapSplitsController: AppController = async (c) => { c, ); - return c.json(200); + return c.newResponse(null, { status: 204 }); }; export const getZapSplitsController: AppController = async (c) => { @@ -346,5 +346,5 @@ export const updateInstanceController: AppController = async (c) => { c, ); - return c.json(204); + return c.newResponse(null, { status: 204 }); }; diff --git a/packages/ditto/controllers/api/pleroma.ts b/packages/ditto/controllers/api/pleroma.ts index 976c2c0a..302eaca6 100644 --- a/packages/ditto/controllers/api/pleroma.ts +++ b/packages/ditto/controllers/api/pleroma.ts @@ -98,7 +98,7 @@ const pleromaAdminTagController: AppController = async (c) => { ); } - return new Response(null, { status: 204 }); + return c.newResponse(null, { status: 204 }); }; const pleromaAdminUntagController: AppController = async (c) => { @@ -121,7 +121,7 @@ const pleromaAdminUntagController: AppController = async (c) => { ); } - return new Response(null, { status: 204 }); + return c.newResponse(null, { status: 204 }); }; const pleromaAdminSuggestSchema = z.object({ @@ -137,7 +137,7 @@ const pleromaAdminSuggestController: AppController = async (c) => { await updateUser(pubkey, { suggested: true }, c); } - return new Response(null, { status: 204 }); + return c.newResponse(null, { status: 204 }); }; const pleromaAdminUnsuggestController: AppController = async (c) => { @@ -149,7 +149,7 @@ const pleromaAdminUnsuggestController: AppController = async (c) => { await updateUser(pubkey, { suggested: false }, c); } - return new Response(null, { status: 204 }); + return c.newResponse(null, { status: 204 }); }; export { From 403d0ac5c2ddc5459e03d699669b63413ad20489 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 10:03:01 -0600 Subject: [PATCH 27/99] Add logi log to translate controller --- packages/ditto/controllers/api/translate.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/ditto/controllers/api/translate.ts b/packages/ditto/controllers/api/translate.ts index 7395ff2f..8a99edde 100644 --- a/packages/ditto/controllers/api/translate.ts +++ b/packages/ditto/controllers/api/translate.ts @@ -1,4 +1,5 @@ import { cachedTranslationsSizeGauge } from '@ditto/metrics'; +import { logi } from '@soapbox/logi'; import { LanguageCode } from 'iso-639-1'; import { z } from 'zod'; @@ -9,6 +10,7 @@ import { getEvent } from '@/queries.ts'; import { localeSchema } from '@/schema.ts'; import { parseBody } from '@/utils/api.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; +import { errorJson } from '@/utils/log.ts'; const translateSchema = z.object({ lang: localeSchema, @@ -140,6 +142,7 @@ const translateController: AppController = async (c) => { if (e instanceof Error && e.message.includes('not supported')) { return c.json({ error: `Translation of source language '${event.language}' not supported` }, 422); } + logi({ level: 'error', ns: 'ditto.translate', error: errorJson(e) }); return c.json({ error: 'Service Unavailable' }, 503); } }; From d791a9b35079be5d25acfcd214ab0917bdd35dfe Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 10:35:27 -0600 Subject: [PATCH 28/99] Fix DeepL Response parsing, mock DeepL tests so they can always run without API keys --- packages/translators/DeepLTranslator.test.ts | 69 ++++++++++++++------ packages/translators/DeepLTranslator.ts | 17 ++++- packages/translators/schema.test.ts | 7 ++ 3 files changed, 70 insertions(+), 23 deletions(-) create mode 100644 packages/translators/schema.test.ts diff --git a/packages/translators/DeepLTranslator.test.ts b/packages/translators/DeepLTranslator.test.ts index ae1565c9..8e37e44b 100644 --- a/packages/translators/DeepLTranslator.test.ts +++ b/packages/translators/DeepLTranslator.test.ts @@ -1,21 +1,20 @@ -import { DittoConf } from '@ditto/conf'; import { detectLanguage } from '@ditto/lang'; import { assert, assertEquals } from '@std/assert'; import { DeepLTranslator } from './DeepLTranslator.ts'; -const { - deeplBaseUrl: baseUrl, - deeplApiKey: apiKey, - translationProvider, -} = new DittoConf(Deno.env); - -const deepl = 'deepl'; - -Deno.test('DeepL translation with source language omitted', { - ignore: !(translationProvider === deepl && apiKey), -}, async () => { - const translator = new DeepLTranslator({ fetch: fetch, baseUrl, apiKey: apiKey! }); +Deno.test('DeepL translation with source language omitted', async () => { + const translator = mockDeepL({ + translations: [ + { detected_source_language: 'PT', text: 'Good morning friends' }, + { detected_source_language: 'PT', text: 'My name is Patrick' }, + { + detected_source_language: 'PT', + text: + 'I will live in America, I promise. But first, I should mention that lande is interpreting this text as Italian, how strange.', + }, + ], + }); const data = await translator.translate( [ @@ -33,10 +32,18 @@ Deno.test('DeepL translation with source language omitted', { assertEquals(detectLanguage(data.results[2], 0), 'en'); }); -Deno.test('DeepL translation with source language set', { - ignore: !(translationProvider === deepl && apiKey), -}, async () => { - const translator = new DeepLTranslator({ fetch: fetch, baseUrl, apiKey: apiKey as string }); +Deno.test('DeepL translation with source language set', async () => { + const translator = mockDeepL({ + translations: [ + { detected_source_language: 'PT', text: 'Good morning friends' }, + { detected_source_language: 'PT', text: 'My name is Patrick' }, + { + detected_source_language: 'PT', + text: + 'I will live in America, I promise. But first, I should mention that lande is interpreting this text as Italian, how strange.', + }, + ], + }); const data = await translator.translate( [ @@ -54,10 +61,16 @@ Deno.test('DeepL translation with source language set', { assertEquals(detectLanguage(data.results[2], 0), 'en'); }); -Deno.test("DeepL translation doesn't alter Nostr URIs", { - ignore: !(translationProvider === deepl && apiKey), -}, async () => { - const translator = new DeepLTranslator({ fetch: fetch, baseUrl, apiKey: apiKey as string }); +Deno.test("DeepL translation doesn't alter Nostr URIs", async () => { + const translator = mockDeepL({ + translations: [ + { + detected_source_language: 'EN', + text: + 'Graças ao trabalho de nostr:nprofile1qy2hwumn8ghj7erfw36x7tnsw43z7un9d3shjqpqgujeqakgt7fyp6zjggxhyy7ft623qtcaay5lkc8n8gkry4cvnrzqep59se e nostr:nprofile1qy2hwumn8ghj7erfw36x7tnsw43z7un9d3shjqpqe6tnvlr46lv3lwdu80r07kanhk6jcxy5r07w9umgv9kuhu9dl5hsz44l8s , agora é possível filtrar o feed global por idioma no #Ditto!', + }, + ], + }); const patrick = 'nostr:nprofile1qy2hwumn8ghj7erfw36x7tnsw43z7un9d3shjqpqgujeqakgt7fyp6zjggxhyy7ft623qtcaay5lkc8n8gkry4cvnrzqep59se'; @@ -72,3 +85,17 @@ Deno.test("DeepL translation doesn't alter Nostr URIs", { assert(output.includes(patrick)); assert(output.includes(danidfra)); }); + +interface DeepLResponse { + translations: { + detected_source_language: string; + text: string; + }[]; +} + +function mockDeepL(json: DeepLResponse): DeepLTranslator { + return new DeepLTranslator({ + apiKey: 'deepl', + fetch: () => Promise.resolve(new Response(JSON.stringify(json))), + }); +} diff --git a/packages/translators/DeepLTranslator.ts b/packages/translators/DeepLTranslator.ts index f4b6f918..93da8ad7 100644 --- a/packages/translators/DeepLTranslator.ts +++ b/packages/translators/DeepLTranslator.ts @@ -72,7 +72,13 @@ export class DeepLTranslator implements DittoTranslator { const json = await response.json(); if (!response.ok) { - throw new Error(json['message']); + const result = DeepLTranslator.errorSchema().safeParse(json); + + if (result.success) { + throw new Error(result.data.message); + } else { + throw new Error(`Unexpected DeepL error: ${response.statusText} (${response.status})`); + } } return DeepLTranslator.schema().parse(json); @@ -84,10 +90,17 @@ export class DeepLTranslator implements DittoTranslator { return z.object({ translations: z.array( z.object({ - detected_source_language: languageSchema, + detected_source_language: z.string().transform((val) => val.toLowerCase()).pipe(languageSchema), text: z.string(), }), ), }); } + + /** DeepL error response schema. */ + private static errorSchema() { + return z.object({ + message: z.string(), + }); + } } diff --git a/packages/translators/schema.test.ts b/packages/translators/schema.test.ts new file mode 100644 index 00000000..6d37992c --- /dev/null +++ b/packages/translators/schema.test.ts @@ -0,0 +1,7 @@ +import { assertEquals } from '@std/assert'; + +import { languageSchema } from './schema.ts'; + +Deno.test('languageSchema', () => { + assertEquals(languageSchema.safeParse('en').success, true); +}); From 2150259abad023234cf8a9aa92ee8c59f37eff74 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 10:36:43 -0600 Subject: [PATCH 29/99] languageSchema does not lowercase the code --- packages/translators/schema.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/translators/schema.test.ts b/packages/translators/schema.test.ts index 6d37992c..4ca84adc 100644 --- a/packages/translators/schema.test.ts +++ b/packages/translators/schema.test.ts @@ -3,5 +3,6 @@ import { assertEquals } from '@std/assert'; import { languageSchema } from './schema.ts'; Deno.test('languageSchema', () => { - assertEquals(languageSchema.safeParse('en').success, true); + assertEquals(languageSchema.safeParse('pt').success, true); + assertEquals(languageSchema.safeParse('PT').success, false); }); From 1afb09e60495477425373203bb7f3cf3662b6b71 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 10:39:31 -0600 Subject: [PATCH 30/99] DittoTranslator: source_lang -> sourceLang --- packages/ditto/controllers/api/translate.ts | 2 +- packages/translators/DeepLTranslator.test.ts | 4 ++-- packages/translators/DeepLTranslator.ts | 4 ++-- packages/translators/DittoTranslator.ts | 2 +- packages/translators/LibreTranslateTranslator.test.ts | 4 ++-- packages/translators/LibreTranslateTranslator.ts | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/ditto/controllers/api/translate.ts b/packages/ditto/controllers/api/translate.ts index 8a99edde..de183e23 100644 --- a/packages/ditto/controllers/api/translate.ts +++ b/packages/ditto/controllers/api/translate.ts @@ -132,7 +132,7 @@ const translateController: AppController = async (c) => { } } - mastodonTranslation.detected_source_language = data.source_lang; + mastodonTranslation.detected_source_language = data.sourceLang; translationCache.set(cacheKey, mastodonTranslation); cachedTranslationsSizeGauge.set(translationCache.size); diff --git a/packages/translators/DeepLTranslator.test.ts b/packages/translators/DeepLTranslator.test.ts index 8e37e44b..a688f135 100644 --- a/packages/translators/DeepLTranslator.test.ts +++ b/packages/translators/DeepLTranslator.test.ts @@ -26,7 +26,7 @@ Deno.test('DeepL translation with source language omitted', async () => { 'en', ); - assertEquals(data.source_lang, 'pt'); + assertEquals(data.sourceLang, 'pt'); assertEquals(detectLanguage(data.results[0], 0), 'en'); assertEquals(detectLanguage(data.results[1], 0), 'en'); assertEquals(detectLanguage(data.results[2], 0), 'en'); @@ -55,7 +55,7 @@ Deno.test('DeepL translation with source language set', async () => { 'en', ); - assertEquals(data.source_lang, 'pt'); + assertEquals(data.sourceLang, 'pt'); assertEquals(detectLanguage(data.results[0], 0), 'en'); assertEquals(detectLanguage(data.results[1], 0), 'en'); assertEquals(detectLanguage(data.results[2], 0), 'en'); diff --git a/packages/translators/DeepLTranslator.ts b/packages/translators/DeepLTranslator.ts index 93da8ad7..673c6e07 100644 --- a/packages/translators/DeepLTranslator.ts +++ b/packages/translators/DeepLTranslator.ts @@ -32,12 +32,12 @@ export class DeepLTranslator implements DittoTranslator { source: LanguageCode | undefined, dest: LanguageCode, opts?: { signal?: AbortSignal }, - ): Promise<{ results: string[]; source_lang: LanguageCode }> { + ): Promise<{ results: string[]; sourceLang: LanguageCode }> { const { translations } = await this.translateMany(texts, source, dest, opts); return { results: translations.map((value) => value.text), - source_lang: translations[0]?.detected_source_language, + sourceLang: translations[0]?.detected_source_language, }; } diff --git a/packages/translators/DittoTranslator.ts b/packages/translators/DittoTranslator.ts index 7e5e1d50..2a9fb7db 100644 --- a/packages/translators/DittoTranslator.ts +++ b/packages/translators/DittoTranslator.ts @@ -14,5 +14,5 @@ export interface DittoTranslator { targetLanguage: LanguageCode, /** Custom options. */ opts?: { signal?: AbortSignal }, - ): Promise<{ results: string[]; source_lang: LanguageCode }>; + ): Promise<{ results: string[]; sourceLang: LanguageCode }>; } diff --git a/packages/translators/LibreTranslateTranslator.test.ts b/packages/translators/LibreTranslateTranslator.test.ts index fc6c0a55..ca8c1d79 100644 --- a/packages/translators/LibreTranslateTranslator.test.ts +++ b/packages/translators/LibreTranslateTranslator.test.ts @@ -27,7 +27,7 @@ Deno.test('LibreTranslate translation with source language omitted', { 'ca', ); - assertEquals(data.source_lang, 'pt'); + assertEquals(data.sourceLang, 'pt'); assertEquals(detectLanguage(data.results[0], 0), 'ca'); assertEquals(detectLanguage(data.results[1], 0), 'ca'); assertEquals(detectLanguage(data.results[2], 0), 'ca'); @@ -48,7 +48,7 @@ Deno.test('LibreTranslate translation with source language set', { 'ca', ); - assertEquals(data.source_lang, 'pt'); + assertEquals(data.sourceLang, 'pt'); assertEquals(detectLanguage(data.results[0], 0), 'ca'); assertEquals(detectLanguage(data.results[1], 0), 'ca'); assertEquals(detectLanguage(data.results[2], 0), 'ca'); diff --git a/packages/translators/LibreTranslateTranslator.ts b/packages/translators/LibreTranslateTranslator.ts index b75f9b54..a8145223 100644 --- a/packages/translators/LibreTranslateTranslator.ts +++ b/packages/translators/LibreTranslateTranslator.ts @@ -32,14 +32,14 @@ export class LibreTranslateTranslator implements DittoTranslator { source: LanguageCode | undefined, dest: LanguageCode, opts?: { signal?: AbortSignal }, - ): Promise<{ results: string[]; source_lang: LanguageCode }> { + ): Promise<{ results: string[]; sourceLang: LanguageCode }> { const translations = await Promise.all( texts.map((text) => this.translateOne(text, source, dest, 'html', { signal: opts?.signal })), ); return { results: translations.map((value) => value.translatedText), - source_lang: (translations[0]?.detectedLanguage?.language ?? source) as LanguageCode, // cast is ok + sourceLang: (translations[0]?.detectedLanguage?.language ?? source) as LanguageCode, // cast is ok }; } From 91f9bd944210afe4d6869cae3c6c20d8db0bf3c7 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 11:04:41 -0600 Subject: [PATCH 31/99] Add mock LibreTranslate tests --- .../LibreTranslateTranslator.test.ts | 68 ++++++++++++++----- .../translators/LibreTranslateTranslator.ts | 25 +++++-- 2 files changed, 71 insertions(+), 22 deletions(-) diff --git a/packages/translators/LibreTranslateTranslator.test.ts b/packages/translators/LibreTranslateTranslator.test.ts index ca8c1d79..94da0ec0 100644 --- a/packages/translators/LibreTranslateTranslator.test.ts +++ b/packages/translators/LibreTranslateTranslator.test.ts @@ -1,21 +1,10 @@ -import { DittoConf } from '@ditto/conf'; import { detectLanguage } from '@ditto/lang'; import { assertEquals } from '@std/assert'; import { LibreTranslateTranslator } from './LibreTranslateTranslator.ts'; -const { - libretranslateBaseUrl: baseUrl, - libretranslateApiKey: apiKey, - translationProvider, -} = new DittoConf(Deno.env); - -const libretranslate = 'libretranslate'; - -Deno.test('LibreTranslate translation with source language omitted', { - ignore: !(translationProvider === libretranslate && apiKey), -}, async () => { - const translator = new LibreTranslateTranslator({ fetch: fetch, baseUrl, apiKey: apiKey! }); +Deno.test('LibreTranslate translation with source language omitted', async () => { + const translator = mockLibreTranslate(); const data = await translator.translate( [ @@ -33,10 +22,8 @@ Deno.test('LibreTranslate translation with source language omitted', { assertEquals(detectLanguage(data.results[2], 0), 'ca'); }); -Deno.test('LibreTranslate translation with source language set', { - ignore: !(translationProvider === libretranslate && apiKey), -}, async () => { - const translator = new LibreTranslateTranslator({ fetch: fetch, baseUrl, apiKey: apiKey! }); +Deno.test('LibreTranslate translation with source language set', async () => { + const translator = mockLibreTranslate(); const data = await translator.translate( [ @@ -53,3 +40,50 @@ Deno.test('LibreTranslate translation with source language set', { assertEquals(detectLanguage(data.results[1], 0), 'ca'); assertEquals(detectLanguage(data.results[2], 0), 'ca'); }); + +function mockLibreTranslate(): LibreTranslateTranslator { + return new LibreTranslateTranslator({ + apiKey: 'libretranslate', + fetch: async (input, init) => { + const req = new Request(input, init); + const body = await req.json(); + + switch (body.q) { + case 'Bom dia amigos': + return jsonResponse({ + detectedLanguage: { language: 'pt' }, + translatedText: 'Bon dia, amics.', + }); + case 'Meu nome é Patrick, um nome belo ou feio? A questão é mais profunda do que parece.': + return jsonResponse({ + detectedLanguage: { language: 'pt' }, + translatedText: 'Em dic Patrick, un nom molt o lleig? La pregunta és més profunda del que sembla.', + }); + case 'A respiração é mais importante do que comer e tomar agua.': + return jsonResponse({ + detectedLanguage: { language: 'pt' }, + translatedText: 'La respiració és més important que menjar i prendre aigua.', + }); + } + + return new Response(JSON.stringify({ error: 'Not found' }), { status: 404 }); + }, + }); +} + +interface LibreTranslateResponse { + translatedText: string; + detectedLanguage?: { + language: string; + }; +} + +function jsonResponse(json: LibreTranslateResponse): Response { + const body = JSON.stringify(json); + + return new Response(body, { + headers: { + 'Content-Type': 'application/json', + }, + }); +} diff --git a/packages/translators/LibreTranslateTranslator.ts b/packages/translators/LibreTranslateTranslator.ts index a8145223..cc978e90 100644 --- a/packages/translators/LibreTranslateTranslator.ts +++ b/packages/translators/LibreTranslateTranslator.ts @@ -71,12 +71,20 @@ export class LibreTranslateTranslator implements DittoTranslator { const response = await this.fetch(request); const json = await response.json(); - if (!response.ok) { - throw new Error(json['error']); - } - const data = LibreTranslateTranslator.schema().parse(json); - return data; + console.log(json); + + if (!response.ok) { + const result = LibreTranslateTranslator.errorSchema().safeParse(json); + + if (result.success) { + throw new Error(result.data.error); + } else { + throw new Error(`Unexpected LibreTranslate error: ${response.statusText} (${response.status})`); + } + } + + return LibreTranslateTranslator.schema().parse(json); } /** Libretranslate response schema. @@ -90,4 +98,11 @@ export class LibreTranslateTranslator implements DittoTranslator { }).optional(), }); } + + /** Libretranslate error response schema. */ + private static errorSchema() { + return z.object({ + error: z.string(), + }); + } } From 5c0a35077642554e348a5ca16d9734a236b67e63 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 11:19:50 -0600 Subject: [PATCH 32/99] Add @ditto/router package --- deno.json | 1 + packages/router/DittoApp.test.ts | 23 +++++++++++++ packages/router/DittoApp.ts | 21 ++++++++++++ packages/router/DittoEnv.ts | 20 +++++++++++ packages/router/DittoMiddleware.ts | 5 +++ packages/router/DittoRoute.test.ts | 12 +++++++ packages/router/DittoRoute.ts | 53 ++++++++++++++++++++++++++++++ packages/router/deno.json | 7 ++++ packages/router/mod.ts | 4 +++ 9 files changed, 146 insertions(+) create mode 100644 packages/router/DittoApp.test.ts create mode 100644 packages/router/DittoApp.ts create mode 100644 packages/router/DittoEnv.ts create mode 100644 packages/router/DittoMiddleware.ts create mode 100644 packages/router/DittoRoute.test.ts create mode 100644 packages/router/DittoRoute.ts create mode 100644 packages/router/deno.json create mode 100644 packages/router/mod.ts diff --git a/deno.json b/deno.json index a3f06bd5..4a34db67 100644 --- a/deno.json +++ b/deno.json @@ -9,6 +9,7 @@ "./packages/metrics", "./packages/policies", "./packages/ratelimiter", + "./packages/router", "./packages/translators", "./packages/uploaders" ], diff --git a/packages/router/DittoApp.test.ts b/packages/router/DittoApp.test.ts new file mode 100644 index 00000000..83da5bca --- /dev/null +++ b/packages/router/DittoApp.test.ts @@ -0,0 +1,23 @@ +import { DittoConf } from '@ditto/conf'; +import { DittoDB } from '@ditto/db'; +import { Hono } from '@hono/hono'; +import { MockRelay } from '@nostrify/nostrify/test'; + +import { DittoApp } from './DittoApp.ts'; +import { DittoRoute } from './DittoRoute.ts'; + +Deno.test('DittoApp', async () => { + await using db = DittoDB.create('memory://'); + const conf = new DittoConf(new Map()); + const relay = new MockRelay(); + + const app = new DittoApp({ conf, db, relay }); + + const hono = new Hono(); + const route = new DittoRoute(); + + app.route('/', route); + + // @ts-expect-error Passing a non-DittoRoute to route. + app.route('/', hono); +}); diff --git a/packages/router/DittoApp.ts b/packages/router/DittoApp.ts new file mode 100644 index 00000000..3309f65d --- /dev/null +++ b/packages/router/DittoApp.ts @@ -0,0 +1,21 @@ +import { Hono } from '@hono/hono'; + +import type { HonoOptions } from '@hono/hono/hono-base'; +import type { DittoEnv } from './DittoEnv.ts'; + +export class DittoApp extends Hono { + // @ts-ignore Require a DittoRoute for type safety. + declare route: (path: string, app: Hono) => Hono; + + constructor(vars: Omit, opts: HonoOptions = {}) { + super(opts); + + this.use((c, next) => { + c.set('db', vars.db); + c.set('conf', vars.conf); + c.set('relay', vars.relay); + c.set('signal', c.req.raw.signal); + return next(); + }); + } +} diff --git a/packages/router/DittoEnv.ts b/packages/router/DittoEnv.ts new file mode 100644 index 00000000..761bc3f8 --- /dev/null +++ b/packages/router/DittoEnv.ts @@ -0,0 +1,20 @@ +import type { DittoConf } from '@ditto/conf'; +import type { DittoDatabase } from '@ditto/db'; +import type { Env } from '@hono/hono'; +import type { NRelay } from '@nostrify/nostrify'; + +export interface DittoEnv extends Env { + Variables: { + /** Ditto site configuration. */ + conf: DittoConf; + /** Relay store. */ + relay: NRelay; + /** + * Database object. + * @deprecated Store data as Nostr events instead. + */ + db: DittoDatabase; + /** Abort signal for the request. */ + signal: AbortSignal; + }; +} diff --git a/packages/router/DittoMiddleware.ts b/packages/router/DittoMiddleware.ts new file mode 100644 index 00000000..1483ca90 --- /dev/null +++ b/packages/router/DittoMiddleware.ts @@ -0,0 +1,5 @@ +import type { MiddlewareHandler } from '@hono/hono'; +import type { DittoEnv } from './DittoEnv.ts'; + +// deno-lint-ignore ban-types +export type DittoMiddleware = MiddlewareHandler; diff --git a/packages/router/DittoRoute.test.ts b/packages/router/DittoRoute.test.ts new file mode 100644 index 00000000..737019c4 --- /dev/null +++ b/packages/router/DittoRoute.test.ts @@ -0,0 +1,12 @@ +import { assertEquals } from '@std/assert'; + +import { DittoRoute } from './DittoRoute.ts'; + +Deno.test('DittoRoute', async () => { + const route = new DittoRoute(); + const response = await route.request('/'); + const body = await response.json(); + + assertEquals(response.status, 500); + assertEquals(body, { error: 'Missing required variable: db' }); +}); diff --git a/packages/router/DittoRoute.ts b/packages/router/DittoRoute.ts new file mode 100644 index 00000000..369fb858 --- /dev/null +++ b/packages/router/DittoRoute.ts @@ -0,0 +1,53 @@ +import { type ErrorHandler, Hono } from '@hono/hono'; +import { HTTPException } from '@hono/hono/http-exception'; + +import type { HonoOptions } from '@hono/hono/hono-base'; +import type { DittoEnv } from './DittoEnv.ts'; + +/** + * Ditto base route class. + * Ensures that required variables are set for type safety. + */ +export class DittoRoute extends Hono { + constructor(opts: HonoOptions = {}) { + super(opts); + + this.use((c, next) => { + this.assertVars(c.var); + return next(); + }); + + this.onError(this._errorHandler); + } + + private assertVars(vars: Partial): DittoEnv['Variables'] { + if (!vars.db) this.throwMissingVar('db'); + if (!vars.conf) this.throwMissingVar('conf'); + if (!vars.relay) this.throwMissingVar('relay'); + if (!vars.signal) this.throwMissingVar('signal'); + + return { + ...vars, + db: vars.db, + conf: vars.conf, + relay: vars.relay, + signal: vars.signal, + }; + } + + private throwMissingVar(name: string): never { + throw new HTTPException(500, { message: `Missing required variable: ${name}` }); + } + + private _errorHandler: ErrorHandler = (error, c) => { + if (error instanceof HTTPException) { + if (error.res) { + return error.res; + } else { + return c.json({ error: error.message }, error.status); + } + } + + return c.json({ error: 'Something went wrong' }, 500); + }; +} diff --git a/packages/router/deno.json b/packages/router/deno.json new file mode 100644 index 00000000..8321baaf --- /dev/null +++ b/packages/router/deno.json @@ -0,0 +1,7 @@ +{ + "name": "@ditto/router", + "version": "1.1.0", + "exports": { + ".": "./mod.ts" + } +} diff --git a/packages/router/mod.ts b/packages/router/mod.ts new file mode 100644 index 00000000..8e9d1d46 --- /dev/null +++ b/packages/router/mod.ts @@ -0,0 +1,4 @@ +export { DittoApp } from './DittoApp.ts'; +export { DittoRoute } from './DittoRoute.ts'; + +export type { DittoEnv } from './DittoEnv.ts'; From c7624e99d70df96de254f20ee410a2315a61d366 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 11:26:27 -0600 Subject: [PATCH 33/99] Swap the names of DittoDB and DittoDatabase --- packages/db/DittoDB.test.ts | 6 -- packages/db/DittoDB.ts | 76 ++++--------------------- packages/db/DittoDatabase.test.ts | 6 ++ packages/db/DittoDatabase.ts | 74 ++++++++++++++++++++---- packages/db/adapters/DittoPglite.ts | 4 +- packages/db/adapters/DittoPostgres.ts | 4 +- packages/db/mod.ts | 4 +- packages/ditto/storages.ts | 12 ++-- packages/ditto/storages/DittoPgStore.ts | 4 +- packages/ditto/test.ts | 6 +- packages/ditto/workers/policy.worker.ts | 4 +- 11 files changed, 100 insertions(+), 100 deletions(-) delete mode 100644 packages/db/DittoDB.test.ts create mode 100644 packages/db/DittoDatabase.test.ts diff --git a/packages/db/DittoDB.test.ts b/packages/db/DittoDB.test.ts deleted file mode 100644 index 1a283319..00000000 --- a/packages/db/DittoDB.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { DittoDB } from './DittoDB.ts'; - -Deno.test('DittoDB', async () => { - const db = DittoDB.create('memory://'); - await DittoDB.migrate(db.kysely); -}); diff --git a/packages/db/DittoDB.ts b/packages/db/DittoDB.ts index f3442808..99ab4c70 100644 --- a/packages/db/DittoDB.ts +++ b/packages/db/DittoDB.ts @@ -1,69 +1,15 @@ -import fs from 'node:fs/promises'; -import path from 'node:path'; +import type { Kysely } from 'kysely'; -import { logi } from '@soapbox/logi'; -import { FileMigrationProvider, type Kysely, Migrator } from 'kysely'; - -import { DittoPglite } from './adapters/DittoPglite.ts'; -import { DittoPostgres } from './adapters/DittoPostgres.ts'; - -import type { JsonValue } from '@std/json'; -import type { DittoDatabase, DittoDatabaseOpts } from './DittoDatabase.ts'; import type { DittoTables } from './DittoTables.ts'; -export class DittoDB { - /** Open a new database connection. */ - static create(databaseUrl: string, opts?: DittoDatabaseOpts): DittoDatabase { - const { protocol } = new URL(databaseUrl); - - switch (protocol) { - case 'file:': - case 'memory:': - return DittoPglite.create(databaseUrl, opts); - case 'postgres:': - case 'postgresql:': - return DittoPostgres.create(databaseUrl, opts); - default: - throw new Error('Unsupported database URL.'); - } - } - - /** Migrate the database to the latest version. */ - static async migrate(kysely: Kysely) { - const migrator = new Migrator({ - db: kysely, - provider: new FileMigrationProvider({ - fs, - path, - migrationFolder: new URL(import.meta.resolve('./migrations')).pathname, - }), - }); - - logi({ level: 'info', ns: 'ditto.db.migration', msg: 'Running migrations...', state: 'started' }); - const { results, error } = await migrator.migrateToLatest(); - - if (error) { - logi({ - level: 'fatal', - ns: 'ditto.db.migration', - msg: 'Migration failed.', - state: 'failed', - results: results as unknown as JsonValue, - error: error instanceof Error ? error : null, - }); - Deno.exit(1); - } else { - if (!results?.length) { - logi({ level: 'info', ns: 'ditto.db.migration', msg: 'Everything up-to-date.', state: 'skipped' }); - } else { - logi({ - level: 'info', - ns: 'ditto.db.migration', - msg: 'Migrations finished!', - state: 'migrated', - results: results as unknown as JsonValue, - }); - } - } - } +export interface DittoDB extends AsyncDisposable { + readonly kysely: Kysely; + readonly poolSize: number; + readonly availableConnections: number; + listen(channel: string, callback: (payload: string) => void): void; +} + +export interface DittoDBOpts { + poolSize?: number; + debug?: 0 | 1 | 2 | 3 | 4 | 5; } diff --git a/packages/db/DittoDatabase.test.ts b/packages/db/DittoDatabase.test.ts new file mode 100644 index 00000000..a91affd5 --- /dev/null +++ b/packages/db/DittoDatabase.test.ts @@ -0,0 +1,6 @@ +import { DittoDatabase } from './DittoDatabase.ts'; + +Deno.test('DittoDatabase', async () => { + const db = DittoDatabase.create('memory://'); + await DittoDatabase.migrate(db.kysely); +}); diff --git a/packages/db/DittoDatabase.ts b/packages/db/DittoDatabase.ts index ebe97cec..916402dd 100644 --- a/packages/db/DittoDatabase.ts +++ b/packages/db/DittoDatabase.ts @@ -1,15 +1,69 @@ -import type { Kysely } from 'kysely'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { logi } from '@soapbox/logi'; +import { FileMigrationProvider, type Kysely, Migrator } from 'kysely'; + +import { DittoPglite } from './adapters/DittoPglite.ts'; +import { DittoPostgres } from './adapters/DittoPostgres.ts'; + +import type { JsonValue } from '@std/json'; +import type { DittoDB, DittoDBOpts } from './DittoDB.ts'; import type { DittoTables } from './DittoTables.ts'; -export interface DittoDatabase extends AsyncDisposable { - readonly kysely: Kysely; - readonly poolSize: number; - readonly availableConnections: number; - listen(channel: string, callback: (payload: string) => void): void; -} +export class DittoDatabase { + /** Open a new database connection. */ + static create(databaseUrl: string, opts?: DittoDBOpts): DittoDB { + const { protocol } = new URL(databaseUrl); -export interface DittoDatabaseOpts { - poolSize?: number; - debug?: 0 | 1 | 2 | 3 | 4 | 5; + switch (protocol) { + case 'file:': + case 'memory:': + return DittoPglite.create(databaseUrl, opts); + case 'postgres:': + case 'postgresql:': + return DittoPostgres.create(databaseUrl, opts); + default: + throw new Error('Unsupported database URL.'); + } + } + + /** Migrate the database to the latest version. */ + static async migrate(kysely: Kysely) { + const migrator = new Migrator({ + db: kysely, + provider: new FileMigrationProvider({ + fs, + path, + migrationFolder: new URL(import.meta.resolve('./migrations')).pathname, + }), + }); + + logi({ level: 'info', ns: 'ditto.db.migration', msg: 'Running migrations...', state: 'started' }); + const { results, error } = await migrator.migrateToLatest(); + + if (error) { + logi({ + level: 'fatal', + ns: 'ditto.db.migration', + msg: 'Migration failed.', + state: 'failed', + results: results as unknown as JsonValue, + error: error instanceof Error ? error : null, + }); + Deno.exit(1); + } else { + if (!results?.length) { + logi({ level: 'info', ns: 'ditto.db.migration', msg: 'Everything up-to-date.', state: 'skipped' }); + } else { + logi({ + level: 'info', + ns: 'ditto.db.migration', + msg: 'Migrations finished!', + state: 'migrated', + results: results as unknown as JsonValue, + }); + } + } + } } diff --git a/packages/db/adapters/DittoPglite.ts b/packages/db/adapters/DittoPglite.ts index 5e7e6ca4..9a4ad657 100644 --- a/packages/db/adapters/DittoPglite.ts +++ b/packages/db/adapters/DittoPglite.ts @@ -6,11 +6,11 @@ import { Kysely } from 'kysely'; import { KyselyLogger } from '../KyselyLogger.ts'; import { isWorker } from '../utils/worker.ts'; -import type { DittoDatabase, DittoDatabaseOpts } from '../DittoDatabase.ts'; +import type { DittoDB, DittoDBOpts } from '../DittoDB.ts'; import type { DittoTables } from '../DittoTables.ts'; export class DittoPglite { - static create(databaseUrl: string, opts?: DittoDatabaseOpts): DittoDatabase { + static create(databaseUrl: string, opts?: DittoDBOpts): DittoDB { const url = new URL(databaseUrl); if (url.protocol === 'file:' && isWorker()) { diff --git a/packages/db/adapters/DittoPostgres.ts b/packages/db/adapters/DittoPostgres.ts index b62a878b..6657a8d6 100644 --- a/packages/db/adapters/DittoPostgres.ts +++ b/packages/db/adapters/DittoPostgres.ts @@ -14,11 +14,11 @@ import postgres from 'postgres'; import { KyselyLogger } from '../KyselyLogger.ts'; -import type { DittoDatabase, DittoDatabaseOpts } from '../DittoDatabase.ts'; +import type { DittoDB, DittoDBOpts } from '../DittoDB.ts'; import type { DittoTables } from '../DittoTables.ts'; export class DittoPostgres { - static create(databaseUrl: string, opts?: DittoDatabaseOpts): DittoDatabase { + static create(databaseUrl: string, opts?: DittoDBOpts): DittoDB { const pg = postgres(databaseUrl, { max: opts?.poolSize }); const kysely = new Kysely({ diff --git a/packages/db/mod.ts b/packages/db/mod.ts index 39521f20..14c7669c 100644 --- a/packages/db/mod.ts +++ b/packages/db/mod.ts @@ -1,4 +1,4 @@ -export { DittoDB } from './DittoDB.ts'; +export { DittoDatabase } from './DittoDatabase.ts'; -export type { DittoDatabase } from './DittoDatabase.ts'; +export type { DittoDB } from './DittoDB.ts'; export type { DittoTables } from './DittoTables.ts'; diff --git a/packages/ditto/storages.ts b/packages/ditto/storages.ts index ff7b2954..dedd4081 100644 --- a/packages/ditto/storages.ts +++ b/packages/ditto/storages.ts @@ -1,5 +1,5 @@ // deno-lint-ignore-file require-await -import { type DittoDatabase, DittoDB } from '@ditto/db'; +import { DittoDatabase, type DittoDB } from '@ditto/db'; import { NPool, NRelay1 } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; @@ -12,25 +12,25 @@ import { seedZapSplits } from '@/utils/zap-split.ts'; export class Storages { private static _db: Promise | undefined; - private static _database: Promise | undefined; + private static _database: Promise | undefined; private static _admin: Promise | undefined; private static _client: Promise> | undefined; - public static async database(): Promise { + public static async database(): Promise { if (!this._database) { this._database = (async () => { - const db = DittoDB.create(Conf.databaseUrl, { + const db = DittoDatabase.create(Conf.databaseUrl, { poolSize: Conf.pg.poolSize, debug: Conf.pgliteDebug, }); - await DittoDB.migrate(db.kysely); + await DittoDatabase.migrate(db.kysely); return db; })(); } return this._database; } - public static async kysely(): Promise { + public static async kysely(): Promise { const { kysely } = await this.database(); return kysely; } diff --git a/packages/ditto/storages/DittoPgStore.ts b/packages/ditto/storages/DittoPgStore.ts index a921a309..98fad50b 100644 --- a/packages/ditto/storages/DittoPgStore.ts +++ b/packages/ditto/storages/DittoPgStore.ts @@ -1,6 +1,6 @@ // deno-lint-ignore-file require-await -import { DittoDatabase, DittoTables } from '@ditto/db'; +import { type DittoDB, type DittoTables } from '@ditto/db'; import { detectLanguage } from '@ditto/lang'; import { NPostgres, NPostgresSchema } from '@nostrify/db'; import { dbEventsCounter, internalSubscriptionsSizeGauge } from '@ditto/metrics'; @@ -50,7 +50,7 @@ interface TagConditionOpts { /** Options for the EventsDB store. */ interface DittoPgStoreOpts { /** Kysely instance to use. */ - db: DittoDatabase; + db: DittoDB; /** Pubkey of the admin account. */ pubkey: string; /** Timeout in milliseconds for database queries. */ diff --git a/packages/ditto/test.ts b/packages/ditto/test.ts index c363963f..c245eb21 100644 --- a/packages/ditto/test.ts +++ b/packages/ditto/test.ts @@ -1,4 +1,4 @@ -import { DittoDB } from '@ditto/db'; +import { DittoDatabase } from '@ditto/db'; import { NostrEvent } from '@nostrify/nostrify'; import { Conf } from '@/config.ts'; @@ -13,9 +13,9 @@ export async function eventFixture(name: string): Promise { /** Create a database for testing. It uses `DATABASE_URL`, or creates an in-memory database by default. */ export async function createTestDB(opts?: { pure?: boolean }) { - const db = DittoDB.create(Conf.databaseUrl, { poolSize: 1 }); + const db = DittoDatabase.create(Conf.databaseUrl, { poolSize: 1 }); - await DittoDB.migrate(db.kysely); + await DittoDatabase.migrate(db.kysely); const store = new DittoPgStore({ db, diff --git a/packages/ditto/workers/policy.worker.ts b/packages/ditto/workers/policy.worker.ts index 852c24b5..89ca0158 100644 --- a/packages/ditto/workers/policy.worker.ts +++ b/packages/ditto/workers/policy.worker.ts @@ -1,4 +1,4 @@ -import { DittoDB } from '@ditto/db'; +import { DittoDatabase } from '@ditto/db'; import '@soapbox/safe-fetch/load'; import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify'; import { ReadOnlyPolicy } from '@nostrify/policies'; @@ -30,7 +30,7 @@ export class CustomPolicy implements NPolicy { async init({ path, databaseUrl, pubkey }: PolicyInit): Promise { const Policy = (await import(path)).default; - const db = DittoDB.create(databaseUrl, { poolSize: 1 }); + const db = DittoDatabase.create(databaseUrl, { poolSize: 1 }); const store = new DittoPgStore({ db, From 5231c8a94f3512a9a670a27cdce62ecfc3521f38 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 11:32:04 -0600 Subject: [PATCH 34/99] Rename DittoDatabase to DittoPolyPg --- packages/db/DittoDatabase.test.ts | 6 ------ packages/db/adapters/DittoPolyPg.test.ts | 6 ++++++ .../{DittoDatabase.ts => adapters/DittoPolyPg.ts} | 13 +++++++------ packages/db/mod.ts | 2 +- packages/ditto/storages.ts | 6 +++--- packages/ditto/test.ts | 6 +++--- packages/ditto/workers/policy.worker.ts | 4 ++-- 7 files changed, 22 insertions(+), 21 deletions(-) delete mode 100644 packages/db/DittoDatabase.test.ts create mode 100644 packages/db/adapters/DittoPolyPg.test.ts rename packages/db/{DittoDatabase.ts => adapters/DittoPolyPg.ts} (83%) diff --git a/packages/db/DittoDatabase.test.ts b/packages/db/DittoDatabase.test.ts deleted file mode 100644 index a91affd5..00000000 --- a/packages/db/DittoDatabase.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { DittoDatabase } from './DittoDatabase.ts'; - -Deno.test('DittoDatabase', async () => { - const db = DittoDatabase.create('memory://'); - await DittoDatabase.migrate(db.kysely); -}); diff --git a/packages/db/adapters/DittoPolyPg.test.ts b/packages/db/adapters/DittoPolyPg.test.ts new file mode 100644 index 00000000..539a6ed0 --- /dev/null +++ b/packages/db/adapters/DittoPolyPg.test.ts @@ -0,0 +1,6 @@ +import { DittoPolyPg } from './DittoPolyPg.ts'; + +Deno.test('DittoPolyPg', async () => { + const db = DittoPolyPg.create('memory://'); + await DittoPolyPg.migrate(db.kysely); +}); diff --git a/packages/db/DittoDatabase.ts b/packages/db/adapters/DittoPolyPg.ts similarity index 83% rename from packages/db/DittoDatabase.ts rename to packages/db/adapters/DittoPolyPg.ts index 916402dd..9befe788 100644 --- a/packages/db/DittoDatabase.ts +++ b/packages/db/adapters/DittoPolyPg.ts @@ -4,14 +4,15 @@ import path from 'node:path'; import { logi } from '@soapbox/logi'; import { FileMigrationProvider, type Kysely, Migrator } from 'kysely'; -import { DittoPglite } from './adapters/DittoPglite.ts'; -import { DittoPostgres } from './adapters/DittoPostgres.ts'; +import { DittoPglite } from './DittoPglite.ts'; +import { DittoPostgres } from './DittoPostgres.ts'; import type { JsonValue } from '@std/json'; -import type { DittoDB, DittoDBOpts } from './DittoDB.ts'; -import type { DittoTables } from './DittoTables.ts'; +import type { DittoDB, DittoDBOpts } from '../DittoDB.ts'; +import type { DittoTables } from '../DittoTables.ts'; -export class DittoDatabase { +/** Creates either a PGlite or Postgres connection depending on the databaseUrl. */ +export class DittoPolyPg { /** Open a new database connection. */ static create(databaseUrl: string, opts?: DittoDBOpts): DittoDB { const { protocol } = new URL(databaseUrl); @@ -51,7 +52,7 @@ export class DittoDatabase { results: results as unknown as JsonValue, error: error instanceof Error ? error : null, }); - Deno.exit(1); + throw new Error('Migration failed.'); } else { if (!results?.length) { logi({ level: 'info', ns: 'ditto.db.migration', msg: 'Everything up-to-date.', state: 'skipped' }); diff --git a/packages/db/mod.ts b/packages/db/mod.ts index 14c7669c..49100cd6 100644 --- a/packages/db/mod.ts +++ b/packages/db/mod.ts @@ -1,4 +1,4 @@ -export { DittoDatabase } from './DittoDatabase.ts'; +export { DittoPolyPg } from './adapters/DittoPolyPg.ts'; export type { DittoDB } from './DittoDB.ts'; export type { DittoTables } from './DittoTables.ts'; diff --git a/packages/ditto/storages.ts b/packages/ditto/storages.ts index dedd4081..320714f7 100644 --- a/packages/ditto/storages.ts +++ b/packages/ditto/storages.ts @@ -1,5 +1,5 @@ // deno-lint-ignore-file require-await -import { DittoDatabase, type DittoDB } from '@ditto/db'; +import { type DittoDB, DittoPolyPg } from '@ditto/db'; import { NPool, NRelay1 } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; @@ -19,11 +19,11 @@ export class Storages { public static async database(): Promise { if (!this._database) { this._database = (async () => { - const db = DittoDatabase.create(Conf.databaseUrl, { + const db = DittoPolyPg.create(Conf.databaseUrl, { poolSize: Conf.pg.poolSize, debug: Conf.pgliteDebug, }); - await DittoDatabase.migrate(db.kysely); + await DittoPolyPg.migrate(db.kysely); return db; })(); } diff --git a/packages/ditto/test.ts b/packages/ditto/test.ts index c245eb21..eb472ffa 100644 --- a/packages/ditto/test.ts +++ b/packages/ditto/test.ts @@ -1,4 +1,4 @@ -import { DittoDatabase } from '@ditto/db'; +import { DittoPolyPg } from '@ditto/db'; import { NostrEvent } from '@nostrify/nostrify'; import { Conf } from '@/config.ts'; @@ -13,9 +13,9 @@ export async function eventFixture(name: string): Promise { /** Create a database for testing. It uses `DATABASE_URL`, or creates an in-memory database by default. */ export async function createTestDB(opts?: { pure?: boolean }) { - const db = DittoDatabase.create(Conf.databaseUrl, { poolSize: 1 }); + const db = DittoPolyPg.create(Conf.databaseUrl, { poolSize: 1 }); - await DittoDatabase.migrate(db.kysely); + await DittoPolyPg.migrate(db.kysely); const store = new DittoPgStore({ db, diff --git a/packages/ditto/workers/policy.worker.ts b/packages/ditto/workers/policy.worker.ts index 89ca0158..49fc75ef 100644 --- a/packages/ditto/workers/policy.worker.ts +++ b/packages/ditto/workers/policy.worker.ts @@ -1,4 +1,4 @@ -import { DittoDatabase } from '@ditto/db'; +import { DittoPolyPg } from '@ditto/db'; import '@soapbox/safe-fetch/load'; import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify'; import { ReadOnlyPolicy } from '@nostrify/policies'; @@ -30,7 +30,7 @@ export class CustomPolicy implements NPolicy { async init({ path, databaseUrl, pubkey }: PolicyInit): Promise { const Policy = (await import(path)).default; - const db = DittoDatabase.create(databaseUrl, { poolSize: 1 }); + const db = DittoPolyPg.create(databaseUrl, { poolSize: 1 }); const store = new DittoPgStore({ db, From 44c4b3188c532f0ee5d7fa29201eb7ccdca4be3d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 11:35:37 -0600 Subject: [PATCH 35/99] DittoPolyPg: fix path to migrations --- packages/db/adapters/DittoPolyPg.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/db/adapters/DittoPolyPg.ts b/packages/db/adapters/DittoPolyPg.ts index 9befe788..623ee9fc 100644 --- a/packages/db/adapters/DittoPolyPg.ts +++ b/packages/db/adapters/DittoPolyPg.ts @@ -36,7 +36,7 @@ export class DittoPolyPg { provider: new FileMigrationProvider({ fs, path, - migrationFolder: new URL(import.meta.resolve('./migrations')).pathname, + migrationFolder: new URL(import.meta.resolve('../migrations')).pathname, }), }); From 0841563d6981953822ca8589cd9116b0d44c66df Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 12:04:52 -0600 Subject: [PATCH 36/99] Remove AdminSigner, Conf.pubkey, Conf.nsec, add Conf.signer --- packages/api/middleware/confMw.test.ts | 2 +- packages/conf/DittoConf.test.ts | 15 ++++--- packages/conf/DittoConf.ts | 42 ++++++++++--------- packages/ditto/controllers/api/accounts.ts | 4 +- packages/ditto/controllers/api/admin.ts | 22 +++++++--- packages/ditto/controllers/api/ditto.ts | 26 +++++++----- packages/ditto/controllers/api/instance.ts | 4 +- .../ditto/controllers/api/notifications.ts | 2 +- packages/ditto/controllers/api/pleroma.ts | 8 ++-- packages/ditto/controllers/api/reports.ts | 4 +- packages/ditto/controllers/api/statuses.ts | 2 +- packages/ditto/controllers/api/suggestions.ts | 17 +++++--- packages/ditto/controllers/api/timelines.ts | 2 +- packages/ditto/controllers/api/trends.ts | 6 +-- .../ditto/controllers/nostr/relay-info.ts | 2 +- packages/ditto/middleware/auth98Middleware.ts | 2 +- packages/ditto/pipeline.ts | 16 ++++--- packages/ditto/signers/AdminSigner.ts | 9 ---- packages/ditto/storages.ts | 4 +- packages/ditto/storages/AdminStore.ts | 6 ++- packages/ditto/storages/hydrate.bench.ts | 2 +- packages/ditto/storages/hydrate.ts | 17 ++++---- packages/ditto/test.ts | 2 +- packages/ditto/trends.ts | 3 +- packages/ditto/utils/api.ts | 5 +-- packages/ditto/utils/connect.ts | 2 +- packages/ditto/utils/instance.ts | 2 +- packages/ditto/utils/nip05.ts | 2 +- packages/ditto/utils/outbox.ts | 2 +- packages/ditto/utils/pleroma.ts | 6 +-- packages/ditto/utils/zap-split.ts | 5 +-- .../ditto/views/mastodon/notifications.ts | 4 +- packages/ditto/workers/policy.ts | 2 +- scripts/admin-event.ts | 4 +- scripts/admin-role.ts | 4 +- scripts/setup-kind0.ts | 3 +- 36 files changed, 135 insertions(+), 125 deletions(-) delete mode 100644 packages/ditto/signers/AdminSigner.ts diff --git a/packages/api/middleware/confMw.test.ts b/packages/api/middleware/confMw.test.ts index 5eac707c..350a585f 100644 --- a/packages/api/middleware/confMw.test.ts +++ b/packages/api/middleware/confMw.test.ts @@ -10,7 +10,7 @@ Deno.test('confMw', async () => { const app = new Hono(); - app.get('/', confMw(env), (c) => c.text(c.var.conf.pubkey)); + app.get('/', confMw(env), async (c) => c.text(await c.var.conf.signer.getPublicKey())); const response = await app.request('/'); const body = await response.text(); diff --git a/packages/conf/DittoConf.test.ts b/packages/conf/DittoConf.test.ts index c2e87c46..b6c2b707 100644 --- a/packages/conf/DittoConf.test.ts +++ b/packages/conf/DittoConf.test.ts @@ -9,12 +9,11 @@ Deno.test('DittoConfig', async (t) => { const config = new DittoConf(env); - await t.step('nsec', () => { - assertEquals(config.nsec, 'nsec19shyxpuzd0cq2p5078fwnws7tyykypud6z205fzhlmlrs2vpz6hs83zwkw'); - }); - - await t.step('pubkey', () => { - assertEquals(config.pubkey, '1ba0c5ed1bbbf3b7eb0d7843ba16836a0201ea68a76bafcba507358c45911ff6'); + await t.step('signer', async () => { + assertEquals( + await config.signer.getPublicKey(), + '1ba0c5ed1bbbf3b7eb0d7843ba16836a0201ea68a76bafcba507358c45911ff6', + ); }); }); @@ -22,8 +21,8 @@ Deno.test('DittoConfig defaults', async (t) => { const env = new Map(); const config = new DittoConf(env); - await t.step('nsec throws', () => { - assertThrows(() => config.nsec); + await t.step('signer throws', () => { + assertThrows(() => config.signer); }); await t.step('port', () => { diff --git a/packages/conf/DittoConf.ts b/packages/conf/DittoConf.ts index 456e9cd2..b7f5be79 100644 --- a/packages/conf/DittoConf.ts +++ b/packages/conf/DittoConf.ts @@ -1,10 +1,11 @@ import os from 'node:os'; import path from 'node:path'; -import ISO6391, { type LanguageCode } from 'iso-639-1'; -import { getPublicKey, nip19 } from 'nostr-tools'; +import { NSecSigner } from '@nostrify/nostrify'; import { decodeBase64 } from '@std/encoding/base64'; import { encodeBase64Url } from '@std/encoding/base64url'; +import ISO6391, { type LanguageCode } from 'iso-639-1'; +import { nip19 } from 'nostr-tools'; import { getEcdsaPublicKey } from './utils/crypto.ts'; import { optionalBooleanSchema, optionalNumberSchema } from './utils/schema.ts'; @@ -14,35 +15,36 @@ import { mergeURLPath } from './utils/url.ts'; export class DittoConf { constructor(private env: { get(key: string): string | undefined }) {} - /** Cached parsed admin pubkey value. */ - private _pubkey: string | undefined; + /** Cached parsed admin signer. */ + private _signer: NSecSigner | undefined; /** Cached parsed VAPID public key value. */ private _vapidPublicKey: Promise | undefined; - /** Ditto admin secret key in nip19 format. This is the way it's configured by an admin. */ - get nsec(): `nsec1${string}` { - const value = this.env.get('DITTO_NSEC'); - if (!value) { + /** + * Ditto admin secret key in hex format. + * @deprecated Use `signer` instead. TODO: handle auth tokens. + */ + get seckey(): Uint8Array { + const nsec = this.env.get('DITTO_NSEC'); + + if (!nsec) { throw new Error('Missing DITTO_NSEC'); } - if (!value.startsWith('nsec1')) { + + if (!nsec.startsWith('nsec1')) { throw new Error('Invalid DITTO_NSEC'); } - return value as `nsec1${string}`; + + return nip19.decode(nsec as `nsec1${string}`).data; } - /** Ditto admin secret key in hex format. */ - get seckey(): Uint8Array { - return nip19.decode(this.nsec).data; - } - - /** Ditto admin public key in hex format. */ - get pubkey(): string { - if (!this._pubkey) { - this._pubkey = getPublicKey(this.seckey); + /** Ditto admin signer. */ + get signer(): NSecSigner { + if (!this._signer) { + this._signer = new NSecSigner(this.seckey); } - return this._pubkey; + return this._signer; } /** Port to use when serving the HTTP server. */ diff --git a/packages/ditto/controllers/api/accounts.ts b/packages/ditto/controllers/api/accounts.ts index 8a1b9e3d..27710063 100644 --- a/packages/ditto/controllers/api/accounts.ts +++ b/packages/ditto/controllers/api/accounts.ts @@ -211,7 +211,9 @@ const accountStatusesController: AppController = async (c) => { const [[author], [user]] = await Promise.all([ store.query([{ kinds: [0], authors: [pubkey], limit: 1 }], { signal }), - store.query([{ kinds: [30382], authors: [conf.pubkey], '#d': [pubkey], limit: 1 }], { signal }), + store.query([{ kinds: [30382], authors: [await conf.signer.getPublicKey()], '#d': [pubkey], limit: 1 }], { + signal, + }), ]); if (author) { diff --git a/packages/ditto/controllers/api/admin.ts b/packages/ditto/controllers/api/admin.ts index 1e3b4615..9e9ba5d0 100644 --- a/packages/ditto/controllers/api/admin.ts +++ b/packages/ditto/controllers/api/admin.ts @@ -43,13 +43,15 @@ const adminAccountsController: AppController = async (c) => { staff, } = adminAccountQuerySchema.parse(c.req.query()); + const adminPubkey = await conf.signer.getPublicKey(); + if (pending) { if (disabled || silenced || suspended || sensitized) { return c.json([]); } const orig = await store.query( - [{ kinds: [30383], authors: [conf.pubkey], '#k': ['3036'], '#n': ['pending'], ...params }], + [{ kinds: [30383], authors: [adminPubkey], '#k': ['3036'], '#n': ['pending'], ...params }], { signal }, ); @@ -86,7 +88,10 @@ const adminAccountsController: AppController = async (c) => { n.push('moderator'); } - const events = await store.query([{ kinds: [30382], authors: [conf.pubkey], '#n': n, ...params }], { signal }); + const events = await store.query( + [{ kinds: [30382], authors: [adminPubkey], '#n': n, ...params }], + { signal }, + ); const pubkeys = new Set( events @@ -157,9 +162,11 @@ const adminActionController: AppController = async (c) => { } if (data.type === 'revoke_name') { n.revoke_name = true; - store.remove([{ kinds: [30360], authors: [conf.pubkey], '#p': [authorId] }]).catch((e: unknown) => { - logi({ level: 'error', ns: 'ditto.api.admin.account.action', type: data.type, error: errorJson(e) }); - }); + store.remove([{ kinds: [30360], authors: [await conf.signer.getPublicKey()], '#p': [authorId] }]).catch( + (e: unknown) => { + logi({ level: 'error', ns: 'ditto.api.admin.account.action', type: data.type, error: errorJson(e) }); + }, + ); } await updateUser(authorId, n, c); @@ -185,7 +192,10 @@ const adminApproveController: AppController = async (c) => { return c.json({ error: 'Invalid NIP-05' }, 400); } - const [existing] = await store.query([{ kinds: [30360], authors: [conf.pubkey], '#d': [r], limit: 1 }]); + const [existing] = await store.query([ + { kinds: [30360], authors: [await conf.signer.getPublicKey()], '#d': [r], limit: 1 }, + ]); + if (existing) { return c.json({ error: 'NIP-05 already granted to another user' }, 400); } diff --git a/packages/ditto/controllers/api/ditto.ts b/packages/ditto/controllers/api/ditto.ts index b9fef08a..752124dc 100644 --- a/packages/ditto/controllers/api/ditto.ts +++ b/packages/ditto/controllers/api/ditto.ts @@ -9,7 +9,6 @@ import { createEvent, paginated, parseBody, updateAdminEvent } from '@/utils/api import { getInstanceMetadata } from '@/utils/instance.ts'; import { deleteTag } from '@/utils/tags.ts'; import { DittoZapSplits, getZapSplits } from '@/utils/zap-split.ts'; -import { AdminSigner } from '@/signers/AdminSigner.ts'; import { screenshotsSchema } from '@/schemas/nostr.ts'; import { booleanParamSchema, percentageSchema, wsUrlSchema } from '@/schema.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; @@ -33,7 +32,7 @@ export const adminRelaysController: AppController = async (c) => { const store = await Storages.db(); const [event] = await store.query([ - { kinds: [10002], authors: [conf.pubkey], limit: 1 }, + { kinds: [10002], authors: [await conf.signer.getPublicKey()], limit: 1 }, ]); if (!event) { @@ -44,10 +43,11 @@ export const adminRelaysController: AppController = async (c) => { }; export const adminSetRelaysController: AppController = async (c) => { + const { conf } = c.var; const store = await Storages.db(); const relays = relaySchema.array().parse(await c.req.json()); - const event = await new AdminSigner().signEvent({ + const event = await conf.signer.signEvent({ kind: 10002, tags: relays.map(({ url, marker }) => marker ? ['r', url, marker] : ['r', url]), content: '', @@ -98,7 +98,7 @@ export const nameRequestController: AppController = async (c) => { ['r', name], ['L', 'nip05.domain'], ['l', name.split('@')[1], 'nip05.domain'], - ['p', conf.pubkey], + ['p', await conf.signer.getPublicKey()], ], }, c); @@ -124,7 +124,7 @@ export const nameRequestsController: AppController = async (c) => { const filter: NostrFilter = { kinds: [30383], - authors: [conf.pubkey], + authors: [await conf.signer.getPublicKey()], '#k': ['3036'], '#p': [pubkey], ...params, @@ -179,7 +179,9 @@ export const updateZapSplitsController: AppController = async (c) => { return c.json({ error: result.error }, 400); } - const dittoZapSplit = await getZapSplits(store, conf.pubkey); + const adminPubkey = await conf.signer.getPublicKey(); + + const dittoZapSplit = await getZapSplits(store, adminPubkey); if (!dittoZapSplit) { return c.json({ error: 'Zap split not activated, restart the server.' }, 404); } @@ -192,7 +194,7 @@ export const updateZapSplitsController: AppController = async (c) => { } await updateListAdminEvent( - { kinds: [30078], authors: [conf.pubkey], '#d': ['pub.ditto.zapSplits'], limit: 1 }, + { kinds: [30078], authors: [adminPubkey], '#d': ['pub.ditto.zapSplits'], limit: 1 }, (tags) => pubkeys.reduce((accumulator, pubkey) => { return addTag(accumulator, ['p', pubkey, data[pubkey].weight.toString(), data[pubkey].message]); @@ -215,7 +217,9 @@ export const deleteZapSplitsController: AppController = async (c) => { return c.json({ error: result.error }, 400); } - const dittoZapSplit = await getZapSplits(store, conf.pubkey); + const adminPubkey = await conf.signer.getPublicKey(); + + const dittoZapSplit = await getZapSplits(store, adminPubkey); if (!dittoZapSplit) { return c.json({ error: 'Zap split not activated, restart the server.' }, 404); } @@ -223,7 +227,7 @@ export const deleteZapSplitsController: AppController = async (c) => { const { data } = result; await updateListAdminEvent( - { kinds: [30078], authors: [conf.pubkey], '#d': ['pub.ditto.zapSplits'], limit: 1 }, + { kinds: [30078], authors: [adminPubkey], '#d': ['pub.ditto.zapSplits'], limit: 1 }, (tags) => data.reduce((accumulator, currentValue) => { return deleteTag(accumulator, ['p', currentValue]); @@ -238,7 +242,7 @@ export const getZapSplitsController: AppController = async (c) => { const { conf } = c.var; const store = c.get('store'); - const dittoZapSplit: DittoZapSplits | undefined = await getZapSplits(store, conf.pubkey) ?? {}; + const dittoZapSplit: DittoZapSplits | undefined = await getZapSplits(store, await conf.signer.getPublicKey()) ?? {}; if (!dittoZapSplit) { return c.json({ error: 'Zap split not activated, restart the server.' }, 404); } @@ -311,7 +315,7 @@ export const updateInstanceController: AppController = async (c) => { const { conf } = c.var; const body = await parseBody(c.req.raw); const result = updateInstanceSchema.safeParse(body); - const pubkey = conf.pubkey; + const pubkey = await conf.signer.getPublicKey(); if (!result.success) { return c.json(result.error, 422); diff --git a/packages/ditto/controllers/api/instance.ts b/packages/ditto/controllers/api/instance.ts index d17a91c1..8c3c6e4c 100644 --- a/packages/ditto/controllers/api/instance.ts +++ b/packages/ditto/controllers/api/instance.ts @@ -68,7 +68,7 @@ const instanceV1Controller: AppController = async (c) => { version, email: meta.email, nostr: { - pubkey: conf.pubkey, + pubkey: await conf.signer.getPublicKey(), relay: `${wsProtocol}//${host}/relay`, }, rules: [], @@ -141,7 +141,7 @@ const instanceV2Controller: AppController = async (c) => { }, }, nostr: { - pubkey: conf.pubkey, + pubkey: await conf.signer.getPublicKey(), relay: `${wsProtocol}//${host}/relay`, }, pleroma: { diff --git a/packages/ditto/controllers/api/notifications.ts b/packages/ditto/controllers/api/notifications.ts index fd8b5720..dfd4a03c 100644 --- a/packages/ditto/controllers/api/notifications.ts +++ b/packages/ditto/controllers/api/notifications.ts @@ -68,7 +68,7 @@ const notificationsController: AppController = async (c) => { } if (types.has('ditto:name_grant') && !account_id) { - filters.push({ kinds: [30360], authors: [conf.pubkey], '#p': [pubkey], ...params }); + filters.push({ kinds: [30360], authors: [await conf.signer.getPublicKey()], '#p': [pubkey], ...params }); } return renderNotifications(filters, types, params, c); diff --git a/packages/ditto/controllers/api/pleroma.ts b/packages/ditto/controllers/api/pleroma.ts index 302eaca6..721347f3 100644 --- a/packages/ditto/controllers/api/pleroma.ts +++ b/packages/ditto/controllers/api/pleroma.ts @@ -2,7 +2,6 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { configSchema, elixirTupleSchema } from '@/schemas/pleroma-api.ts'; -import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Storages } from '@/storages.ts'; import { createAdminEvent, updateAdminEvent, updateUser } from '@/utils/api.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; @@ -34,7 +33,6 @@ const configController: AppController = async (c) => { /** Pleroma admin config controller. */ const updateConfigController: AppController = async (c) => { const { conf } = c.var; - const { pubkey } = conf; const store = await Storages.db(); const configs = await getPleromaConfigs(store, c.req.raw.signal); @@ -44,7 +42,7 @@ const updateConfigController: AppController = async (c) => { await createAdminEvent({ kind: 30078, - content: await new AdminSigner().nip44.encrypt(pubkey, JSON.stringify(configs)), + content: await conf.signer.nip44.encrypt(await conf.signer.getPublicKey(), JSON.stringify(configs)), tags: [ ['d', 'pub.ditto.pleroma.config'], ['encrypted', 'nip44'], @@ -77,7 +75,7 @@ const pleromaAdminTagController: AppController = async (c) => { if (!pubkey) continue; await updateAdminEvent( - { kinds: [30382], authors: [conf.pubkey], '#d': [pubkey], limit: 1 }, + { kinds: [30382], authors: [await conf.signer.getPublicKey()], '#d': [pubkey], limit: 1 }, (prev) => { const tags = prev?.tags ?? [['d', pubkey]]; @@ -110,7 +108,7 @@ const pleromaAdminUntagController: AppController = async (c) => { if (!pubkey) continue; await updateAdminEvent( - { kinds: [30382], authors: [conf.pubkey], '#d': [pubkey], limit: 1 }, + { kinds: [30382], authors: [await conf.signer.getPublicKey()], '#d': [pubkey], limit: 1 }, (prev) => ({ kind: 30382, content: prev?.content ?? '', diff --git a/packages/ditto/controllers/api/reports.ts b/packages/ditto/controllers/api/reports.ts index b25e7233..11285825 100644 --- a/packages/ditto/controllers/api/reports.ts +++ b/packages/ditto/controllers/api/reports.ts @@ -36,7 +36,7 @@ const reportController: AppController = async (c) => { const tags = [ ['p', account_id, category], - ['P', conf.pubkey], + ['P', await conf.signer.getPublicKey()], ]; for (const status of status_ids) { @@ -70,7 +70,7 @@ const adminReportsController: AppController = async (c) => { const filter: NostrFilter = { kinds: [30383], - authors: [conf.pubkey], + authors: [await conf.signer.getPublicKey()], '#k': ['1984'], ...params, }; diff --git a/packages/ditto/controllers/api/statuses.ts b/packages/ditto/controllers/api/statuses.ts index 7c2276c7..6aa53308 100644 --- a/packages/ditto/controllers/api/statuses.ts +++ b/packages/ditto/controllers/api/statuses.ts @@ -196,7 +196,7 @@ const createStatusController: AppController = async (c) => { if (conf.zapSplitsEnabled) { const meta = n.json().pipe(n.metadata()).catch({}).parse(author?.content); const lnurl = getLnurl(meta); - const dittoZapSplit = await getZapSplits(store, conf.pubkey); + const dittoZapSplit = await getZapSplits(store, await conf.signer.getPublicKey()); if (lnurl && dittoZapSplit) { const totalSplit = Object.values(dittoZapSplit).reduce((total, { weight }) => total + weight, 0); for (const zapPubkey in dittoZapSplit) { diff --git a/packages/ditto/controllers/api/suggestions.ts b/packages/ditto/controllers/api/suggestions.ts index 0a85b95b..5dbf0d14 100644 --- a/packages/ditto/controllers/api/suggestions.ts +++ b/packages/ditto/controllers/api/suggestions.ts @@ -31,8 +31,8 @@ async function renderV2Suggestions(c: AppContext, params: { offset: number; limi const pubkey = await signer?.getPublicKey(); const filters: NostrFilter[] = [ - { kinds: [30382], authors: [conf.pubkey], '#n': ['suggested'], limit }, - { kinds: [1985], '#L': ['pub.ditto.trends'], '#l': [`#p`], authors: [conf.pubkey], limit: 1 }, + { kinds: [30382], authors: [await conf.signer.getPublicKey()], '#n': ['suggested'], limit }, + { kinds: [1985], '#L': ['pub.ditto.trends'], '#l': [`#p`], authors: [await conf.signer.getPublicKey()], limit: 1 }, ]; if (pubkey) { @@ -41,13 +41,20 @@ async function renderV2Suggestions(c: AppContext, params: { offset: number; limi } const events = await store.query(filters, { signal }); + const adminPubkey = await conf.signer.getPublicKey(); const [userEvents, followsEvent, mutesEvent, trendingEvent] = [ - events.filter((event) => matchFilter({ kinds: [30382], authors: [conf.pubkey], '#n': ['suggested'] }, event)), + events.filter((event) => matchFilter({ kinds: [30382], authors: [adminPubkey], '#n': ['suggested'] }, event)), pubkey ? events.find((event) => matchFilter({ kinds: [3], authors: [pubkey] }, event)) : undefined, pubkey ? events.find((event) => matchFilter({ kinds: [10000], authors: [pubkey] }, event)) : undefined, events.find((event) => - matchFilter({ kinds: [1985], '#L': ['pub.ditto.trends'], '#l': [`#p`], authors: [conf.pubkey], limit: 1 }, event) + matchFilter({ + kinds: [1985], + '#L': ['pub.ditto.trends'], + '#l': [`#p`], + authors: [adminPubkey], + limit: 1, + }, event) ), ]; @@ -95,7 +102,7 @@ export const localSuggestionsController: AppController = async (c) => { const store = c.get('store'); const grants = await store.query( - [{ kinds: [30360], authors: [conf.pubkey], ...params }], + [{ kinds: [30360], authors: [await conf.signer.getPublicKey()], ...params }], { signal }, ); diff --git a/packages/ditto/controllers/api/timelines.ts b/packages/ditto/controllers/api/timelines.ts index e8b8987a..a6f872b9 100644 --- a/packages/ditto/controllers/api/timelines.ts +++ b/packages/ditto/controllers/api/timelines.ts @@ -95,7 +95,7 @@ const suggestedTimelineController: AppController = async (c) => { const params = c.get('pagination'); const [follows] = await store.query( - [{ kinds: [3], authors: [conf.pubkey], limit: 1 }], + [{ kinds: [3], authors: [await conf.signer.getPublicKey()], limit: 1 }], ); const authors = [...getTagSet(follows?.tags ?? [], 'p')]; diff --git a/packages/ditto/controllers/api/trends.ts b/packages/ditto/controllers/api/trends.ts index 88ea335e..f14cf0b7 100644 --- a/packages/ditto/controllers/api/trends.ts +++ b/packages/ditto/controllers/api/trends.ts @@ -53,7 +53,7 @@ const trendingTagsController: AppController = async (c) => { async function getTrendingHashtags(conf: DittoConf) { const store = await Storages.db(); - const trends = await getTrendingTags(store, 't', conf.pubkey); + const trends = await getTrendingTags(store, 't', await conf.signer.getPublicKey()); return trends.map((trend) => { const hashtag = trend.value; @@ -106,7 +106,7 @@ const trendingLinksController: AppController = async (c) => { async function getTrendingLinks(conf: DittoConf) { const store = await Storages.db(); - const trends = await getTrendingTags(store, 'r', conf.pubkey); + const trends = await getTrendingTags(store, 'r', await conf.signer.getPublicKey()); return Promise.all(trends.map(async (trend) => { const link = trend.value; @@ -148,7 +148,7 @@ const trendingStatusesController: AppController = async (c) => { kinds: [1985], '#L': ['pub.ditto.trends'], '#l': ['#e'], - authors: [conf.pubkey], + authors: [await conf.signer.getPublicKey()], until, limit: 1, }]); diff --git a/packages/ditto/controllers/nostr/relay-info.ts b/packages/ditto/controllers/nostr/relay-info.ts index 54576b38..d4721cdc 100644 --- a/packages/ditto/controllers/nostr/relay-info.ts +++ b/packages/ditto/controllers/nostr/relay-info.ts @@ -14,7 +14,7 @@ const relayInfoController: AppController = async (c) => { return c.json({ name: meta.name, description: meta.about, - pubkey: conf.pubkey, + pubkey: await conf.signer.getPublicKey(), contact: meta.email, supported_nips: [1, 5, 9, 11, 16, 45, 50, 46, 98], software: 'Ditto', diff --git a/packages/ditto/middleware/auth98Middleware.ts b/packages/ditto/middleware/auth98Middleware.ts index 889e5ea9..573853f0 100644 --- a/packages/ditto/middleware/auth98Middleware.ts +++ b/packages/ditto/middleware/auth98Middleware.ts @@ -40,7 +40,7 @@ function requireRole(role: UserRole, opts?: ParseAuthRequestOpts): AppMiddleware const [user] = await store.query([{ kinds: [30382], - authors: [conf.pubkey], + authors: [await conf.signer.getPublicKey()], '#d': [proof.pubkey], limit: 1, }]); diff --git a/packages/ditto/pipeline.ts b/packages/ditto/pipeline.ts index 07be1bd9..602d0e2b 100644 --- a/packages/ditto/pipeline.ts +++ b/packages/ditto/pipeline.ts @@ -11,7 +11,6 @@ import { Conf } from '@/config.ts'; import { DittoPush } from '@/DittoPush.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { RelayError } from '@/RelayError.ts'; -import { AdminSigner } from '@/signers/AdminSigner.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; import { eventAge, Time } from '@/utils.ts'; @@ -83,7 +82,7 @@ async function handleEvent(event: DittoEvent, opts: PipelineOpts): Promise } // Ensure the event doesn't violate the policy. - if (event.pubkey !== Conf.pubkey) { + if (event.pubkey !== await Conf.signer.getPublicKey()) { await policyFilter(event, opts.signal); } @@ -297,11 +296,12 @@ async function webPush(event: NostrEvent): Promise { } async function generateSetEvents(event: NostrEvent): Promise { - const tagsAdmin = event.tags.some(([name, value]) => ['p', 'P'].includes(name) && value === Conf.pubkey); + const signer = Conf.signer; + const pubkey = await signer.getPublicKey(); + + const tagsAdmin = event.tags.some(([name, value]) => ['p', 'P'].includes(name) && value === pubkey); if (event.kind === 1984 && tagsAdmin) { - const signer = new AdminSigner(); - const rel = await signer.signEvent({ kind: 30383, content: '', @@ -310,8 +310,8 @@ async function generateSetEvents(event: NostrEvent): Promise { ['p', event.pubkey], ['k', '1984'], ['n', 'open'], - ...[...getTagSet(event.tags, 'p')].map((pubkey) => ['P', pubkey]), - ...[...getTagSet(event.tags, 'e')].map((pubkey) => ['e', pubkey]), + ...[...getTagSet(event.tags, 'p')].map((value) => ['P', value]), + ...[...getTagSet(event.tags, 'e')].map((value) => ['e', value]), ], created_at: Math.floor(Date.now() / 1000), }); @@ -320,8 +320,6 @@ async function generateSetEvents(event: NostrEvent): Promise { } if (event.kind === 3036 && tagsAdmin) { - const signer = new AdminSigner(); - const rel = await signer.signEvent({ kind: 30383, content: '', diff --git a/packages/ditto/signers/AdminSigner.ts b/packages/ditto/signers/AdminSigner.ts deleted file mode 100644 index 5aea2e21..00000000 --- a/packages/ditto/signers/AdminSigner.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { NSecSigner } from '@nostrify/nostrify'; -import { Conf } from '@/config.ts'; - -/** Sign events as the Ditto server. */ -export class AdminSigner extends NSecSigner { - constructor() { - super(Conf.seckey); - } -} diff --git a/packages/ditto/storages.ts b/packages/ditto/storages.ts index 320714f7..7b77a037 100644 --- a/packages/ditto/storages.ts +++ b/packages/ditto/storages.ts @@ -42,7 +42,7 @@ export class Storages { const db = await this.database(); const store = new DittoPgStore({ db, - pubkey: Conf.pubkey, + pubkey: await Conf.signer.getPublicKey(), timeout: Conf.db.timeouts.default, notify: Conf.notifyEnabled, }); @@ -68,7 +68,7 @@ export class Storages { const db = await this.db(); const [relayList] = await db.query([ - { kinds: [10002], authors: [Conf.pubkey], limit: 1 }, + { kinds: [10002], authors: [await Conf.signer.getPublicKey()], limit: 1 }, ]); const tags = relayList?.tags ?? []; diff --git a/packages/ditto/storages/AdminStore.ts b/packages/ditto/storages/AdminStore.ts index 4ebe2743..ae03c59d 100644 --- a/packages/ditto/storages/AdminStore.ts +++ b/packages/ditto/storages/AdminStore.ts @@ -18,15 +18,17 @@ export class AdminStore implements NStore { const users = await this.store.query([{ kinds: [30382], - authors: [Conf.pubkey], + authors: [await Conf.signer.getPublicKey()], '#d': [...pubkeys], limit: pubkeys.size, }]); + const adminPubkey = await Conf.signer.getPublicKey(); + return events.filter((event) => { const user = users.find( ({ kind, pubkey, tags }) => - kind === 30382 && pubkey === Conf.pubkey && tags.find(([name]) => name === 'd')?.[1] === event.pubkey, + kind === 30382 && pubkey === adminPubkey && tags.find(([name]) => name === 'd')?.[1] === event.pubkey, ); const n = getTagSet(user?.tags ?? [], 'n'); diff --git a/packages/ditto/storages/hydrate.bench.ts b/packages/ditto/storages/hydrate.bench.ts index 026b1f81..4da8afbf 100644 --- a/packages/ditto/storages/hydrate.bench.ts +++ b/packages/ditto/storages/hydrate.bench.ts @@ -10,5 +10,5 @@ const testStats = JSON.parse(await Deno.readTextFile('fixtures/stats.json')); const events = testEvents.slice(0, 20); Deno.bench('assembleEvents with home feed', () => { - assembleEvents(events, testEvents, testStats); + assembleEvents('', events, testEvents, testStats); }); diff --git a/packages/ditto/storages/hydrate.ts b/packages/ditto/storages/hydrate.ts index 0836bd76..96341a1f 100644 --- a/packages/ditto/storages/hydrate.ts +++ b/packages/ditto/storages/hydrate.ts @@ -79,15 +79,18 @@ async function hydrateEvents(opts: HydrateOpts): Promise { // Dedupe events. const results = [...new Map(cache.map((event) => [event.id, event])).values()]; + const admin = await Conf.signer.getPublicKey(); + // First connect all the events to each-other, then connect the connected events to the original list. - assembleEvents(results, results, stats); - assembleEvents(events, results, stats); + assembleEvents(admin, results, results, stats); + assembleEvents(admin, events, results, stats); return events; } /** Connect the events in list `b` to the DittoEvent fields in list `a`. */ export function assembleEvents( + admin: string, a: DittoEvent[], b: DittoEvent[], stats: { @@ -96,8 +99,6 @@ export function assembleEvents( favicons: Record; }, ): DittoEvent[] { - const admin = Conf.pubkey; - const authorStats = stats.authors.reduce((result, { pubkey, ...stat }) => { result[pubkey] = { ...stat, @@ -316,7 +317,7 @@ async function gatherProfiles({ events, store, signal }: HydrateOpts): Promise { +async function gatherUsers({ events, store, signal }: HydrateOpts): Promise { const pubkeys = new Set(events.map((event) => event.pubkey)); if (!pubkeys.size) { @@ -324,13 +325,13 @@ function gatherUsers({ events, store, signal }: HydrateOpts): Promise { +async function gatherInfo({ events, store, signal }: HydrateOpts): Promise { const ids = new Set(); for (const event of events) { @@ -344,7 +345,7 @@ function gatherInfo({ events, store, signal }: HydrateOpts): Promise { - const signer = new AdminSigner(); + const signer = Conf.signer; const event = await signer.signEvent({ content: '', @@ -126,7 +125,7 @@ function updateEventInfo(id: string, n: Record, c: AppContext): } async function updateNames(k: number, d: string, n: Record, c: AppContext): Promise { - const signer = new AdminSigner(); + const signer = Conf.signer; const admin = await signer.getPublicKey(); return updateAdminEvent( diff --git a/packages/ditto/utils/connect.ts b/packages/ditto/utils/connect.ts index 7726fa89..095b93c4 100644 --- a/packages/ditto/utils/connect.ts +++ b/packages/ditto/utils/connect.ts @@ -20,7 +20,7 @@ export async function getClientConnectUri(signal?: AbortSignal): Promise url: Conf.localDomain, }; - uri.host = Conf.pubkey; + uri.host = await Conf.signer.getPublicKey(); uri.searchParams.set('relay', Conf.relay); uri.searchParams.set('metadata', JSON.stringify(metadata)); diff --git a/packages/ditto/utils/instance.ts b/packages/ditto/utils/instance.ts index c0b9c0d4..3f746e07 100644 --- a/packages/ditto/utils/instance.ts +++ b/packages/ditto/utils/instance.ts @@ -18,7 +18,7 @@ export interface InstanceMetadata extends NostrMetadata { /** Get and parse instance metadata from the kind 0 of the admin user. */ export async function getInstanceMetadata(store: NStore, signal?: AbortSignal): Promise { const [event] = await store.query( - [{ kinds: [0], authors: [Conf.pubkey], limit: 1 }], + [{ kinds: [0], authors: [await Conf.signer.getPublicKey()], limit: 1 }], { signal }, ); diff --git a/packages/ditto/utils/nip05.ts b/packages/ditto/utils/nip05.ts index 798fabdf..6c53c18c 100644 --- a/packages/ditto/utils/nip05.ts +++ b/packages/ditto/utils/nip05.ts @@ -57,7 +57,7 @@ export async function localNip05Lookup(store: NStore, localpart: string): Promis const [grant] = await store.query([{ kinds: [30360], '#d': [`${localpart}@${Conf.url.host}`], - authors: [Conf.pubkey], + authors: [await Conf.signer.getPublicKey()], limit: 1, }]); diff --git a/packages/ditto/utils/outbox.ts b/packages/ditto/utils/outbox.ts index 891cccb8..074518bc 100644 --- a/packages/ditto/utils/outbox.ts +++ b/packages/ditto/utils/outbox.ts @@ -6,7 +6,7 @@ export async function getRelays(store: NStore, pubkey: string): Promise(); const events = await store.query([ - { kinds: [10002], authors: [pubkey, Conf.pubkey], limit: 2 }, + { kinds: [10002], authors: [pubkey, await Conf.signer.getPublicKey()], limit: 2 }, ]); for (const event of events) { diff --git a/packages/ditto/utils/pleroma.ts b/packages/ditto/utils/pleroma.ts index 05c35b7c..db3ca6a1 100644 --- a/packages/ditto/utils/pleroma.ts +++ b/packages/ditto/utils/pleroma.ts @@ -2,11 +2,11 @@ import { NSchema as n, NStore } from '@nostrify/nostrify'; import { Conf } from '@/config.ts'; import { configSchema } from '@/schemas/pleroma-api.ts'; -import { AdminSigner } from '@/signers/AdminSigner.ts'; import { PleromaConfigDB } from '@/utils/PleromaConfigDB.ts'; export async function getPleromaConfigs(store: NStore, signal?: AbortSignal): Promise { - const { pubkey } = Conf; + const signer = Conf.signer; + const pubkey = await signer.getPublicKey(); const [event] = await store.query([{ kinds: [30078], @@ -20,7 +20,7 @@ export async function getPleromaConfigs(store: NStore, signal?: AbortSignal): Pr } try { - const decrypted = await new AdminSigner().nip44.decrypt(Conf.pubkey, event.content); + const decrypted = await signer.nip44.decrypt(pubkey, event.content); const configs = n.json().pipe(configSchema.array()).catch([]).parse(decrypted); return new PleromaConfigDB(configs); } catch (_e) { diff --git a/packages/ditto/utils/zap-split.ts b/packages/ditto/utils/zap-split.ts index e5df1538..85b6f056 100644 --- a/packages/ditto/utils/zap-split.ts +++ b/packages/ditto/utils/zap-split.ts @@ -1,4 +1,3 @@ -import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Conf } from '@/config.ts'; import { NSchema as n, NStore } from '@nostrify/nostrify'; import { nostrNow } from '@/utils.ts'; @@ -38,13 +37,13 @@ export async function getZapSplits(store: NStore, pubkey: string): Promise name === 'p' && value === opts.viewerPubkey); if (event.kind === 1 && mentioned) { @@ -29,7 +29,7 @@ function renderNotification(event: DittoEvent, opts: RenderNotificationOpts) { return renderReaction(event, opts); } - if (event.kind === 30360 && event.pubkey === Conf.pubkey) { + if (event.kind === 30360 && event.pubkey === await Conf.signer.getPublicKey()) { return renderNameGrant(event); } diff --git a/packages/ditto/workers/policy.ts b/packages/ditto/workers/policy.ts index 7b3d23b0..02de539c 100644 --- a/packages/ditto/workers/policy.ts +++ b/packages/ditto/workers/policy.ts @@ -48,7 +48,7 @@ class PolicyWorker implements NPolicy { await this.worker.init({ path: Conf.policy, databaseUrl: Conf.databaseUrl, - pubkey: Conf.pubkey, + pubkey: await Conf.signer.getPublicKey(), }); logi({ diff --git a/scripts/admin-event.ts b/scripts/admin-event.ts index 70f8ed48..aec9e145 100644 --- a/scripts/admin-event.ts +++ b/scripts/admin-event.ts @@ -1,12 +1,12 @@ import { JsonParseStream } from '@std/json/json-parse-stream'; import { TextLineStream } from '@std/streams/text-line-stream'; -import { AdminSigner } from '../packages/ditto/signers/AdminSigner.ts'; +import { Conf } from '../packages/ditto/config.ts'; import { Storages } from '../packages/ditto/storages.ts'; import { type EventStub } from '../packages/ditto/utils/api.ts'; import { nostrNow } from '../packages/ditto/utils.ts'; -const signer = new AdminSigner(); +const signer = Conf.signer; const store = await Storages.db(); const readable = Deno.stdin.readable diff --git a/scripts/admin-role.ts b/scripts/admin-role.ts index 369440c9..4da9610e 100644 --- a/scripts/admin-role.ts +++ b/scripts/admin-role.ts @@ -1,7 +1,7 @@ import { NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { AdminSigner } from '../packages/ditto/signers/AdminSigner.ts'; +import { Conf } from '../packages/ditto/config.ts'; import { Storages } from '../packages/ditto/storages.ts'; import { nostrNow } from '../packages/ditto/utils.ts'; @@ -20,7 +20,7 @@ if (!['admin', 'user'].includes(role)) { Deno.exit(1); } -const signer = new AdminSigner(); +const signer = Conf.signer; const admin = await signer.getPublicKey(); const [existing] = await store.query([{ diff --git a/scripts/setup-kind0.ts b/scripts/setup-kind0.ts index ff7cbd1a..85f7a6ca 100644 --- a/scripts/setup-kind0.ts +++ b/scripts/setup-kind0.ts @@ -1,7 +1,6 @@ import { Command } from 'commander'; import { NostrEvent } from 'nostr-tools'; -import { AdminSigner } from '../packages/ditto/signers/AdminSigner.ts'; import { nostrNow } from '../packages/ditto/utils.ts'; import { Conf } from '../packages/ditto/config.ts'; import { Storages } from '../packages/ditto/storages.ts'; @@ -36,7 +35,7 @@ if (import.meta.main) { content.picture = image; content.website = Conf.localDomain; - const signer = new AdminSigner(); + const signer = Conf.signer; const bare: Omit = { created_at: nostrNow(), kind: 0, From 67aec57990f52d121a240e6ca144205208265bfc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 14:29:22 -0600 Subject: [PATCH 37/99] Rename @ditto/api to @ditto/mastoapi, start using the new router and middleware in app --- deno.json | 2 +- packages/api/middleware/confMw.test.ts | 19 --- packages/api/middleware/confMw.ts | 15 -- .../api/middleware/confRequiredMw.test.ts | 22 --- packages/api/middleware/confRequiredMw.ts | 15 -- packages/api/middleware/mod.ts | 2 - packages/ditto/app.ts | 55 +++--- packages/ditto/controllers/api/accounts.ts | 104 ++++++------ packages/ditto/controllers/api/admin.ts | 51 +++--- packages/ditto/controllers/api/bookmarks.ts | 9 +- packages/ditto/controllers/api/captcha.ts | 4 +- packages/ditto/controllers/api/cashu.test.ts | 8 +- packages/ditto/controllers/api/cashu.ts | 135 ++++++++------- packages/ditto/controllers/api/ditto.ts | 59 +++---- packages/ditto/controllers/api/markers.ts | 8 +- packages/ditto/controllers/api/media.ts | 5 +- packages/ditto/controllers/api/mutes.ts | 9 +- .../ditto/controllers/api/notifications.ts | 25 +-- packages/ditto/controllers/api/pleroma.ts | 16 +- packages/ditto/controllers/api/push.ts | 4 +- packages/ditto/controllers/api/reactions.ts | 29 ++-- packages/ditto/controllers/api/reports.ts | 56 +++--- packages/ditto/controllers/api/search.ts | 24 +-- packages/ditto/controllers/api/statuses.ts | 159 ++++++++---------- packages/ditto/controllers/api/streaming.ts | 7 +- packages/ditto/controllers/api/suggestions.ts | 44 +++-- packages/ditto/controllers/api/timelines.ts | 25 ++- packages/ditto/controllers/api/translate.ts | 5 +- packages/ditto/controllers/api/trends.ts | 17 +- packages/ditto/controllers/frontend.ts | 12 +- packages/ditto/controllers/manifest.ts | 5 +- .../ditto/controllers/nostr/relay-info.ts | 7 +- packages/ditto/controllers/nostr/relay.ts | 15 +- .../ditto/controllers/well-known/nostr.ts | 6 +- packages/ditto/middleware/auth98Middleware.ts | 21 ++- .../ditto/middleware/paginationMiddleware.ts | 49 ------ packages/ditto/middleware/requireSigner.ts | 29 ---- packages/ditto/middleware/signerMiddleware.ts | 75 --------- packages/ditto/middleware/storeMiddleware.ts | 28 --- .../ditto/middleware/uploaderMiddleware.ts | 3 +- packages/ditto/pipeline.ts | 2 +- packages/ditto/queries.ts | 16 +- packages/ditto/storages/hydrate.test.ts | 12 +- packages/ditto/storages/hydrate.ts | 34 ++-- packages/ditto/utils/api.ts | 2 +- packages/ditto/views.ts | 28 ++- packages/mastoapi/auth/aes.bench.ts | 18 ++ packages/mastoapi/auth/aes.test.ts | 15 ++ packages/mastoapi/auth/aes.ts | 17 ++ packages/mastoapi/auth/token.bench.ts | 11 ++ packages/mastoapi/auth/token.test.ts | 18 ++ packages/mastoapi/auth/token.ts | 30 ++++ packages/{api => mastoapi}/deno.json | 2 +- packages/mastoapi/middleware/mod.ts | 2 + .../middleware/paginationMiddleware.ts | 81 +++++++++ .../mastoapi/middleware/userMiddleware.ts | 128 ++++++++++++++ .../mastoapi/pagination/link-header.test.ts | 34 ++++ packages/mastoapi/pagination/link-header.ts | 39 +++++ packages/mastoapi/pagination/paginate.test.ts | 0 packages/mastoapi/pagination/paginate.ts | 43 +++++ packages/mastoapi/pagination/schema.test.ts | 23 +++ packages/mastoapi/pagination/schema.ts | 14 ++ packages/mastoapi/signers/ConnectSigner.ts | 124 ++++++++++++++ packages/mastoapi/signers/ReadOnlySigner.ts | 18 ++ packages/router/DittoApp.test.ts | 4 +- packages/router/DittoEnv.ts | 4 +- packages/router/mod.ts | 1 + 67 files changed, 1134 insertions(+), 769 deletions(-) delete mode 100644 packages/api/middleware/confMw.test.ts delete mode 100644 packages/api/middleware/confMw.ts delete mode 100644 packages/api/middleware/confRequiredMw.test.ts delete mode 100644 packages/api/middleware/confRequiredMw.ts delete mode 100644 packages/api/middleware/mod.ts delete mode 100644 packages/ditto/middleware/paginationMiddleware.ts delete mode 100644 packages/ditto/middleware/requireSigner.ts delete mode 100644 packages/ditto/middleware/signerMiddleware.ts delete mode 100644 packages/ditto/middleware/storeMiddleware.ts create mode 100644 packages/mastoapi/auth/aes.bench.ts create mode 100644 packages/mastoapi/auth/aes.test.ts create mode 100644 packages/mastoapi/auth/aes.ts create mode 100644 packages/mastoapi/auth/token.bench.ts create mode 100644 packages/mastoapi/auth/token.test.ts create mode 100644 packages/mastoapi/auth/token.ts rename packages/{api => mastoapi}/deno.json (75%) create mode 100644 packages/mastoapi/middleware/mod.ts create mode 100644 packages/mastoapi/middleware/paginationMiddleware.ts create mode 100644 packages/mastoapi/middleware/userMiddleware.ts create mode 100644 packages/mastoapi/pagination/link-header.test.ts create mode 100644 packages/mastoapi/pagination/link-header.ts create mode 100644 packages/mastoapi/pagination/paginate.test.ts create mode 100644 packages/mastoapi/pagination/paginate.ts create mode 100644 packages/mastoapi/pagination/schema.test.ts create mode 100644 packages/mastoapi/pagination/schema.ts create mode 100644 packages/mastoapi/signers/ConnectSigner.ts create mode 100644 packages/mastoapi/signers/ReadOnlySigner.ts diff --git a/deno.json b/deno.json index 4a34db67..4466b7b3 100644 --- a/deno.json +++ b/deno.json @@ -1,11 +1,11 @@ { "version": "1.1.0", "workspace": [ - "./packages/api", "./packages/conf", "./packages/db", "./packages/ditto", "./packages/lang", + "./packages/mastoapi", "./packages/metrics", "./packages/policies", "./packages/ratelimiter", diff --git a/packages/api/middleware/confMw.test.ts b/packages/api/middleware/confMw.test.ts deleted file mode 100644 index 350a585f..00000000 --- a/packages/api/middleware/confMw.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Hono } from '@hono/hono'; -import { assertEquals } from '@std/assert'; - -import { confMw } from './confMw.ts'; - -Deno.test('confMw', async () => { - const env = new Map([ - ['DITTO_NSEC', 'nsec19shyxpuzd0cq2p5078fwnws7tyykypud6z205fzhlmlrs2vpz6hs83zwkw'], - ]); - - const app = new Hono(); - - app.get('/', confMw(env), async (c) => c.text(await c.var.conf.signer.getPublicKey())); - - const response = await app.request('/'); - const body = await response.text(); - - assertEquals(body, '1ba0c5ed1bbbf3b7eb0d7843ba16836a0201ea68a76bafcba507358c45911ff6'); -}); diff --git a/packages/api/middleware/confMw.ts b/packages/api/middleware/confMw.ts deleted file mode 100644 index ebfdfe4b..00000000 --- a/packages/api/middleware/confMw.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { DittoConf } from '@ditto/conf'; - -import type { MiddlewareHandler } from '@hono/hono'; - -/** Set Ditto config. */ -export function confMw( - env: { get(key: string): string | undefined }, -): MiddlewareHandler<{ Variables: { conf: DittoConf } }> { - const conf = new DittoConf(env); - - return async (c, next) => { - c.set('conf', conf); - await next(); - }; -} diff --git a/packages/api/middleware/confRequiredMw.test.ts b/packages/api/middleware/confRequiredMw.test.ts deleted file mode 100644 index 9dfcc096..00000000 --- a/packages/api/middleware/confRequiredMw.test.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { Hono } from '@hono/hono'; -import { assertEquals } from '@std/assert'; - -import { confMw } from './confMw.ts'; -import { confRequiredMw } from './confRequiredMw.ts'; - -Deno.test('confRequiredMw', async (t) => { - const app = new Hono(); - - app.get('/without', confRequiredMw, (c) => c.text('ok')); - app.get('/with', confMw(new Map()), confRequiredMw, (c) => c.text('ok')); - - await t.step('without conf returns 500', async () => { - const response = await app.request('/without'); - assertEquals(response.status, 500); - }); - - await t.step('with conf returns 200', async () => { - const response = await app.request('/with'); - assertEquals(response.status, 200); - }); -}); diff --git a/packages/api/middleware/confRequiredMw.ts b/packages/api/middleware/confRequiredMw.ts deleted file mode 100644 index dc4d661d..00000000 --- a/packages/api/middleware/confRequiredMw.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { HTTPException } from '@hono/hono/http-exception'; - -import type { DittoConf } from '@ditto/conf'; -import type { MiddlewareHandler } from '@hono/hono'; - -/** Throws an error if conf isn't set. */ -export const confRequiredMw: MiddlewareHandler<{ Variables: { conf: DittoConf } }> = async (c, next) => { - const { conf } = c.var; - - if (!conf) { - throw new HTTPException(500, { message: 'Ditto config not set in request.' }); - } - - await next(); -}; diff --git a/packages/api/middleware/mod.ts b/packages/api/middleware/mod.ts deleted file mode 100644 index 54a1b35c..00000000 --- a/packages/api/middleware/mod.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { confMw } from './confMw.ts'; -export { confRequiredMw } from './confRequiredMw.ts'; diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index 88bfa7f9..9944426c 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -1,16 +1,18 @@ -import { confMw } from '@ditto/api/middleware'; -import { type DittoConf } from '@ditto/conf'; -import { DittoTables } from '@ditto/db'; +import { DittoConf } from '@ditto/conf'; +import { DittoDB } from '@ditto/db'; +import { paginationMiddleware, userMiddleware } from '@ditto/mastoapi/middleware'; +import { DittoApp, type DittoEnv } from '@ditto/router'; import { type DittoTranslator } from '@ditto/translators'; -import { type Context, Env as HonoEnv, Handler, Hono, Input as HonoInput, MiddlewareHandler } from '@hono/hono'; +import { type Context, Handler, Input as HonoInput, MiddlewareHandler } from '@hono/hono'; import { every } from '@hono/hono/combine'; import { cors } from '@hono/hono/cors'; import { serveStatic } from '@hono/hono/deno'; -import { NostrEvent, NostrSigner, NStore, NUploader } from '@nostrify/nostrify'; -import { Kysely } from 'kysely'; +import { NostrEvent, NostrSigner, NRelay, NUploader } from '@nostrify/nostrify'; import '@/startup.ts'; +import { Conf } from '@/config.ts'; +import { Storages } from '@/storages.ts'; import { Time } from '@/utils/time.ts'; import { @@ -140,34 +142,33 @@ import { cacheControlMiddleware } from '@/middleware/cacheControlMiddleware.ts'; import { cspMiddleware } from '@/middleware/cspMiddleware.ts'; import { metricsMiddleware } from '@/middleware/metricsMiddleware.ts'; import { notActivitypubMiddleware } from '@/middleware/notActivitypubMiddleware.ts'; -import { paginationMiddleware } from '@/middleware/paginationMiddleware.ts'; import { rateLimitMiddleware } from '@/middleware/rateLimitMiddleware.ts'; -import { requireSigner } from '@/middleware/requireSigner.ts'; -import { signerMiddleware } from '@/middleware/signerMiddleware.ts'; -import { storeMiddleware } from '@/middleware/storeMiddleware.ts'; import { uploaderMiddleware } from '@/middleware/uploaderMiddleware.ts'; import { translatorMiddleware } from '@/middleware/translatorMiddleware.ts'; import { logiMiddleware } from '@/middleware/logiMiddleware.ts'; -export interface AppEnv extends HonoEnv { +export interface AppEnv extends DittoEnv { Variables: { conf: DittoConf; - /** Signer to get the logged-in user's pubkey, relays, and to sign events, or `undefined` if the user isn't logged in. */ - signer?: NostrSigner; /** Uploader for the user to upload files. */ uploader?: NUploader; /** NIP-98 signed event proving the pubkey is owned by the user. */ proof?: NostrEvent; /** Kysely instance for the database. */ - kysely: Kysely; - /** Storage for the user, might filter out unwanted content. */ - store: NStore; + db: DittoDB; + /** Base database store. No content filtering. */ + relay: NRelay; /** Normalized pagination params. */ pagination: { since?: number; until?: number; limit: number }; - /** Normalized list pagination params. */ - listPagination: { offset: number; limit: number }; /** Translation service. */ translator?: DittoTranslator; + signal: AbortSignal; + user?: { + /** Signer to get the logged-in user's pubkey, relays, and to sign events, or `undefined` if the user isn't logged in. */ + signer: NostrSigner; + /** User's relay. Might filter out unwanted content. */ + relay: NRelay; + }; }; } @@ -176,21 +177,29 @@ type AppMiddleware = MiddlewareHandler; // deno-lint-ignore no-explicit-any type AppController

= Handler>; -const app = new Hono({ strict: false }); +const app = new DittoApp({ + conf: Conf, + db: await Storages.database(), + relay: await Storages.db(), +}, { + strict: false, +}); /** User-provided files in the gitignored `public/` directory. */ const publicFiles = serveStatic({ root: './public/' }); /** Static files provided by the Ditto repo, checked into git. */ const staticFiles = serveStatic({ root: new URL('./static/', import.meta.url).pathname }); -app.use(confMw(Deno.env), cacheControlMiddleware({ noStore: true })); +app.use(cacheControlMiddleware({ noStore: true })); const ratelimit = every( rateLimitMiddleware(30, Time.seconds(5), false), rateLimitMiddleware(300, Time.minutes(5), false), ); -app.use('/api/*', metricsMiddleware, ratelimit, paginationMiddleware, logiMiddleware); +const requireSigner = userMiddleware({ privileged: false, required: true }); + +app.use('/api/*', metricsMiddleware, ratelimit, paginationMiddleware(), logiMiddleware); app.use('/.well-known/*', metricsMiddleware, ratelimit, logiMiddleware); app.use('/nodeinfo/*', metricsMiddleware, ratelimit, logiMiddleware); app.use('/oauth/*', metricsMiddleware, ratelimit, logiMiddleware); @@ -201,10 +210,8 @@ app.get('/relay', metricsMiddleware, ratelimit, relayController); app.use( cspMiddleware(), cors({ origin: '*', exposeHeaders: ['link'] }), - signerMiddleware, uploaderMiddleware, auth98Middleware(), - storeMiddleware, ); app.get('/metrics', metricsController); @@ -251,7 +258,7 @@ app.post('/oauth/revoke', revokeTokenController); app.post('/oauth/authorize', oauthAuthorizeController); app.get('/oauth/authorize', oauthController); -app.post('/api/v1/accounts', requireProof({ pow: 20 }), createAccountController); +app.post('/api/v1/accounts', requireProof(), createAccountController); app.get('/api/v1/accounts/verify_credentials', requireSigner, verifyCredentialsController); app.patch('/api/v1/accounts/update_credentials', requireSigner, updateCredentialsController); app.get('/api/v1/accounts/search', accountSearchController); diff --git a/packages/ditto/controllers/api/accounts.ts b/packages/ditto/controllers/api/accounts.ts index 27710063..24f7d5af 100644 --- a/packages/ditto/controllers/api/accounts.ts +++ b/packages/ditto/controllers/api/accounts.ts @@ -26,7 +26,9 @@ const createAccountSchema = z.object({ }); const createAccountController: AppController = async (c) => { - const pubkey = await c.get('signer')?.getPublicKey()!; + const { user } = c.var; + + const pubkey = await user!.signer.getPublicKey(); const result = createAccountSchema.safeParse(await c.req.json()); if (!result.success) { @@ -46,15 +48,15 @@ const createAccountController: AppController = async (c) => { }; const verifyCredentialsController: AppController = async (c) => { - const signer = c.get('signer')!; - const pubkey = await signer.getPublicKey(); + const { relay, user } = c.var; - const store = await Storages.db(); + const signer = user!.signer; + const pubkey = await signer.getPublicKey(); const [author, [settingsEvent]] = await Promise.all([ getAuthor(pubkey, { signal: AbortSignal.timeout(5000) }), - store.query([{ + relay.query([{ kinds: [30078], authors: [pubkey], '#d': ['pub.ditto.pleroma_settings_store'], @@ -115,12 +117,10 @@ const accountSearchQuerySchema = z.object({ }); const accountSearchController: AppController = async (c) => { - const { store } = c.var; - const { signal } = c.req.raw; - const { limit } = c.get('pagination'); + const { db, relay, user, pagination, signal } = c.var; + const { limit } = pagination; - const kysely = await Storages.kysely(); - const viewerPubkey = await c.get('signer')?.getPublicKey(); + const viewerPubkey = await user?.signer.getPublicKey(); const result = accountSearchQuerySchema.safeParse(c.req.query()); @@ -144,8 +144,8 @@ const accountSearchController: AppController = async (c) => { events.push(event); } else { const following = viewerPubkey ? await getFollowedPubkeys(viewerPubkey) : new Set(); - const authors = [...await getPubkeysBySearch(kysely, { q: query, limit, offset: 0, following })]; - const profiles = await store.query([{ kinds: [0], authors, limit }], { signal }); + const authors = [...await getPubkeysBySearch(db.kysely, { q: query, limit, offset: 0, following })]; + const profiles = await relay.query([{ kinds: [0], authors, limit }], { signal }); for (const pubkey of authors) { const profile = profiles.find((event) => event.pubkey === pubkey); @@ -155,14 +155,16 @@ const accountSearchController: AppController = async (c) => { } } - const accounts = await hydrateEvents({ events, store, signal }) + const accounts = await hydrateEvents({ events, relay, signal }) .then((events) => events.map((event) => renderAccount(event))); return c.json(accounts); }; const relationshipsController: AppController = async (c) => { - const pubkey = await c.get('signer')?.getPublicKey()!; + const { user } = c.var; + + const pubkey = await user!.signer.getPublicKey(); const ids = z.array(z.string()).safeParse(c.req.queries('id[]')); if (!ids.success) { @@ -201,17 +203,17 @@ const accountStatusesQuerySchema = z.object({ }); const accountStatusesController: AppController = async (c) => { + const { conf, user, signal } = c.var; + const pubkey = c.req.param('pubkey'); - const { conf } = c.var; const { since, until } = c.var.pagination; const { pinned, limit, exclude_replies, tagged, only_media } = accountStatusesQuerySchema.parse(c.req.query()); - const { signal } = c.req.raw; - const store = await Storages.db(); + const { relay } = c.var; - const [[author], [user]] = await Promise.all([ - store.query([{ kinds: [0], authors: [pubkey], limit: 1 }], { signal }), - store.query([{ kinds: [30382], authors: [await conf.signer.getPublicKey()], '#d': [pubkey], limit: 1 }], { + const [[author], [userEvent]] = await Promise.all([ + relay.query([{ kinds: [0], authors: [pubkey], limit: 1 }], { signal }), + relay.query([{ kinds: [30382], authors: [await conf.signer.getPublicKey()], '#d': [pubkey], limit: 1 }], { signal, }), ]); @@ -220,14 +222,14 @@ const accountStatusesController: AppController = async (c) => { assertAuthenticated(c, author); } - const names = getTagSet(user?.tags ?? [], 'n'); + const names = getTagSet(userEvent?.tags ?? [], 'n'); if (names.has('disabled')) { return c.json([]); } if (pinned) { - const [pinEvent] = await store.query([{ kinds: [10001], authors: [pubkey], limit: 1 }], { signal }); + const [pinEvent] = await relay.query([{ kinds: [10001], authors: [pubkey], limit: 1 }], { signal }); if (pinEvent) { const pinnedEventIds = getTagSet(pinEvent.tags, 'e'); return renderStatuses(c, [...pinnedEventIds].reverse()); @@ -264,8 +266,8 @@ const accountStatusesController: AppController = async (c) => { const opts = { signal, limit, timeout: conf.db.timeouts.timelines }; - const events = await store.query([filter], opts) - .then((events) => hydrateEvents({ events, store, signal })) + const events = await relay.query([filter], opts) + .then((events) => hydrateEvents({ events, relay, signal })) .then((events) => { if (exclude_replies) { return events.filter((event) => { @@ -276,7 +278,7 @@ const accountStatusesController: AppController = async (c) => { return events; }); - const viewerPubkey = await c.get('signer')?.getPublicKey(); + const viewerPubkey = await user?.signer.getPublicKey(); const statuses = await Promise.all( events.map((event) => { @@ -303,12 +305,11 @@ const updateCredentialsSchema = z.object({ }); const updateCredentialsController: AppController = async (c) => { - const signer = c.get('signer')!; - const pubkey = await signer.getPublicKey(); + const { relay, user, signal } = c.var; + + const pubkey = await user!.signer.getPublicKey(); const body = await parseBody(c.req.raw); const result = updateCredentialsSchema.safeParse(body); - const store = await Storages.db(); - const signal = c.req.raw.signal; if (!result.success) { return c.json(result.error, 422); @@ -318,7 +319,7 @@ const updateCredentialsController: AppController = async (c) => { let event: NostrEvent | undefined; if (keys.length === 1 && keys[0] === 'pleroma_settings_store') { - event = (await store.query([{ kinds: [0], authors: [pubkey] }]))[0]; + event = (await relay.query([{ kinds: [0], authors: [pubkey] }]))[0]; } else { event = await updateEvent( { kinds: [0], authors: [pubkey], limit: 1 }, @@ -374,7 +375,7 @@ const updateCredentialsController: AppController = async (c) => { let account: MastodonAccount; if (event) { - await hydrateEvents({ events: [event], store, signal }); + await hydrateEvents({ events: [event], relay, signal }); account = await renderAccount(event, { withSource: true, settingsStore }); } else { account = await accountFromPubkey(pubkey, { withSource: true, settingsStore }); @@ -393,7 +394,9 @@ const updateCredentialsController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/accounts/#follow */ const followController: AppController = async (c) => { - const sourcePubkey = await c.get('signer')?.getPublicKey()!; + const { user } = c.var; + + const sourcePubkey = await user!.signer.getPublicKey(); const targetPubkey = c.req.param('pubkey'); await updateListEvent( @@ -410,7 +413,9 @@ const followController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/accounts/#unfollow */ const unfollowController: AppController = async (c) => { - const sourcePubkey = await c.get('signer')?.getPublicKey()!; + const { user } = c.var; + + const sourcePubkey = await user!.signer.getPublicKey(); const targetPubkey = c.req.param('pubkey'); await updateListEvent( @@ -447,7 +452,9 @@ const unblockController: AppController = (c) => { /** https://docs.joinmastodon.org/methods/accounts/#mute */ const muteController: AppController = async (c) => { - const sourcePubkey = await c.get('signer')?.getPublicKey()!; + const { user } = c.var; + + const sourcePubkey = await user!.signer.getPublicKey(); const targetPubkey = c.req.param('pubkey'); await updateListEvent( @@ -462,7 +469,9 @@ const muteController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/accounts/#unmute */ const unmuteController: AppController = async (c) => { - const sourcePubkey = await c.get('signer')?.getPublicKey()!; + const { user } = c.var; + + const sourcePubkey = await user!.signer.getPublicKey(); const targetPubkey = c.req.param('pubkey'); await updateListEvent( @@ -476,14 +485,12 @@ const unmuteController: AppController = async (c) => { }; const favouritesController: AppController = async (c) => { - const pubkey = await c.get('signer')?.getPublicKey()!; - const params = c.get('pagination'); - const { signal } = c.req.raw; + const { relay, user, pagination, signal } = c.var; - const store = await Storages.db(); + const pubkey = await user!.signer.getPublicKey(); - const events7 = await store.query( - [{ kinds: [7], authors: [pubkey], ...params }], + const events7 = await relay.query( + [{ kinds: [7], authors: [pubkey], ...pagination }], { signal }, ); @@ -491,10 +498,10 @@ const favouritesController: AppController = async (c) => { .map((event) => event.tags.find((tag) => tag[0] === 'e')?.[1]) .filter((id): id is string => !!id); - const events1 = await store.query([{ kinds: [1, 20], ids }], { signal }) - .then((events) => hydrateEvents({ events, store, signal })); + const events1 = await relay.query([{ kinds: [1, 20], ids }], { signal }) + .then((events) => hydrateEvents({ events, relay, signal })); - const viewerPubkey = await c.get('signer')?.getPublicKey(); + const viewerPubkey = await user?.signer.getPublicKey(); const statuses = await Promise.all( events1.map((event) => renderStatus(event, { viewerPubkey })), @@ -503,16 +510,15 @@ const favouritesController: AppController = async (c) => { }; const familiarFollowersController: AppController = async (c) => { - const store = await Storages.db(); - const signer = c.get('signer')!; - const pubkey = await signer.getPublicKey(); + const { relay, user } = c.var; + const pubkey = await user!.signer.getPublicKey(); const ids = z.array(z.string()).parse(c.req.queries('id[]')); const follows = await getFollowedPubkeys(pubkey); const results = await Promise.all(ids.map(async (id) => { - const followLists = await store.query([{ kinds: [3], authors: [...follows], '#p': [id] }]) - .then((events) => hydrateEvents({ events, store })); + const followLists = await relay.query([{ kinds: [3], authors: [...follows], '#p': [id] }]) + .then((events) => hydrateEvents({ events, relay })); const accounts = await Promise.all( followLists.map((event) => event.author ? renderAccount(event.author) : accountFromPubkey(event.pubkey)), diff --git a/packages/ditto/controllers/api/admin.ts b/packages/ditto/controllers/api/admin.ts index 9e9ba5d0..0568cd57 100644 --- a/packages/ditto/controllers/api/admin.ts +++ b/packages/ditto/controllers/api/admin.ts @@ -4,7 +4,6 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { booleanParamSchema } from '@/schema.ts'; -import { Storages } from '@/storages.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { createAdminEvent, paginated, parseBody, updateEventInfo, updateUser } from '@/utils/api.ts'; import { renderNameRequest } from '@/views/ditto.ts'; @@ -29,10 +28,8 @@ const adminAccountQuerySchema = z.object({ }); const adminAccountsController: AppController = async (c) => { - const { conf } = c.var; - const store = await Storages.db(); - const params = c.get('pagination'); - const { signal } = c.req.raw; + const { conf, relay, signal, pagination } = c.var; + const { local, pending, @@ -50,8 +47,8 @@ const adminAccountsController: AppController = async (c) => { return c.json([]); } - const orig = await store.query( - [{ kinds: [30383], authors: [adminPubkey], '#k': ['3036'], '#n': ['pending'], ...params }], + const orig = await relay.query( + [{ kinds: [30383], authors: [adminPubkey], '#k': ['3036'], '#n': ['pending'], ...pagination }], { signal }, ); @@ -61,8 +58,8 @@ const adminAccountsController: AppController = async (c) => { .filter((id): id is string => !!id), ); - const events = await store.query([{ kinds: [3036], ids: [...ids] }]) - .then((events) => hydrateEvents({ store, events, signal })); + const events = await relay.query([{ kinds: [3036], ids: [...ids] }]) + .then((events) => hydrateEvents({ relay, events, signal })); const nameRequests = await Promise.all(events.map(renderNameRequest)); return paginated(c, orig, nameRequests); @@ -88,8 +85,8 @@ const adminAccountsController: AppController = async (c) => { n.push('moderator'); } - const events = await store.query( - [{ kinds: [30382], authors: [adminPubkey], '#n': n, ...params }], + const events = await relay.query( + [{ kinds: [30382], authors: [adminPubkey], '#n': n, ...pagination }], { signal }, ); @@ -99,8 +96,8 @@ const adminAccountsController: AppController = async (c) => { .filter((pubkey): pubkey is string => !!pubkey), ); - const authors = await store.query([{ kinds: [0], authors: [...pubkeys] }]) - .then((events) => hydrateEvents({ store, events, signal })); + const authors = await relay.query([{ kinds: [0], authors: [...pubkeys] }]) + .then((events) => hydrateEvents({ relay, events, signal })); const accounts = await Promise.all( [...pubkeys].map((pubkey) => { @@ -112,14 +109,14 @@ const adminAccountsController: AppController = async (c) => { return paginated(c, events, accounts); } - const filter: NostrFilter = { kinds: [0], ...params }; + const filter: NostrFilter = { kinds: [0], ...pagination }; if (local) { filter.search = `domain:${conf.url.host}`; } - const events = await store.query([filter], { signal }) - .then((events) => hydrateEvents({ store, events, signal })); + const events = await relay.query([filter], { signal }) + .then((events) => hydrateEvents({ relay, events, signal })); const accounts = await Promise.all(events.map(renderAdminAccount)); return paginated(c, events, accounts); @@ -130,9 +127,9 @@ const adminAccountActionSchema = z.object({ }); const adminActionController: AppController = async (c) => { - const { conf } = c.var; + const { conf, relay } = c.var; + const body = await parseBody(c.req.raw); - const store = await Storages.db(); const result = adminAccountActionSchema.safeParse(body); const authorId = c.req.param('id'); @@ -156,13 +153,13 @@ const adminActionController: AppController = async (c) => { if (data.type === 'suspend') { n.disabled = true; n.suspended = true; - store.remove([{ authors: [authorId] }]).catch((e: unknown) => { + relay.remove!([{ authors: [authorId] }]).catch((e: unknown) => { logi({ level: 'error', ns: 'ditto.api.admin.account.action', type: data.type, error: errorJson(e) }); }); } if (data.type === 'revoke_name') { n.revoke_name = true; - store.remove([{ kinds: [30360], authors: [await conf.signer.getPublicKey()], '#p': [authorId] }]).catch( + relay.remove!([{ kinds: [30360], authors: [await conf.signer.getPublicKey()], '#p': [authorId] }]).catch( (e: unknown) => { logi({ level: 'error', ns: 'ditto.api.admin.account.action', type: data.type, error: errorJson(e) }); }, @@ -177,9 +174,9 @@ const adminActionController: AppController = async (c) => { const adminApproveController: AppController = async (c) => { const { conf } = c.var; const eventId = c.req.param('id'); - const store = await Storages.db(); + const { relay } = c.var; - const [event] = await store.query([{ kinds: [3036], ids: [eventId] }]); + const [event] = await relay.query([{ kinds: [3036], ids: [eventId] }]); if (!event) { return c.json({ error: 'Event not found' }, 404); } @@ -192,7 +189,7 @@ const adminApproveController: AppController = async (c) => { return c.json({ error: 'Invalid NIP-05' }, 400); } - const [existing] = await store.query([ + const [existing] = await relay.query([ { kinds: [30360], authors: [await conf.signer.getPublicKey()], '#d': [r], limit: 1 }, ]); @@ -212,7 +209,7 @@ const adminApproveController: AppController = async (c) => { }, c); await updateEventInfo(eventId, { pending: false, approved: true, rejected: false }, c); - await hydrateEvents({ events: [event], store }); + await hydrateEvents({ events: [event], relay }); const nameRequest = await renderNameRequest(event); return c.json(nameRequest); @@ -220,15 +217,15 @@ const adminApproveController: AppController = async (c) => { const adminRejectController: AppController = async (c) => { const eventId = c.req.param('id'); - const store = await Storages.db(); + const { relay } = c.var; - const [event] = await store.query([{ kinds: [3036], ids: [eventId] }]); + const [event] = await relay.query([{ kinds: [3036], ids: [eventId] }]); if (!event) { return c.json({ error: 'Event not found' }, 404); } await updateEventInfo(eventId, { pending: false, approved: false, rejected: true }, c); - await hydrateEvents({ events: [event], store }); + await hydrateEvents({ events: [event], relay }); const nameRequest = await renderNameRequest(event); return c.json(nameRequest); diff --git a/packages/ditto/controllers/api/bookmarks.ts b/packages/ditto/controllers/api/bookmarks.ts index 6d80b500..e5253986 100644 --- a/packages/ditto/controllers/api/bookmarks.ts +++ b/packages/ditto/controllers/api/bookmarks.ts @@ -1,15 +1,14 @@ import { type AppController } from '@/app.ts'; -import { Storages } from '@/storages.ts'; import { getTagSet } from '@/utils/tags.ts'; import { renderStatuses } from '@/views.ts'; /** https://docs.joinmastodon.org/methods/bookmarks/#get */ const bookmarksController: AppController = async (c) => { - const store = await Storages.db(); - const pubkey = await c.get('signer')?.getPublicKey()!; - const { signal } = c.req.raw; + const { relay, user, signal } = c.var; - const [event10003] = await store.query( + const pubkey = await user!.signer.getPublicKey(); + + const [event10003] = await relay.query( [{ kinds: [10003], authors: [pubkey], limit: 1 }], { signal }, ); diff --git a/packages/ditto/controllers/api/captcha.ts b/packages/ditto/controllers/api/captcha.ts index 7b310e53..790913af 100644 --- a/packages/ditto/controllers/api/captcha.ts +++ b/packages/ditto/controllers/api/captcha.ts @@ -152,9 +152,11 @@ const pointSchema = z.object({ /** Verify the captcha solution and sign an event in the database. */ export const captchaVerifyController: AppController = async (c) => { + const { user } = c.var; + const id = c.req.param('id'); const result = pointSchema.safeParse(await c.req.json()); - const pubkey = await c.get('signer')!.getPublicKey(); + const pubkey = await user!.signer.getPublicKey(); if (!result.success) { return c.json({ error: 'Invalid input' }, { status: 422 }); diff --git a/packages/ditto/controllers/api/cashu.test.ts b/packages/ditto/controllers/api/cashu.test.ts index 57be895d..d82e205e 100644 --- a/packages/ditto/controllers/api/cashu.test.ts +++ b/packages/ditto/controllers/api/cashu.test.ts @@ -1,11 +1,10 @@ -import { confMw } from '@ditto/api/middleware'; import { Env as HonoEnv, Hono } from '@hono/hono'; import { NostrSigner, NSecSigner, NStore } from '@nostrify/nostrify'; import { genEvent } from '@nostrify/nostrify/test'; -import { generateSecretKey, getPublicKey } from 'nostr-tools'; import { bytesToString, stringToBytes } from '@scure/base'; import { stub } from '@std/testing/mock'; import { assertEquals, assertExists, assertObjectMatch } from '@std/assert'; +import { generateSecretKey, getPublicKey } from 'nostr-tools'; import { createTestDB } from '@/test.ts'; @@ -44,7 +43,6 @@ Deno.test('PUT /wallet must be successful', { }, ); - app.use(confMw(new Map())); app.route('/', cashuApp); const response = await app.request('/wallet', { @@ -123,7 +121,6 @@ Deno.test('PUT /wallet must NOT be successful: wrong request body/schema', async }, ); - app.use(confMw(new Map())); app.route('/', cashuApp); const response = await app.request('/wallet', { @@ -162,7 +159,6 @@ Deno.test('PUT /wallet must NOT be successful: wallet already exists', { }, ); - app.use(confMw(new Map())); app.route('/', cashuApp); await db.store.event(genEvent({ kind: 17375 }, sk)); @@ -206,7 +202,6 @@ Deno.test('GET /wallet must be successful', { }, ); - app.use(confMw(new Map())); app.route('/', cashuApp); // Wallet @@ -312,7 +307,6 @@ Deno.test('GET /mints must be successful', async () => { }, ); - app.use(confMw(new Map())); app.route('/', cashuApp); const response = await app.request('/mints', { diff --git a/packages/ditto/controllers/api/cashu.ts b/packages/ditto/controllers/api/cashu.ts index dd753884..60832ac4 100644 --- a/packages/ditto/controllers/api/cashu.ts +++ b/packages/ditto/controllers/api/cashu.ts @@ -1,22 +1,22 @@ import { Proof } from '@cashu/cashu-ts'; -import { confRequiredMw } from '@ditto/api/middleware'; -import { Hono } from '@hono/hono'; +import { userMiddleware } from '@ditto/mastoapi/middleware'; +import { DittoMiddleware, DittoRoute } from '@ditto/router'; import { generateSecretKey, getPublicKey } from 'nostr-tools'; import { bytesToString, stringToBytes } from '@scure/base'; import { z } from 'zod'; import { createEvent, parseBody } from '@/utils/api.ts'; -import { requireNip44Signer } from '@/middleware/requireSigner.ts'; -import { requireStore } from '@/middleware/storeMiddleware.ts'; import { walletSchema } from '@/schema.ts'; import { swapNutzapsMiddleware } from '@/middleware/swapNutzapsMiddleware.ts'; import { isNostrId } from '@/utils.ts'; import { logi } from '@soapbox/logi'; import { errorJson } from '@/utils/log.ts'; +import { SetRequired } from 'type-fest'; +import { NostrSigner } from '@nostrify/nostrify'; type Wallet = z.infer; -const app = new Hono().use('*', confRequiredMw, requireStore); +const app = new DittoRoute(); // app.delete('/wallet') -> 204 @@ -33,6 +33,19 @@ interface Nutzap { recipient_pubkey: string; } +const requireNip44Signer: DittoMiddleware<{ user: { signer: SetRequired } }> = async ( + c, + next, +) => { + const { user } = c.var; + + if (!user?.signer.nip44) { + return c.json({ error: 'User does not have a NIP-44 signer' }, 400); + } + + await next(); +}; + const createCashuWalletAndNutzapInfoSchema = z.object({ mints: z.array(z.string().url()).nonempty().transform((val) => { return [...new Set(val)]; @@ -44,12 +57,11 @@ const createCashuWalletAndNutzapInfoSchema = z.object({ * https://github.com/nostr-protocol/nips/blob/master/60.md * https://github.com/nostr-protocol/nips/blob/master/61.md#nutzap-informational-event */ -app.put('/wallet', requireNip44Signer, async (c) => { - const { conf, signer } = c.var; - const store = c.get('store'); - const pubkey = await signer.getPublicKey(); +app.put('/wallet', userMiddleware({ privileged: false, required: true }), requireNip44Signer, async (c) => { + const { conf, user, relay, signal } = c.var; + + const pubkey = await user.signer.getPublicKey(); const body = await parseBody(c.req.raw); - const { signal } = c.req.raw; const result = createCashuWalletAndNutzapInfoSchema.safeParse(body); if (!result.success) { @@ -58,7 +70,7 @@ app.put('/wallet', requireNip44Signer, async (c) => { const { mints } = result.data; - const [event] = await store.query([{ authors: [pubkey], kinds: [17375] }], { signal }); + const [event] = await relay.query([{ authors: [pubkey], kinds: [17375] }], { signal }); if (event) { return c.json({ error: 'You already have a wallet 😏' }, 400); } @@ -75,7 +87,7 @@ app.put('/wallet', requireNip44Signer, async (c) => { walletContentTags.push(['mint', mint]); } - const encryptedWalletContentTags = await signer.nip44.encrypt(pubkey, JSON.stringify(walletContentTags)); + const encryptedWalletContentTags = await user.signer.nip44.encrypt(pubkey, JSON.stringify(walletContentTags)); // Wallet await createEvent({ @@ -105,58 +117,63 @@ app.put('/wallet', requireNip44Signer, async (c) => { }); /** Gets a wallet, if it exists. */ -app.get('/wallet', requireNip44Signer, swapNutzapsMiddleware, async (c) => { - const { conf, signer } = c.var; - const store = c.get('store'); - const pubkey = await signer.getPublicKey(); - const { signal } = c.req.raw; +app.get( + '/wallet', + userMiddleware({ privileged: false, required: true }), + requireNip44Signer, + swapNutzapsMiddleware, + async (c) => { + const { conf, relay, user, signal } = c.var; - const [event] = await store.query([{ authors: [pubkey], kinds: [17375] }], { signal }); - if (!event) { - return c.json({ error: 'Wallet not found' }, 404); - } + const pubkey = await user.signer.getPublicKey(); - const decryptedContent: string[][] = JSON.parse(await signer.nip44.decrypt(pubkey, event.content)); - - const privkey = decryptedContent.find(([value]) => value === 'privkey')?.[1]; - if (!privkey || !isNostrId(privkey)) { - return c.json({ error: 'Wallet does not contain privkey or privkey is not a valid nostr id.' }, 422); - } - - const p2pk = getPublicKey(stringToBytes('hex', privkey)); - - let balance = 0; - const mints: string[] = []; - - const tokens = await store.query([{ authors: [pubkey], kinds: [7375] }], { signal }); - for (const token of tokens) { - try { - const decryptedContent: { mint: string; proofs: Proof[] } = JSON.parse( - await signer.nip44.decrypt(pubkey, token.content), - ); - - if (!mints.includes(decryptedContent.mint)) { - mints.push(decryptedContent.mint); - } - - balance += decryptedContent.proofs.reduce((accumulator, current) => { - return accumulator + current.amount; - }, 0); - } catch (e) { - logi({ level: 'error', ns: 'ditto.api.cashu.wallet.swap', error: errorJson(e) }); + const [event] = await relay.query([{ authors: [pubkey], kinds: [17375] }], { signal }); + if (!event) { + return c.json({ error: 'Wallet not found' }, 404); } - } - // TODO: maybe change the 'Wallet' type data structure so each mint is a key and the value are the tokens associated with a given mint - const walletEntity: Wallet = { - pubkey_p2pk: p2pk, - mints, - relays: [conf.relay], - balance, - }; + const decryptedContent: string[][] = JSON.parse(await user.signer.nip44.decrypt(pubkey, event.content)); - return c.json(walletEntity, 200); -}); + const privkey = decryptedContent.find(([value]) => value === 'privkey')?.[1]; + if (!privkey || !isNostrId(privkey)) { + return c.json({ error: 'Wallet does not contain privkey or privkey is not a valid nostr id.' }, 422); + } + + const p2pk = getPublicKey(stringToBytes('hex', privkey)); + + let balance = 0; + const mints: string[] = []; + + const tokens = await relay.query([{ authors: [pubkey], kinds: [7375] }], { signal }); + for (const token of tokens) { + try { + const decryptedContent: { mint: string; proofs: Proof[] } = JSON.parse( + await user.signer.nip44.decrypt(pubkey, token.content), + ); + + if (!mints.includes(decryptedContent.mint)) { + mints.push(decryptedContent.mint); + } + + balance += decryptedContent.proofs.reduce((accumulator, current) => { + return accumulator + current.amount; + }, 0); + } catch (e) { + logi({ level: 'error', ns: 'ditto.api.cashu.wallet.swap', error: errorJson(e) }); + } + } + + // TODO: maybe change the 'Wallet' type data structure so each mint is a key and the value are the tokens associated with a given mint + const walletEntity: Wallet = { + pubkey_p2pk: p2pk, + mints, + relays: [conf.relay], + balance, + }; + + return c.json(walletEntity, 200); + }, +); /** Get mints set by the CASHU_MINTS environment variable. */ app.get('/mints', (c) => { diff --git a/packages/ditto/controllers/api/ditto.ts b/packages/ditto/controllers/api/ditto.ts index 752124dc..f67fed32 100644 --- a/packages/ditto/controllers/api/ditto.ts +++ b/packages/ditto/controllers/api/ditto.ts @@ -28,10 +28,9 @@ const relaySchema = z.object({ type RelayEntity = z.infer; export const adminRelaysController: AppController = async (c) => { - const { conf } = c.var; - const store = await Storages.db(); + const { conf, relay } = c.var; - const [event] = await store.query([ + const [event] = await relay.query([ { kinds: [10002], authors: [await conf.signer.getPublicKey()], limit: 1 }, ]); @@ -43,8 +42,7 @@ export const adminRelaysController: AppController = async (c) => { }; export const adminSetRelaysController: AppController = async (c) => { - const { conf } = c.var; - const store = await Storages.db(); + const { conf, relay } = c.var; const relays = relaySchema.array().parse(await c.req.json()); const event = await conf.signer.signEvent({ @@ -54,7 +52,7 @@ export const adminSetRelaysController: AppController = async (c) => { created_at: Math.floor(Date.now() / 1000), }); - await store.event(event); + await relay.event(event); return c.json(renderRelays(event)); }; @@ -79,14 +77,12 @@ const nameRequestSchema = z.object({ }); export const nameRequestController: AppController = async (c) => { - const store = await Storages.db(); - const signer = c.get('signer')!; - const pubkey = await signer.getPublicKey(); - const { conf } = c.var; + const { conf, relay, user } = c.var; + const pubkey = await user!.signer.getPublicKey(); const { name, reason } = nameRequestSchema.parse(await c.req.json()); - const [existing] = await store.query([{ kinds: [3036], authors: [pubkey], '#r': [name], limit: 1 }]); + const [existing] = await relay.query([{ kinds: [3036], authors: [pubkey], '#r': [name], limit: 1 }]); if (existing) { return c.json({ error: 'Name request already exists' }, 400); } @@ -102,7 +98,7 @@ export const nameRequestController: AppController = async (c) => { ], }, c); - await hydrateEvents({ events: [event], store: await Storages.db() }); + await hydrateEvents({ events: [event], relay }); const nameRequest = await renderNameRequest(event); return c.json(nameRequest); @@ -114,10 +110,8 @@ const nameRequestsSchema = z.object({ }); export const nameRequestsController: AppController = async (c) => { - const { conf } = c.var; - const store = await Storages.db(); - const signer = c.get('signer')!; - const pubkey = await signer.getPublicKey(); + const { conf, relay, user, signal } = c.var; + const pubkey = await user!.signer.getPublicKey(); const params = c.get('pagination'); const { approved, rejected } = nameRequestsSchema.parse(c.req.query()); @@ -137,7 +131,7 @@ export const nameRequestsController: AppController = async (c) => { filter['#n'] = ['rejected']; } - const orig = await store.query([filter]); + const orig = await relay.query([filter]); const ids = new Set(); for (const event of orig) { @@ -151,8 +145,8 @@ export const nameRequestsController: AppController = async (c) => { return c.json([]); } - const events = await store.query([{ kinds: [3036], ids: [...ids], authors: [pubkey] }]) - .then((events) => hydrateEvents({ store, events: events, signal: c.req.raw.signal })); + const events = await relay.query([{ kinds: [3036], ids: [...ids], authors: [pubkey] }]) + .then((events) => hydrateEvents({ relay, events: events, signal })); const nameRequests = await Promise.all( events.map((event) => renderNameRequest(event)), @@ -170,10 +164,9 @@ const zapSplitSchema = z.record( ); export const updateZapSplitsController: AppController = async (c) => { - const { conf } = c.var; + const { conf, relay } = c.var; const body = await parseBody(c.req.raw); const result = zapSplitSchema.safeParse(body); - const store = c.get('store'); if (!result.success) { return c.json({ error: result.error }, 400); @@ -181,7 +174,7 @@ export const updateZapSplitsController: AppController = async (c) => { const adminPubkey = await conf.signer.getPublicKey(); - const dittoZapSplit = await getZapSplits(store, adminPubkey); + const dittoZapSplit = await getZapSplits(relay, adminPubkey); if (!dittoZapSplit) { return c.json({ error: 'Zap split not activated, restart the server.' }, 404); } @@ -208,10 +201,9 @@ export const updateZapSplitsController: AppController = async (c) => { const deleteZapSplitSchema = z.array(n.id()).min(1); export const deleteZapSplitsController: AppController = async (c) => { - const { conf } = c.var; + const { conf, relay } = c.var; const body = await parseBody(c.req.raw); const result = deleteZapSplitSchema.safeParse(body); - const store = c.get('store'); if (!result.success) { return c.json({ error: result.error }, 400); @@ -219,7 +211,7 @@ export const deleteZapSplitsController: AppController = async (c) => { const adminPubkey = await conf.signer.getPublicKey(); - const dittoZapSplit = await getZapSplits(store, adminPubkey); + const dittoZapSplit = await getZapSplits(relay, adminPubkey); if (!dittoZapSplit) { return c.json({ error: 'Zap split not activated, restart the server.' }, 404); } @@ -239,10 +231,9 @@ export const deleteZapSplitsController: AppController = async (c) => { }; export const getZapSplitsController: AppController = async (c) => { - const { conf } = c.var; - const store = c.get('store'); + const { conf, relay } = c.var; - const dittoZapSplit: DittoZapSplits | undefined = await getZapSplits(store, await conf.signer.getPublicKey()) ?? {}; + const dittoZapSplit: DittoZapSplits | undefined = await getZapSplits(relay, await conf.signer.getPublicKey()) ?? {}; if (!dittoZapSplit) { return c.json({ error: 'Zap split not activated, restart the server.' }, 404); } @@ -265,11 +256,11 @@ export const getZapSplitsController: AppController = async (c) => { }; export const statusZapSplitsController: AppController = async (c) => { - const store = c.get('store'); - const id = c.req.param('id'); - const { signal } = c.req.raw; + const { relay, signal } = c.var; - const [event] = await store.query([{ kinds: [1, 20], ids: [id], limit: 1 }], { signal }); + const id = c.req.param('id'); + + const [event] = await relay.query([{ kinds: [1, 20], ids: [id], limit: 1 }], { signal }); if (!event) { return c.json({ error: 'Event not found' }, 404); } @@ -278,8 +269,8 @@ export const statusZapSplitsController: AppController = async (c) => { const pubkeys = zapsTag.map((name) => name[1]); - const users = await store.query([{ authors: pubkeys, kinds: [0], limit: pubkeys.length }], { signal }); - await hydrateEvents({ events: users, store, signal }); + const users = await relay.query([{ authors: pubkeys, kinds: [0], limit: pubkeys.length }], { signal }); + await hydrateEvents({ events: users, relay, signal }); const zapSplits = (await Promise.all(pubkeys.map((pubkey) => { const author = (users.find((event) => event.pubkey === pubkey) as DittoEvent | undefined)?.author; diff --git a/packages/ditto/controllers/api/markers.ts b/packages/ditto/controllers/api/markers.ts index 005ebbe5..7e7cb8dd 100644 --- a/packages/ditto/controllers/api/markers.ts +++ b/packages/ditto/controllers/api/markers.ts @@ -14,7 +14,9 @@ interface Marker { } export const markersController: AppController = async (c) => { - const pubkey = await c.get('signer')?.getPublicKey()!; + const { user } = c.var; + + const pubkey = await user!.signer.getPublicKey(); const timelines = c.req.queries('timeline[]') ?? []; const results = await kv.getMany( @@ -37,7 +39,9 @@ const markerDataSchema = z.object({ }); export const updateMarkersController: AppController = async (c) => { - const pubkey = await c.get('signer')?.getPublicKey()!; + const { user } = c.var; + + const pubkey = await user!.signer.getPublicKey(); const record = z.record(z.enum(['home', 'notifications']), markerDataSchema).parse(await parseBody(c.req.raw)); const timelines = Object.keys(record) as Timeline[]; diff --git a/packages/ditto/controllers/api/media.ts b/packages/ditto/controllers/api/media.ts index fc309cdf..c6c6b062 100644 --- a/packages/ditto/controllers/api/media.ts +++ b/packages/ditto/controllers/api/media.ts @@ -21,9 +21,10 @@ const mediaUpdateSchema = z.object({ }); const mediaController: AppController = async (c) => { - const pubkey = await c.get('signer')?.getPublicKey()!; + const { user, signal } = c.var; + + const pubkey = await user!.signer.getPublicKey(); const result = mediaBodySchema.safeParse(await parseBody(c.req.raw)); - const { signal } = c.req.raw; if (!result.success) { return c.json({ error: 'Bad request.', schema: result.error }, 422); diff --git a/packages/ditto/controllers/api/mutes.ts b/packages/ditto/controllers/api/mutes.ts index 90b5f545..9ce9c5e9 100644 --- a/packages/ditto/controllers/api/mutes.ts +++ b/packages/ditto/controllers/api/mutes.ts @@ -1,15 +1,14 @@ import { type AppController } from '@/app.ts'; -import { Storages } from '@/storages.ts'; import { getTagSet } from '@/utils/tags.ts'; import { renderAccounts } from '@/views.ts'; /** https://docs.joinmastodon.org/methods/mutes/#get */ const mutesController: AppController = async (c) => { - const store = await Storages.db(); - const pubkey = await c.get('signer')?.getPublicKey()!; - const { signal } = c.req.raw; + const { relay, user, signal } = c.var; - const [event10000] = await store.query( + const pubkey = await user!.signer.getPublicKey(); + + const [event10000] = await relay.query( [{ kinds: [10000], authors: [pubkey], limit: 1 }], { signal }, ); diff --git a/packages/ditto/controllers/api/notifications.ts b/packages/ditto/controllers/api/notifications.ts index dfd4a03c..f180cf9e 100644 --- a/packages/ditto/controllers/api/notifications.ts +++ b/packages/ditto/controllers/api/notifications.ts @@ -30,8 +30,9 @@ const notificationsSchema = z.object({ }); const notificationsController: AppController = async (c) => { - const { conf } = c.var; - const pubkey = await c.get('signer')?.getPublicKey()!; + const { conf, user } = c.var; + + const pubkey = await user!.signer.getPublicKey(); const params = c.get('pagination'); const types = notificationTypes @@ -75,20 +76,21 @@ const notificationsController: AppController = async (c) => { }; const notificationController: AppController = async (c) => { + const { relay, user } = c.var; + const id = c.req.param('id'); - const pubkey = await c.get('signer')?.getPublicKey()!; - const store = c.get('store'); + const pubkey = await user!.signer.getPublicKey(); // Remove the timestamp from the ID. const eventId = id.replace(/^\d+-/, ''); - const [event] = await store.query([{ ids: [eventId] }]); + const [event] = await relay.query([{ ids: [eventId] }]); if (!event) { return c.json({ error: 'Event not found' }, { status: 404 }); } - await hydrateEvents({ events: [event], store }); + await hydrateEvents({ events: [event], relay }); const notification = await renderNotification(event, { viewerPubkey: pubkey }); @@ -105,16 +107,15 @@ async function renderNotifications( params: DittoPagination, c: AppContext, ) { - const { conf } = c.var; - const store = c.get('store'); - const pubkey = await c.get('signer')?.getPublicKey()!; - const { signal } = c.req.raw; + const { conf, relay, user, signal } = c.var; + + const pubkey = await user!.signer.getPublicKey(); const opts = { signal, limit: params.limit, timeout: conf.db.timeouts.timelines }; - const events = await store + const events = await relay .query(filters, opts) .then((events) => events.filter((event) => event.pubkey !== pubkey)) - .then((events) => hydrateEvents({ events, store, signal })); + .then((events) => hydrateEvents({ events, relay, signal })); if (!events.length) { return c.json([]); diff --git a/packages/ditto/controllers/api/pleroma.ts b/packages/ditto/controllers/api/pleroma.ts index 721347f3..dc4b0c68 100644 --- a/packages/ditto/controllers/api/pleroma.ts +++ b/packages/ditto/controllers/api/pleroma.ts @@ -2,14 +2,14 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { configSchema, elixirTupleSchema } from '@/schemas/pleroma-api.ts'; -import { Storages } from '@/storages.ts'; import { createAdminEvent, updateAdminEvent, updateUser } from '@/utils/api.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; import { getPleromaConfigs } from '@/utils/pleroma.ts'; const frontendConfigController: AppController = async (c) => { - const store = await Storages.db(); - const configDB = await getPleromaConfigs(store, c.req.raw.signal); + const { relay, signal } = c.var; + + const configDB = await getPleromaConfigs(relay, signal); const frontendConfig = configDB.get(':pleroma', ':frontend_configurations'); if (frontendConfig) { @@ -25,17 +25,17 @@ const frontendConfigController: AppController = async (c) => { }; const configController: AppController = async (c) => { - const store = await Storages.db(); - const configs = await getPleromaConfigs(store, c.req.raw.signal); + const { relay, signal } = c.var; + + const configs = await getPleromaConfigs(relay, signal); return c.json({ configs, need_reboot: false }); }; /** Pleroma admin config controller. */ const updateConfigController: AppController = async (c) => { - const { conf } = c.var; + const { conf, relay, signal } = c.var; - const store = await Storages.db(); - const configs = await getPleromaConfigs(store, c.req.raw.signal); + const configs = await getPleromaConfigs(relay, signal); const { configs: newConfigs } = z.object({ configs: z.array(configSchema) }).parse(await c.req.json()); configs.merge(newConfigs); diff --git a/packages/ditto/controllers/api/push.ts b/packages/ditto/controllers/api/push.ts index 79063622..e613c5f8 100644 --- a/packages/ditto/controllers/api/push.ts +++ b/packages/ditto/controllers/api/push.ts @@ -42,7 +42,7 @@ const pushSubscribeSchema = z.object({ }); export const pushSubscribeController: AppController = async (c) => { - const { conf } = c.var; + const { conf, user } = c.var; const vapidPublicKey = await conf.vapidPublicKey; if (!vapidPublicKey) { @@ -52,7 +52,7 @@ export const pushSubscribeController: AppController = async (c) => { const accessToken = getAccessToken(c.req.raw); const kysely = await Storages.kysely(); - const signer = c.get('signer')!; + const signer = user!.signer; const result = pushSubscribeSchema.safeParse(await parseBody(c.req.raw)); diff --git a/packages/ditto/controllers/api/reactions.ts b/packages/ditto/controllers/api/reactions.ts index 0beb985d..a69ba363 100644 --- a/packages/ditto/controllers/api/reactions.ts +++ b/packages/ditto/controllers/api/reactions.ts @@ -1,7 +1,6 @@ import { AppController } from '@/app.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { Storages } from '@/storages.ts'; import { createEvent } from '@/utils/api.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; @@ -11,16 +10,15 @@ import { renderStatus } from '@/views/mastodon/statuses.ts'; * https://docs.pleroma.social/backend/development/API/pleroma_api/#put-apiv1pleromastatusesidreactionsemoji */ const reactionController: AppController = async (c) => { + const { relay, user } = c.var; const id = c.req.param('id'); const emoji = c.req.param('emoji'); - const signer = c.get('signer')!; if (!/^\p{RGI_Emoji}$/v.test(emoji)) { return c.json({ error: 'Invalid emoji' }, 400); } - const store = await Storages.db(); - const [event] = await store.query([{ kinds: [1, 20], ids: [id], limit: 1 }]); + const [event] = await relay.query([{ kinds: [1, 20], ids: [id], limit: 1 }]); if (!event) { return c.json({ error: 'Status not found' }, 404); @@ -33,9 +31,9 @@ const reactionController: AppController = async (c) => { tags: [['e', id], ['p', event.pubkey]], }, c); - await hydrateEvents({ events: [event], store }); + await hydrateEvents({ events: [event], relay }); - const status = await renderStatus(event, { viewerPubkey: await signer.getPublicKey() }); + const status = await renderStatus(event, { viewerPubkey: await user!.signer.getPublicKey() }); return c.json(status); }; @@ -45,17 +43,17 @@ const reactionController: AppController = async (c) => { * https://docs.pleroma.social/backend/development/API/pleroma_api/#delete-apiv1pleromastatusesidreactionsemoji */ const deleteReactionController: AppController = async (c) => { + const { relay, user } = c.var; + const id = c.req.param('id'); const emoji = c.req.param('emoji'); - const signer = c.get('signer')!; - const pubkey = await signer.getPublicKey(); - const store = await Storages.db(); + const pubkey = await user!.signer.getPublicKey(); if (!/^\p{RGI_Emoji}$/v.test(emoji)) { return c.json({ error: 'Invalid emoji' }, 400); } - const [event] = await store.query([ + const [event] = await relay.query([ { kinds: [1, 20], ids: [id], limit: 1 }, ]); @@ -63,7 +61,7 @@ const deleteReactionController: AppController = async (c) => { return c.json({ error: 'Status not found' }, 404); } - const events = await store.query([ + const events = await relay.query([ { kinds: [7], authors: [pubkey], '#e': [id] }, ]); @@ -88,19 +86,20 @@ const deleteReactionController: AppController = async (c) => { * https://docs.pleroma.social/backend/development/API/pleroma_api/#get-apiv1pleromastatusesidreactions */ const reactionsController: AppController = async (c) => { + const { relay, user } = c.var; + const id = c.req.param('id'); - const store = await Storages.db(); - const pubkey = await c.get('signer')?.getPublicKey(); + const pubkey = await user?.signer.getPublicKey(); const emoji = c.req.param('emoji') as string | undefined; if (typeof emoji === 'string' && !/^\p{RGI_Emoji}$/v.test(emoji)) { return c.json({ error: 'Invalid emoji' }, 400); } - const events = await store.query([{ kinds: [7], '#e': [id], limit: 100 }]) + const events = await relay.query([{ kinds: [7], '#e': [id], limit: 100 }]) .then((events) => events.filter(({ content }) => /^\p{RGI_Emoji}$/v.test(content))) .then((events) => events.filter((event) => !emoji || event.content === emoji)) - .then((events) => hydrateEvents({ events, store })); + .then((events) => hydrateEvents({ events, relay })); /** Events grouped by emoji. */ const byEmoji = events.reduce((acc, event) => { diff --git a/packages/ditto/controllers/api/reports.ts b/packages/ditto/controllers/api/reports.ts index 11285825..7c98ce4e 100644 --- a/packages/ditto/controllers/api/reports.ts +++ b/packages/ditto/controllers/api/reports.ts @@ -18,8 +18,8 @@ const reportSchema = z.object({ /** https://docs.joinmastodon.org/methods/reports/#post */ const reportController: AppController = async (c) => { - const { conf } = c.var; - const store = c.get('store'); + const { conf, relay } = c.var; + const body = await parseBody(c.req.raw); const result = reportSchema.safeParse(body); @@ -49,7 +49,7 @@ const reportController: AppController = async (c) => { tags, }, c); - await hydrateEvents({ events: [event], store }); + await hydrateEvents({ events: [event], relay }); return c.json(await renderReport(event)); }; @@ -61,18 +61,16 @@ const adminReportsSchema = z.object({ /** https://docs.joinmastodon.org/methods/admin/reports/#get */ const adminReportsController: AppController = async (c) => { - const { conf } = c.var; - const store = c.get('store'); - const viewerPubkey = await c.get('signer')?.getPublicKey(); + const { conf, relay, user, pagination } = c.var; - const params = c.get('pagination'); + const viewerPubkey = await user?.signer.getPublicKey(); const { resolved, account_id, target_account_id } = adminReportsSchema.parse(c.req.query()); const filter: NostrFilter = { kinds: [30383], authors: [await conf.signer.getPublicKey()], '#k': ['1984'], - ...params, + ...pagination, }; if (typeof resolved === 'boolean') { @@ -85,7 +83,7 @@ const adminReportsController: AppController = async (c) => { filter['#P'] = [target_account_id]; } - const orig = await store.query([filter]); + const orig = await relay.query([filter]); const ids = new Set(); for (const event of orig) { @@ -95,8 +93,8 @@ const adminReportsController: AppController = async (c) => { } } - const events = await store.query([{ kinds: [1984], ids: [...ids] }]) - .then((events) => hydrateEvents({ store, events: events, signal: c.req.raw.signal })); + const events = await relay.query([{ kinds: [1984], ids: [...ids] }]) + .then((events) => hydrateEvents({ relay, events: events, signal: c.req.raw.signal })); const reports = await Promise.all( events.map((event) => renderAdminReport(event, { viewerPubkey })), @@ -107,12 +105,12 @@ const adminReportsController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/admin/reports/#get-one */ const adminReportController: AppController = async (c) => { - const eventId = c.req.param('id'); - const { signal } = c.req.raw; - const store = c.get('store'); - const pubkey = await c.get('signer')?.getPublicKey(); + const { relay, user, signal } = c.var; - const [event] = await store.query([{ + const eventId = c.req.param('id'); + const pubkey = await user?.signer.getPublicKey(); + + const [event] = await relay.query([{ kinds: [1984], ids: [eventId], limit: 1, @@ -122,7 +120,7 @@ const adminReportController: AppController = async (c) => { return c.json({ error: 'Not found' }, 404); } - await hydrateEvents({ events: [event], store, signal }); + await hydrateEvents({ events: [event], relay, signal }); const report = await renderAdminReport(event, { viewerPubkey: pubkey }); return c.json(report); @@ -130,12 +128,12 @@ const adminReportController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/admin/reports/#resolve */ const adminReportResolveController: AppController = async (c) => { - const eventId = c.req.param('id'); - const { signal } = c.req.raw; - const store = c.get('store'); - const pubkey = await c.get('signer')?.getPublicKey(); + const { relay, user, signal } = c.var; - const [event] = await store.query([{ + const eventId = c.req.param('id'); + const pubkey = await user?.signer.getPublicKey(); + + const [event] = await relay.query([{ kinds: [1984], ids: [eventId], limit: 1, @@ -146,19 +144,19 @@ const adminReportResolveController: AppController = async (c) => { } await updateEventInfo(eventId, { open: false, closed: true }, c); - await hydrateEvents({ events: [event], store, signal }); + await hydrateEvents({ events: [event], relay, signal }); const report = await renderAdminReport(event, { viewerPubkey: pubkey }); return c.json(report); }; const adminReportReopenController: AppController = async (c) => { - const eventId = c.req.param('id'); - const { signal } = c.req.raw; - const store = c.get('store'); - const pubkey = await c.get('signer')?.getPublicKey(); + const { relay, user, signal } = c.var; - const [event] = await store.query([{ + const eventId = c.req.param('id'); + const pubkey = await user?.signer.getPublicKey(); + + const [event] = await relay.query([{ kinds: [1984], ids: [eventId], limit: 1, @@ -169,7 +167,7 @@ const adminReportReopenController: AppController = async (c) => { } await updateEventInfo(eventId, { open: true, closed: false }, c); - await hydrateEvents({ events: [event], store, signal }); + await hydrateEvents({ events: [event], relay, signal }); const report = await renderAdminReport(event, { viewerPubkey: pubkey }); return c.json(report); diff --git a/packages/ditto/controllers/api/search.ts b/packages/ditto/controllers/api/search.ts index e890f166..3ce9e0ac 100644 --- a/packages/ditto/controllers/api/search.ts +++ b/packages/ditto/controllers/api/search.ts @@ -26,16 +26,16 @@ const searchQuerySchema = z.object({ type SearchQuery = z.infer & { since?: number; until?: number; limit: number }; const searchController: AppController = async (c) => { + const { user, pagination, signal } = c.var; + const result = searchQuerySchema.safeParse(c.req.query()); - const params = c.get('pagination'); - const { signal } = c.req.raw; - const viewerPubkey = await c.get('signer')?.getPublicKey(); + const viewerPubkey = await user?.signer.getPublicKey(); if (!result.success) { return c.json({ error: 'Bad request', schema: result.error }, 422); } - const event = await lookupEvent({ ...result.data, ...params }, signal); + const event = await lookupEvent({ ...result.data, ...pagination }, signal); const lookup = extractIdentifier(result.data.q); // Render account from pubkey. @@ -54,7 +54,7 @@ const searchController: AppController = async (c) => { events = [event]; } - events.push(...(await searchEvents({ ...result.data, ...params, viewerPubkey }, signal))); + events.push(...(await searchEvents({ ...result.data, ...pagination, viewerPubkey }, signal))); const [accounts, statuses] = await Promise.all([ Promise.all( @@ -78,7 +78,7 @@ const searchController: AppController = async (c) => { }; if (result.data.type === 'accounts') { - return paginatedList(c, { ...result.data, ...params }, body); + return paginatedList(c, { ...result.data, ...pagination }, body); } else { return paginated(c, events, body); } @@ -94,7 +94,7 @@ async function searchEvents( return Promise.resolve([]); } - const store = await Storages.db(); + const relay = await Storages.db(); const filter: NostrFilter = { kinds: typeToKinds(type), @@ -121,9 +121,9 @@ async function searchEvents( } // Query the events. - let events = await store + let events = await relay .query([filter], { signal }) - .then((events) => hydrateEvents({ events, store, signal })); + .then((events) => hydrateEvents({ events, relay, signal })); // When using an authors filter, return the events in the same order as the filter. if (filter.authors) { @@ -150,10 +150,10 @@ function typeToKinds(type: SearchQuery['type']): number[] { /** Resolve a searched value into an event, if applicable. */ async function lookupEvent(query: SearchQuery, signal: AbortSignal): Promise { const filters = await getLookupFilters(query, signal); - const store = await Storages.db(); + const relay = await Storages.db(); - return store.query(filters, { limit: 1, signal }) - .then((events) => hydrateEvents({ events, store, signal })) + return relay.query(filters, { limit: 1, signal }) + .then((events) => hydrateEvents({ events, relay, signal })) .then(([event]) => event); } diff --git a/packages/ditto/controllers/api/statuses.ts b/packages/ditto/controllers/api/statuses.ts index 6aa53308..5b73be9f 100644 --- a/packages/ditto/controllers/api/statuses.ts +++ b/packages/ditto/controllers/api/statuses.ts @@ -9,11 +9,11 @@ import { type AppController } from '@/app.ts'; import { DittoUpload, dittoUploads } from '@/DittoUploads.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { getAncestors, getAuthor, getDescendants, getEvent } from '@/queries.ts'; +import { paginationSchema } from '@/schemas/pagination.ts'; import { addTag, deleteTag } from '@/utils/tags.ts'; import { asyncReplaceAll } from '@/utils/text.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; import { languageSchema } from '@/schema.ts'; -import { Storages } from '@/storages.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { assertAuthenticated, createEvent, paginated, paginatedList, parseBody, updateListEvent } from '@/utils/api.ts'; import { getInvoice, getLnurl } from '@/utils/lnurl.ts'; @@ -46,9 +46,9 @@ const createStatusSchema = z.object({ ); const statusController: AppController = async (c) => { - const id = c.req.param('id'); - const signal = AbortSignal.any([c.req.raw.signal, AbortSignal.timeout(1500)]); + const { user, signal } = c.var; + const id = c.req.param('id'); const event = await getEvent(id, { signal }); if (event?.author) { @@ -56,7 +56,7 @@ const statusController: AppController = async (c) => { } if (event) { - const viewerPubkey = await c.get('signer')?.getPublicKey(); + const viewerPubkey = await user?.signer.getPublicKey(); const status = await renderStatus(event, { viewerPubkey }); return c.json(status); } @@ -65,10 +65,10 @@ const statusController: AppController = async (c) => { }; const createStatusController: AppController = async (c) => { - const { conf } = c.var; + const { conf, relay, user, signal } = c.var; + const body = await parseBody(c.req.raw); const result = createStatusSchema.safeParse(body); - const store = c.get('store'); if (!result.success) { return c.json({ error: 'Bad request', schema: result.error }, 400); @@ -87,14 +87,14 @@ const createStatusController: AppController = async (c) => { const tags: string[][] = []; if (data.in_reply_to_id) { - const [ancestor] = await store.query([{ ids: [data.in_reply_to_id] }]); + const [ancestor] = await relay.query([{ ids: [data.in_reply_to_id] }]); if (!ancestor) { return c.json({ error: 'Original post not found.' }, 404); } const rootId = ancestor.tags.find((tag) => tag[0] === 'e' && tag[3] === 'root')?.[1] ?? ancestor.id; - const root = rootId === ancestor.id ? ancestor : await store.query([{ ids: [rootId] }]).then(([event]) => event); + const root = rootId === ancestor.id ? ancestor : await relay.query([{ ids: [rootId] }]).then(([event]) => event); if (root) { tags.push(['e', root.id, conf.relay, 'root', root.pubkey]); @@ -108,7 +108,7 @@ const createStatusController: AppController = async (c) => { let quoted: DittoEvent | undefined; if (data.quote_id) { - [quoted] = await store.query([{ ids: [data.quote_id] }]); + [quoted] = await relay.query([{ ids: [data.quote_id] }]); if (!quoted) { return c.json({ error: 'Quoted post not found.' }, 404); @@ -190,13 +190,13 @@ const createStatusController: AppController = async (c) => { } } - const pubkey = await c.get('signer')?.getPublicKey()!; + const pubkey = await user!.signer.getPublicKey(); const author = pubkey ? await getAuthor(pubkey) : undefined; if (conf.zapSplitsEnabled) { const meta = n.json().pipe(n.metadata()).catch({}).parse(author?.content); const lnurl = getLnurl(meta); - const dittoZapSplit = await getZapSplits(store, await conf.signer.getPublicKey()); + const dittoZapSplit = await getZapSplits(relay, await conf.signer.getPublicKey()); if (lnurl && dittoZapSplit) { const totalSplit = Object.values(dittoZapSplit).reduce((total, { weight }) => total + weight, 0); for (const zapPubkey in dittoZapSplit) { @@ -256,8 +256,8 @@ const createStatusController: AppController = async (c) => { if (data.quote_id) { await hydrateEvents({ events: [event], - store: await Storages.db(), - signal: c.req.raw.signal, + relay, + signal, }); } @@ -265,11 +265,11 @@ const createStatusController: AppController = async (c) => { }; const deleteStatusController: AppController = async (c) => { - const { conf } = c.var; - const id = c.req.param('id'); - const pubkey = await c.get('signer')?.getPublicKey(); + const { conf, user, signal } = c.var; - const event = await getEvent(id, { signal: c.req.raw.signal }); + const id = c.req.param('id'); + const pubkey = await user?.signer.getPublicKey(); + const event = await getEvent(id, { signal }); if (event) { if (event.pubkey === pubkey) { @@ -289,10 +289,11 @@ const deleteStatusController: AppController = async (c) => { }; const contextController: AppController = async (c) => { + const { relay, user } = c.var; + const id = c.req.param('id'); - const store = c.get('store'); - const [event] = await store.query([{ kinds: [1, 20], ids: [id] }]); - const viewerPubkey = await c.get('signer')?.getPublicKey(); + const [event] = await relay.query([{ kinds: [1, 20], ids: [id] }]); + const viewerPubkey = await user?.signer.getPublicKey(); async function renderStatuses(events: NostrEvent[]) { const statuses = await Promise.all( @@ -303,14 +304,14 @@ const contextController: AppController = async (c) => { if (event) { const [ancestorEvents, descendantEvents] = await Promise.all([ - getAncestors(store, event), - getDescendants(store, event), + getAncestors(relay, event), + getDescendants(relay, event), ]); await hydrateEvents({ events: [...ancestorEvents, ...descendantEvents], signal: c.req.raw.signal, - store, + relay, }); const [ancestors, descendants] = await Promise.all([ @@ -325,10 +326,10 @@ const contextController: AppController = async (c) => { }; const favouriteController: AppController = async (c) => { - const { conf } = c.var; + const { conf, relay, user } = c.var; + const id = c.req.param('id'); - const store = await Storages.db(); - const [target] = await store.query([{ ids: [id], kinds: [1, 20] }]); + const [target] = await relay.query([{ ids: [id], kinds: [1, 20] }]); if (target) { await createEvent({ @@ -340,9 +341,9 @@ const favouriteController: AppController = async (c) => { ], }, c); - await hydrateEvents({ events: [target], store }); + await hydrateEvents({ events: [target], relay }); - const status = await renderStatus(target, { viewerPubkey: await c.get('signer')?.getPublicKey() }); + const status = await renderStatus(target, { viewerPubkey: await user?.signer.getPublicKey() }); if (status) { status.favourited = true; @@ -366,13 +367,10 @@ const favouritedByController: AppController = (c) => { /** https://docs.joinmastodon.org/methods/statuses/#boost */ const reblogStatusController: AppController = async (c) => { - const { conf } = c.var; - const eventId = c.req.param('id'); - const { signal } = c.req.raw; + const { conf, relay, user, signal } = c.var; - const event = await getEvent(eventId, { - kind: 1, - }); + const eventId = c.req.param('id'); + const event = await getEvent(eventId); if (!event) { return c.json({ error: 'Event not found.' }, 404); @@ -388,28 +386,28 @@ const reblogStatusController: AppController = async (c) => { await hydrateEvents({ events: [reblogEvent], - store: await Storages.db(), + relay, signal: signal, }); - const status = await renderReblog(reblogEvent, { viewerPubkey: await c.get('signer')?.getPublicKey() }); + const status = await renderReblog(reblogEvent, { viewerPubkey: await user?.signer.getPublicKey() }); return c.json(status); }; /** https://docs.joinmastodon.org/methods/statuses/#unreblog */ const unreblogStatusController: AppController = async (c) => { - const { conf } = c.var; - const eventId = c.req.param('id'); - const pubkey = await c.get('signer')?.getPublicKey()!; - const store = await Storages.db(); + const { conf, relay, user } = c.var; - const [event] = await store.query([{ ids: [eventId], kinds: [1, 20] }]); + const eventId = c.req.param('id'); + const pubkey = await user!.signer.getPublicKey(); + + const [event] = await relay.query([{ ids: [eventId], kinds: [1, 20] }]); if (!event) { return c.json({ error: 'Record not found' }, 404); } - const [repostEvent] = await store.query( + const [repostEvent] = await relay.query( [{ kinds: [6], authors: [pubkey], '#e': [event.id], limit: 1 }], ); @@ -432,20 +430,20 @@ const rebloggedByController: AppController = (c) => { }; const quotesController: AppController = async (c) => { - const id = c.req.param('id'); - const params = c.get('pagination'); - const store = await Storages.db(); + const { relay, user, pagination } = c.var; - const [event] = await store.query([{ ids: [id], kinds: [1, 20] }]); + const id = c.req.param('id'); + + const [event] = await relay.query([{ ids: [id], kinds: [1, 20] }]); if (!event) { return c.json({ error: 'Event not found.' }, 404); } - const quotes = await store - .query([{ kinds: [1, 20], '#q': [event.id], ...params }]) - .then((events) => hydrateEvents({ events, store })); + const quotes = await relay + .query([{ kinds: [1, 20], '#q': [event.id], ...pagination }]) + .then((events) => hydrateEvents({ events, relay })); - const viewerPubkey = await c.get('signer')?.getPublicKey(); + const viewerPubkey = await user?.signer.getPublicKey(); const statuses = await Promise.all( quotes.map((event) => renderStatus(event, { viewerPubkey })), @@ -460,14 +458,11 @@ const quotesController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/statuses/#bookmark */ const bookmarkController: AppController = async (c) => { - const { conf } = c.var; - const pubkey = await c.get('signer')?.getPublicKey()!; + const { conf, user } = c.var; + const pubkey = await user!.signer.getPublicKey(); const eventId = c.req.param('id'); - const event = await getEvent(eventId, { - kind: 1, - relations: ['author', 'event_stats', 'author_stats'], - }); + const event = await getEvent(eventId); if (event) { await updateListEvent( @@ -488,14 +483,12 @@ const bookmarkController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/statuses/#unbookmark */ const unbookmarkController: AppController = async (c) => { - const { conf } = c.var; - const pubkey = await c.get('signer')?.getPublicKey()!; + const { conf, user } = c.var; + + const pubkey = await user!.signer.getPublicKey(); const eventId = c.req.param('id'); - const event = await getEvent(eventId, { - kind: 1, - relations: ['author', 'event_stats', 'author_stats'], - }); + const event = await getEvent(eventId); if (event) { await updateListEvent( @@ -516,14 +509,12 @@ const unbookmarkController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/statuses/#pin */ const pinController: AppController = async (c) => { - const { conf } = c.var; - const pubkey = await c.get('signer')?.getPublicKey()!; + const { conf, user } = c.var; + + const pubkey = await user!.signer.getPublicKey(); const eventId = c.req.param('id'); - const event = await getEvent(eventId, { - kind: 1, - relations: ['author', 'event_stats', 'author_stats'], - }); + const event = await getEvent(eventId); if (event) { await updateListEvent( @@ -544,14 +535,13 @@ const pinController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/statuses/#unpin */ const unpinController: AppController = async (c) => { - const { conf } = c.var; - const pubkey = await c.get('signer')?.getPublicKey()!; + const { conf, user, signal } = c.var; + + const pubkey = await user!.signer.getPublicKey(); const eventId = c.req.param('id'); - const { signal } = c.req.raw; const event = await getEvent(eventId, { kind: 1, - relations: ['author', 'event_stats', 'author_stats'], signal, }); @@ -580,11 +570,10 @@ const zapSchema = z.object({ }); const zapController: AppController = async (c) => { - const { conf } = c.var; + const { conf, relay, signal } = c.var; + const body = await parseBody(c.req.raw); const result = zapSchema.safeParse(body); - const { signal } = c.req.raw; - const store = c.get('store'); if (!result.success) { return c.json({ error: 'Bad request', schema: result.error }, 400); @@ -611,7 +600,7 @@ const zapController: AppController = async (c) => { ); } } else { - [target] = await store.query([{ authors: [account_id], kinds: [0], limit: 1 }]); + [target] = await relay.query([{ authors: [account_id], kinds: [0], limit: 1 }]); const meta = n.json().pipe(n.metadata()).catch({}).parse(target?.content); lnurl = getLnurl(meta); if (target && lnurl) { @@ -638,19 +627,19 @@ const zapController: AppController = async (c) => { }; const zappedByController: AppController = async (c) => { - const id = c.req.param('id'); - const params = c.get('listPagination'); - const store = await Storages.db(); - const kysely = await Storages.kysely(); + const { db, relay } = c.var; - const zaps = await kysely.selectFrom('event_zaps') + const id = c.req.param('id'); + const { offset, limit } = paginationSchema.parse(c.req.query()); + + const zaps = await db.kysely.selectFrom('event_zaps') .selectAll() .where('target_event_id', '=', id) .orderBy('amount_millisats', 'desc') - .limit(params.limit) - .offset(params.offset).execute(); + .limit(limit) + .offset(offset).execute(); - const authors = await store.query([{ kinds: [0], authors: zaps.map((zap) => zap.sender_pubkey) }]); + const authors = await relay.query([{ kinds: [0], authors: zaps.map((zap) => zap.sender_pubkey) }]); const results = (await Promise.all( zaps.map(async (zap) => { @@ -668,7 +657,7 @@ const zappedByController: AppController = async (c) => { }), )).filter(Boolean); - return paginatedList(c, params, results); + return paginatedList(c, { limit, offset }, results); }; export { diff --git a/packages/ditto/controllers/api/streaming.ts b/packages/ditto/controllers/api/streaming.ts index 4171e1be..b39f1db5 100644 --- a/packages/ditto/controllers/api/streaming.ts +++ b/packages/ditto/controllers/api/streaming.ts @@ -68,7 +68,7 @@ const limiter = new TTLCache(); const connections = new Set(); const streamingController: AppController = async (c) => { - const { conf } = c.var; + const { conf, relay } = c.var; const upgrade = c.req.header('upgrade'); const token = c.req.header('sec-websocket-protocol'); const stream = streamSchema.optional().catch(undefined).parse(c.req.query('stream')); @@ -93,7 +93,6 @@ const streamingController: AppController = async (c) => { const { socket, response } = Deno.upgradeWebSocket(c.req.raw, { protocol: token, idleTimeout: 30 }); - const store = await Storages.db(); const policy = pubkey ? new MuteListPolicy(pubkey, await Storages.admin()) : undefined; function send(e: StreamingEvent) { @@ -108,7 +107,7 @@ const streamingController: AppController = async (c) => { render: (event: NostrEvent) => Promise, ) { try { - for await (const msg of store.req([filter], { signal: controller.signal })) { + for await (const msg of relay.req([filter], { signal: controller.signal })) { if (msg[0] === 'EVENT') { const event = msg[2]; @@ -119,7 +118,7 @@ const streamingController: AppController = async (c) => { } } - await hydrateEvents({ events: [event], store, signal: AbortSignal.timeout(1000) }); + await hydrateEvents({ events: [event], relay, signal: AbortSignal.timeout(1000) }); const result = await render(event); diff --git a/packages/ditto/controllers/api/suggestions.ts b/packages/ditto/controllers/api/suggestions.ts index 5dbf0d14..3af4f678 100644 --- a/packages/ditto/controllers/api/suggestions.ts +++ b/packages/ditto/controllers/api/suggestions.ts @@ -2,33 +2,32 @@ import { NostrFilter } from '@nostrify/nostrify'; import { matchFilter } from 'nostr-tools'; import { AppContext, AppController } from '@/app.ts'; +import { paginationSchema } from '@/schemas/pagination.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { paginated, paginatedList } from '@/utils/api.ts'; import { getTagSet } from '@/utils/tags.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; export const suggestionsV1Controller: AppController = async (c) => { - const signal = c.req.raw.signal; - const params = c.get('listPagination'); - const suggestions = await renderV2Suggestions(c, params, signal); + const { signal } = c.var; + const { offset, limit } = paginationSchema.parse(c.req.query()); + const suggestions = await renderV2Suggestions(c, { offset, limit }, signal); const accounts = suggestions.map(({ account }) => account); - return paginatedList(c, params, accounts); + return paginatedList(c, { offset, limit }, accounts); }; export const suggestionsV2Controller: AppController = async (c) => { - const signal = c.req.raw.signal; - const params = c.get('listPagination'); - const suggestions = await renderV2Suggestions(c, params, signal); - return paginatedList(c, params, suggestions); + const { signal } = c.var; + const { offset, limit } = paginationSchema.parse(c.req.query()); + const suggestions = await renderV2Suggestions(c, { offset, limit }, signal); + return paginatedList(c, { offset, limit }, suggestions); }; async function renderV2Suggestions(c: AppContext, params: { offset: number; limit: number }, signal?: AbortSignal) { - const { conf } = c.var; + const { conf, relay, user } = c.var; const { offset, limit } = params; - const store = c.get('store'); - const signer = c.get('signer'); - const pubkey = await signer?.getPublicKey(); + const pubkey = await user?.signer.getPublicKey(); const filters: NostrFilter[] = [ { kinds: [30382], authors: [await conf.signer.getPublicKey()], '#n': ['suggested'], limit }, @@ -40,7 +39,7 @@ async function renderV2Suggestions(c: AppContext, params: { offset: number; limi filters.push({ kinds: [10000], authors: [pubkey], limit: 1 }); } - const events = await store.query(filters, { signal }); + const events = await relay.query(filters, { signal }); const adminPubkey = await conf.signer.getPublicKey(); const [userEvents, followsEvent, mutesEvent, trendingEvent] = [ @@ -79,11 +78,11 @@ async function renderV2Suggestions(c: AppContext, params: { offset: number; limi const authors = [...pubkeys].slice(offset, offset + limit); - const profiles = await store.query( + const profiles = await relay.query( [{ kinds: [0], authors, limit: authors.length }], { signal }, ) - .then((events) => hydrateEvents({ events, store, signal })); + .then((events) => hydrateEvents({ events, relay, signal })); return Promise.all(authors.map(async (pubkey) => { const profile = profiles.find((event) => event.pubkey === pubkey); @@ -96,13 +95,10 @@ async function renderV2Suggestions(c: AppContext, params: { offset: number; limi } export const localSuggestionsController: AppController = async (c) => { - const { conf } = c.var; - const signal = c.req.raw.signal; - const params = c.get('pagination'); - const store = c.get('store'); + const { conf, relay, pagination, signal } = c.var; - const grants = await store.query( - [{ kinds: [30360], authors: [await conf.signer.getPublicKey()], ...params }], + const grants = await relay.query( + [{ kinds: [30360], authors: [await conf.signer.getPublicKey()], ...pagination }], { signal }, ); @@ -115,11 +111,11 @@ export const localSuggestionsController: AppController = async (c) => { } } - const profiles = await store.query( - [{ kinds: [0], authors: [...pubkeys], search: `domain:${conf.url.host}`, ...params }], + const profiles = await relay.query( + [{ kinds: [0], authors: [...pubkeys], search: `domain:${conf.url.host}`, ...pagination }], { signal }, ) - .then((events) => hydrateEvents({ store, events, signal })); + .then((events) => hydrateEvents({ relay, events, signal })); const suggestions = [...pubkeys].map((pubkey) => { const profile = profiles.find((event) => event.pubkey === pubkey); diff --git a/packages/ditto/controllers/api/timelines.ts b/packages/ditto/controllers/api/timelines.ts index a6f872b9..b8c74f41 100644 --- a/packages/ditto/controllers/api/timelines.ts +++ b/packages/ditto/controllers/api/timelines.ts @@ -15,8 +15,8 @@ const homeQuerySchema = z.object({ }); const homeTimelineController: AppController = async (c) => { - const params = c.get('pagination'); - const pubkey = await c.get('signer')?.getPublicKey()!; + const { user, pagination } = c.var; + const pubkey = await user?.signer.getPublicKey()!; const result = homeQuerySchema.safeParse(c.req.query()); if (!result.success) { @@ -26,7 +26,7 @@ const homeTimelineController: AppController = async (c) => { const { exclude_replies, only_media } = result.data; const authors = [...await getFeedPubkeys(pubkey)]; - const filter: NostrFilter = { authors, kinds: [1, 6, 20], ...params }; + const filter: NostrFilter = { authors, kinds: [1, 6, 20], ...pagination }; const search: string[] = []; @@ -90,35 +90,32 @@ const hashtagTimelineController: AppController = (c) => { }; const suggestedTimelineController: AppController = async (c) => { - const { conf } = c.var; - const store = c.get('store'); - const params = c.get('pagination'); + const { conf, relay, pagination } = c.var; - const [follows] = await store.query( + const [follows] = await relay.query( [{ kinds: [3], authors: [await conf.signer.getPublicKey()], limit: 1 }], ); const authors = [...getTagSet(follows?.tags ?? [], 'p')]; - return renderStatuses(c, [{ authors, kinds: [1, 20], ...params }]); + return renderStatuses(c, [{ authors, kinds: [1, 20], ...pagination }]); }; /** Render statuses for timelines. */ async function renderStatuses(c: AppContext, filters: NostrFilter[]) { - const { conf } = c.var; - const { signal } = c.req.raw; - const store = c.get('store'); + const { conf, relay, user, signal } = c.var; + const opts = { signal, timeout: conf.db.timeouts.timelines }; - const events = await store + const events = await relay .query(filters, opts) - .then((events) => hydrateEvents({ events, store, signal })); + .then((events) => hydrateEvents({ events, relay, signal })); if (!events.length) { return c.json([]); } - const viewerPubkey = await c.get('signer')?.getPublicKey(); + const viewerPubkey = await user?.signer.getPublicKey(); const statuses = (await Promise.all(events.map((event) => { if (event.kind === 6) { diff --git a/packages/ditto/controllers/api/translate.ts b/packages/ditto/controllers/api/translate.ts index de183e23..f9ff4dcd 100644 --- a/packages/ditto/controllers/api/translate.ts +++ b/packages/ditto/controllers/api/translate.ts @@ -17,8 +17,9 @@ const translateSchema = z.object({ }); const translateController: AppController = async (c) => { + const { user, signal } = c.var; + const result = translateSchema.safeParse(await parseBody(c.req.raw)); - const { signal } = c.req.raw; if (!result.success) { return c.json({ error: 'Bad request.', schema: result.error }, 422); @@ -38,7 +39,7 @@ const translateController: AppController = async (c) => { return c.json({ error: 'Record not found' }, 400); } - const viewerPubkey = await c.get('signer')?.getPublicKey(); + const viewerPubkey = await user?.signer.getPublicKey(); if (lang.toLowerCase() === event?.language?.toLowerCase()) { return c.json({ error: 'Source and target languages are the same. No translation needed.' }, 400); diff --git a/packages/ditto/controllers/api/trends.ts b/packages/ditto/controllers/api/trends.ts index f14cf0b7..5af88557 100644 --- a/packages/ditto/controllers/api/trends.ts +++ b/packages/ditto/controllers/api/trends.ts @@ -52,8 +52,8 @@ const trendingTagsController: AppController = async (c) => { }; async function getTrendingHashtags(conf: DittoConf) { - const store = await Storages.db(); - const trends = await getTrendingTags(store, 't', await conf.signer.getPublicKey()); + const relay = await Storages.db(); + const trends = await getTrendingTags(relay, 't', await conf.signer.getPublicKey()); return trends.map((trend) => { const hashtag = trend.value; @@ -105,8 +105,8 @@ const trendingLinksController: AppController = async (c) => { }; async function getTrendingLinks(conf: DittoConf) { - const store = await Storages.db(); - const trends = await getTrendingTags(store, 'r', await conf.signer.getPublicKey()); + const relay = await Storages.db(); + const trends = await getTrendingTags(relay, 'r', await conf.signer.getPublicKey()); return Promise.all(trends.map(async (trend) => { const link = trend.value; @@ -140,11 +140,10 @@ async function getTrendingLinks(conf: DittoConf) { } const trendingStatusesController: AppController = async (c) => { - const { conf } = c.var; - const store = await Storages.db(); + const { conf, relay } = c.var; const { limit, offset, until } = paginationSchema.parse(c.req.query()); - const [label] = await store.query([{ + const [label] = await relay.query([{ kinds: [1985], '#L': ['pub.ditto.trends'], '#l': ['#e'], @@ -162,8 +161,8 @@ const trendingStatusesController: AppController = async (c) => { return c.json([]); } - const results = await store.query([{ kinds: [1, 20], ids }]) - .then((events) => hydrateEvents({ events, store })); + const results = await relay.query([{ kinds: [1, 20], ids }]) + .then((events) => hydrateEvents({ events, relay })); // Sort events in the order they appear in the label. const events = ids diff --git a/packages/ditto/controllers/frontend.ts b/packages/ditto/controllers/frontend.ts index ec9f11a5..d19a20cb 100644 --- a/packages/ditto/controllers/frontend.ts +++ b/packages/ditto/controllers/frontend.ts @@ -1,7 +1,6 @@ import { logi } from '@soapbox/logi'; import { AppMiddleware } from '@/app.ts'; -import { Storages } from '@/storages.ts'; import { getPathParams, MetadataEntities } from '@/utils/og-metadata.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; import { errorJson } from '@/utils/log.ts'; @@ -10,11 +9,14 @@ import { renderMetadata } from '@/views/meta.ts'; import { getAuthor, getEvent } from '@/queries.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; import { renderAccount } from '@/views/mastodon/accounts.ts'; +import { NStore } from '@nostrify/nostrify'; /** Placeholder to find & replace with metadata. */ const META_PLACEHOLDER = '' as const; export const frontendController: AppMiddleware = async (c) => { + const { relay } = c.var; + c.header('Cache-Control', 'max-age=86400, s-maxage=30, public, stale-if-error=604800'); try { @@ -23,7 +25,7 @@ export const frontendController: AppMiddleware = async (c) => { if (content.includes(META_PLACEHOLDER)) { const params = getPathParams(c.req.path); try { - const entities = await getEntities(params ?? {}); + const entities = await getEntities(relay, params ?? {}); const meta = renderMetadata(c.req.url, entities); return c.html(content.replace(META_PLACEHOLDER, meta)); } catch (e) { @@ -37,11 +39,9 @@ export const frontendController: AppMiddleware = async (c) => { } }; -async function getEntities(params: { acct?: string; statusId?: string }): Promise { - const store = await Storages.db(); - +async function getEntities(relay: NStore, params: { acct?: string; statusId?: string }): Promise { const entities: MetadataEntities = { - instance: await getInstanceMetadata(store), + instance: await getInstanceMetadata(relay), }; if (params.statusId) { diff --git a/packages/ditto/controllers/manifest.ts b/packages/ditto/controllers/manifest.ts index 2e75de04..70d42dea 100644 --- a/packages/ditto/controllers/manifest.ts +++ b/packages/ditto/controllers/manifest.ts @@ -1,10 +1,11 @@ import { AppController } from '@/app.ts'; -import { Storages } from '@/storages.ts'; import { WebManifestCombined } from '@/types/webmanifest.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; export const manifestController: AppController = async (c) => { - const meta = await getInstanceMetadata(await Storages.db(), c.req.raw.signal); + const { relay, signal } = c.var; + + const meta = await getInstanceMetadata(relay, signal); const manifest: WebManifestCombined = { description: meta.about, diff --git a/packages/ditto/controllers/nostr/relay-info.ts b/packages/ditto/controllers/nostr/relay-info.ts index d4721cdc..50702c23 100644 --- a/packages/ditto/controllers/nostr/relay-info.ts +++ b/packages/ditto/controllers/nostr/relay-info.ts @@ -1,13 +1,12 @@ import denoJson from 'deno.json' with { type: 'json' }; import { AppController } from '@/app.ts'; -import { Storages } from '@/storages.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; const relayInfoController: AppController = async (c) => { - const { conf } = c.var; - const store = await Storages.db(); - const meta = await getInstanceMetadata(store, c.req.raw.signal); + const { conf, relay, signal } = c.var; + + const meta = await getInstanceMetadata(relay, signal); c.res.headers.set('access-control-allow-origin', '*'); diff --git a/packages/ditto/controllers/nostr/relay.ts b/packages/ditto/controllers/nostr/relay.ts index 0284ce64..191aed36 100644 --- a/packages/ditto/controllers/nostr/relay.ts +++ b/packages/ditto/controllers/nostr/relay.ts @@ -18,7 +18,7 @@ import { AppController } from '@/app.ts'; import { relayInfoController } from '@/controllers/nostr/relay-info.ts'; import * as pipeline from '@/pipeline.ts'; import { RelayError } from '@/RelayError.ts'; -import { Storages } from '@/storages.ts'; +import { type DittoPgStore } from '@/storages/DittoPgStore.ts'; import { errorJson } from '@/utils/log.ts'; import { purifyEvent } from '@/utils/purify.ts'; import { Time } from '@/utils/time.ts'; @@ -42,7 +42,7 @@ const limiters = { const connections = new Set(); /** Set up the Websocket connection. */ -function connectStream(socket: WebSocket, ip: string | undefined, conf: DittoConf) { +function connectStream(conf: DittoConf, relay: DittoPgStore, socket: WebSocket, ip: string | undefined) { const controllers = new Map(); if (ip) { @@ -133,10 +133,8 @@ function connectStream(socket: WebSocket, ip: string | undefined, conf: DittoCon controllers.get(subId)?.abort(); controllers.set(subId, controller); - const store = await Storages.db(); - try { - for await (const [verb, , ...rest] of store.req(filters, { limit: 100, timeout: conf.db.timeouts.relay })) { + for await (const [verb, , ...rest] of relay.req(filters, { limit: 100, timeout: conf.db.timeouts.relay })) { send([verb, subId, ...rest] as NostrRelayMsg); } } catch (e) { @@ -185,8 +183,7 @@ function connectStream(socket: WebSocket, ip: string | undefined, conf: DittoCon /** Handle COUNT. Return the number of events matching the filters. */ async function handleCount([_, subId, ...filters]: NostrClientCOUNT): Promise { if (rateLimited(limiters.req)) return; - const store = await Storages.db(); - const { count } = await store.count(filters, { timeout: conf.db.timeouts.relay }); + const { count } = await relay.count(filters, { timeout: conf.db.timeouts.relay }); send(['COUNT', subId, { count, approximate: false }]); } @@ -199,7 +196,7 @@ function connectStream(socket: WebSocket, ip: string | undefined, conf: DittoCon } const relayController: AppController = (c, next) => { - const { conf } = c.var; + const { conf, relay } = c.var; const upgrade = c.req.header('upgrade'); // NIP-11: https://github.com/nostr-protocol/nips/blob/master/11.md @@ -218,7 +215,7 @@ const relayController: AppController = (c, next) => { } const { socket, response } = Deno.upgradeWebSocket(c.req.raw, { idleTimeout: 30 }); - connectStream(socket, ip, conf); + connectStream(conf, relay as DittoPgStore, socket, ip); return response; }; diff --git a/packages/ditto/controllers/well-known/nostr.ts b/packages/ditto/controllers/well-known/nostr.ts index 4fd366e7..ee442788 100644 --- a/packages/ditto/controllers/well-known/nostr.ts +++ b/packages/ditto/controllers/well-known/nostr.ts @@ -12,17 +12,17 @@ const emptyResult: NostrJson = { names: {}, relays: {} }; * https://github.com/nostr-protocol/nips/blob/master/05.md */ const nostrController: AppController = async (c) => { + const { relay } = c.var; + // If there are no query parameters, this will always return an empty result. if (!Object.entries(c.req.queries()).length) { c.header('Cache-Control', 'max-age=31536000, public, immutable, stale-while-revalidate=86400'); return c.json(emptyResult); } - const store = c.get('store'); - const result = nameSchema.safeParse(c.req.query('name')); const name = result.success ? result.data : undefined; - const pointer = name ? await localNip05Lookup(store, name) : undefined; + const pointer = name ? await localNip05Lookup(relay, name) : undefined; if (!name || !pointer) { // Not found, cache for 5 minutes. diff --git a/packages/ditto/middleware/auth98Middleware.ts b/packages/ditto/middleware/auth98Middleware.ts index 573853f0..18fce5fd 100644 --- a/packages/ditto/middleware/auth98Middleware.ts +++ b/packages/ditto/middleware/auth98Middleware.ts @@ -22,8 +22,13 @@ function auth98Middleware(opts: ParseAuthRequestOpts = {}): AppMiddleware { const result = await parseAuthRequest(req, opts); if (result.success) { - c.set('signer', new ReadOnlySigner(result.data.pubkey)); - c.set('proof', result.data); + const user = { + relay: c.var.relay, + signer: new ReadOnlySigner(result.data.pubkey), + ...c.var.user, + }; + + c.set('user', user); } await next(); @@ -71,7 +76,7 @@ function withProof( opts?: ParseAuthRequestOpts, ): AppMiddleware { return async (c, next) => { - const signer = c.get('signer'); + const signer = c.var.user?.signer; const pubkey = await signer?.getPublicKey(); const proof = c.get('proof') || await obtainProof(c, opts); @@ -84,7 +89,13 @@ function withProof( c.set('proof', proof); if (!signer) { - c.set('signer', new ReadOnlySigner(proof.pubkey)); + const user = { + relay: c.var.relay, + signer: new ReadOnlySigner(proof.pubkey), + ...c.var.user, + }; + + c.set('user', user); } await handler(c, proof, next); @@ -96,7 +107,7 @@ function withProof( /** Get the proof over Nostr Connect. */ async function obtainProof(c: AppContext, opts?: ParseAuthRequestOpts) { - const signer = c.get('signer'); + const signer = c.var.user?.signer; if (!signer) { throw new HTTPException(401, { res: c.json({ error: 'No way to sign Nostr event' }, 401), diff --git a/packages/ditto/middleware/paginationMiddleware.ts b/packages/ditto/middleware/paginationMiddleware.ts deleted file mode 100644 index b1f1e2f3..00000000 --- a/packages/ditto/middleware/paginationMiddleware.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { AppMiddleware } from '@/app.ts'; -import { paginationSchema } from '@/schemas/pagination.ts'; -import { Storages } from '@/storages.ts'; - -/** Fixes compatibility with Mastodon apps by that don't use `Link` headers. */ -export const paginationMiddleware: AppMiddleware = async (c, next) => { - const pagination = paginationSchema.parse(c.req.query()); - - const { - max_id: maxId, - min_id: minId, - since, - until, - } = pagination; - - if ((maxId && !until) || (minId && !since)) { - const ids: string[] = []; - - if (maxId) ids.push(maxId); - if (minId) ids.push(minId); - - if (ids.length) { - const store = await Storages.db(); - - const events = await store.query( - [{ ids, limit: ids.length }], - { signal: c.req.raw.signal }, - ); - - for (const event of events) { - if (!until && maxId === event.id) pagination.until = event.created_at; - if (!since && minId === event.id) pagination.since = event.created_at; - } - } - } - - c.set('pagination', { - since: pagination.since, - until: pagination.until, - limit: pagination.limit, - }); - - c.set('listPagination', { - limit: pagination.limit, - offset: pagination.offset, - }); - - await next(); -}; diff --git a/packages/ditto/middleware/requireSigner.ts b/packages/ditto/middleware/requireSigner.ts deleted file mode 100644 index 7733b26f..00000000 --- a/packages/ditto/middleware/requireSigner.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { MiddlewareHandler } from '@hono/hono'; -import { HTTPException } from '@hono/hono/http-exception'; -import { NostrSigner } from '@nostrify/nostrify'; -import { SetRequired } from 'type-fest'; - -/** Throw a 401 if a signer isn't set. */ -export const requireSigner: MiddlewareHandler<{ Variables: { signer: NostrSigner } }> = async (c, next) => { - if (!c.get('signer')) { - throw new HTTPException(401, { message: 'No pubkey provided' }); - } - - await next(); -}; - -/** Throw a 401 if a NIP-44 signer isn't set. */ -export const requireNip44Signer: MiddlewareHandler<{ Variables: { signer: SetRequired } }> = - async (c, next) => { - const signer = c.get('signer'); - - if (!signer) { - throw new HTTPException(401, { message: 'No pubkey provided' }); - } - - if (!signer.nip44) { - throw new HTTPException(401, { message: 'No NIP-44 signer provided' }); - } - - await next(); - }; diff --git a/packages/ditto/middleware/signerMiddleware.ts b/packages/ditto/middleware/signerMiddleware.ts deleted file mode 100644 index deea86b3..00000000 --- a/packages/ditto/middleware/signerMiddleware.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { type DittoConf } from '@ditto/conf'; -import { MiddlewareHandler } from '@hono/hono'; -import { HTTPException } from '@hono/hono/http-exception'; -import { NostrSigner, NSecSigner } from '@nostrify/nostrify'; -import { nip19 } from 'nostr-tools'; - -import { ConnectSigner } from '@/signers/ConnectSigner.ts'; -import { ReadOnlySigner } from '@/signers/ReadOnlySigner.ts'; -import { Storages } from '@/storages.ts'; -import { aesDecrypt } from '@/utils/aes.ts'; -import { getTokenHash } from '@/utils/auth.ts'; - -/** We only accept "Bearer" type. */ -const BEARER_REGEX = new RegExp(`^Bearer (${nip19.BECH32_REGEX.source})$`); - -/** Make a `signer` object available to all controllers, or unset if the user isn't logged in. */ -export const signerMiddleware: MiddlewareHandler<{ Variables: { signer: NostrSigner; conf: DittoConf } }> = async ( - c, - next, -) => { - const { conf } = c.var; - const header = c.req.header('authorization'); - const match = header?.match(BEARER_REGEX); - - if (match) { - const [_, bech32] = match; - - if (bech32.startsWith('token1')) { - try { - const kysely = await Storages.kysely(); - const tokenHash = await getTokenHash(bech32 as `token1${string}`); - - const { pubkey: userPubkey, bunker_pubkey: bunkerPubkey, nip46_sk_enc, nip46_relays } = await kysely - .selectFrom('auth_tokens') - .select(['pubkey', 'bunker_pubkey', 'nip46_sk_enc', 'nip46_relays']) - .where('token_hash', '=', tokenHash) - .executeTakeFirstOrThrow(); - - const nep46Seckey = await aesDecrypt(conf.seckey, nip46_sk_enc); - - c.set( - 'signer', - new ConnectSigner({ - bunkerPubkey, - userPubkey, - signer: new NSecSigner(nep46Seckey), - relays: nip46_relays, - }), - ); - } catch { - throw new HTTPException(401); - } - } else { - try { - const decoded = nip19.decode(bech32!); - - switch (decoded.type) { - case 'npub': - c.set('signer', new ReadOnlySigner(decoded.data)); - break; - case 'nprofile': - c.set('signer', new ReadOnlySigner(decoded.data.pubkey)); - break; - case 'nsec': - c.set('signer', new NSecSigner(decoded.data)); - break; - } - } catch { - throw new HTTPException(401); - } - } - } - - await next(); -}; diff --git a/packages/ditto/middleware/storeMiddleware.ts b/packages/ditto/middleware/storeMiddleware.ts deleted file mode 100644 index f69712a3..00000000 --- a/packages/ditto/middleware/storeMiddleware.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { MiddlewareHandler } from '@hono/hono'; -import { NostrSigner, NStore } from '@nostrify/nostrify'; - -import { UserStore } from '@/storages/UserStore.ts'; -import { Storages } from '@/storages.ts'; - -export const requireStore: MiddlewareHandler<{ Variables: { store: NStore } }> = async (c, next) => { - if (!c.get('store')) { - throw new Error('Store is required'); - } - await next(); -}; - -/** Store middleware. */ -export const storeMiddleware: MiddlewareHandler<{ Variables: { signer?: NostrSigner; store: NStore } }> = async ( - c, - next, -) => { - const pubkey = await c.get('signer')?.getPublicKey(); - - if (pubkey) { - const store = new UserStore(pubkey, await Storages.admin()); - c.set('store', store); - } else { - c.set('store', await Storages.admin()); - } - await next(); -}; diff --git a/packages/ditto/middleware/uploaderMiddleware.ts b/packages/ditto/middleware/uploaderMiddleware.ts index 10cd3d2b..2a3cffd3 100644 --- a/packages/ditto/middleware/uploaderMiddleware.ts +++ b/packages/ditto/middleware/uploaderMiddleware.ts @@ -6,7 +6,8 @@ import { AppMiddleware } from '@/app.ts'; /** Set an uploader for the user. */ export const uploaderMiddleware: AppMiddleware = async (c, next) => { - const { signer, conf } = c.var; + const { user, conf } = c.var; + const signer = user?.signer; switch (conf.uploader) { case 's3': diff --git a/packages/ditto/pipeline.ts b/packages/ditto/pipeline.ts index 602d0e2b..815229a0 100644 --- a/packages/ditto/pipeline.ts +++ b/packages/ditto/pipeline.ts @@ -137,7 +137,7 @@ function isProtectedEvent(event: NostrEvent): boolean { /** Hydrate the event with the user, if applicable. */ async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { - await hydrateEvents({ events: [event], store: await Storages.db(), signal }); + await hydrateEvents({ events: [event], relay: await Storages.db(), signal }); } /** Maybe store the event, if eligible. */ diff --git a/packages/ditto/queries.ts b/packages/ditto/queries.ts index f60d3daa..d4f0cb11 100644 --- a/packages/ditto/queries.ts +++ b/packages/ditto/queries.ts @@ -19,13 +19,13 @@ interface GetEventOpts { /** * Get a Nostr event by its ID. - * @deprecated Use `store.query` directly. + * @deprecated Use `relay.query` directly. */ const getEvent = async ( id: string, opts: GetEventOpts = {}, ): Promise => { - const store = await Storages.db(); + const relay = await Storages.db(); const { kind, signal = AbortSignal.timeout(1000) } = opts; const filter: NostrFilter = { ids: [id], limit: 1 }; @@ -33,23 +33,23 @@ const getEvent = async ( filter.kinds = [kind]; } - return await store.query([filter], { limit: 1, signal }) - .then((events) => hydrateEvents({ events, store, signal })) + return await relay.query([filter], { limit: 1, signal }) + .then((events) => hydrateEvents({ events, relay, signal })) .then(([event]) => event); }; /** * Get a Nostr `set_medatadata` event for a user's pubkey. - * @deprecated Use `store.query` directly. + * @deprecated Use `relay.query` directly. */ async function getAuthor(pubkey: string, opts: GetEventOpts = {}): Promise { - const store = await Storages.db(); + const relay = await Storages.db(); const { signal = AbortSignal.timeout(1000) } = opts; - const events = await store.query([{ authors: [pubkey], kinds: [0], limit: 1 }], { limit: 1, signal }); + const events = await relay.query([{ authors: [pubkey], kinds: [0], limit: 1 }], { limit: 1, signal }); const event = events[0] ?? fallbackAuthor(pubkey); - await hydrateEvents({ events: [event], store, signal }); + await hydrateEvents({ events: [event], relay, signal }); return event; } diff --git a/packages/ditto/storages/hydrate.test.ts b/packages/ditto/storages/hydrate.test.ts index 1527f321..ebafa6af 100644 --- a/packages/ditto/storages/hydrate.test.ts +++ b/packages/ditto/storages/hydrate.test.ts @@ -18,7 +18,7 @@ Deno.test('hydrateEvents(): author --- WITHOUT stats', async () => { await hydrateEvents({ events: [event1], - store: relay, + relay, kysely: db.kysely, }); @@ -43,7 +43,7 @@ Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { await hydrateEvents({ events: [event6], - store: relay, + relay, kysely: db.kysely, }); @@ -72,7 +72,7 @@ Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { await hydrateEvents({ events: [event1quoteRepost], - store: relay, + relay, kysely: db.kysely, }); @@ -102,7 +102,7 @@ Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () await hydrateEvents({ events: [event6], - store: relay, + relay, kysely: db.kysely, }); @@ -131,7 +131,7 @@ Deno.test('hydrateEvents(): report pubkey and post // kind 1984 --- WITHOUT stat await hydrateEvents({ events: [reportEvent], - store: relay, + relay, kysely: db.kysely, }); @@ -161,7 +161,7 @@ Deno.test('hydrateEvents(): zap sender, zap amount, zapped post // kind 9735 --- await hydrateEvents({ events: [zapReceipt], - store: relay, + relay, kysely: db.kysely, }); diff --git a/packages/ditto/storages/hydrate.ts b/packages/ditto/storages/hydrate.ts index 96341a1f..5bf51f96 100644 --- a/packages/ditto/storages/hydrate.ts +++ b/packages/ditto/storages/hydrate.ts @@ -15,14 +15,14 @@ import { Storages } from '@/storages.ts'; interface HydrateOpts { events: DittoEvent[]; - store: NStore; + relay: NStore; signal?: AbortSignal; kysely?: Kysely; } /** Hydrate events using the provided storage. */ async function hydrateEvents(opts: HydrateOpts): Promise { - const { events, store, signal, kysely = await Storages.kysely() } = opts; + const { events, relay, signal, kysely = await Storages.kysely() } = opts; if (!events.length) { return events; @@ -30,23 +30,23 @@ async function hydrateEvents(opts: HydrateOpts): Promise { const cache = [...events]; - for (const event of await gatherRelatedEvents({ events: cache, store, signal })) { + for (const event of await gatherRelatedEvents({ events: cache, relay, signal })) { cache.push(event); } - for (const event of await gatherQuotes({ events: cache, store, signal })) { + for (const event of await gatherQuotes({ events: cache, relay, signal })) { cache.push(event); } - for (const event of await gatherProfiles({ events: cache, store, signal })) { + for (const event of await gatherProfiles({ events: cache, relay, signal })) { cache.push(event); } - for (const event of await gatherUsers({ events: cache, store, signal })) { + for (const event of await gatherUsers({ events: cache, relay, signal })) { cache.push(event); } - for (const event of await gatherInfo({ events: cache, store, signal })) { + for (const event of await gatherInfo({ events: cache, relay, signal })) { cache.push(event); } @@ -199,7 +199,7 @@ export function assembleEvents( } /** Collect event targets (eg reposts, quote posts, reacted posts, etc.) */ -function gatherRelatedEvents({ events, store, signal }: HydrateOpts): Promise { +function gatherRelatedEvents({ events, relay, signal }: HydrateOpts): Promise { const ids = new Set(); for (const event of events) { @@ -234,14 +234,14 @@ function gatherRelatedEvents({ events, store, signal }: HydrateOpts): Promise { +function gatherQuotes({ events, relay, signal }: HydrateOpts): Promise { const ids = new Set(); for (const event of events) { @@ -253,14 +253,14 @@ function gatherQuotes({ events, store, signal }: HydrateOpts): Promise { +async function gatherProfiles({ events, relay, signal }: HydrateOpts): Promise { const pubkeys = new Set(); for (const event of events) { @@ -300,7 +300,7 @@ async function gatherProfiles({ events, store, signal }: HydrateOpts): Promise { +async function gatherUsers({ events, relay, signal }: HydrateOpts): Promise { const pubkeys = new Set(events.map((event) => event.pubkey)); if (!pubkeys.size) { return Promise.resolve([]); } - return store.query( + return relay.query( [{ kinds: [30382], authors: [await Conf.signer.getPublicKey()], '#d': [...pubkeys], limit: pubkeys.size }], { signal }, ); } /** Collect info events from the events. */ -async function gatherInfo({ events, store, signal }: HydrateOpts): Promise { +async function gatherInfo({ events, relay, signal }: HydrateOpts): Promise { const ids = new Set(); for (const event of events) { @@ -344,7 +344,7 @@ async function gatherInfo({ events, store, signal }: HydrateOpts): Promise + !c.var.user && author.tags.some(([name, value, ns]) => name === 'l' && value === '!no-unauthenticated' && ns === 'com.atproto.label.defs#selfLabel' diff --git a/packages/ditto/views.ts b/packages/ditto/views.ts index 562043db..879c3196 100644 --- a/packages/ditto/views.ts +++ b/packages/ditto/views.ts @@ -1,7 +1,7 @@ import { NostrEvent, NostrFilter } from '@nostrify/nostrify'; import { AppContext } from '@/app.ts'; -import { Storages } from '@/storages.ts'; +import { paginationSchema } from '@/schemas/pagination.ts'; import { renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; import { paginated, paginatedList } from '@/utils/api.ts'; @@ -20,13 +20,12 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], opts?: } const { signal = AbortSignal.timeout(1000), filterFn } = opts ?? {}; + const { relay } = c.var; - const store = await Storages.db(); - - const events = await store.query(filters, { signal }) + const events = await relay.query(filters, { signal }) // Deduplicate by author. .then((events) => Array.from(new Map(events.map((event) => [event.pubkey, event])).values())) - .then((events) => hydrateEvents({ events, store, signal })) + .then((events) => hydrateEvents({ events, relay, signal })) .then((events) => filterFn ? events.filter(filterFn) : events); const accounts = await Promise.all( @@ -43,14 +42,13 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], opts?: } async function renderAccounts(c: AppContext, pubkeys: string[]) { - const { offset, limit } = c.get('listPagination'); + const { offset, limit } = paginationSchema.parse(c.req.query()); const authors = pubkeys.reverse().slice(offset, offset + limit); - const store = await Storages.db(); - const signal = c.req.raw.signal; + const { relay, signal } = c.var; - const events = await store.query([{ kinds: [0], authors }], { signal }) - .then((events) => hydrateEvents({ events, store, signal })); + const events = await relay.query([{ kinds: [0], authors }], { signal }) + .then((events) => hydrateEvents({ events, relay, signal })); const accounts = await Promise.all( authors.map((pubkey) => { @@ -72,11 +70,11 @@ async function renderStatuses(c: AppContext, ids: string[], signal = AbortSignal return c.json([]); } - const store = await Storages.db(); - const { limit } = c.get('pagination'); + const { user, relay, pagination } = c.var; + const { limit } = pagination; - const events = await store.query([{ kinds: [1, 20], ids, limit }], { signal }) - .then((events) => hydrateEvents({ events, store, signal })); + const events = await relay.query([{ kinds: [1, 20], ids, limit }], { signal }) + .then((events) => hydrateEvents({ events, relay, signal })); if (!events.length) { return c.json([]); @@ -84,7 +82,7 @@ async function renderStatuses(c: AppContext, ids: string[], signal = AbortSignal const sortedEvents = [...events].sort((a, b) => ids.indexOf(a.id) - ids.indexOf(b.id)); - const viewerPubkey = await c.get('signer')?.getPublicKey(); + const viewerPubkey = await user?.signer.getPublicKey(); const statuses = await Promise.all( sortedEvents.map((event) => renderStatus(event, { viewerPubkey })), diff --git a/packages/mastoapi/auth/aes.bench.ts b/packages/mastoapi/auth/aes.bench.ts new file mode 100644 index 00000000..3b46f436 --- /dev/null +++ b/packages/mastoapi/auth/aes.bench.ts @@ -0,0 +1,18 @@ +import { generateSecretKey } from 'nostr-tools'; + +import { aesDecrypt, aesEncrypt } from './aes.ts'; + +Deno.bench('aesEncrypt', async (b) => { + const sk = generateSecretKey(); + const decrypted = generateSecretKey(); + b.start(); + await aesEncrypt(sk, decrypted); +}); + +Deno.bench('aesDecrypt', async (b) => { + const sk = generateSecretKey(); + const decrypted = generateSecretKey(); + const encrypted = await aesEncrypt(sk, decrypted); + b.start(); + await aesDecrypt(sk, encrypted); +}); diff --git a/packages/mastoapi/auth/aes.test.ts b/packages/mastoapi/auth/aes.test.ts new file mode 100644 index 00000000..ee735731 --- /dev/null +++ b/packages/mastoapi/auth/aes.test.ts @@ -0,0 +1,15 @@ +import { assertEquals } from '@std/assert'; +import { encodeHex } from '@std/encoding/hex'; +import { generateSecretKey } from 'nostr-tools'; + +import { aesDecrypt, aesEncrypt } from './aes.ts'; + +Deno.test('aesDecrypt & aesEncrypt', async () => { + const sk = generateSecretKey(); + const data = generateSecretKey(); + + const encrypted = await aesEncrypt(sk, data); + const decrypted = await aesDecrypt(sk, encrypted); + + assertEquals(encodeHex(decrypted), encodeHex(data)); +}); diff --git a/packages/mastoapi/auth/aes.ts b/packages/mastoapi/auth/aes.ts new file mode 100644 index 00000000..983fc39c --- /dev/null +++ b/packages/mastoapi/auth/aes.ts @@ -0,0 +1,17 @@ +/** Encrypt data with AES-GCM and a secret key. */ +export async function aesEncrypt(sk: Uint8Array, plaintext: Uint8Array): Promise { + const secretKey = await crypto.subtle.importKey('raw', sk, { name: 'AES-GCM' }, false, ['encrypt']); + const iv = crypto.getRandomValues(new Uint8Array(12)); + const buffer = await crypto.subtle.encrypt({ name: 'AES-GCM', iv }, secretKey, plaintext); + + return new Uint8Array([...iv, ...new Uint8Array(buffer)]); +} + +/** Decrypt data with AES-GCM and a secret key. */ +export async function aesDecrypt(sk: Uint8Array, ciphertext: Uint8Array): Promise { + const secretKey = await crypto.subtle.importKey('raw', sk, { name: 'AES-GCM' }, false, ['decrypt']); + const iv = ciphertext.slice(0, 12); + const buffer = await crypto.subtle.decrypt({ name: 'AES-GCM', iv }, secretKey, ciphertext.slice(12)); + + return new Uint8Array(buffer); +} diff --git a/packages/mastoapi/auth/token.bench.ts b/packages/mastoapi/auth/token.bench.ts new file mode 100644 index 00000000..5df41d0f --- /dev/null +++ b/packages/mastoapi/auth/token.bench.ts @@ -0,0 +1,11 @@ +import { generateToken, getTokenHash } from './token.ts'; + +Deno.bench('generateToken', async () => { + await generateToken(); +}); + +Deno.bench('getTokenHash', async (b) => { + const { token } = await generateToken(); + b.start(); + await getTokenHash(token); +}); diff --git a/packages/mastoapi/auth/token.test.ts b/packages/mastoapi/auth/token.test.ts new file mode 100644 index 00000000..6f002267 --- /dev/null +++ b/packages/mastoapi/auth/token.test.ts @@ -0,0 +1,18 @@ +import { assertEquals } from '@std/assert'; +import { decodeHex, encodeHex } from '@std/encoding/hex'; + +import { generateToken, getTokenHash } from './token.ts'; + +Deno.test('generateToken', async () => { + const sk = decodeHex('a0968751df8fd42f362213f08751911672f2a037113b392403bbb7dd31b71c95'); + + const { token, hash } = await generateToken(sk); + + assertEquals(token, 'token15ztgw5wl3l2z7d3zz0cgw5v3zee09gphzyanjfqrhwma6vdhrj2sauwknd'); + assertEquals(encodeHex(hash), 'ab4c4ead4d1c72a38fffd45b999937b7e3f25f867b19aaf252df858e77b66a8a'); +}); + +Deno.test('getTokenHash', async () => { + const hash = await getTokenHash('token15ztgw5wl3l2z7d3zz0cgw5v3zee09gphzyanjfqrhwma6vdhrj2sauwknd'); + assertEquals(encodeHex(hash), 'ab4c4ead4d1c72a38fffd45b999937b7e3f25f867b19aaf252df858e77b66a8a'); +}); diff --git a/packages/mastoapi/auth/token.ts b/packages/mastoapi/auth/token.ts new file mode 100644 index 00000000..8d71ed6f --- /dev/null +++ b/packages/mastoapi/auth/token.ts @@ -0,0 +1,30 @@ +import { bech32 } from '@scure/base'; +import { generateSecretKey } from 'nostr-tools'; + +/** + * Generate an auth token for the API. + * + * Returns a bech32 encoded API token and the SHA-256 hash of the bytes. + * The token should be presented to the user, but only the hash should be stored in the database. + */ +export async function generateToken(sk = generateSecretKey()): Promise<{ token: `token1${string}`; hash: Uint8Array }> { + const words = bech32.toWords(sk); + const token = bech32.encode('token', words); + + const buffer = await crypto.subtle.digest('SHA-256', sk); + const hash = new Uint8Array(buffer); + + return { token, hash }; +} + +/** + * Get the SHA-256 hash of an API token. + * First decodes from bech32 then hashes the bytes. + * Used to identify the user in the database by the hash of their token. + */ +export async function getTokenHash(token: `token1${string}`): Promise { + const { bytes: sk } = bech32.decodeToBytes(token); + const buffer = await crypto.subtle.digest('SHA-256', sk); + + return new Uint8Array(buffer); +} diff --git a/packages/api/deno.json b/packages/mastoapi/deno.json similarity index 75% rename from packages/api/deno.json rename to packages/mastoapi/deno.json index a8bbb3f5..f9abac55 100644 --- a/packages/api/deno.json +++ b/packages/mastoapi/deno.json @@ -1,5 +1,5 @@ { - "name": "@ditto/api", + "name": "@ditto/mastoapi", "version": "1.1.0", "exports": { "./middleware": "./middleware/mod.ts" diff --git a/packages/mastoapi/middleware/mod.ts b/packages/mastoapi/middleware/mod.ts new file mode 100644 index 00000000..7cdd6748 --- /dev/null +++ b/packages/mastoapi/middleware/mod.ts @@ -0,0 +1,2 @@ +export { paginationMiddleware } from './paginationMiddleware.ts'; +export { userMiddleware } from './userMiddleware.ts'; diff --git a/packages/mastoapi/middleware/paginationMiddleware.ts b/packages/mastoapi/middleware/paginationMiddleware.ts new file mode 100644 index 00000000..cca64229 --- /dev/null +++ b/packages/mastoapi/middleware/paginationMiddleware.ts @@ -0,0 +1,81 @@ +import { paginated, paginatedList } from '../pagination/paginate.ts'; +import { paginationSchema } from '../pagination/schema.ts'; + +import type { DittoMiddleware } from '@ditto/router'; +import type { NostrEvent } from '@nostrify/nostrify'; + +interface Pagination { + since?: number; + until?: number; + limit: number; +} + +interface ListPagination { + limit: number; + offset: number; +} + +type HeaderRecord = Record; +type PaginateFn = (events: NostrEvent[], body: object | unknown[], headers?: HeaderRecord) => Response; +type ListPaginateFn = (params: ListPagination, body: object | unknown[], headers?: HeaderRecord) => Response; + +/** Fixes compatibility with Mastodon apps by that don't use `Link` headers. */ +// @ts-ignore Types are right. +export function paginationMiddleware(): DittoMiddleware<{ pagination: Pagination; paginate: PaginateFn }>; +export function paginationMiddleware( + type: 'list', +): DittoMiddleware<{ pagination: ListPagination; paginate: ListPaginateFn }>; +export function paginationMiddleware( + type?: string, +): DittoMiddleware<{ pagination?: Pagination | ListPagination; paginate: PaginateFn | ListPaginateFn }> { + return async (c, next) => { + const { relay } = c.var; + + const pagination = paginationSchema.parse(c.req.query()); + + const { + max_id: maxId, + min_id: minId, + since, + until, + } = pagination; + + if ((maxId && !until) || (minId && !since)) { + const ids: string[] = []; + + if (maxId) ids.push(maxId); + if (minId) ids.push(minId); + + if (ids.length) { + const events = await relay.query( + [{ ids, limit: ids.length }], + { signal: c.req.raw.signal }, + ); + + for (const event of events) { + if (!until && maxId === event.id) pagination.until = event.created_at; + if (!since && minId === event.id) pagination.since = event.created_at; + } + } + } + + if (type === 'list') { + c.set('pagination', { + limit: pagination.limit, + offset: pagination.offset, + }); + const fn: ListPaginateFn = (params, body, headers) => paginatedList(c, params, body, headers); + c.set('paginate', fn); + } else { + c.set('pagination', { + since: pagination.since, + until: pagination.until, + limit: pagination.limit, + }); + const fn: PaginateFn = (events, body, headers) => paginated(c, events, body, headers); + c.set('paginate', fn); + } + + await next(); + }; +} diff --git a/packages/mastoapi/middleware/userMiddleware.ts b/packages/mastoapi/middleware/userMiddleware.ts new file mode 100644 index 00000000..29a7b6f3 --- /dev/null +++ b/packages/mastoapi/middleware/userMiddleware.ts @@ -0,0 +1,128 @@ +import { HTTPException } from '@hono/hono/http-exception'; +import { type NostrSigner, type NRelay, NSecSigner } from '@nostrify/nostrify'; +import { nip19 } from 'nostr-tools'; + +import { aesDecrypt } from '../auth/aes.ts'; +import { getTokenHash } from '../auth/token.ts'; +import { ConnectSigner } from '../signers/ConnectSigner.ts'; +import { ReadOnlySigner } from '../signers/ReadOnlySigner.ts'; + +import type { DittoConf } from '@ditto/conf'; +import type { DittoDB } from '@ditto/db'; +import type { DittoMiddleware } from '@ditto/router'; + +interface User { + signer: NostrSigner; + relay: NRelay; +} + +/** We only accept "Bearer" type. */ +const BEARER_REGEX = new RegExp(`^Bearer (${nip19.BECH32_REGEX.source})$`); + +export function userMiddleware(opts: { privileged: true; required: false }): never; +// @ts-ignore The types are right. +export function userMiddleware(opts: { privileged: false; required: true }): DittoMiddleware<{ user: User }>; +export function userMiddleware(opts: { privileged: true; required?: boolean }): DittoMiddleware<{ user: User }>; +export function userMiddleware(opts: { privileged: false; required?: boolean }): DittoMiddleware<{ user?: User }>; +export function userMiddleware(opts: { privileged: boolean; required?: boolean }): DittoMiddleware<{ user?: User }> { + const { privileged, required = privileged } = opts; + + if (privileged && !required) { + throw new Error('Privileged middleware requires authorization.'); + } + + return async (c, next) => { + const header = c.req.header('authorization'); + + if (!header && required) { + throw new HTTPException(403, { message: 'Authorization required.' }); + } + + if (header) { + const user: User = { + signer: await getSigner(header, c.var), + relay: c.var.relay, // TODO: set user's relay + }; + + c.set('user', user); + } + + if (privileged) { + // TODO: add back nip98 auth + throw new HTTPException(500); + } + + await next(); + }; +} + +interface GetSignerOpts { + db: DittoDB; + conf: DittoConf; + relay: NRelay; +} + +function getSigner(header: string, opts: GetSignerOpts): NostrSigner | Promise { + const match = header.match(BEARER_REGEX); + + if (!match) { + throw new HTTPException(400, { message: 'Invalid Authorization header.' }); + } + + const [_, bech32] = match; + + if (isToken(bech32)) { + return getSignerFromToken(bech32, opts); + } else { + return getSignerFromNip19(bech32); + } +} + +function isToken(value: string): value is `token1${string}` { + return value.startsWith('token1'); +} + +async function getSignerFromToken(token: `token1${string}`, opts: GetSignerOpts): Promise { + const { conf, db, relay } = opts; + + try { + const tokenHash = await getTokenHash(token); + + const row = await db.kysely + .selectFrom('auth_tokens') + .select(['pubkey', 'bunker_pubkey', 'nip46_sk_enc', 'nip46_relays']) + .where('token_hash', '=', tokenHash) + .executeTakeFirstOrThrow(); + + const nep46Seckey = await aesDecrypt(conf.seckey, row.nip46_sk_enc); + + return new ConnectSigner({ + bunkerPubkey: row.bunker_pubkey, + userPubkey: row.pubkey, + signer: new NSecSigner(nep46Seckey), + relays: row.nip46_relays, + relay, + }); + } catch { + throw new HTTPException(401, { message: 'Token is wrong or expired.' }); + } +} + +function getSignerFromNip19(bech32: string): NostrSigner { + try { + const decoded = nip19.decode(bech32); + + switch (decoded.type) { + case 'npub': + return new ReadOnlySigner(decoded.data); + case 'nprofile': + return new ReadOnlySigner(decoded.data.pubkey); + case 'nsec': + return new NSecSigner(decoded.data); + } + } catch { + // fallthrough + } + + throw new HTTPException(401, { message: 'Invalid NIP-19 identifier in Authorization header.' }); +} diff --git a/packages/mastoapi/pagination/link-header.test.ts b/packages/mastoapi/pagination/link-header.test.ts new file mode 100644 index 00000000..db41eaa0 --- /dev/null +++ b/packages/mastoapi/pagination/link-header.test.ts @@ -0,0 +1,34 @@ +import { genEvent } from '@nostrify/nostrify/test'; +import { assertEquals } from '@std/assert'; + +import { buildLinkHeader, buildListLinkHeader } from './link-header.ts'; + +Deno.test('buildLinkHeader', () => { + const url = 'https://ditto.test/api/v1/events'; + + const events = [ + genEvent({ created_at: 1 }), + genEvent({ created_at: 2 }), + genEvent({ created_at: 3 }), + ]; + + const link = buildLinkHeader(url, events); + + assertEquals( + link?.toString(), + '; rel="next", ; rel="prev"', + ); +}); + +Deno.test('buildListLinkHeader', () => { + const url = 'https://ditto.test/api/v1/tags'; + + const params = { offset: 0, limit: 3 }; + + const link = buildListLinkHeader(url, params); + + assertEquals( + link?.toString(), + '; rel="next", ; rel="prev"', + ); +}); diff --git a/packages/mastoapi/pagination/link-header.ts b/packages/mastoapi/pagination/link-header.ts new file mode 100644 index 00000000..648b4aab --- /dev/null +++ b/packages/mastoapi/pagination/link-header.ts @@ -0,0 +1,39 @@ +import type { NostrEvent } from '@nostrify/nostrify'; + +/** Build HTTP Link header for Mastodon API pagination. */ +export function buildLinkHeader(url: string, events: NostrEvent[]): string | undefined { + if (events.length <= 1) return; + + const firstEvent = events[0]; + const lastEvent = events[events.length - 1]; + + const { pathname, search } = new URL(url); + + const next = new URL(pathname + search, url); + const prev = new URL(pathname + search, url); + + next.searchParams.set('until', String(lastEvent.created_at)); + prev.searchParams.set('since', String(firstEvent.created_at)); + + return `<${next}>; rel="next", <${prev}>; rel="prev"`; +} + +/** Build HTTP Link header for paginating Nostr lists. */ +export function buildListLinkHeader( + url: string, + params: { offset: number; limit: number }, +): string | undefined { + const { pathname, search } = new URL(url); + const { offset, limit } = params; + + const next = new URL(pathname + search, url); + const prev = new URL(pathname + search, url); + + next.searchParams.set('offset', String(offset + limit)); + prev.searchParams.set('offset', String(Math.max(offset - limit, 0))); + + next.searchParams.set('limit', String(limit)); + prev.searchParams.set('limit', String(limit)); + + return `<${next}>; rel="next", <${prev}>; rel="prev"`; +} diff --git a/packages/mastoapi/pagination/paginate.test.ts b/packages/mastoapi/pagination/paginate.test.ts new file mode 100644 index 00000000..e69de29b diff --git a/packages/mastoapi/pagination/paginate.ts b/packages/mastoapi/pagination/paginate.ts new file mode 100644 index 00000000..2da2e478 --- /dev/null +++ b/packages/mastoapi/pagination/paginate.ts @@ -0,0 +1,43 @@ +import { buildLinkHeader, buildListLinkHeader } from './link-header.ts'; + +import type { Context } from '@hono/hono'; +import type { NostrEvent } from '@nostrify/nostrify'; + +type HeaderRecord = Record; + +/** Return results with pagination headers. Assumes chronological sorting of events. */ +export function paginated( + c: Context, + events: NostrEvent[], + body: object | unknown[], + headers: HeaderRecord = {}, +): Response { + const link = buildLinkHeader(c.req.url, events); + + if (link) { + headers.link = link; + } + + // Filter out undefined entities. + const results = Array.isArray(body) ? body.filter(Boolean) : body; + return c.json(results, 200, headers); +} + +/** paginate a list of tags. */ +export function paginatedList( + c: Context, + params: { offset: number; limit: number }, + body: object | unknown[], + headers: HeaderRecord = {}, +): Response { + const link = buildListLinkHeader(c.req.url, params); + const hasMore = Array.isArray(body) ? body.length > 0 : true; + + if (link) { + headers.link = hasMore ? link : link.split(', ').find((link) => link.endsWith('; rel="prev"'))!; + } + + // Filter out undefined entities. + const results = Array.isArray(body) ? body.filter(Boolean) : body; + return c.json(results, 200, headers); +} diff --git a/packages/mastoapi/pagination/schema.test.ts b/packages/mastoapi/pagination/schema.test.ts new file mode 100644 index 00000000..94be9091 --- /dev/null +++ b/packages/mastoapi/pagination/schema.test.ts @@ -0,0 +1,23 @@ +import { assertEquals } from '@std/assert'; + +import { paginationSchema } from './schema.ts'; + +Deno.test('paginationSchema', () => { + const pagination = paginationSchema.parse({ + limit: '10', + offset: '20', + max_id: '1', + min_id: '2', + since: '3', + until: '4', + }); + + assertEquals(pagination, { + limit: 10, + offset: 20, + max_id: '1', + min_id: '2', + since: 3, + until: 4, + }); +}); diff --git a/packages/mastoapi/pagination/schema.ts b/packages/mastoapi/pagination/schema.ts new file mode 100644 index 00000000..89e3c5f6 --- /dev/null +++ b/packages/mastoapi/pagination/schema.ts @@ -0,0 +1,14 @@ +import { z } from 'zod'; + +/** Schema to parse pagination query params. */ +export const paginationSchema = z.object({ + max_id: z.string().transform((val) => { + if (!val.includes('-')) return val; + return val.split('-')[1]; + }).optional().catch(undefined), + min_id: z.string().optional().catch(undefined), + since: z.coerce.number().nonnegative().optional().catch(undefined), + until: z.coerce.number().nonnegative().optional().catch(undefined), + limit: z.coerce.number().catch(20).transform((value) => Math.min(Math.max(value, 0), 40)), + offset: z.coerce.number().nonnegative().catch(0), +}); diff --git a/packages/mastoapi/signers/ConnectSigner.ts b/packages/mastoapi/signers/ConnectSigner.ts new file mode 100644 index 00000000..e3671413 --- /dev/null +++ b/packages/mastoapi/signers/ConnectSigner.ts @@ -0,0 +1,124 @@ +// deno-lint-ignore-file require-await +import { HTTPException } from '@hono/hono/http-exception'; +import { NConnectSigner, type NostrEvent, type NostrSigner, type NRelay } from '@nostrify/nostrify'; + +interface ConnectSignerOpts { + relay: NRelay; + bunkerPubkey: string; + userPubkey: string; + signer: NostrSigner; + relays?: string[]; +} + +/** + * NIP-46 signer. + * + * Simple extension of nostrify's `NConnectSigner`, with our options to keep it DRY. + */ +export class ConnectSigner implements NostrSigner { + private signer: Promise; + + constructor(private opts: ConnectSignerOpts) { + this.signer = this.init(opts.signer); + } + + async init(signer: NostrSigner): Promise { + return new NConnectSigner({ + encryption: 'nip44', + pubkey: this.opts.bunkerPubkey, + relay: this.opts.relay, + signer, + timeout: 60_000, + }); + } + + async signEvent(event: Omit): Promise { + const signer = await this.signer; + try { + return await signer.signEvent(event); + } catch (e) { + if (e instanceof Error && e.name === 'AbortError') { + throw new HTTPException(408, { message: 'The event was not signed quickly enough' }); + } else { + throw e; + } + } + } + + readonly nip04 = { + encrypt: async (pubkey: string, plaintext: string): Promise => { + const signer = await this.signer; + try { + return await signer.nip04.encrypt(pubkey, plaintext); + } catch (e) { + if (e instanceof Error && e.name === 'AbortError') { + throw new HTTPException(408, { + message: 'Text was not encrypted quickly enough', + }); + } else { + throw e; + } + } + }, + + decrypt: async (pubkey: string, ciphertext: string): Promise => { + const signer = await this.signer; + try { + return await signer.nip04.decrypt(pubkey, ciphertext); + } catch (e) { + if (e instanceof Error && e.name === 'AbortError') { + throw new HTTPException(408, { + message: 'Text was not decrypted quickly enough', + }); + } else { + throw e; + } + } + }, + }; + + readonly nip44 = { + encrypt: async (pubkey: string, plaintext: string): Promise => { + const signer = await this.signer; + try { + return await signer.nip44.encrypt(pubkey, plaintext); + } catch (e) { + if (e instanceof Error && e.name === 'AbortError') { + throw new HTTPException(408, { + message: 'Text was not encrypted quickly enough', + }); + } else { + throw e; + } + } + }, + + decrypt: async (pubkey: string, ciphertext: string): Promise => { + const signer = await this.signer; + try { + return await signer.nip44.decrypt(pubkey, ciphertext); + } catch (e) { + if (e instanceof Error && e.name === 'AbortError') { + throw new HTTPException(408, { + message: 'Text was not decrypted quickly enough', + }); + } else { + throw e; + } + } + }, + }; + + // Prevent unnecessary NIP-46 round-trips. + async getPublicKey(): Promise { + return this.opts.userPubkey; + } + + /** Get the user's relays if they passed in an `nprofile` auth token. */ + async getRelays(): Promise> { + return this.opts.relays?.reduce>((acc, relay) => { + acc[relay] = { read: true, write: true }; + return acc; + }, {}) ?? {}; + } +} diff --git a/packages/mastoapi/signers/ReadOnlySigner.ts b/packages/mastoapi/signers/ReadOnlySigner.ts new file mode 100644 index 00000000..74740b03 --- /dev/null +++ b/packages/mastoapi/signers/ReadOnlySigner.ts @@ -0,0 +1,18 @@ +// deno-lint-ignore-file require-await +import { HTTPException } from '@hono/hono/http-exception'; + +import type { NostrEvent, NostrSigner } from '@nostrify/nostrify'; + +export class ReadOnlySigner implements NostrSigner { + constructor(private pubkey: string) {} + + async signEvent(): Promise { + throw new HTTPException(401, { + message: 'Log in with Nostr Connect to sign events', + }); + } + + async getPublicKey(): Promise { + return this.pubkey; + } +} diff --git a/packages/router/DittoApp.test.ts b/packages/router/DittoApp.test.ts index 83da5bca..329b9dbc 100644 --- a/packages/router/DittoApp.test.ts +++ b/packages/router/DittoApp.test.ts @@ -1,5 +1,5 @@ import { DittoConf } from '@ditto/conf'; -import { DittoDB } from '@ditto/db'; +import { DittoPolyPg } from '@ditto/db'; import { Hono } from '@hono/hono'; import { MockRelay } from '@nostrify/nostrify/test'; @@ -7,7 +7,7 @@ import { DittoApp } from './DittoApp.ts'; import { DittoRoute } from './DittoRoute.ts'; Deno.test('DittoApp', async () => { - await using db = DittoDB.create('memory://'); + await using db = DittoPolyPg.create('memory://'); const conf = new DittoConf(new Map()); const relay = new MockRelay(); diff --git a/packages/router/DittoEnv.ts b/packages/router/DittoEnv.ts index 761bc3f8..7f399e62 100644 --- a/packages/router/DittoEnv.ts +++ b/packages/router/DittoEnv.ts @@ -1,5 +1,5 @@ import type { DittoConf } from '@ditto/conf'; -import type { DittoDatabase } from '@ditto/db'; +import type { DittoDB } from '@ditto/db'; import type { Env } from '@hono/hono'; import type { NRelay } from '@nostrify/nostrify'; @@ -13,7 +13,7 @@ export interface DittoEnv extends Env { * Database object. * @deprecated Store data as Nostr events instead. */ - db: DittoDatabase; + db: DittoDB; /** Abort signal for the request. */ signal: AbortSignal; }; diff --git a/packages/router/mod.ts b/packages/router/mod.ts index 8e9d1d46..a4361da6 100644 --- a/packages/router/mod.ts +++ b/packages/router/mod.ts @@ -2,3 +2,4 @@ export { DittoApp } from './DittoApp.ts'; export { DittoRoute } from './DittoRoute.ts'; export type { DittoEnv } from './DittoEnv.ts'; +export type { DittoMiddleware } from './DittoMiddleware.ts'; From e1bf86eb21b3674047c55c329996419d4f594ad3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 14:45:44 -0600 Subject: [PATCH 38/99] Make auth middleware work again (in a hacky way for now) --- packages/ditto/app.ts | 56 ++++++++++--------- packages/ditto/controllers/api/cashu.ts | 2 + packages/ditto/middleware/auth98Middleware.ts | 7 +-- .../ditto/middleware/swapNutzapsMiddleware.ts | 36 +++++------- packages/ditto/utils/api.ts | 8 +-- 5 files changed, 54 insertions(+), 55 deletions(-) diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index 9944426c..8b291a66 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -7,6 +7,7 @@ import { type Context, Handler, Input as HonoInput, MiddlewareHandler } from '@h import { every } from '@hono/hono/combine'; import { cors } from '@hono/hono/cors'; import { serveStatic } from '@hono/hono/deno'; +import { createFactory } from '@hono/hono/factory'; import { NostrEvent, NostrSigner, NRelay, NUploader } from '@nostrify/nostrify'; import '@/startup.ts'; @@ -137,7 +138,7 @@ import { metricsController } from '@/controllers/metrics.ts'; import { manifestController } from '@/controllers/manifest.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; import { nostrController } from '@/controllers/well-known/nostr.ts'; -import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts'; +import { auth98Middleware, requireProof as _requireProof, requireRole } from '@/middleware/auth98Middleware.ts'; import { cacheControlMiddleware } from '@/middleware/cacheControlMiddleware.ts'; import { cspMiddleware } from '@/middleware/cspMiddleware.ts'; import { metricsMiddleware } from '@/middleware/metricsMiddleware.ts'; @@ -197,7 +198,10 @@ const ratelimit = every( rateLimitMiddleware(300, Time.minutes(5), false), ); +const factory = createFactory(); const requireSigner = userMiddleware({ privileged: false, required: true }); +const requireAdmin = factory.createHandlers(requireSigner, requireRole('admin')); +const requireProof = factory.createHandlers(requireSigner, _requireProof()); app.use('/api/*', metricsMiddleware, ratelimit, paginationMiddleware(), logiMiddleware); app.use('/.well-known/*', metricsMiddleware, ratelimit, logiMiddleware); @@ -258,7 +262,7 @@ app.post('/oauth/revoke', revokeTokenController); app.post('/oauth/authorize', oauthAuthorizeController); app.get('/oauth/authorize', oauthController); -app.post('/api/v1/accounts', requireProof(), createAccountController); +app.post('/api/v1/accounts', ...requireProof, createAccountController); app.get('/api/v1/accounts/verify_credentials', requireSigner, verifyCredentialsController); app.patch('/api/v1/accounts/update_credentials', requireSigner, updateCredentialsController); app.get('/api/v1/accounts/search', accountSearchController); @@ -372,25 +376,25 @@ app.get('/api/v1/bookmarks', requireSigner, bookmarksController); app.get('/api/v1/blocks', requireSigner, blocksController); app.get('/api/v1/mutes', requireSigner, mutesController); -app.get('/api/v1/markers', requireProof(), markersController); -app.post('/api/v1/markers', requireProof(), updateMarkersController); +app.get('/api/v1/markers', ...requireProof, markersController); +app.post('/api/v1/markers', ...requireProof, updateMarkersController); app.get('/api/v1/push/subscription', requireSigner, getSubscriptionController); -app.post('/api/v1/push/subscription', requireProof(), pushSubscribeController); +app.post('/api/v1/push/subscription', ...requireProof, pushSubscribeController); app.get('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/reactions', reactionsController); app.get('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/reactions/:emoji', reactionsController); app.put('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/reactions/:emoji', requireSigner, reactionController); app.delete('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/reactions/:emoji', requireSigner, deleteReactionController); -app.get('/api/v1/pleroma/admin/config', requireRole('admin'), configController); -app.post('/api/v1/pleroma/admin/config', requireRole('admin'), updateConfigController); -app.delete('/api/v1/pleroma/admin/statuses/:id', requireRole('admin'), pleromaAdminDeleteStatusController); +app.get('/api/v1/pleroma/admin/config', ...requireAdmin, configController); +app.post('/api/v1/pleroma/admin/config', ...requireAdmin, updateConfigController); +app.delete('/api/v1/pleroma/admin/statuses/:id', ...requireAdmin, pleromaAdminDeleteStatusController); -app.get('/api/v1/admin/ditto/relays', requireRole('admin'), adminRelaysController); -app.put('/api/v1/admin/ditto/relays', requireRole('admin'), adminSetRelaysController); +app.get('/api/v1/admin/ditto/relays', ...requireAdmin, adminRelaysController); +app.put('/api/v1/admin/ditto/relays', ...requireAdmin, adminSetRelaysController); -app.put('/api/v1/admin/ditto/instance', requireRole('admin'), updateInstanceController); +app.put('/api/v1/admin/ditto/instance', ...requireAdmin, updateInstanceController); app.post('/api/v1/ditto/names', requireSigner, nameRequestController); app.get('/api/v1/ditto/names', requireSigner, nameRequestsController); @@ -399,7 +403,7 @@ app.get('/api/v1/ditto/captcha', rateLimitMiddleware(3, Time.minutes(1)), captch app.post( '/api/v1/ditto/captcha/:id/verify', rateLimitMiddleware(8, Time.minutes(1)), - requireProof(), + ...requireProof, captchaVerifyController, ); @@ -410,8 +414,8 @@ app.get( ); app.get('/api/v1/ditto/:id{[0-9a-f]{64}}/zap_splits', statusZapSplitsController); -app.put('/api/v1/admin/ditto/zap_splits', requireRole('admin'), updateZapSplitsController); -app.delete('/api/v1/admin/ditto/zap_splits', requireRole('admin'), deleteZapSplitsController); +app.put('/api/v1/admin/ditto/zap_splits', ...requireAdmin, updateZapSplitsController); +app.delete('/api/v1/admin/ditto/zap_splits', ...requireAdmin, deleteZapSplitsController); app.post('/api/v1/ditto/zap', requireSigner, zapController); app.get('/api/v1/ditto/statuses/:id{[0-9a-f]{64}}/zapped_by', zappedByController); @@ -419,35 +423,35 @@ app.get('/api/v1/ditto/statuses/:id{[0-9a-f]{64}}/zapped_by', zappedByController app.route('/api/v1/ditto/cashu', cashuApp); app.post('/api/v1/reports', requireSigner, reportController); -app.get('/api/v1/admin/reports', requireSigner, requireRole('admin'), adminReportsController); -app.get('/api/v1/admin/reports/:id{[0-9a-f]{64}}', requireSigner, requireRole('admin'), adminReportController); +app.get('/api/v1/admin/reports', requireSigner, ...requireAdmin, adminReportsController); +app.get('/api/v1/admin/reports/:id{[0-9a-f]{64}}', requireSigner, ...requireAdmin, adminReportController); app.post( '/api/v1/admin/reports/:id{[0-9a-f]{64}}/resolve', requireSigner, - requireRole('admin'), + ...requireAdmin, adminReportResolveController, ); app.post( '/api/v1/admin/reports/:id{[0-9a-f]{64}}/reopen', requireSigner, - requireRole('admin'), + ...requireAdmin, adminReportReopenController, ); -app.get('/api/v1/admin/accounts', requireRole('admin'), adminAccountsController); -app.post('/api/v1/admin/accounts/:id{[0-9a-f]{64}}/action', requireSigner, requireRole('admin'), adminActionController); +app.get('/api/v1/admin/accounts', ...requireAdmin, adminAccountsController); +app.post('/api/v1/admin/accounts/:id{[0-9a-f]{64}}/action', requireSigner, ...requireAdmin, adminActionController); app.post( '/api/v1/admin/accounts/:id{[0-9a-f]{64}}/approve', requireSigner, - requireRole('admin'), + ...requireAdmin, adminApproveController, ); -app.post('/api/v1/admin/accounts/:id{[0-9a-f]{64}}/reject', requireSigner, requireRole('admin'), adminRejectController); +app.post('/api/v1/admin/accounts/:id{[0-9a-f]{64}}/reject', requireSigner, ...requireAdmin, adminRejectController); -app.put('/api/v1/pleroma/admin/users/tag', requireRole('admin'), pleromaAdminTagController); -app.delete('/api/v1/pleroma/admin/users/tag', requireRole('admin'), pleromaAdminUntagController); -app.patch('/api/v1/pleroma/admin/users/suggest', requireRole('admin'), pleromaAdminSuggestController); -app.patch('/api/v1/pleroma/admin/users/unsuggest', requireRole('admin'), pleromaAdminUnsuggestController); +app.put('/api/v1/pleroma/admin/users/tag', ...requireAdmin, pleromaAdminTagController); +app.delete('/api/v1/pleroma/admin/users/tag', ...requireAdmin, pleromaAdminUntagController); +app.patch('/api/v1/pleroma/admin/users/suggest', ...requireAdmin, pleromaAdminSuggestController); +app.patch('/api/v1/pleroma/admin/users/unsuggest', ...requireAdmin, pleromaAdminUnsuggestController); // Not (yet) implemented. app.get('/api/v1/custom_emojis', emptyArrayController); diff --git a/packages/ditto/controllers/api/cashu.ts b/packages/ditto/controllers/api/cashu.ts index 60832ac4..16a28b09 100644 --- a/packages/ditto/controllers/api/cashu.ts +++ b/packages/ditto/controllers/api/cashu.ts @@ -93,6 +93,7 @@ app.put('/wallet', userMiddleware({ privileged: false, required: true }), requir await createEvent({ kind: 17375, content: encryptedWalletContentTags, + // @ts-ignore kill me }, c); // Nutzap information @@ -103,6 +104,7 @@ app.put('/wallet', userMiddleware({ privileged: false, required: true }), requir ['relay', conf.relay], // TODO: add more relays once things get more stable ['pubkey', p2pk], ], + // @ts-ignore kill me }, c); // TODO: hydrate wallet and add a 'balance' field when a 'renderWallet' view function is created diff --git a/packages/ditto/middleware/auth98Middleware.ts b/packages/ditto/middleware/auth98Middleware.ts index 18fce5fd..48ac5875 100644 --- a/packages/ditto/middleware/auth98Middleware.ts +++ b/packages/ditto/middleware/auth98Middleware.ts @@ -3,7 +3,6 @@ import { NostrEvent } from '@nostrify/nostrify'; import { type AppContext, type AppMiddleware } from '@/app.ts'; import { ReadOnlySigner } from '@/signers/ReadOnlySigner.ts'; -import { Storages } from '@/storages.ts'; import { localRequest } from '@/utils/api.ts'; import { buildAuthEventTemplate, @@ -40,10 +39,9 @@ type UserRole = 'user' | 'admin'; /** Require the user to prove their role before invoking the controller. */ function requireRole(role: UserRole, opts?: ParseAuthRequestOpts): AppMiddleware { return withProof(async (c, proof, next) => { - const { conf } = c.var; - const store = await Storages.db(); + const { conf, relay } = c.var; - const [user] = await store.query([{ + const [user] = await relay.query([{ kinds: [30382], authors: [await conf.signer.getPublicKey()], '#d': [proof.pubkey], @@ -108,6 +106,7 @@ function withProof( /** Get the proof over Nostr Connect. */ async function obtainProof(c: AppContext, opts?: ParseAuthRequestOpts) { const signer = c.var.user?.signer; + if (!signer) { throw new HTTPException(401, { res: c.json({ error: 'No way to sign Nostr event' }, 401), diff --git a/packages/ditto/middleware/swapNutzapsMiddleware.ts b/packages/ditto/middleware/swapNutzapsMiddleware.ts index aa68c1c1..79bdf01e 100644 --- a/packages/ditto/middleware/swapNutzapsMiddleware.ts +++ b/packages/ditto/middleware/swapNutzapsMiddleware.ts @@ -1,13 +1,12 @@ import { CashuMint, CashuWallet, getEncodedToken, type Proof } from '@cashu/cashu-ts'; -import { type DittoConf } from '@ditto/conf'; import { MiddlewareHandler } from '@hono/hono'; import { HTTPException } from '@hono/hono/http-exception'; import { getPublicKey } from 'nostr-tools'; -import { NostrFilter, NostrSigner, NSchema as n, NStore } from '@nostrify/nostrify'; -import { SetRequired } from 'type-fest'; +import { NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { stringToBytes } from '@scure/base'; import { logi } from '@soapbox/logi'; +import { AppEnv } from '@/app.ts'; import { isNostrId } from '@/utils.ts'; import { errorJson } from '@/utils/log.ts'; import { createEvent } from '@/utils/api.ts'; @@ -17,33 +16,28 @@ import { z } from 'zod'; * Swap nutzaps into wallet (create new events) if the user has a wallet, otheriwse, just fallthrough. * Errors are only thrown if 'signer' and 'store' middlewares are not set. */ -export const swapNutzapsMiddleware: MiddlewareHandler< - { Variables: { signer: SetRequired; store: NStore; conf: DittoConf } } -> = async (c, next) => { - const { conf } = c.var; - const signer = c.get('signer'); - const store = c.get('store'); +export const swapNutzapsMiddleware: MiddlewareHandler = async (c, next) => { + const { conf, relay, user, signal } = c.var; - if (!signer) { + if (!user) { throw new HTTPException(401, { message: 'No pubkey provided' }); } - if (!signer.nip44) { + if (!user.signer.nip44) { throw new HTTPException(401, { message: 'No NIP-44 signer provided' }); } - if (!store) { + if (!relay) { throw new HTTPException(401, { message: 'No store provided' }); } - const { signal } = c.req.raw; - const pubkey = await signer.getPublicKey(); - const [wallet] = await store.query([{ authors: [pubkey], kinds: [17375] }], { signal }); + const pubkey = await user.signer.getPublicKey(); + const [wallet] = await relay.query([{ authors: [pubkey], kinds: [17375] }], { signal }); if (wallet) { let decryptedContent: string; try { - decryptedContent = await signer.nip44.decrypt(pubkey, wallet.content); + decryptedContent = await user.signer.nip44.decrypt(pubkey, wallet.content); } catch (e) { logi({ level: 'error', @@ -68,7 +62,7 @@ export const swapNutzapsMiddleware: MiddlewareHandler< } const p2pk = getPublicKey(stringToBytes('hex', privkey)); - const [nutzapInformation] = await store.query([{ authors: [pubkey], kinds: [10019] }], { signal }); + const [nutzapInformation] = await relay.query([{ authors: [pubkey], kinds: [10019] }], { signal }); if (!nutzapInformation) { return c.json({ error: 'You need to have a nutzap information event so we can get the mints.' }, 400); } @@ -88,14 +82,14 @@ export const swapNutzapsMiddleware: MiddlewareHandler< const nutzapsFilter: NostrFilter = { kinds: [9321], '#p': [pubkey], '#u': mints }; - const [nutzapHistory] = await store.query([{ kinds: [7376], authors: [pubkey] }], { signal }); + const [nutzapHistory] = await relay.query([{ kinds: [7376], authors: [pubkey] }], { signal }); if (nutzapHistory) { nutzapsFilter.since = nutzapHistory.created_at; } const mintsToProofs: { [key: string]: { proofs: Proof[]; redeemed: string[][] } } = {}; - const nutzaps = await store.query([nutzapsFilter], { signal }); + const nutzaps = await relay.query([nutzapsFilter], { signal }); for (const event of nutzaps) { try { @@ -154,7 +148,7 @@ export const swapNutzapsMiddleware: MiddlewareHandler< const unspentProofs = await createEvent({ kind: 7375, - content: await signer.nip44.encrypt( + content: await user.signer.nip44.encrypt( pubkey, JSON.stringify({ mint, @@ -169,7 +163,7 @@ export const swapNutzapsMiddleware: MiddlewareHandler< await createEvent({ kind: 7376, - content: await signer.nip44.encrypt( + content: await user.signer.nip44.encrypt( pubkey, JSON.stringify([ ['direction', 'in'], diff --git a/packages/ditto/utils/api.ts b/packages/ditto/utils/api.ts index 0ac80a73..37a38d6a 100644 --- a/packages/ditto/utils/api.ts +++ b/packages/ditto/utils/api.ts @@ -18,16 +18,16 @@ import { purifyEvent } from '@/utils/purify.ts'; type EventStub = TypeFest.SetOptional; /** Publish an event through the pipeline. */ -async function createEvent(t: EventStub, c: Context): Promise { - const signer = c.get('signer'); +async function createEvent(t: EventStub, c: AppContext): Promise { + const { user } = c.var; - if (!signer) { + if (!user) { throw new HTTPException(401, { res: c.json({ error: 'No way to sign Nostr event' }, 401), }); } - const event = await signer.signEvent({ + const event = await user.signer.signEvent({ content: '', created_at: nostrNow(), tags: [], From 33786d2e5db07acebd11d25b3c9e82ae43ac7e3e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 18:48:44 -0600 Subject: [PATCH 39/99] Fix cashu tests, sorta --- packages/ditto/controllers/api/cashu.test.ts | 103 ++++++------------ .../mastoapi/middleware/userMiddleware.ts | 6 +- 2 files changed, 33 insertions(+), 76 deletions(-) diff --git a/packages/ditto/controllers/api/cashu.test.ts b/packages/ditto/controllers/api/cashu.test.ts index d82e205e..f8d37ca0 100644 --- a/packages/ditto/controllers/api/cashu.test.ts +++ b/packages/ditto/controllers/api/cashu.test.ts @@ -1,53 +1,39 @@ -import { Env as HonoEnv, Hono } from '@hono/hono'; -import { NostrSigner, NSecSigner, NStore } from '@nostrify/nostrify'; +import { DittoConf } from '@ditto/conf'; +import { DittoApp } from '@ditto/router'; +import { NSecSigner } from '@nostrify/nostrify'; import { genEvent } from '@nostrify/nostrify/test'; import { bytesToString, stringToBytes } from '@scure/base'; import { stub } from '@std/testing/mock'; import { assertEquals, assertExists, assertObjectMatch } from '@std/assert'; -import { generateSecretKey, getPublicKey } from 'nostr-tools'; +import { generateSecretKey, getPublicKey, nip19 } from 'nostr-tools'; import { createTestDB } from '@/test.ts'; import cashuApp from '@/controllers/api/cashu.ts'; import { walletSchema } from '@/schema.ts'; -interface AppEnv extends HonoEnv { - Variables: { - /** Signer to get the logged-in user's pubkey, relays, and to sign events. */ - signer: NostrSigner; - /** Storage for the user, might filter out unwanted content. */ - store: NStore; - }; -} - Deno.test('PUT /wallet must be successful', { sanitizeOps: false, sanitizeResources: false, }, async () => { using _mock = mockFetch(); await using db = await createTestDB(); - const store = db.store; + const relay = db.store; const sk = generateSecretKey(); const signer = new NSecSigner(sk); const nostrPrivateKey = bytesToString('hex', sk); - const app = new Hono().use( - async (c, next) => { - c.set('signer', signer); - await next(); - }, - async (c, next) => { - c.set('store', store); - await next(); - }, - ); + const app = new DittoApp({ db, relay, conf: new DittoConf(new Map()) }); app.route('/', cashuApp); const response = await app.request('/wallet', { method: 'PUT', - headers: [['content-type', 'application/json']], + headers: { + 'authorization': `Bearer ${nip19.nsecEncode(sk)}`, + 'content-type': 'application/json', + }, body: JSON.stringify({ mints: [ 'https://houston.mint.com', @@ -61,7 +47,7 @@ Deno.test('PUT /wallet must be successful', { const pubkey = await signer.getPublicKey(); - const [wallet] = await store.query([{ authors: [pubkey], kinds: [17375] }]); + const [wallet] = await relay.query([{ authors: [pubkey], kinds: [17375] }]); assertExists(wallet); assertEquals(wallet.kind, 17375); @@ -88,7 +74,7 @@ Deno.test('PUT /wallet must be successful', { ]); assertEquals(data.balance, 0); - const [nutzap_info] = await store.query([{ authors: [pubkey], kinds: [10019] }]); + const [nutzap_info] = await relay.query([{ authors: [pubkey], kinds: [10019] }]); assertExists(nutzap_info); assertEquals(nutzap_info.kind, 10019); @@ -105,27 +91,19 @@ Deno.test('PUT /wallet must be successful', { Deno.test('PUT /wallet must NOT be successful: wrong request body/schema', async () => { using _mock = mockFetch(); await using db = await createTestDB(); - const store = db.store; - + const relay = db.store; const sk = generateSecretKey(); - const signer = new NSecSigner(sk); - const app = new Hono().use( - async (c, next) => { - c.set('signer', signer); - await next(); - }, - async (c, next) => { - c.set('store', store); - await next(); - }, - ); + const app = new DittoApp({ db, relay, conf: new DittoConf(new Map()) }); app.route('/', cashuApp); const response = await app.request('/wallet', { method: 'PUT', - headers: [['content-type', 'application/json']], + headers: { + 'authorization': `Bearer ${nip19.nsecEncode(sk)}`, + 'content-type': 'application/json', + }, body: JSON.stringify({ mints: [], // no mints should throw an error }), @@ -143,21 +121,10 @@ Deno.test('PUT /wallet must NOT be successful: wallet already exists', { }, async () => { using _mock = mockFetch(); await using db = await createTestDB(); - const store = db.store; - + const relay = db.store; const sk = generateSecretKey(); - const signer = new NSecSigner(sk); - const app = new Hono().use( - async (c, next) => { - c.set('signer', signer); - await next(); - }, - async (c, next) => { - c.set('store', store); - await next(); - }, - ); + const app = new DittoApp({ db, relay, conf: new DittoConf(new Map()) }); app.route('/', cashuApp); @@ -165,7 +132,10 @@ Deno.test('PUT /wallet must NOT be successful: wallet already exists', { const response = await app.request('/wallet', { method: 'PUT', - headers: [['content-type', 'application/json']], + headers: { + 'authorization': `Bearer ${nip19.nsecEncode(sk)}`, + 'content-type': 'application/json', + }, body: JSON.stringify({ mints: ['https://mint.heart.com'], }), @@ -183,7 +153,7 @@ Deno.test('GET /wallet must be successful', { }, async () => { using _mock = mockFetch(); await using db = await createTestDB(); - const store = db.store; + const relay = db.store; const sk = generateSecretKey(); const signer = new NSecSigner(sk); @@ -191,16 +161,7 @@ Deno.test('GET /wallet must be successful', { const privkey = bytesToString('hex', sk); const p2pk = getPublicKey(stringToBytes('hex', privkey)); - const app = new Hono().use( - async (c, next) => { - c.set('signer', signer); - await next(); - }, - async (c, next) => { - c.set('store', store); - await next(); - }, - ); + const app = new DittoApp({ db, relay, conf: new DittoConf(new Map()) }); app.route('/', cashuApp); @@ -282,6 +243,9 @@ Deno.test('GET /wallet must be successful', { const response = await app.request('/wallet', { method: 'GET', + headers: { + 'authorization': `Bearer ${nip19.nsecEncode(sk)}`, + }, }); const body = await response.json(); @@ -298,14 +262,9 @@ Deno.test('GET /wallet must be successful', { Deno.test('GET /mints must be successful', async () => { using _mock = mockFetch(); await using db = await createTestDB(); - const store = db.store; + const relay = db.store; - const app = new Hono().use( - async (c, next) => { - c.set('store', store); - await next(); - }, - ); + const app = new DittoApp({ db, relay, conf: new DittoConf(new Map()) }); app.route('/', cashuApp); diff --git a/packages/mastoapi/middleware/userMiddleware.ts b/packages/mastoapi/middleware/userMiddleware.ts index 29a7b6f3..4a88e325 100644 --- a/packages/mastoapi/middleware/userMiddleware.ts +++ b/packages/mastoapi/middleware/userMiddleware.ts @@ -34,10 +34,6 @@ export function userMiddleware(opts: { privileged: boolean; required?: boolean } return async (c, next) => { const header = c.req.header('authorization'); - if (!header && required) { - throw new HTTPException(403, { message: 'Authorization required.' }); - } - if (header) { const user: User = { signer: await getSigner(header, c.var), @@ -45,6 +41,8 @@ export function userMiddleware(opts: { privileged: boolean; required?: boolean } }; c.set('user', user); + } else if (required) { + throw new HTTPException(403, { message: 'Authorization required.' }); } if (privileged) { From 8a978b088bdc0e516d23f5427bd80b355f43d551 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 18:57:55 -0600 Subject: [PATCH 40/99] Use the user's store in a few places where it matters --- packages/ditto/controllers/api/notifications.ts | 3 ++- packages/ditto/controllers/api/timelines.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/ditto/controllers/api/notifications.ts b/packages/ditto/controllers/api/notifications.ts index f180cf9e..f0435bc4 100644 --- a/packages/ditto/controllers/api/notifications.ts +++ b/packages/ditto/controllers/api/notifications.ts @@ -107,8 +107,9 @@ async function renderNotifications( params: DittoPagination, c: AppContext, ) { - const { conf, relay, user, signal } = c.var; + const { conf, user, signal } = c.var; + const relay = user!.relay; const pubkey = await user!.signer.getPublicKey(); const opts = { signal, limit: params.limit, timeout: conf.db.timeouts.timelines }; diff --git a/packages/ditto/controllers/api/timelines.ts b/packages/ditto/controllers/api/timelines.ts index b8c74f41..5ef83856 100644 --- a/packages/ditto/controllers/api/timelines.ts +++ b/packages/ditto/controllers/api/timelines.ts @@ -103,8 +103,9 @@ const suggestedTimelineController: AppController = async (c) => { /** Render statuses for timelines. */ async function renderStatuses(c: AppContext, filters: NostrFilter[]) { - const { conf, relay, user, signal } = c.var; + const { conf, user, signal } = c.var; + const relay = user?.relay ?? c.var.relay; const opts = { signal, timeout: conf.db.timeouts.timelines }; const events = await relay From 8f49b99935c2ce001980c3d8b295c9056be49d94 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 20:03:31 -0600 Subject: [PATCH 41/99] Consolidate AdminStore and UserStore --- packages/ditto/storages/AdminStore.ts | 43 ----------- packages/ditto/storages/UserStore.test.ts | 10 +-- packages/ditto/storages/UserStore.ts | 73 ++++++++++++------- .../mastoapi/middleware/userMiddleware.ts | 10 ++- 4 files changed, 60 insertions(+), 76 deletions(-) delete mode 100644 packages/ditto/storages/AdminStore.ts diff --git a/packages/ditto/storages/AdminStore.ts b/packages/ditto/storages/AdminStore.ts deleted file mode 100644 index ae03c59d..00000000 --- a/packages/ditto/storages/AdminStore.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { NostrEvent, NostrFilter, NStore } from '@nostrify/nostrify'; - -import { Conf } from '@/config.ts'; -import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { getTagSet } from '@/utils/tags.ts'; - -/** A store that prevents banned users from being displayed. */ -export class AdminStore implements NStore { - constructor(private store: NStore) {} - - async event(event: NostrEvent, opts?: { signal?: AbortSignal }): Promise { - return await this.store.event(event, opts); - } - - async query(filters: NostrFilter[], opts: { signal?: AbortSignal; limit?: number } = {}): Promise { - const events = await this.store.query(filters, opts); - const pubkeys = new Set(events.map((event) => event.pubkey)); - - const users = await this.store.query([{ - kinds: [30382], - authors: [await Conf.signer.getPublicKey()], - '#d': [...pubkeys], - limit: pubkeys.size, - }]); - - const adminPubkey = await Conf.signer.getPublicKey(); - - return events.filter((event) => { - const user = users.find( - ({ kind, pubkey, tags }) => - kind === 30382 && pubkey === adminPubkey && tags.find(([name]) => name === 'd')?.[1] === event.pubkey, - ); - - const n = getTagSet(user?.tags ?? [], 'n'); - - if (n.has('disabled')) { - return false; - } - - return true; - }); - } -} diff --git a/packages/ditto/storages/UserStore.test.ts b/packages/ditto/storages/UserStore.test.ts index d04ece07..56ec1254 100644 --- a/packages/ditto/storages/UserStore.test.ts +++ b/packages/ditto/storages/UserStore.test.ts @@ -14,9 +14,8 @@ Deno.test('query events of users that are not muted', async () => { const blockEventCopy = structuredClone(blockEvent); const event1authorUserMeCopy = structuredClone(event1authorUserMe); - const db = new MockRelay(); - - const store = new UserStore(userBlackCopy.pubkey, db); + const relay = new MockRelay(); + const store = new UserStore({ relay, userPubkey: userBlackCopy.pubkey }); await store.event(blockEventCopy); await store.event(userBlackCopy); @@ -30,9 +29,8 @@ Deno.test('user never muted anyone', async () => { const userBlackCopy = structuredClone(userBlack); const userMeCopy = structuredClone(userMe); - const db = new MockRelay(); - - const store = new UserStore(userBlackCopy.pubkey, db); + const relay = new MockRelay(); + const store = new UserStore({ relay, userPubkey: userBlackCopy.pubkey }); await store.event(userBlackCopy); await store.event(userMeCopy); diff --git a/packages/ditto/storages/UserStore.ts b/packages/ditto/storages/UserStore.ts index 2449d8c1..0533917c 100644 --- a/packages/ditto/storages/UserStore.ts +++ b/packages/ditto/storages/UserStore.ts @@ -1,43 +1,66 @@ -import { NostrEvent, NostrFilter, NStore } from '@nostrify/nostrify'; +import { NostrEvent, NostrFilter, NostrRelayCLOSED, NostrRelayEOSE, NostrRelayEVENT, NRelay } from '@nostrify/nostrify'; -import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { getTagSet } from '@/utils/tags.ts'; +interface UserStoreOpts { + relay: NRelay; + userPubkey: string; + adminPubkey?: string; +} -export class UserStore implements NStore { - private promise: Promise | undefined; +export class UserStore implements NRelay { + constructor(private opts: UserStoreOpts) {} - constructor(private pubkey: string, private store: NStore) {} + req( + filters: NostrFilter[], + opts?: { signal?: AbortSignal }, + ): AsyncIterable { + // TODO: support req maybe? It would be inefficient. + return this.opts.relay.req(filters, opts); + } async event(event: NostrEvent, opts?: { signal?: AbortSignal }): Promise { - return await this.store.event(event, opts); + return await this.opts.relay.event(event, opts); } /** * Query events that `pubkey` did not mute * https://github.com/nostr-protocol/nips/blob/master/51.md#standard-lists */ - async query(filters: NostrFilter[], opts: { signal?: AbortSignal; limit?: number } = {}): Promise { - const events = await this.store.query(filters, opts); - const pubkeys = await this.getMutedPubkeys(); + async query(filters: NostrFilter[], opts: { signal?: AbortSignal; limit?: number } = {}): Promise { + const { relay, userPubkey, adminPubkey } = this.opts; + + const mutes = new Set(); + const [muteList] = await this.opts.relay.query([{ authors: [userPubkey], kinds: [10000], limit: 1 }]); + + for (const [name, value] of muteList?.tags ?? []) { + if (name === 'p') { + mutes.add(value); + } + } + + const events = await relay.query(filters, opts); + + const users = adminPubkey + ? await relay.query([{ + kinds: [30382], + authors: [adminPubkey], + '#d': [...events.map(({ pubkey }) => pubkey)], + }]) + : []; return events.filter((event) => { - return event.kind === 0 || !pubkeys.has(event.pubkey); + const user = users.find((user) => user.tags.find(([name]) => name === 'd')?.[1] === event.pubkey); + + for (const [name, value] of user?.tags ?? []) { + if (name === 'n' && value === 'disabled') { + return false; + } + } + + return event.kind === 0 || !mutes.has(event.pubkey); }); } - private async getMuteList(): Promise { - if (!this.promise) { - this.promise = this.store.query([{ authors: [this.pubkey], kinds: [10000], limit: 1 }]); - } - const [muteList] = await this.promise; - return muteList; - } - - private async getMutedPubkeys(): Promise> { - const mutedPubkeysEvent = await this.getMuteList(); - if (!mutedPubkeysEvent) { - return new Set(); - } - return getTagSet(mutedPubkeysEvent.tags, 'p'); + close(): Promise { + return this.opts.relay.close(); } } diff --git a/packages/mastoapi/middleware/userMiddleware.ts b/packages/mastoapi/middleware/userMiddleware.ts index 4a88e325..a86fecc5 100644 --- a/packages/mastoapi/middleware/userMiddleware.ts +++ b/packages/mastoapi/middleware/userMiddleware.ts @@ -35,9 +35,15 @@ export function userMiddleware(opts: { privileged: boolean; required?: boolean } const header = c.req.header('authorization'); if (header) { + const { relay, conf } = c.var; + + const signer = await getSigner(header, c.var); + const userPubkey = await signer.getPublicKey(); + const adminPubkey = await conf.signer.getPublicKey(); + const user: User = { - signer: await getSigner(header, c.var), - relay: c.var.relay, // TODO: set user's relay + signer, + relay: new UserStore({ relay, userPubkey, adminPubkey }), }; c.set('user', user); From f83925331af56f284adccbfcef3554aa3cef112a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 20:04:57 -0600 Subject: [PATCH 42/99] Apply the UserStore to the userMiddleware --- packages/mastoapi/middleware/userMiddleware.ts | 1 + packages/{ditto => mastoapi}/storages/UserStore.test.ts | 4 ++-- packages/{ditto => mastoapi}/storages/UserStore.ts | 9 ++++++++- 3 files changed, 11 insertions(+), 3 deletions(-) rename packages/{ditto => mastoapi}/storages/UserStore.test.ts (96%) rename packages/{ditto => mastoapi}/storages/UserStore.ts (92%) diff --git a/packages/mastoapi/middleware/userMiddleware.ts b/packages/mastoapi/middleware/userMiddleware.ts index a86fecc5..71a375eb 100644 --- a/packages/mastoapi/middleware/userMiddleware.ts +++ b/packages/mastoapi/middleware/userMiddleware.ts @@ -6,6 +6,7 @@ import { aesDecrypt } from '../auth/aes.ts'; import { getTokenHash } from '../auth/token.ts'; import { ConnectSigner } from '../signers/ConnectSigner.ts'; import { ReadOnlySigner } from '../signers/ReadOnlySigner.ts'; +import { UserStore } from '../storages/UserStore.ts'; import type { DittoConf } from '@ditto/conf'; import type { DittoDB } from '@ditto/db'; diff --git a/packages/ditto/storages/UserStore.test.ts b/packages/mastoapi/storages/UserStore.test.ts similarity index 96% rename from packages/ditto/storages/UserStore.test.ts rename to packages/mastoapi/storages/UserStore.test.ts index 56ec1254..c9aa3329 100644 --- a/packages/ditto/storages/UserStore.test.ts +++ b/packages/mastoapi/storages/UserStore.test.ts @@ -1,7 +1,7 @@ import { MockRelay } from '@nostrify/nostrify/test'; - import { assertEquals } from '@std/assert'; -import { UserStore } from '@/storages/UserStore.ts'; + +import { UserStore } from './UserStore.ts'; import userBlack from '~/fixtures/events/kind-0-black.json' with { type: 'json' }; import userMe from '~/fixtures/events/event-0-makes-repost-with-quote-repost.json' with { type: 'json' }; diff --git a/packages/ditto/storages/UserStore.ts b/packages/mastoapi/storages/UserStore.ts similarity index 92% rename from packages/ditto/storages/UserStore.ts rename to packages/mastoapi/storages/UserStore.ts index 0533917c..dec77916 100644 --- a/packages/ditto/storages/UserStore.ts +++ b/packages/mastoapi/storages/UserStore.ts @@ -1,4 +1,11 @@ -import { NostrEvent, NostrFilter, NostrRelayCLOSED, NostrRelayEOSE, NostrRelayEVENT, NRelay } from '@nostrify/nostrify'; +import type { + NostrEvent, + NostrFilter, + NostrRelayCLOSED, + NostrRelayEOSE, + NostrRelayEVENT, + NRelay, +} from '@nostrify/nostrify'; interface UserStoreOpts { relay: NRelay; From 5ad7f1d5d7c3c1b1319fbb3b67139d8a977c7312 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 13:27:19 -0600 Subject: [PATCH 43/99] userMiddleware -> tokenMiddleware --- packages/mastoapi/middleware/mod.ts | 2 +- .../{userMiddleware.ts => tokenMiddleware.ts} | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) rename packages/mastoapi/middleware/{userMiddleware.ts => tokenMiddleware.ts} (86%) diff --git a/packages/mastoapi/middleware/mod.ts b/packages/mastoapi/middleware/mod.ts index 7cdd6748..e4c346e1 100644 --- a/packages/mastoapi/middleware/mod.ts +++ b/packages/mastoapi/middleware/mod.ts @@ -1,2 +1,2 @@ export { paginationMiddleware } from './paginationMiddleware.ts'; -export { userMiddleware } from './userMiddleware.ts'; +export { tokenMiddleware } from './tokenMiddleware.ts'; diff --git a/packages/mastoapi/middleware/userMiddleware.ts b/packages/mastoapi/middleware/tokenMiddleware.ts similarity index 86% rename from packages/mastoapi/middleware/userMiddleware.ts rename to packages/mastoapi/middleware/tokenMiddleware.ts index 71a375eb..9b666ed1 100644 --- a/packages/mastoapi/middleware/userMiddleware.ts +++ b/packages/mastoapi/middleware/tokenMiddleware.ts @@ -20,12 +20,12 @@ interface User { /** We only accept "Bearer" type. */ const BEARER_REGEX = new RegExp(`^Bearer (${nip19.BECH32_REGEX.source})$`); -export function userMiddleware(opts: { privileged: true; required: false }): never; +export function tokenMiddleware(opts: { privileged: true; required: false }): never; // @ts-ignore The types are right. -export function userMiddleware(opts: { privileged: false; required: true }): DittoMiddleware<{ user: User }>; -export function userMiddleware(opts: { privileged: true; required?: boolean }): DittoMiddleware<{ user: User }>; -export function userMiddleware(opts: { privileged: false; required?: boolean }): DittoMiddleware<{ user?: User }>; -export function userMiddleware(opts: { privileged: boolean; required?: boolean }): DittoMiddleware<{ user?: User }> { +export function tokenMiddleware(opts: { privileged: false; required: true }): DittoMiddleware<{ user: User }>; +export function tokenMiddleware(opts: { privileged: true; required?: boolean }): DittoMiddleware<{ user: User }>; +export function tokenMiddleware(opts: { privileged: false; required?: boolean }): DittoMiddleware<{ user?: User }>; +export function tokenMiddleware(opts: { privileged: boolean; required?: boolean }): DittoMiddleware<{ user?: User }> { const { privileged, required = privileged } = opts; if (privileged && !required) { From 438ab0921697da3af7eae72a225f9db26b6cd655 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 14:52:17 -0600 Subject: [PATCH 44/99] Split userMiddleware into tokenMiddleware and a new userMiddleware --- packages/ditto/app.ts | 5 +- packages/ditto/controllers/api/cashu.test.ts | 21 +++- packages/ditto/controllers/api/cashu.ts | 103 +++++++----------- packages/mastoapi/middleware/User.ts | 6 + packages/mastoapi/middleware/mod.ts | 3 + .../mastoapi/middleware/tokenMiddleware.ts | 26 +---- .../mastoapi/middleware/userMiddleware.ts | 27 +++++ 7 files changed, 97 insertions(+), 94 deletions(-) create mode 100644 packages/mastoapi/middleware/User.ts create mode 100644 packages/mastoapi/middleware/userMiddleware.ts diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index 8b291a66..459baf6b 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -1,6 +1,6 @@ import { DittoConf } from '@ditto/conf'; import { DittoDB } from '@ditto/db'; -import { paginationMiddleware, userMiddleware } from '@ditto/mastoapi/middleware'; +import { paginationMiddleware, tokenMiddleware, userMiddleware } from '@ditto/mastoapi/middleware'; import { DittoApp, type DittoEnv } from '@ditto/router'; import { type DittoTranslator } from '@ditto/translators'; import { type Context, Handler, Input as HonoInput, MiddlewareHandler } from '@hono/hono'; @@ -199,7 +199,7 @@ const ratelimit = every( ); const factory = createFactory(); -const requireSigner = userMiddleware({ privileged: false, required: true }); +const requireSigner = userMiddleware(); const requireAdmin = factory.createHandlers(requireSigner, requireRole('admin')); const requireProof = factory.createHandlers(requireSigner, _requireProof()); @@ -214,6 +214,7 @@ app.get('/relay', metricsMiddleware, ratelimit, relayController); app.use( cspMiddleware(), cors({ origin: '*', exposeHeaders: ['link'] }), + tokenMiddleware(), uploaderMiddleware, auth98Middleware(), ); diff --git a/packages/ditto/controllers/api/cashu.test.ts b/packages/ditto/controllers/api/cashu.test.ts index f8d37ca0..67530b6a 100644 --- a/packages/ditto/controllers/api/cashu.test.ts +++ b/packages/ditto/controllers/api/cashu.test.ts @@ -1,5 +1,6 @@ import { DittoConf } from '@ditto/conf'; -import { DittoApp } from '@ditto/router'; +import { type User } from '@ditto/mastoapi/middleware'; +import { DittoApp, DittoMiddleware } from '@ditto/router'; import { NSecSigner } from '@nostrify/nostrify'; import { genEvent } from '@nostrify/nostrify/test'; import { bytesToString, stringToBytes } from '@scure/base'; @@ -12,6 +13,13 @@ import { createTestDB } from '@/test.ts'; import cashuApp from '@/controllers/api/cashu.ts'; import { walletSchema } from '@/schema.ts'; +function testUserMiddleware(user: User): DittoMiddleware<{ user: User }> { + return async (c, next) => { + c.set('user', user); + await next(); + }; +} + Deno.test('PUT /wallet must be successful', { sanitizeOps: false, sanitizeResources: false, @@ -26,12 +34,12 @@ Deno.test('PUT /wallet must be successful', { const app = new DittoApp({ db, relay, conf: new DittoConf(new Map()) }); + app.use(testUserMiddleware({ signer, relay })); app.route('/', cashuApp); const response = await app.request('/wallet', { method: 'PUT', headers: { - 'authorization': `Bearer ${nip19.nsecEncode(sk)}`, 'content-type': 'application/json', }, body: JSON.stringify({ @@ -93,15 +101,16 @@ Deno.test('PUT /wallet must NOT be successful: wrong request body/schema', async await using db = await createTestDB(); const relay = db.store; const sk = generateSecretKey(); + const signer = new NSecSigner(sk); const app = new DittoApp({ db, relay, conf: new DittoConf(new Map()) }); + app.use(testUserMiddleware({ signer, relay })); app.route('/', cashuApp); const response = await app.request('/wallet', { method: 'PUT', headers: { - 'authorization': `Bearer ${nip19.nsecEncode(sk)}`, 'content-type': 'application/json', }, body: JSON.stringify({ @@ -123,9 +132,11 @@ Deno.test('PUT /wallet must NOT be successful: wallet already exists', { await using db = await createTestDB(); const relay = db.store; const sk = generateSecretKey(); + const signer = new NSecSigner(sk); const app = new DittoApp({ db, relay, conf: new DittoConf(new Map()) }); + app.use(testUserMiddleware({ signer, relay })); app.route('/', cashuApp); await db.store.event(genEvent({ kind: 17375 }, sk)); @@ -163,6 +174,7 @@ Deno.test('GET /wallet must be successful', { const app = new DittoApp({ db, relay, conf: new DittoConf(new Map()) }); + app.use(testUserMiddleware({ signer, relay })); app.route('/', cashuApp); // Wallet @@ -243,9 +255,6 @@ Deno.test('GET /wallet must be successful', { const response = await app.request('/wallet', { method: 'GET', - headers: { - 'authorization': `Bearer ${nip19.nsecEncode(sk)}`, - }, }); const body = await response.json(); diff --git a/packages/ditto/controllers/api/cashu.ts b/packages/ditto/controllers/api/cashu.ts index 16a28b09..9c7dcab0 100644 --- a/packages/ditto/controllers/api/cashu.ts +++ b/packages/ditto/controllers/api/cashu.ts @@ -1,6 +1,6 @@ import { Proof } from '@cashu/cashu-ts'; import { userMiddleware } from '@ditto/mastoapi/middleware'; -import { DittoMiddleware, DittoRoute } from '@ditto/router'; +import { DittoRoute } from '@ditto/router'; import { generateSecretKey, getPublicKey } from 'nostr-tools'; import { bytesToString, stringToBytes } from '@scure/base'; import { z } from 'zod'; @@ -11,8 +11,6 @@ import { swapNutzapsMiddleware } from '@/middleware/swapNutzapsMiddleware.ts'; import { isNostrId } from '@/utils.ts'; import { logi } from '@soapbox/logi'; import { errorJson } from '@/utils/log.ts'; -import { SetRequired } from 'type-fest'; -import { NostrSigner } from '@nostrify/nostrify'; type Wallet = z.infer; @@ -33,19 +31,6 @@ interface Nutzap { recipient_pubkey: string; } -const requireNip44Signer: DittoMiddleware<{ user: { signer: SetRequired } }> = async ( - c, - next, -) => { - const { user } = c.var; - - if (!user?.signer.nip44) { - return c.json({ error: 'User does not have a NIP-44 signer' }, 400); - } - - await next(); -}; - const createCashuWalletAndNutzapInfoSchema = z.object({ mints: z.array(z.string().url()).nonempty().transform((val) => { return [...new Set(val)]; @@ -57,7 +42,7 @@ const createCashuWalletAndNutzapInfoSchema = z.object({ * https://github.com/nostr-protocol/nips/blob/master/60.md * https://github.com/nostr-protocol/nips/blob/master/61.md#nutzap-informational-event */ -app.put('/wallet', userMiddleware({ privileged: false, required: true }), requireNip44Signer, async (c) => { +app.put('/wallet', userMiddleware('nip44'), async (c) => { const { conf, user, relay, signal } = c.var; const pubkey = await user.signer.getPublicKey(); @@ -119,63 +104,57 @@ app.put('/wallet', userMiddleware({ privileged: false, required: true }), requir }); /** Gets a wallet, if it exists. */ -app.get( - '/wallet', - userMiddleware({ privileged: false, required: true }), - requireNip44Signer, - swapNutzapsMiddleware, - async (c) => { - const { conf, relay, user, signal } = c.var; +app.get('/wallet', userMiddleware('nip44'), swapNutzapsMiddleware, async (c) => { + const { conf, relay, user, signal } = c.var; - const pubkey = await user.signer.getPublicKey(); + const pubkey = await user.signer.getPublicKey(); - const [event] = await relay.query([{ authors: [pubkey], kinds: [17375] }], { signal }); - if (!event) { - return c.json({ error: 'Wallet not found' }, 404); - } + const [event] = await relay.query([{ authors: [pubkey], kinds: [17375] }], { signal }); + if (!event) { + return c.json({ error: 'Wallet not found' }, 404); + } - const decryptedContent: string[][] = JSON.parse(await user.signer.nip44.decrypt(pubkey, event.content)); + const decryptedContent: string[][] = JSON.parse(await user.signer.nip44.decrypt(pubkey, event.content)); - const privkey = decryptedContent.find(([value]) => value === 'privkey')?.[1]; - if (!privkey || !isNostrId(privkey)) { - return c.json({ error: 'Wallet does not contain privkey or privkey is not a valid nostr id.' }, 422); - } + const privkey = decryptedContent.find(([value]) => value === 'privkey')?.[1]; + if (!privkey || !isNostrId(privkey)) { + return c.json({ error: 'Wallet does not contain privkey or privkey is not a valid nostr id.' }, 422); + } - const p2pk = getPublicKey(stringToBytes('hex', privkey)); + const p2pk = getPublicKey(stringToBytes('hex', privkey)); - let balance = 0; - const mints: string[] = []; + let balance = 0; + const mints: string[] = []; - const tokens = await relay.query([{ authors: [pubkey], kinds: [7375] }], { signal }); - for (const token of tokens) { - try { - const decryptedContent: { mint: string; proofs: Proof[] } = JSON.parse( - await user.signer.nip44.decrypt(pubkey, token.content), - ); + const tokens = await relay.query([{ authors: [pubkey], kinds: [7375] }], { signal }); + for (const token of tokens) { + try { + const decryptedContent: { mint: string; proofs: Proof[] } = JSON.parse( + await user.signer.nip44.decrypt(pubkey, token.content), + ); - if (!mints.includes(decryptedContent.mint)) { - mints.push(decryptedContent.mint); - } - - balance += decryptedContent.proofs.reduce((accumulator, current) => { - return accumulator + current.amount; - }, 0); - } catch (e) { - logi({ level: 'error', ns: 'ditto.api.cashu.wallet.swap', error: errorJson(e) }); + if (!mints.includes(decryptedContent.mint)) { + mints.push(decryptedContent.mint); } + + balance += decryptedContent.proofs.reduce((accumulator, current) => { + return accumulator + current.amount; + }, 0); + } catch (e) { + logi({ level: 'error', ns: 'ditto.api.cashu.wallet.swap', error: errorJson(e) }); } + } - // TODO: maybe change the 'Wallet' type data structure so each mint is a key and the value are the tokens associated with a given mint - const walletEntity: Wallet = { - pubkey_p2pk: p2pk, - mints, - relays: [conf.relay], - balance, - }; + // TODO: maybe change the 'Wallet' type data structure so each mint is a key and the value are the tokens associated with a given mint + const walletEntity: Wallet = { + pubkey_p2pk: p2pk, + mints, + relays: [conf.relay], + balance, + }; - return c.json(walletEntity, 200); - }, -); + return c.json(walletEntity, 200); +}); /** Get mints set by the CASHU_MINTS environment variable. */ app.get('/mints', (c) => { diff --git a/packages/mastoapi/middleware/User.ts b/packages/mastoapi/middleware/User.ts new file mode 100644 index 00000000..ac38b8de --- /dev/null +++ b/packages/mastoapi/middleware/User.ts @@ -0,0 +1,6 @@ +import type { NostrSigner, NRelay } from '@nostrify/nostrify'; + +export interface User { + signer: S; + relay: R; +} diff --git a/packages/mastoapi/middleware/mod.ts b/packages/mastoapi/middleware/mod.ts index e4c346e1..fb6ffb59 100644 --- a/packages/mastoapi/middleware/mod.ts +++ b/packages/mastoapi/middleware/mod.ts @@ -1,2 +1,5 @@ export { paginationMiddleware } from './paginationMiddleware.ts'; export { tokenMiddleware } from './tokenMiddleware.ts'; +export { userMiddleware } from './userMiddleware.ts'; + +export type { User } from './User.ts'; diff --git a/packages/mastoapi/middleware/tokenMiddleware.ts b/packages/mastoapi/middleware/tokenMiddleware.ts index 9b666ed1..407548ed 100644 --- a/packages/mastoapi/middleware/tokenMiddleware.ts +++ b/packages/mastoapi/middleware/tokenMiddleware.ts @@ -11,27 +11,12 @@ import { UserStore } from '../storages/UserStore.ts'; import type { DittoConf } from '@ditto/conf'; import type { DittoDB } from '@ditto/db'; import type { DittoMiddleware } from '@ditto/router'; - -interface User { - signer: NostrSigner; - relay: NRelay; -} +import type { User } from './User.ts'; /** We only accept "Bearer" type. */ const BEARER_REGEX = new RegExp(`^Bearer (${nip19.BECH32_REGEX.source})$`); -export function tokenMiddleware(opts: { privileged: true; required: false }): never; -// @ts-ignore The types are right. -export function tokenMiddleware(opts: { privileged: false; required: true }): DittoMiddleware<{ user: User }>; -export function tokenMiddleware(opts: { privileged: true; required?: boolean }): DittoMiddleware<{ user: User }>; -export function tokenMiddleware(opts: { privileged: false; required?: boolean }): DittoMiddleware<{ user?: User }>; -export function tokenMiddleware(opts: { privileged: boolean; required?: boolean }): DittoMiddleware<{ user?: User }> { - const { privileged, required = privileged } = opts; - - if (privileged && !required) { - throw new Error('Privileged middleware requires authorization.'); - } - +export function tokenMiddleware(): DittoMiddleware<{ user?: User }> { return async (c, next) => { const header = c.req.header('authorization'); @@ -48,13 +33,6 @@ export function tokenMiddleware(opts: { privileged: boolean; required?: boolean }; c.set('user', user); - } else if (required) { - throw new HTTPException(403, { message: 'Authorization required.' }); - } - - if (privileged) { - // TODO: add back nip98 auth - throw new HTTPException(500); } await next(); diff --git a/packages/mastoapi/middleware/userMiddleware.ts b/packages/mastoapi/middleware/userMiddleware.ts new file mode 100644 index 00000000..5b18e718 --- /dev/null +++ b/packages/mastoapi/middleware/userMiddleware.ts @@ -0,0 +1,27 @@ +import { HTTPException } from '@hono/hono/http-exception'; + +import type { DittoMiddleware } from '@ditto/router'; +import type { NostrSigner } from '@nostrify/nostrify'; +import type { SetRequired } from 'type-fest'; +import type { User } from './User.ts'; + +type Nip44Signer = SetRequired; + +export function userMiddleware(): DittoMiddleware<{ user: User }>; +// @ts-ignore Types are right. +export function userMiddleware(enc: 'nip44'): DittoMiddleware<{ user: User }>; +export function userMiddleware(enc?: 'nip04' | 'nip44'): DittoMiddleware<{ user: User }> { + return async (c, next) => { + const { user } = c.var; + + if (!user) { + throw new HTTPException(403, { message: 'Authorization required.' }); + } + + if (enc && !user.signer[enc]) { + throw new HTTPException(403, { message: `User does not have a ${enc} signer` }); + } + + await next(); + }; +} From d0c7cc7a45a29cc31350f05a038f1d41623a699a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 15:05:54 -0600 Subject: [PATCH 45/99] Improve cashu test --- packages/ditto/controllers/api/cashu.test.ts | 107 ++++++++----------- 1 file changed, 46 insertions(+), 61 deletions(-) diff --git a/packages/ditto/controllers/api/cashu.test.ts b/packages/ditto/controllers/api/cashu.test.ts index 67530b6a..bb80128f 100644 --- a/packages/ditto/controllers/api/cashu.test.ts +++ b/packages/ditto/controllers/api/cashu.test.ts @@ -13,30 +13,15 @@ import { createTestDB } from '@/test.ts'; import cashuApp from '@/controllers/api/cashu.ts'; import { walletSchema } from '@/schema.ts'; -function testUserMiddleware(user: User): DittoMiddleware<{ user: User }> { - return async (c, next) => { - c.set('user', user); - await next(); - }; -} - Deno.test('PUT /wallet must be successful', { sanitizeOps: false, sanitizeResources: false, }, async () => { - using _mock = mockFetch(); - await using db = await createTestDB(); - const relay = db.store; + await using test = await createTestApp(); - const sk = generateSecretKey(); - const signer = new NSecSigner(sk); + const { app, signer, sk, relay } = test; const nostrPrivateKey = bytesToString('hex', sk); - const app = new DittoApp({ db, relay, conf: new DittoConf(new Map()) }); - - app.use(testUserMiddleware({ signer, relay })); - app.route('/', cashuApp); - const response = await app.request('/wallet', { method: 'PUT', headers: { @@ -97,16 +82,8 @@ Deno.test('PUT /wallet must be successful', { }); Deno.test('PUT /wallet must NOT be successful: wrong request body/schema', async () => { - using _mock = mockFetch(); - await using db = await createTestDB(); - const relay = db.store; - const sk = generateSecretKey(); - const signer = new NSecSigner(sk); - - const app = new DittoApp({ db, relay, conf: new DittoConf(new Map()) }); - - app.use(testUserMiddleware({ signer, relay })); - app.route('/', cashuApp); + await using test = await createTestApp(); + const { app } = test; const response = await app.request('/wallet', { method: 'PUT', @@ -128,18 +105,10 @@ Deno.test('PUT /wallet must NOT be successful: wallet already exists', { sanitizeOps: false, sanitizeResources: false, }, async () => { - using _mock = mockFetch(); - await using db = await createTestDB(); - const relay = db.store; - const sk = generateSecretKey(); - const signer = new NSecSigner(sk); + await using test = await createTestApp(); + const { app, sk, relay } = test; - const app = new DittoApp({ db, relay, conf: new DittoConf(new Map()) }); - - app.use(testUserMiddleware({ signer, relay })); - app.route('/', cashuApp); - - await db.store.event(genEvent({ kind: 17375 }, sk)); + await relay.event(genEvent({ kind: 17375 }, sk)); const response = await app.request('/wallet', { method: 'PUT', @@ -162,23 +131,15 @@ Deno.test('GET /wallet must be successful', { sanitizeOps: false, sanitizeResources: false, }, async () => { - using _mock = mockFetch(); - await using db = await createTestDB(); - const relay = db.store; + await using test = await createTestApp(); + const { app, sk, relay, signer } = test; - const sk = generateSecretKey(); - const signer = new NSecSigner(sk); const pubkey = await signer.getPublicKey(); const privkey = bytesToString('hex', sk); const p2pk = getPublicKey(stringToBytes('hex', privkey)); - const app = new DittoApp({ db, relay, conf: new DittoConf(new Map()) }); - - app.use(testUserMiddleware({ signer, relay })); - app.route('/', cashuApp); - // Wallet - await db.store.event(genEvent({ + await relay.event(genEvent({ kind: 17375, content: await signer.nip44.encrypt( pubkey, @@ -190,7 +151,7 @@ Deno.test('GET /wallet must be successful', { }, sk)); // Nutzap information - await db.store.event(genEvent({ + await relay.event(genEvent({ kind: 10019, tags: [ ['pubkey', p2pk], @@ -199,7 +160,7 @@ Deno.test('GET /wallet must be successful', { }, sk)); // Unspent proofs - await db.store.event(genEvent({ + await relay.event(genEvent({ kind: 7375, content: await signer.nip44.encrypt( pubkey, @@ -240,7 +201,7 @@ Deno.test('GET /wallet must be successful', { // Nutzap const senderSk = generateSecretKey(); - await db.store.event(genEvent({ + await relay.event(genEvent({ kind: 9321, content: 'Nice post!', tags: [ @@ -269,13 +230,8 @@ Deno.test('GET /wallet must be successful', { }); Deno.test('GET /mints must be successful', async () => { - using _mock = mockFetch(); - await using db = await createTestDB(); - const relay = db.store; - - const app = new DittoApp({ db, relay, conf: new DittoConf(new Map()) }); - - app.route('/', cashuApp); + await using test = await createTestApp(); + const { app } = test; const response = await app.request('/mints', { method: 'GET', @@ -287,13 +243,42 @@ Deno.test('GET /mints must be successful', async () => { assertEquals(body, { mints: [] }); }); -function mockFetch() { +async function createTestApp() { + const conf = new DittoConf(new Map()); + + const db = await createTestDB(); + const relay = db.store; + + const sk = generateSecretKey(); + const signer = new NSecSigner(sk); + + const app = new DittoApp({ db, relay, conf }); + + app.use(testUserMiddleware({ signer, relay })); + app.route('/', cashuApp); + const mock = stub(globalThis, 'fetch', () => { return Promise.resolve(new Response()); }); + return { - [Symbol.dispose]: () => { + app, + db, + conf, + sk, + signer, + relay, + [Symbol.asyncDispose]: async () => { mock.restore(); + await db[Symbol.asyncDispose](); + await relay[Symbol.asyncDispose](); }, }; } + +function testUserMiddleware(user: User): DittoMiddleware<{ user: User }> { + return async (c, next) => { + c.set('user', user); + await next(); + }; +} From e5657d67c0dc9ec5408ee0d5b761c646b8911f82 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 15:08:37 -0600 Subject: [PATCH 46/99] app -> route --- packages/ditto/controllers/api/cashu.test.ts | 42 ++++++++++---------- packages/ditto/controllers/api/cashu.ts | 10 ++--- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/packages/ditto/controllers/api/cashu.test.ts b/packages/ditto/controllers/api/cashu.test.ts index bb80128f..22e9a38f 100644 --- a/packages/ditto/controllers/api/cashu.test.ts +++ b/packages/ditto/controllers/api/cashu.test.ts @@ -10,19 +10,19 @@ import { generateSecretKey, getPublicKey, nip19 } from 'nostr-tools'; import { createTestDB } from '@/test.ts'; -import cashuApp from '@/controllers/api/cashu.ts'; +import cashuRoute from './cashu.ts'; import { walletSchema } from '@/schema.ts'; Deno.test('PUT /wallet must be successful', { sanitizeOps: false, sanitizeResources: false, }, async () => { - await using test = await createTestApp(); + await using test = await createTestRoute(); - const { app, signer, sk, relay } = test; + const { route, signer, sk, relay } = test; const nostrPrivateKey = bytesToString('hex', sk); - const response = await app.request('/wallet', { + const response = await route.request('/wallet', { method: 'PUT', headers: { 'content-type': 'application/json', @@ -82,10 +82,10 @@ Deno.test('PUT /wallet must be successful', { }); Deno.test('PUT /wallet must NOT be successful: wrong request body/schema', async () => { - await using test = await createTestApp(); - const { app } = test; + await using test = await createTestRoute(); + const { route } = test; - const response = await app.request('/wallet', { + const response = await route.request('/wallet', { method: 'PUT', headers: { 'content-type': 'application/json', @@ -105,12 +105,12 @@ Deno.test('PUT /wallet must NOT be successful: wallet already exists', { sanitizeOps: false, sanitizeResources: false, }, async () => { - await using test = await createTestApp(); - const { app, sk, relay } = test; + await using test = await createTestRoute(); + const { route, sk, relay } = test; await relay.event(genEvent({ kind: 17375 }, sk)); - const response = await app.request('/wallet', { + const response = await route.request('/wallet', { method: 'PUT', headers: { 'authorization': `Bearer ${nip19.nsecEncode(sk)}`, @@ -131,8 +131,8 @@ Deno.test('GET /wallet must be successful', { sanitizeOps: false, sanitizeResources: false, }, async () => { - await using test = await createTestApp(); - const { app, sk, relay, signer } = test; + await using test = await createTestRoute(); + const { route, sk, relay, signer } = test; const pubkey = await signer.getPublicKey(); const privkey = bytesToString('hex', sk); @@ -214,7 +214,7 @@ Deno.test('GET /wallet must be successful', { ], }, senderSk)); - const response = await app.request('/wallet', { + const response = await route.request('/wallet', { method: 'GET', }); @@ -230,10 +230,10 @@ Deno.test('GET /wallet must be successful', { }); Deno.test('GET /mints must be successful', async () => { - await using test = await createTestApp(); - const { app } = test; + await using test = await createTestRoute(); + const { route } = test; - const response = await app.request('/mints', { + const response = await route.request('/mints', { method: 'GET', }); @@ -243,7 +243,7 @@ Deno.test('GET /mints must be successful', async () => { assertEquals(body, { mints: [] }); }); -async function createTestApp() { +async function createTestRoute() { const conf = new DittoConf(new Map()); const db = await createTestDB(); @@ -252,17 +252,17 @@ async function createTestApp() { const sk = generateSecretKey(); const signer = new NSecSigner(sk); - const app = new DittoApp({ db, relay, conf }); + const route = new DittoApp({ db, relay, conf }); - app.use(testUserMiddleware({ signer, relay })); - app.route('/', cashuApp); + route.use(testUserMiddleware({ signer, relay })); + route.route('/', cashuRoute); const mock = stub(globalThis, 'fetch', () => { return Promise.resolve(new Response()); }); return { - app, + route, db, conf, sk, diff --git a/packages/ditto/controllers/api/cashu.ts b/packages/ditto/controllers/api/cashu.ts index 9c7dcab0..2d3a1519 100644 --- a/packages/ditto/controllers/api/cashu.ts +++ b/packages/ditto/controllers/api/cashu.ts @@ -14,7 +14,7 @@ import { errorJson } from '@/utils/log.ts'; type Wallet = z.infer; -const app = new DittoRoute(); +const route = new DittoRoute(); // app.delete('/wallet') -> 204 @@ -42,7 +42,7 @@ const createCashuWalletAndNutzapInfoSchema = z.object({ * https://github.com/nostr-protocol/nips/blob/master/60.md * https://github.com/nostr-protocol/nips/blob/master/61.md#nutzap-informational-event */ -app.put('/wallet', userMiddleware('nip44'), async (c) => { +route.put('/wallet', userMiddleware('nip44'), async (c) => { const { conf, user, relay, signal } = c.var; const pubkey = await user.signer.getPublicKey(); @@ -104,7 +104,7 @@ app.put('/wallet', userMiddleware('nip44'), async (c) => { }); /** Gets a wallet, if it exists. */ -app.get('/wallet', userMiddleware('nip44'), swapNutzapsMiddleware, async (c) => { +route.get('/wallet', userMiddleware('nip44'), swapNutzapsMiddleware, async (c) => { const { conf, relay, user, signal } = c.var; const pubkey = await user.signer.getPublicKey(); @@ -157,7 +157,7 @@ app.get('/wallet', userMiddleware('nip44'), swapNutzapsMiddleware, async (c) => }); /** Get mints set by the CASHU_MINTS environment variable. */ -app.get('/mints', (c) => { +route.get('/mints', (c) => { const { conf } = c.var; // TODO: Return full Mint information: https://github.com/cashubtc/nuts/blob/main/06.md @@ -166,4 +166,4 @@ app.get('/mints', (c) => { return c.json({ mints }, 200); }); -export default app; +export default route; From 72851bc5365d967c97c249287379e44a6ffe66d8 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 15:08:58 -0600 Subject: [PATCH 47/99] Remove AdminStore from storages --- packages/ditto/storages.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/packages/ditto/storages.ts b/packages/ditto/storages.ts index 7b77a037..d5d0f029 100644 --- a/packages/ditto/storages.ts +++ b/packages/ditto/storages.ts @@ -5,7 +5,6 @@ import { logi } from '@soapbox/logi'; import { Conf } from '@/config.ts'; import { wsUrlSchema } from '@/schema.ts'; -import { AdminStore } from '@/storages/AdminStore.ts'; import { DittoPgStore } from '@/storages/DittoPgStore.ts'; import { getRelays } from '@/utils/outbox.ts'; import { seedZapSplits } from '@/utils/zap-split.ts'; @@ -13,7 +12,6 @@ import { seedZapSplits } from '@/utils/zap-split.ts'; export class Storages { private static _db: Promise | undefined; private static _database: Promise | undefined; - private static _admin: Promise | undefined; private static _client: Promise> | undefined; public static async database(): Promise { @@ -53,14 +51,6 @@ export class Storages { return this._db; } - /** Admin user storage. */ - public static async admin(): Promise { - if (!this._admin) { - this._admin = Promise.resolve(new AdminStore(await this.db())); - } - return this._admin; - } - /** Relay pool storage. */ public static async client(): Promise> { if (!this._client) { From f0add87c6db2f0c573ecec07201bf223d97295a1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 15:35:03 -0600 Subject: [PATCH 48/99] Create @ditto/nip98 package --- deno.json | 1 + packages/ditto/middleware/auth98Middleware.ts | 7 +---- packages/ditto/schema.ts | 13 ---------- packages/ditto/schemas/nostr.ts | 16 +----------- packages/nip98/deno.json | 7 +++++ packages/{ditto/utils => nip98}/nip98.ts | 26 ++++++++++++++----- packages/nip98/schema.ts | 20 ++++++++++++++ 7 files changed, 49 insertions(+), 41 deletions(-) create mode 100644 packages/nip98/deno.json rename packages/{ditto/utils => nip98}/nip98.ts (79%) create mode 100644 packages/nip98/schema.ts diff --git a/deno.json b/deno.json index 4466b7b3..20d87204 100644 --- a/deno.json +++ b/deno.json @@ -7,6 +7,7 @@ "./packages/lang", "./packages/mastoapi", "./packages/metrics", + "./packages/nip98", "./packages/policies", "./packages/ratelimiter", "./packages/router", diff --git a/packages/ditto/middleware/auth98Middleware.ts b/packages/ditto/middleware/auth98Middleware.ts index 48ac5875..6cab6566 100644 --- a/packages/ditto/middleware/auth98Middleware.ts +++ b/packages/ditto/middleware/auth98Middleware.ts @@ -1,15 +1,10 @@ +import { buildAuthEventTemplate, parseAuthRequest, type ParseAuthRequestOpts, validateAuthEvent } from '@ditto/nip98'; import { HTTPException } from '@hono/hono/http-exception'; import { NostrEvent } from '@nostrify/nostrify'; import { type AppContext, type AppMiddleware } from '@/app.ts'; import { ReadOnlySigner } from '@/signers/ReadOnlySigner.ts'; import { localRequest } from '@/utils/api.ts'; -import { - buildAuthEventTemplate, - parseAuthRequest, - type ParseAuthRequestOpts, - validateAuthEvent, -} from '@/utils/nip98.ts'; /** * NIP-98 auth. diff --git a/packages/ditto/schema.ts b/packages/ditto/schema.ts index 30b4520a..56c9b998 100644 --- a/packages/ditto/schema.ts +++ b/packages/ditto/schema.ts @@ -13,18 +13,6 @@ function filteredArray(schema: T) { )); } -/** https://developer.mozilla.org/en-US/docs/Glossary/Base64#the_unicode_problem */ -const decode64Schema = z.string().transform((value, ctx) => { - try { - const binString = atob(value); - const bytes = Uint8Array.from(binString, (m) => m.codePointAt(0)!); - return new TextDecoder().decode(bytes); - } catch (_e) { - ctx.addIssue({ code: z.ZodIssueCode.custom, message: 'Invalid base64', fatal: true }); - return z.NEVER; - } -}); - /** Parses a hashtag, eg `#yolo`. */ const hashtagSchema = z.string().regex(/^\w{1,30}$/); @@ -96,7 +84,6 @@ const walletSchema = z.object({ export { booleanParamSchema, - decode64Schema, fileSchema, filteredArray, hashtagSchema, diff --git a/packages/ditto/schemas/nostr.ts b/packages/ditto/schemas/nostr.ts index 05cd0f31..558e6c13 100644 --- a/packages/ditto/schemas/nostr.ts +++ b/packages/ditto/schemas/nostr.ts @@ -1,14 +1,8 @@ import { NSchema as n } from '@nostrify/nostrify'; -import { getEventHash, verifyEvent } from 'nostr-tools'; import { z } from 'zod'; import { safeUrlSchema, sizesSchema } from '@/schema.ts'; -/** Nostr event schema that also verifies the event's signature. */ -const signedEventSchema = n.event() - .refine((event) => event.id === getEventHash(event), 'Event ID does not match hash') - .refine(verifyEvent, 'Event signature is invalid'); - /** Kind 0 standardized fields extended with Ditto custom fields. */ const metadataSchema = n.metadata().and(z.object({ fields: z.tuple([z.string(), z.string()]).array().optional().catch(undefined), @@ -68,12 +62,4 @@ const emojiTagSchema = z.tuple([z.literal('emoji'), z.string(), z.string().url() /** NIP-30 custom emoji tag. */ type EmojiTag = z.infer; -export { - type EmojiTag, - emojiTagSchema, - metadataSchema, - relayInfoDocSchema, - screenshotsSchema, - serverMetaSchema, - signedEventSchema, -}; +export { type EmojiTag, emojiTagSchema, metadataSchema, relayInfoDocSchema, screenshotsSchema, serverMetaSchema }; diff --git a/packages/nip98/deno.json b/packages/nip98/deno.json new file mode 100644 index 00000000..108e1bb8 --- /dev/null +++ b/packages/nip98/deno.json @@ -0,0 +1,7 @@ +{ + "name": "@ditto/nip98", + "version": "1.0.0", + "exports": { + ".": "./nip98.ts" + } +} diff --git a/packages/ditto/utils/nip98.ts b/packages/nip98/nip98.ts similarity index 79% rename from packages/ditto/utils/nip98.ts rename to packages/nip98/nip98.ts index f83fcddb..e8574c86 100644 --- a/packages/ditto/utils/nip98.ts +++ b/packages/nip98/nip98.ts @@ -1,11 +1,8 @@ -import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; +import { type NostrEvent, NSchema as n } from '@nostrify/nostrify'; import { encodeHex } from '@std/encoding/hex'; -import { EventTemplate, nip13 } from 'nostr-tools'; +import { type EventTemplate, nip13 } from 'nostr-tools'; -import { decode64Schema } from '@/schema.ts'; -import { signedEventSchema } from '@/schemas/nostr.ts'; -import { eventAge, findTag, nostrNow } from '@/utils.ts'; -import { Time } from '@/utils/time.ts'; +import { decode64Schema, signedEventSchema } from './schema.ts'; /** Decode a Nostr event from a base64 encoded string. */ const decode64EventSchema = decode64Schema.pipe(n.json()).pipe(signedEventSchema); @@ -32,7 +29,7 @@ async function parseAuthRequest(req: Request, opts: ParseAuthRequestOpts = {}) { /** Compare the auth event with the request, returning a zod SafeParse type. */ function validateAuthEvent(req: Request, event: NostrEvent, opts: ParseAuthRequestOpts = {}) { - const { maxAge = Time.minutes(1), validatePayload = true, pow = 0 } = opts; + const { maxAge = 60_000, validatePayload = true, pow = 0 } = opts; const schema = signedEventSchema .refine((event) => event.kind === 27235, 'Event must be kind 27235') @@ -87,4 +84,19 @@ function tagValue(event: NostrEvent, tagName: string): string | undefined { return findTag(event.tags, tagName)?.[1]; } +/** Get the current time in Nostr format. */ +const nostrNow = (): number => Math.floor(Date.now() / 1000); + +/** Convenience function to convert Nostr dates into native Date objects. */ +const nostrDate = (seconds: number): Date => new Date(seconds * 1000); + +/** Return the event's age in milliseconds. */ +function eventAge(event: NostrEvent): number { + return Date.now() - nostrDate(event.created_at).getTime(); +} + +function findTag(tags: string[][], name: string): string[] | undefined { + return tags.find((tag) => tag[0] === name); +} + export { buildAuthEventTemplate, parseAuthRequest, type ParseAuthRequestOpts, validateAuthEvent }; diff --git a/packages/nip98/schema.ts b/packages/nip98/schema.ts new file mode 100644 index 00000000..a0cf627c --- /dev/null +++ b/packages/nip98/schema.ts @@ -0,0 +1,20 @@ +import { NSchema as n } from '@nostrify/nostrify'; +import { getEventHash, verifyEvent } from 'nostr-tools'; +import z from 'zod'; + +/** https://developer.mozilla.org/en-US/docs/Glossary/Base64#the_unicode_problem */ +export const decode64Schema = z.string().transform((value, ctx) => { + try { + const binString = atob(value); + const bytes = Uint8Array.from(binString, (m) => m.codePointAt(0)!); + return new TextDecoder().decode(bytes); + } catch (_e) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: 'Invalid base64', fatal: true }); + return z.NEVER; + } +}); + +/** Nostr event schema that also verifies the event's signature. */ +export const signedEventSchema = n.event() + .refine((event) => event.id === getEventHash(event), 'Event ID does not match hash') + .refine(verifyEvent, 'Event signature is invalid'); From adeff1cae519b552a62ba9fac518c91cfab05cfd Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 15:53:29 -0600 Subject: [PATCH 49/99] tokenMiddleware: support nip98 auth --- packages/ditto/app.ts | 3 +- packages/mastoapi/middleware/User.ts | 1 + .../mastoapi/middleware/tokenMiddleware.ts | 90 ++++++++++++------- 3 files changed, 59 insertions(+), 35 deletions(-) diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index 459baf6b..3333b9eb 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -138,7 +138,7 @@ import { metricsController } from '@/controllers/metrics.ts'; import { manifestController } from '@/controllers/manifest.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; import { nostrController } from '@/controllers/well-known/nostr.ts'; -import { auth98Middleware, requireProof as _requireProof, requireRole } from '@/middleware/auth98Middleware.ts'; +import { requireProof as _requireProof, requireRole } from '@/middleware/auth98Middleware.ts'; import { cacheControlMiddleware } from '@/middleware/cacheControlMiddleware.ts'; import { cspMiddleware } from '@/middleware/cspMiddleware.ts'; import { metricsMiddleware } from '@/middleware/metricsMiddleware.ts'; @@ -216,7 +216,6 @@ app.use( cors({ origin: '*', exposeHeaders: ['link'] }), tokenMiddleware(), uploaderMiddleware, - auth98Middleware(), ); app.get('/metrics', metricsController); diff --git a/packages/mastoapi/middleware/User.ts b/packages/mastoapi/middleware/User.ts index ac38b8de..8fd61f96 100644 --- a/packages/mastoapi/middleware/User.ts +++ b/packages/mastoapi/middleware/User.ts @@ -3,4 +3,5 @@ import type { NostrSigner, NRelay } from '@nostrify/nostrify'; export interface User { signer: S; relay: R; + verified?: boolean; } diff --git a/packages/mastoapi/middleware/tokenMiddleware.ts b/packages/mastoapi/middleware/tokenMiddleware.ts index 407548ed..4796b5d4 100644 --- a/packages/mastoapi/middleware/tokenMiddleware.ts +++ b/packages/mastoapi/middleware/tokenMiddleware.ts @@ -1,5 +1,6 @@ +import { parseAuthRequest } from '@ditto/nip98'; import { HTTPException } from '@hono/hono/http-exception'; -import { type NostrSigner, type NRelay, NSecSigner } from '@nostrify/nostrify'; +import { type NostrSigner, NSecSigner } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; import { aesDecrypt } from '../auth/aes.ts'; @@ -8,14 +9,10 @@ import { ConnectSigner } from '../signers/ConnectSigner.ts'; import { ReadOnlySigner } from '../signers/ReadOnlySigner.ts'; import { UserStore } from '../storages/UserStore.ts'; -import type { DittoConf } from '@ditto/conf'; -import type { DittoDB } from '@ditto/db'; -import type { DittoMiddleware } from '@ditto/router'; +import type { DittoEnv, DittoMiddleware } from '@ditto/router'; +import type { Context } from '@hono/hono'; import type { User } from './User.ts'; -/** We only accept "Bearer" type. */ -const BEARER_REGEX = new RegExp(`^Bearer (${nip19.BECH32_REGEX.source})$`); - export function tokenMiddleware(): DittoMiddleware<{ user?: User }> { return async (c, next) => { const header = c.req.header('authorization'); @@ -23,13 +20,15 @@ export function tokenMiddleware(): DittoMiddleware<{ user?: User }> { if (header) { const { relay, conf } = c.var; - const signer = await getSigner(header, c.var); + const auth = parseAuthorization(header); + const signer = await getSigner(c, auth); const userPubkey = await signer.getPublicKey(); const adminPubkey = await conf.signer.getPublicKey(); const user: User = { signer, relay: new UserStore({ relay, userPubkey, adminPubkey }), + verified: auth.realm === 'Nostr', }; c.set('user', user); @@ -39,34 +38,26 @@ export function tokenMiddleware(): DittoMiddleware<{ user?: User }> { }; } -interface GetSignerOpts { - db: DittoDB; - conf: DittoConf; - relay: NRelay; -} - -function getSigner(header: string, opts: GetSignerOpts): NostrSigner | Promise { - const match = header.match(BEARER_REGEX); - - if (!match) { - throw new HTTPException(400, { message: 'Invalid Authorization header.' }); - } - - const [_, bech32] = match; - - if (isToken(bech32)) { - return getSignerFromToken(bech32, opts); - } else { - return getSignerFromNip19(bech32); +function getSigner(c: Context, auth: Authorization): NostrSigner | Promise { + switch (auth.realm) { + case 'Bearer': { + if (isToken(auth.token)) { + return getSignerFromToken(c, auth.token); + } else { + return getSignerFromNip19(auth.token); + } + } + case 'Nostr': { + return getSignerFromNip98(c); + } + default: { + throw new HTTPException(400, { message: 'Unsupported Authorization realm.' }); + } } } -function isToken(value: string): value is `token1${string}` { - return value.startsWith('token1'); -} - -async function getSignerFromToken(token: `token1${string}`, opts: GetSignerOpts): Promise { - const { conf, db, relay } = opts; +async function getSignerFromToken(c: Context, token: `token1${string}`): Promise { + const { conf, db, relay } = c.var; try { const tokenHash = await getTokenHash(token); @@ -109,3 +100,36 @@ function getSignerFromNip19(bech32: string): NostrSigner { throw new HTTPException(401, { message: 'Invalid NIP-19 identifier in Authorization header.' }); } + +async function getSignerFromNip98(c: Context): Promise { + const { conf } = c.var; + + const req = Object.create(c.req.raw, { + url: { value: conf.local(c.req.url) }, + }); + + const result = await parseAuthRequest(req); + + if (result.success) { + return new ReadOnlySigner(result.data.pubkey); + } else { + throw new HTTPException(401, { message: 'Invalid NIP-98 event in Authorization header.' }); + } +} + +interface Authorization { + realm: string; + token: string; +} + +function parseAuthorization(header: string): Authorization { + const [realm, ...parts] = header.split(' '); + return { + realm, + token: parts.join(' '), + }; +} + +function isToken(value: string): value is `token1${string}` { + return value.startsWith('token1'); +} From 806bfc1b45c090ed09883f22bfe51a16035808dd Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 16:54:38 -0600 Subject: [PATCH 50/99] Delete auth98Middleware, replace with userMiddleware --- packages/ditto/app.ts | 146 +++++++++--------- packages/ditto/controllers/api/cashu.ts | 4 +- packages/ditto/middleware/auth98Middleware.ts | 121 --------------- packages/ditto/utils/api.ts | 9 -- packages/mastoapi/middleware/User.ts | 1 - .../mastoapi/middleware/tokenMiddleware.ts | 1 - .../mastoapi/middleware/userMiddleware.ts | 58 ++++++- 7 files changed, 133 insertions(+), 207 deletions(-) delete mode 100644 packages/ditto/middleware/auth98Middleware.ts diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index 3333b9eb..3b1c3f48 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -7,7 +7,6 @@ import { type Context, Handler, Input as HonoInput, MiddlewareHandler } from '@h import { every } from '@hono/hono/combine'; import { cors } from '@hono/hono/cors'; import { serveStatic } from '@hono/hono/deno'; -import { createFactory } from '@hono/hono/factory'; import { NostrEvent, NostrSigner, NRelay, NUploader } from '@nostrify/nostrify'; import '@/startup.ts'; @@ -138,7 +137,6 @@ import { metricsController } from '@/controllers/metrics.ts'; import { manifestController } from '@/controllers/manifest.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; import { nostrController } from '@/controllers/well-known/nostr.ts'; -import { requireProof as _requireProof, requireRole } from '@/middleware/auth98Middleware.ts'; import { cacheControlMiddleware } from '@/middleware/cacheControlMiddleware.ts'; import { cspMiddleware } from '@/middleware/cspMiddleware.ts'; import { metricsMiddleware } from '@/middleware/metricsMiddleware.ts'; @@ -198,11 +196,6 @@ const ratelimit = every( rateLimitMiddleware(300, Time.minutes(5), false), ); -const factory = createFactory(); -const requireSigner = userMiddleware(); -const requireAdmin = factory.createHandlers(requireSigner, requireRole('admin')); -const requireProof = factory.createHandlers(requireSigner, _requireProof()); - app.use('/api/*', metricsMiddleware, ratelimit, paginationMiddleware(), logiMiddleware); app.use('/.well-known/*', metricsMiddleware, ratelimit, logiMiddleware); app.use('/nodeinfo/*', metricsMiddleware, ratelimit, logiMiddleware); @@ -262,27 +255,27 @@ app.post('/oauth/revoke', revokeTokenController); app.post('/oauth/authorize', oauthAuthorizeController); app.get('/oauth/authorize', oauthController); -app.post('/api/v1/accounts', ...requireProof, createAccountController); -app.get('/api/v1/accounts/verify_credentials', requireSigner, verifyCredentialsController); -app.patch('/api/v1/accounts/update_credentials', requireSigner, updateCredentialsController); +app.post('/api/v1/accounts', userMiddleware({ verify: true }), createAccountController); +app.get('/api/v1/accounts/verify_credentials', userMiddleware(), verifyCredentialsController); +app.patch('/api/v1/accounts/update_credentials', userMiddleware(), updateCredentialsController); app.get('/api/v1/accounts/search', accountSearchController); app.get('/api/v1/accounts/lookup', accountLookupController); -app.get('/api/v1/accounts/relationships', requireSigner, relationshipsController); -app.get('/api/v1/accounts/familiar_followers', requireSigner, familiarFollowersController); -app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/block', requireSigner, blockController); -app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/unblock', requireSigner, unblockController); -app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/mute', requireSigner, muteController); -app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/unmute', requireSigner, unmuteController); +app.get('/api/v1/accounts/relationships', userMiddleware(), relationshipsController); +app.get('/api/v1/accounts/familiar_followers', userMiddleware(), familiarFollowersController); +app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/block', userMiddleware(), blockController); +app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/unblock', userMiddleware(), unblockController); +app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/mute', userMiddleware(), muteController); +app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/unmute', userMiddleware(), unmuteController); app.post( '/api/v1/accounts/:pubkey{[0-9a-f]{64}}/follow', rateLimitMiddleware(2, Time.seconds(1)), - requireSigner, + userMiddleware(), followController, ); app.post( '/api/v1/accounts/:pubkey{[0-9a-f]{64}}/unfollow', rateLimitMiddleware(2, Time.seconds(1)), - requireSigner, + userMiddleware(), unfollowController, ); app.get( @@ -306,22 +299,22 @@ app.get('/api/v1/statuses/:id{[0-9a-f]{64}}/favourited_by', favouritedByControll app.get('/api/v1/statuses/:id{[0-9a-f]{64}}/reblogged_by', rebloggedByController); app.get('/api/v1/statuses/:id{[0-9a-f]{64}}/context', contextController); app.get('/api/v1/statuses/:id{[0-9a-f]{64}}', statusController); -app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/favourite', requireSigner, favouriteController); -app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/bookmark', requireSigner, bookmarkController); -app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unbookmark', requireSigner, unbookmarkController); -app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/pin', requireSigner, pinController); -app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unpin', requireSigner, unpinController); +app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/favourite', userMiddleware(), favouriteController); +app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/bookmark', userMiddleware(), bookmarkController); +app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unbookmark', userMiddleware(), unbookmarkController); +app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/pin', userMiddleware(), pinController); +app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unpin', userMiddleware(), unpinController); app.post( '/api/v1/statuses/:id{[0-9a-f]{64}}/translate', - requireSigner, + userMiddleware(), rateLimitMiddleware(15, Time.minutes(1)), translatorMiddleware, translateController, ); -app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/reblog', requireSigner, reblogStatusController); -app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unreblog', requireSigner, unreblogStatusController); -app.post('/api/v1/statuses', requireSigner, createStatusController); -app.delete('/api/v1/statuses/:id{[0-9a-f]{64}}', requireSigner, deleteStatusController); +app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/reblog', userMiddleware(), reblogStatusController); +app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unreblog', userMiddleware(), unreblogStatusController); +app.post('/api/v1/statuses', userMiddleware(), createStatusController); +app.delete('/api/v1/statuses/:id{[0-9a-f]{64}}', userMiddleware(), deleteStatusController); app.get('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/quotes', quotesController); @@ -332,7 +325,7 @@ app.put( ); app.post('/api/v2/media', mediaController); -app.get('/api/v1/timelines/home', rateLimitMiddleware(8, Time.seconds(30)), requireSigner, homeTimelineController); +app.get('/api/v1/timelines/home', rateLimitMiddleware(8, Time.seconds(30)), userMiddleware(), homeTimelineController); app.get('/api/v1/timelines/public', rateLimitMiddleware(8, Time.seconds(30)), publicTimelineController); app.get('/api/v1/timelines/tag/:hashtag', rateLimitMiddleware(8, Time.seconds(30)), hashtagTimelineController); app.get('/api/v1/timelines/suggested', rateLimitMiddleware(8, Time.seconds(30)), suggestedTimelineController); @@ -368,42 +361,42 @@ app.get('/api/v1/suggestions', suggestionsV1Controller); app.get('/api/v2/suggestions', suggestionsV2Controller); app.get('/api/v2/ditto/suggestions/local', localSuggestionsController); -app.get('/api/v1/notifications', rateLimitMiddleware(8, Time.seconds(30)), requireSigner, notificationsController); -app.get('/api/v1/notifications/:id', requireSigner, notificationController); +app.get('/api/v1/notifications', rateLimitMiddleware(8, Time.seconds(30)), userMiddleware(), notificationsController); +app.get('/api/v1/notifications/:id', userMiddleware(), notificationController); -app.get('/api/v1/favourites', requireSigner, favouritesController); -app.get('/api/v1/bookmarks', requireSigner, bookmarksController); -app.get('/api/v1/blocks', requireSigner, blocksController); -app.get('/api/v1/mutes', requireSigner, mutesController); +app.get('/api/v1/favourites', userMiddleware(), favouritesController); +app.get('/api/v1/bookmarks', userMiddleware(), bookmarksController); +app.get('/api/v1/blocks', userMiddleware(), blocksController); +app.get('/api/v1/mutes', userMiddleware(), mutesController); -app.get('/api/v1/markers', ...requireProof, markersController); -app.post('/api/v1/markers', ...requireProof, updateMarkersController); +app.get('/api/v1/markers', userMiddleware({ verify: true }), markersController); +app.post('/api/v1/markers', userMiddleware({ verify: true }), updateMarkersController); -app.get('/api/v1/push/subscription', requireSigner, getSubscriptionController); -app.post('/api/v1/push/subscription', ...requireProof, pushSubscribeController); +app.get('/api/v1/push/subscription', userMiddleware(), getSubscriptionController); +app.post('/api/v1/push/subscription', userMiddleware({ verify: true }), pushSubscribeController); app.get('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/reactions', reactionsController); app.get('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/reactions/:emoji', reactionsController); -app.put('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/reactions/:emoji', requireSigner, reactionController); -app.delete('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/reactions/:emoji', requireSigner, deleteReactionController); +app.put('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), reactionController); +app.delete('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), deleteReactionController); -app.get('/api/v1/pleroma/admin/config', ...requireAdmin, configController); -app.post('/api/v1/pleroma/admin/config', ...requireAdmin, updateConfigController); -app.delete('/api/v1/pleroma/admin/statuses/:id', ...requireAdmin, pleromaAdminDeleteStatusController); +app.get('/api/v1/pleroma/admin/config', userMiddleware({ role: 'admin' }), configController); +app.post('/api/v1/pleroma/admin/config', userMiddleware({ role: 'admin' }), updateConfigController); +app.delete('/api/v1/pleroma/admin/statuses/:id', userMiddleware({ role: 'admin' }), pleromaAdminDeleteStatusController); -app.get('/api/v1/admin/ditto/relays', ...requireAdmin, adminRelaysController); -app.put('/api/v1/admin/ditto/relays', ...requireAdmin, adminSetRelaysController); +app.get('/api/v1/admin/ditto/relays', userMiddleware({ role: 'admin' }), adminRelaysController); +app.put('/api/v1/admin/ditto/relays', userMiddleware({ role: 'admin' }), adminSetRelaysController); -app.put('/api/v1/admin/ditto/instance', ...requireAdmin, updateInstanceController); +app.put('/api/v1/admin/ditto/instance', userMiddleware({ role: 'admin' }), updateInstanceController); -app.post('/api/v1/ditto/names', requireSigner, nameRequestController); -app.get('/api/v1/ditto/names', requireSigner, nameRequestsController); +app.post('/api/v1/ditto/names', userMiddleware(), nameRequestController); +app.get('/api/v1/ditto/names', userMiddleware(), nameRequestsController); app.get('/api/v1/ditto/captcha', rateLimitMiddleware(3, Time.minutes(1)), captchaController); app.post( '/api/v1/ditto/captcha/:id/verify', rateLimitMiddleware(8, Time.minutes(1)), - ...requireProof, + userMiddleware({ verify: true }), captchaVerifyController, ); @@ -414,44 +407,59 @@ app.get( ); app.get('/api/v1/ditto/:id{[0-9a-f]{64}}/zap_splits', statusZapSplitsController); -app.put('/api/v1/admin/ditto/zap_splits', ...requireAdmin, updateZapSplitsController); -app.delete('/api/v1/admin/ditto/zap_splits', ...requireAdmin, deleteZapSplitsController); +app.put('/api/v1/admin/ditto/zap_splits', userMiddleware({ role: 'admin' }), updateZapSplitsController); +app.delete('/api/v1/admin/ditto/zap_splits', userMiddleware({ role: 'admin' }), deleteZapSplitsController); -app.post('/api/v1/ditto/zap', requireSigner, zapController); +app.post('/api/v1/ditto/zap', userMiddleware(), zapController); app.get('/api/v1/ditto/statuses/:id{[0-9a-f]{64}}/zapped_by', zappedByController); app.route('/api/v1/ditto/cashu', cashuApp); -app.post('/api/v1/reports', requireSigner, reportController); -app.get('/api/v1/admin/reports', requireSigner, ...requireAdmin, adminReportsController); -app.get('/api/v1/admin/reports/:id{[0-9a-f]{64}}', requireSigner, ...requireAdmin, adminReportController); +app.post('/api/v1/reports', userMiddleware(), reportController); +app.get('/api/v1/admin/reports', userMiddleware(), userMiddleware({ role: 'admin' }), adminReportsController); +app.get( + '/api/v1/admin/reports/:id{[0-9a-f]{64}}', + userMiddleware(), + userMiddleware({ role: 'admin' }), + adminReportController, +); app.post( '/api/v1/admin/reports/:id{[0-9a-f]{64}}/resolve', - requireSigner, - ...requireAdmin, + userMiddleware(), + userMiddleware({ role: 'admin' }), adminReportResolveController, ); app.post( '/api/v1/admin/reports/:id{[0-9a-f]{64}}/reopen', - requireSigner, - ...requireAdmin, + userMiddleware(), + userMiddleware({ role: 'admin' }), adminReportReopenController, ); -app.get('/api/v1/admin/accounts', ...requireAdmin, adminAccountsController); -app.post('/api/v1/admin/accounts/:id{[0-9a-f]{64}}/action', requireSigner, ...requireAdmin, adminActionController); +app.get('/api/v1/admin/accounts', userMiddleware({ role: 'admin' }), adminAccountsController); +app.post( + '/api/v1/admin/accounts/:id{[0-9a-f]{64}}/action', + userMiddleware(), + userMiddleware({ role: 'admin' }), + adminActionController, +); app.post( '/api/v1/admin/accounts/:id{[0-9a-f]{64}}/approve', - requireSigner, - ...requireAdmin, + userMiddleware(), + userMiddleware({ role: 'admin' }), adminApproveController, ); -app.post('/api/v1/admin/accounts/:id{[0-9a-f]{64}}/reject', requireSigner, ...requireAdmin, adminRejectController); +app.post( + '/api/v1/admin/accounts/:id{[0-9a-f]{64}}/reject', + userMiddleware(), + userMiddleware({ role: 'admin' }), + adminRejectController, +); -app.put('/api/v1/pleroma/admin/users/tag', ...requireAdmin, pleromaAdminTagController); -app.delete('/api/v1/pleroma/admin/users/tag', ...requireAdmin, pleromaAdminUntagController); -app.patch('/api/v1/pleroma/admin/users/suggest', ...requireAdmin, pleromaAdminSuggestController); -app.patch('/api/v1/pleroma/admin/users/unsuggest', ...requireAdmin, pleromaAdminUnsuggestController); +app.put('/api/v1/pleroma/admin/users/tag', userMiddleware({ role: 'admin' }), pleromaAdminTagController); +app.delete('/api/v1/pleroma/admin/users/tag', userMiddleware({ role: 'admin' }), pleromaAdminUntagController); +app.patch('/api/v1/pleroma/admin/users/suggest', userMiddleware({ role: 'admin' }), pleromaAdminSuggestController); +app.patch('/api/v1/pleroma/admin/users/unsuggest', userMiddleware({ role: 'admin' }), pleromaAdminUnsuggestController); // Not (yet) implemented. app.get('/api/v1/custom_emojis', emptyArrayController); diff --git a/packages/ditto/controllers/api/cashu.ts b/packages/ditto/controllers/api/cashu.ts index 2d3a1519..bb39397c 100644 --- a/packages/ditto/controllers/api/cashu.ts +++ b/packages/ditto/controllers/api/cashu.ts @@ -42,7 +42,7 @@ const createCashuWalletAndNutzapInfoSchema = z.object({ * https://github.com/nostr-protocol/nips/blob/master/60.md * https://github.com/nostr-protocol/nips/blob/master/61.md#nutzap-informational-event */ -route.put('/wallet', userMiddleware('nip44'), async (c) => { +route.put('/wallet', userMiddleware({ enc: 'nip44' }), async (c) => { const { conf, user, relay, signal } = c.var; const pubkey = await user.signer.getPublicKey(); @@ -104,7 +104,7 @@ route.put('/wallet', userMiddleware('nip44'), async (c) => { }); /** Gets a wallet, if it exists. */ -route.get('/wallet', userMiddleware('nip44'), swapNutzapsMiddleware, async (c) => { +route.get('/wallet', userMiddleware({ enc: 'nip44' }), swapNutzapsMiddleware, async (c) => { const { conf, relay, user, signal } = c.var; const pubkey = await user.signer.getPublicKey(); diff --git a/packages/ditto/middleware/auth98Middleware.ts b/packages/ditto/middleware/auth98Middleware.ts deleted file mode 100644 index 6cab6566..00000000 --- a/packages/ditto/middleware/auth98Middleware.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { buildAuthEventTemplate, parseAuthRequest, type ParseAuthRequestOpts, validateAuthEvent } from '@ditto/nip98'; -import { HTTPException } from '@hono/hono/http-exception'; -import { NostrEvent } from '@nostrify/nostrify'; - -import { type AppContext, type AppMiddleware } from '@/app.ts'; -import { ReadOnlySigner } from '@/signers/ReadOnlySigner.ts'; -import { localRequest } from '@/utils/api.ts'; - -/** - * NIP-98 auth. - * https://github.com/nostr-protocol/nips/blob/master/98.md - */ -function auth98Middleware(opts: ParseAuthRequestOpts = {}): AppMiddleware { - return async (c, next) => { - const req = localRequest(c); - const result = await parseAuthRequest(req, opts); - - if (result.success) { - const user = { - relay: c.var.relay, - signer: new ReadOnlySigner(result.data.pubkey), - ...c.var.user, - }; - - c.set('user', user); - } - - await next(); - }; -} - -type UserRole = 'user' | 'admin'; - -/** Require the user to prove their role before invoking the controller. */ -function requireRole(role: UserRole, opts?: ParseAuthRequestOpts): AppMiddleware { - return withProof(async (c, proof, next) => { - const { conf, relay } = c.var; - - const [user] = await relay.query([{ - kinds: [30382], - authors: [await conf.signer.getPublicKey()], - '#d': [proof.pubkey], - limit: 1, - }]); - - if (user && matchesRole(user, role)) { - await next(); - } else { - throw new HTTPException(401); - } - }, opts); -} - -/** Require the user to demonstrate they own the pubkey by signing an event. */ -function requireProof(opts?: ParseAuthRequestOpts): AppMiddleware { - return withProof(async (_c, _proof, next) => { - await next(); - }, opts); -} - -/** Check whether the user fulfills the role. */ -function matchesRole(user: NostrEvent, role: UserRole): boolean { - return user.tags.some(([tag, value]) => tag === 'n' && value === role); -} - -/** HOC to obtain proof in middleware. */ -function withProof( - handler: (c: AppContext, proof: NostrEvent, next: () => Promise) => Promise, - opts?: ParseAuthRequestOpts, -): AppMiddleware { - return async (c, next) => { - const signer = c.var.user?.signer; - const pubkey = await signer?.getPublicKey(); - const proof = c.get('proof') || await obtainProof(c, opts); - - // Prevent people from accidentally using the wrong account. This has no other security implications. - if (proof && pubkey && pubkey !== proof.pubkey) { - throw new HTTPException(401, { message: 'Pubkey mismatch' }); - } - - if (proof) { - c.set('proof', proof); - - if (!signer) { - const user = { - relay: c.var.relay, - signer: new ReadOnlySigner(proof.pubkey), - ...c.var.user, - }; - - c.set('user', user); - } - - await handler(c, proof, next); - } else { - throw new HTTPException(401, { message: 'No proof' }); - } - }; -} - -/** Get the proof over Nostr Connect. */ -async function obtainProof(c: AppContext, opts?: ParseAuthRequestOpts) { - const signer = c.var.user?.signer; - - if (!signer) { - throw new HTTPException(401, { - res: c.json({ error: 'No way to sign Nostr event' }, 401), - }); - } - - const req = localRequest(c); - const reqEvent = await buildAuthEventTemplate(req, opts); - const resEvent = await signer.signEvent(reqEvent); - const result = await validateAuthEvent(req, resEvent, opts); - - if (result.success) { - return result.data; - } -} - -export { auth98Middleware, requireProof, requireRole }; diff --git a/packages/ditto/utils/api.ts b/packages/ditto/utils/api.ts index 37a38d6a..591c1852 100644 --- a/packages/ditto/utils/api.ts +++ b/packages/ditto/utils/api.ts @@ -1,4 +1,3 @@ -import { type Context } from '@hono/hono'; import { HTTPException } from '@hono/hono/http-exception'; import { NostrEvent, NostrFilter } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; @@ -257,13 +256,6 @@ function paginatedList( return c.json(results, 200, headers); } -/** Rewrite the URL of the request object to use the local domain. */ -function localRequest(c: Context): Request { - return Object.create(c.req.raw, { - url: { value: Conf.local(c.req.url) }, - }); -} - /** Actors with Bluesky's `!no-unauthenticated` self-label should require authorization to view. */ function assertAuthenticated(c: AppContext, author: NostrEvent): void { if ( @@ -282,7 +274,6 @@ export { createAdminEvent, createEvent, type EventStub, - localRequest, paginated, paginatedList, parseBody, diff --git a/packages/mastoapi/middleware/User.ts b/packages/mastoapi/middleware/User.ts index 8fd61f96..ac38b8de 100644 --- a/packages/mastoapi/middleware/User.ts +++ b/packages/mastoapi/middleware/User.ts @@ -3,5 +3,4 @@ import type { NostrSigner, NRelay } from '@nostrify/nostrify'; export interface User { signer: S; relay: R; - verified?: boolean; } diff --git a/packages/mastoapi/middleware/tokenMiddleware.ts b/packages/mastoapi/middleware/tokenMiddleware.ts index 4796b5d4..ad174c72 100644 --- a/packages/mastoapi/middleware/tokenMiddleware.ts +++ b/packages/mastoapi/middleware/tokenMiddleware.ts @@ -28,7 +28,6 @@ export function tokenMiddleware(): DittoMiddleware<{ user?: User }> { const user: User = { signer, relay: new UserStore({ relay, userPubkey, adminPubkey }), - verified: auth.realm === 'Nostr', }; c.set('user', user); diff --git a/packages/mastoapi/middleware/userMiddleware.ts b/packages/mastoapi/middleware/userMiddleware.ts index 5b18e718..1afef59a 100644 --- a/packages/mastoapi/middleware/userMiddleware.ts +++ b/packages/mastoapi/middleware/userMiddleware.ts @@ -1,18 +1,29 @@ +import { buildAuthEventTemplate, validateAuthEvent } from '@ditto/nip98'; import { HTTPException } from '@hono/hono/http-exception'; import type { DittoMiddleware } from '@ditto/router'; -import type { NostrSigner } from '@nostrify/nostrify'; +import type { NostrEvent, NostrSigner } from '@nostrify/nostrify'; import type { SetRequired } from 'type-fest'; import type { User } from './User.ts'; type Nip44Signer = SetRequired; +interface UserMiddlewareOpts { + enc?: 'nip04' | 'nip44'; + role?: string; + verify?: boolean; +} + export function userMiddleware(): DittoMiddleware<{ user: User }>; // @ts-ignore Types are right. -export function userMiddleware(enc: 'nip44'): DittoMiddleware<{ user: User }>; -export function userMiddleware(enc?: 'nip04' | 'nip44'): DittoMiddleware<{ user: User }> { +export function userMiddleware( + opts: UserMiddlewareOpts & { enc: 'nip44' }, +): DittoMiddleware<{ user: User }>; +export function userMiddleware(opts: UserMiddlewareOpts): DittoMiddleware<{ user: User }>; +export function userMiddleware(opts: UserMiddlewareOpts = {}): DittoMiddleware<{ user: User }> { return async (c, next) => { - const { user } = c.var; + const { conf, user, relay } = c.var; + const { enc, role, verify } = opts; if (!user) { throw new HTTPException(403, { message: 'Authorization required.' }); @@ -22,6 +33,45 @@ export function userMiddleware(enc?: 'nip04' | 'nip44'): DittoMiddleware<{ user: throw new HTTPException(403, { message: `User does not have a ${enc} signer` }); } + if (role || verify) { + const req = setRequestUrl(c.req.raw, conf.local(c.req.url)); + const reqEvent = await buildAuthEventTemplate(req); + const resEvent = await user.signer.signEvent(reqEvent); + const result = await validateAuthEvent(req, resEvent); + + if (!result.success) { + throw new HTTPException(403, { message: 'Verification failed.' }); + } + + // Prevent people from accidentally using the wrong account. This has no other security implications. + if (result.data.pubkey !== await user.signer.getPublicKey()) { + throw new HTTPException(401, { message: 'Pubkey mismatch' }); + } + + if (role) { + const [user] = await relay.query([{ + kinds: [30382], + authors: [await conf.signer.getPublicKey()], + '#d': [result.data.pubkey], + limit: 1, + }]); + + if (!user || !matchesRole(user, role)) { + throw new HTTPException(403, { message: `Must have ${role} role.` }); + } + } + } + await next(); }; } + +/** Rewrite the URL of the request object. */ +function setRequestUrl(req: Request, url: string): Request { + return Object.create(req, { url: { value: url } }); +} + +/** Check whether the user fulfills the role. */ +function matchesRole(user: NostrEvent, role: string): boolean { + return user.tags.some(([tag, value]) => tag === 'n' && value === role); +} From 26e87b396205aaf3b28e3bf000ca05e2c28c0a06 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 17:44:56 -0600 Subject: [PATCH 51/99] tokenMiddleware: pass token to streaming API --- packages/ditto/app.ts | 9 ++++- packages/ditto/controllers/api/streaming.ts | 36 +++---------------- .../mastoapi/middleware/tokenMiddleware.ts | 6 ++-- packages/policies/MuteListPolicy.ts | 2 +- 4 files changed, 17 insertions(+), 36 deletions(-) diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index 3b1c3f48..2b90f132 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -196,12 +196,19 @@ const ratelimit = every( rateLimitMiddleware(300, Time.minutes(5), false), ); +const socketTokenMiddleware = tokenMiddleware((c) => { + const token = c.req.header('sec-websocket-protocol'); + if (token) { + return `Bearer ${token}`; + } +}); + app.use('/api/*', metricsMiddleware, ratelimit, paginationMiddleware(), logiMiddleware); app.use('/.well-known/*', metricsMiddleware, ratelimit, logiMiddleware); app.use('/nodeinfo/*', metricsMiddleware, ratelimit, logiMiddleware); app.use('/oauth/*', metricsMiddleware, ratelimit, logiMiddleware); -app.get('/api/v1/streaming', metricsMiddleware, ratelimit, streamingController); +app.get('/api/v1/streaming', socketTokenMiddleware, metricsMiddleware, ratelimit, streamingController); app.get('/relay', metricsMiddleware, ratelimit, relayController); app.use( diff --git a/packages/ditto/controllers/api/streaming.ts b/packages/ditto/controllers/api/streaming.ts index b39f1db5..cdd8dae3 100644 --- a/packages/ditto/controllers/api/streaming.ts +++ b/packages/ditto/controllers/api/streaming.ts @@ -12,13 +12,10 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { getFeedPubkeys } from '@/queries.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { Storages } from '@/storages.ts'; -import { getTokenHash } from '@/utils/auth.ts'; import { errorJson } from '@/utils/log.ts'; -import { bech32ToPubkey, Time } from '@/utils.ts'; +import { Time } from '@/utils.ts'; import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts'; import { renderNotification } from '@/views/mastodon/notifications.ts'; -import { HTTPException } from '@hono/hono/http-exception'; /** * Streaming timelines/categories. @@ -68,7 +65,7 @@ const limiter = new TTLCache(); const connections = new Set(); const streamingController: AppController = async (c) => { - const { conf, relay } = c.var; + const { conf, relay, user } = c.var; const upgrade = c.req.header('upgrade'); const token = c.req.header('sec-websocket-protocol'); const stream = streamSchema.optional().catch(undefined).parse(c.req.query('stream')); @@ -78,11 +75,6 @@ const streamingController: AppController = async (c) => { return c.text('Please use websocket protocol', 400); } - const pubkey = token ? await getTokenPubkey(token) : undefined; - if (token && !pubkey) { - return c.json({ error: 'Invalid access token' }, 401); - } - const ip = c.req.header('x-real-ip'); if (ip) { const count = limiter.get(ip) ?? 0; @@ -93,7 +85,8 @@ const streamingController: AppController = async (c) => { const { socket, response } = Deno.upgradeWebSocket(c.req.raw, { protocol: token, idleTimeout: 30 }); - const policy = pubkey ? new MuteListPolicy(pubkey, await Storages.admin()) : undefined; + const pubkey = await user?.signer.getPublicKey(); + const policy = pubkey ? new MuteListPolicy(pubkey, relay) : undefined; function send(e: StreamingEvent) { if (socket.readyState === WebSocket.OPEN) { @@ -229,25 +222,4 @@ async function topicToFilter( } } -async function getTokenPubkey(token: string): Promise { - if (token.startsWith('token1')) { - const kysely = await Storages.kysely(); - const tokenHash = await getTokenHash(token as `token1${string}`); - - const row = await kysely - .selectFrom('auth_tokens') - .select('pubkey') - .where('token_hash', '=', tokenHash) - .executeTakeFirst(); - - if (!row) { - throw new HTTPException(401, { message: 'Invalid access token' }); - } - - return row.pubkey; - } else { - return bech32ToPubkey(token); - } -} - export { streamingController }; diff --git a/packages/mastoapi/middleware/tokenMiddleware.ts b/packages/mastoapi/middleware/tokenMiddleware.ts index ad174c72..d4f8a05b 100644 --- a/packages/mastoapi/middleware/tokenMiddleware.ts +++ b/packages/mastoapi/middleware/tokenMiddleware.ts @@ -13,9 +13,11 @@ import type { DittoEnv, DittoMiddleware } from '@ditto/router'; import type { Context } from '@hono/hono'; import type { User } from './User.ts'; -export function tokenMiddleware(): DittoMiddleware<{ user?: User }> { +type CredentialsFn = (c: Context) => string | undefined; + +export function tokenMiddleware(fn?: CredentialsFn): DittoMiddleware<{ user?: User }> { return async (c, next) => { - const header = c.req.header('authorization'); + const header = fn ? fn(c) : c.req.header('authorization'); if (header) { const { relay, conf } = c.var; diff --git a/packages/policies/MuteListPolicy.ts b/packages/policies/MuteListPolicy.ts index d880c57d..1025e75b 100644 --- a/packages/policies/MuteListPolicy.ts +++ b/packages/policies/MuteListPolicy.ts @@ -15,7 +15,7 @@ export class MuteListPolicy implements NPolicy { } if (pubkeys.has(event.pubkey)) { - return ['OK', event.id, false, 'blocked: Your account has been deactivated.']; + return ['OK', event.id, false, 'blocked: account blocked']; } return ['OK', event.id, true, '']; From 6b1aadc24c75d939033be5aef94509220100839d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 18:46:43 -0600 Subject: [PATCH 52/99] nip98: add explicit types to exported functions --- packages/nip98/nip98.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/nip98/nip98.ts b/packages/nip98/nip98.ts index e8574c86..b0815f91 100644 --- a/packages/nip98/nip98.ts +++ b/packages/nip98/nip98.ts @@ -4,6 +4,8 @@ import { type EventTemplate, nip13 } from 'nostr-tools'; import { decode64Schema, signedEventSchema } from './schema.ts'; +import type { z } from 'zod'; + /** Decode a Nostr event from a base64 encoded string. */ const decode64EventSchema = decode64Schema.pipe(n.json()).pipe(signedEventSchema); @@ -18,7 +20,10 @@ interface ParseAuthRequestOpts { /** Parse the auth event from a Request, returning a zod SafeParse type. */ // deno-lint-ignore require-await -async function parseAuthRequest(req: Request, opts: ParseAuthRequestOpts = {}) { +async function parseAuthRequest( + req: Request, + opts: ParseAuthRequestOpts = {}, +): Promise | z.SafeParseError> { const header = req.headers.get('authorization'); const base64 = header?.match(/^Nostr (.+)$/)?.[1]; const result = decode64EventSchema.safeParse(base64); @@ -28,7 +33,11 @@ async function parseAuthRequest(req: Request, opts: ParseAuthRequestOpts = {}) { } /** Compare the auth event with the request, returning a zod SafeParse type. */ -function validateAuthEvent(req: Request, event: NostrEvent, opts: ParseAuthRequestOpts = {}) { +function validateAuthEvent( + req: Request, + event: NostrEvent, + opts: ParseAuthRequestOpts = {}, +): Promise> { const { maxAge = 60_000, validatePayload = true, pow = 0 } = opts; const schema = signedEventSchema From d2abb1f1e48725d71895523a5b48e177ad3f7b01 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 18:59:14 -0600 Subject: [PATCH 53/99] Fix MuteListPolicy test --- packages/policies/MuteListPolicy.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/policies/MuteListPolicy.test.ts b/packages/policies/MuteListPolicy.test.ts index d07c4472..21c29cbc 100644 --- a/packages/policies/MuteListPolicy.test.ts +++ b/packages/policies/MuteListPolicy.test.ts @@ -25,7 +25,7 @@ Deno.test('block event: muted user cannot post', async () => { const ok = await policy.call(event1authorUserMeCopy); - assertEquals(ok, ['OK', event1authorUserMeCopy.id, false, 'blocked: Your account has been deactivated.']); + assertEquals(ok, ['OK', event1authorUserMeCopy.id, false, 'blocked: account blocked']); }); Deno.test('allow event: user is NOT muted because there is no muted event', async () => { From 82446e3ef151e671dd4af4b7d71f552b8a2158c0 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 20:06:39 -0600 Subject: [PATCH 54/99] Add method and pathname to ditto.http error --- packages/ditto/controllers/error.ts | 5 ++++- packages/ditto/middleware/logiMiddleware.ts | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/ditto/controllers/error.ts b/packages/ditto/controllers/error.ts index 50962fcc..a00a530b 100644 --- a/packages/ditto/controllers/error.ts +++ b/packages/ditto/controllers/error.ts @@ -5,6 +5,9 @@ import { logi } from '@soapbox/logi'; import { errorJson } from '@/utils/log.ts'; export const errorHandler: ErrorHandler = (err, c) => { + const { method } = c.req; + const { pathname } = new URL(c.req.url); + c.header('Cache-Control', 'no-store'); if (err instanceof HTTPException) { @@ -19,7 +22,7 @@ export const errorHandler: ErrorHandler = (err, c) => { return c.json({ error: 'The server was unable to respond in a timely manner' }, 500); } - logi({ level: 'error', ns: 'ditto.http', msg: 'Unhandled error', error: errorJson(err) }); + logi({ level: 'error', ns: 'ditto.http', msg: 'Unhandled error', method, pathname, error: errorJson(err) }); return c.json({ error: 'Something went wrong' }, 500); }; diff --git a/packages/ditto/middleware/logiMiddleware.ts b/packages/ditto/middleware/logiMiddleware.ts index 26233f27..be17e3bb 100644 --- a/packages/ditto/middleware/logiMiddleware.ts +++ b/packages/ditto/middleware/logiMiddleware.ts @@ -12,8 +12,8 @@ export const logiMiddleware: MiddlewareHandler = async (c, next) => { await next(); const end = new Date(); - const delta = (end.getTime() - start.getTime()) / 1000; + const duration = (end.getTime() - start.getTime()) / 1000; const level = c.res.status >= 500 ? 'error' : 'info'; - logi({ level, ns: 'ditto.http.response', method, pathname, status: c.res.status, delta }); + logi({ level, ns: 'ditto.http.response', method, pathname, status: c.res.status, duration }); }; From 5fec5deb063850d15f97c9638725495041c7b6cd Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 20:22:58 -0600 Subject: [PATCH 55/99] publishEvent: publish to pool in background, catch errors and log --- packages/ditto/utils/api.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/ditto/utils/api.ts b/packages/ditto/utils/api.ts index f2aa4ab2..9873fb2c 100644 --- a/packages/ditto/utils/api.ts +++ b/packages/ditto/utils/api.ts @@ -12,6 +12,7 @@ import { RelayError } from '@/RelayError.ts'; import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; import { parseFormData } from '@/utils/formdata.ts'; +import { errorJson } from '@/utils/log.ts'; import { purifyEvent } from '@/utils/purify.ts'; /** EventTemplate with defaults. */ @@ -158,9 +159,16 @@ async function updateNames(k: number, d: string, n: Record, c: async function publishEvent(event: NostrEvent, c: AppContext): Promise { logi({ level: 'info', ns: 'ditto.event', source: 'api', id: event.id, kind: event.kind }); try { - await pipeline.handleEvent(event, { source: 'api', signal: c.req.raw.signal }); - const client = await Storages.client(); - await client.event(purifyEvent(event)); + const promise = pipeline.handleEvent(event, { source: 'api', signal: c.req.raw.signal }); + + promise.then(async () => { + const client = await Storages.client(); + await client.event(purifyEvent(event)); + }).catch((e: unknown) => { + logi({ level: 'error', ns: 'ditto.pool', id: event.id, kind: event.kind, error: errorJson(e) }); + }); + + await promise; } catch (e) { if (e instanceof RelayError) { throw new HTTPException(422, { From 8437da1200024b9d8af6aea51d12207a85676098 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 20:33:44 -0600 Subject: [PATCH 56/99] Fix error handling in nameRequestController --- packages/ditto/controllers/api/ditto.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ditto/controllers/api/ditto.ts b/packages/ditto/controllers/api/ditto.ts index 752124dc..33db6fc7 100644 --- a/packages/ditto/controllers/api/ditto.ts +++ b/packages/ditto/controllers/api/ditto.ts @@ -84,7 +84,13 @@ export const nameRequestController: AppController = async (c) => { const pubkey = await signer.getPublicKey(); const { conf } = c.var; - const { name, reason } = nameRequestSchema.parse(await c.req.json()); + const result = nameRequestSchema.safeParse(await c.req.json()); + + if (!result.success) { + return c.json({ error: 'Invalid username', schema: result.error }, 400); + } + + const { name, reason } = result.data; const [existing] = await store.query([{ kinds: [3036], authors: [pubkey], '#r': [name], limit: 1 }]); if (existing) { From 4cfb6543c774d6156d1ea8d0b978c05b4d6ea47c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 20:50:09 -0600 Subject: [PATCH 57/99] Don't lowercase nip05 name before fetching (for now) --- packages/ditto/pipeline.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ditto/pipeline.ts b/packages/ditto/pipeline.ts index 602d0e2b..42b68d1b 100644 --- a/packages/ditto/pipeline.ts +++ b/packages/ditto/pipeline.ts @@ -190,7 +190,7 @@ async function updateAuthorData(event: NostrEvent, signal: AbortSignal): Promise if (nip05) { const tld = tldts.parse(nip05); if (tld.isIcann && !tld.isIp && !tld.isPrivate) { - const pointer = await nip05Cache.fetch(nip05.toLowerCase(), { signal }); + const pointer = await nip05Cache.fetch(nip05, { signal }); if (pointer.pubkey === event.pubkey) { updates.nip05 = nip05; updates.nip05_domain = tld.domain; From d4fc10fe3e6f58ebaa6d83bca256adacccd221b6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 21:09:22 -0600 Subject: [PATCH 58/99] Add userMiddleware tests --- packages/db/adapters/DummyDB.test.ts | 9 ++ packages/db/adapters/DummyDB.ts | 29 ++++++ packages/db/mod.ts | 3 + .../middleware/userMiddleware.test.ts | 99 +++++++++++++++++++ .../mastoapi/middleware/userMiddleware.ts | 8 +- 5 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 packages/db/adapters/DummyDB.test.ts create mode 100644 packages/db/adapters/DummyDB.ts create mode 100644 packages/mastoapi/middleware/userMiddleware.test.ts diff --git a/packages/db/adapters/DummyDB.test.ts b/packages/db/adapters/DummyDB.test.ts new file mode 100644 index 00000000..c725ab51 --- /dev/null +++ b/packages/db/adapters/DummyDB.test.ts @@ -0,0 +1,9 @@ +import { assertEquals } from '@std/assert'; +import { DummyDB } from './DummyDB.ts'; + +Deno.test('DummyDB', async () => { + const db = DummyDB.create(); + const rows = await db.kysely.selectFrom('nostr_events').selectAll().execute(); + + assertEquals(rows, []); +}); diff --git a/packages/db/adapters/DummyDB.ts b/packages/db/adapters/DummyDB.ts new file mode 100644 index 00000000..51c29b10 --- /dev/null +++ b/packages/db/adapters/DummyDB.ts @@ -0,0 +1,29 @@ +import { DummyDriver, Kysely, PostgresAdapter, PostgresIntrospector, PostgresQueryCompiler } from 'kysely'; + +import type { DittoDB } from '../DittoDB.ts'; +import type { DittoTables } from '../DittoTables.ts'; + +export class DummyDB implements DittoDB { + readonly kysely: Kysely; + readonly poolSize = 0; + readonly availableConnections = 0; + + constructor() { + this.kysely = new Kysely({ + dialect: { + createAdapter: () => new PostgresAdapter(), + createDriver: () => new DummyDriver(), + createIntrospector: (db) => new PostgresIntrospector(db), + createQueryCompiler: () => new PostgresQueryCompiler(), + }, + }); + } + + listen(): void { + // noop + } + + [Symbol.asyncDispose](): Promise { + return Promise.resolve(); + } +} diff --git a/packages/db/mod.ts b/packages/db/mod.ts index 49100cd6..2766e524 100644 --- a/packages/db/mod.ts +++ b/packages/db/mod.ts @@ -1,4 +1,7 @@ +export { DittoPglite } from './adapters/DittoPglite.ts'; export { DittoPolyPg } from './adapters/DittoPolyPg.ts'; +export { DittoPostgres } from './adapters/DittoPostgres.ts'; +export { DummyDB } from './adapters/DummyDB.ts'; export type { DittoDB } from './DittoDB.ts'; export type { DittoTables } from './DittoTables.ts'; diff --git a/packages/mastoapi/middleware/userMiddleware.test.ts b/packages/mastoapi/middleware/userMiddleware.test.ts new file mode 100644 index 00000000..a72a5677 --- /dev/null +++ b/packages/mastoapi/middleware/userMiddleware.test.ts @@ -0,0 +1,99 @@ +import { DittoConf } from '@ditto/conf'; +import { DummyDB } from '@ditto/db'; +import { DittoApp, type DittoMiddleware } from '@ditto/router'; +import { type NostrSigner, NSecSigner } from '@nostrify/nostrify'; +import { MockRelay } from '@nostrify/nostrify/test'; +import { assertEquals } from '@std/assert'; +import { generateSecretKey, nip19 } from 'nostr-tools'; + +import { userMiddleware } from './userMiddleware.ts'; +import { ReadOnlySigner } from '../signers/ReadOnlySigner.ts'; + +import type { User } from './User.ts'; + +Deno.test('no user 401', async () => { + const { app } = testApp(); + const response = await app.use(userMiddleware()).request('/'); + assertEquals(response.status, 401); +}); + +Deno.test('unsupported signer 400', async () => { + const { app, relay } = testApp(); + const signer = new ReadOnlySigner('0461fcbecc4c3374439932d6b8f11269ccdb7cc973ad7a50ae362db135a474dd'); + + const response = await app + .use(setUser({ signer, relay })) + .use(userMiddleware({ enc: 'nip44' })) + .use((c, next) => { + c.var.user.signer.nip44.encrypt; // test that the type is set + return next(); + }) + .request('/'); + + assertEquals(response.status, 400); +}); + +Deno.test('with user 200', async () => { + const { app, user } = testApp(); + + const response = await app + .use(setUser(user)) + .use(userMiddleware()) + .get('/', (c) => c.text('ok')) + .request('/'); + + assertEquals(response.status, 200); +}); + +Deno.test('user and role 403', async () => { + const { app, user } = testApp(); + + const response = await app + .use(setUser(user)) + .use(userMiddleware({ role: 'admin' })) + .request('/'); + + assertEquals(response.status, 403); +}); + +Deno.test('admin role 200', async () => { + const { conf, app, user, relay } = testApp(); + + const event = await conf.signer.signEvent({ + kind: 30382, + tags: [ + ['d', await user.signer.getPublicKey()], + ['n', 'admin'], + ], + content: '', + created_at: Math.floor(Date.now() / 1000), + }); + + await relay.event(event); + + const response = await app + .use(setUser(user)) + .use(userMiddleware({ role: 'admin' })) + .get('/', (c) => c.text('ok')) + .request('/'); + + assertEquals(response.status, 200); +}); + +function testApp() { + const relay = new MockRelay(); + const signer = new NSecSigner(generateSecretKey()); + const conf = new DittoConf(new Map([['DITTO_NSEC', nip19.nsecEncode(generateSecretKey())]])); + const db = new DummyDB(); + const app = new DittoApp({ conf, relay, db }); + const user = { signer, relay }; + + return { app, relay, conf, db, user }; +} + +function setUser(user: User): DittoMiddleware<{ user: User }> { + return async (c, next) => { + c.set('user', user); + await next(); + }; +} diff --git a/packages/mastoapi/middleware/userMiddleware.ts b/packages/mastoapi/middleware/userMiddleware.ts index 1afef59a..8308172d 100644 --- a/packages/mastoapi/middleware/userMiddleware.ts +++ b/packages/mastoapi/middleware/userMiddleware.ts @@ -26,11 +26,11 @@ export function userMiddleware(opts: UserMiddlewareOpts = {}): DittoMiddleware<{ const { enc, role, verify } = opts; if (!user) { - throw new HTTPException(403, { message: 'Authorization required.' }); + throw new HTTPException(401, { message: 'Authorization required' }); } if (enc && !user.signer[enc]) { - throw new HTTPException(403, { message: `User does not have a ${enc} signer` }); + throw new HTTPException(400, { message: `User does not have a ${enc} signer` }); } if (role || verify) { @@ -40,7 +40,7 @@ export function userMiddleware(opts: UserMiddlewareOpts = {}): DittoMiddleware<{ const result = await validateAuthEvent(req, resEvent); if (!result.success) { - throw new HTTPException(403, { message: 'Verification failed.' }); + throw new HTTPException(401, { message: 'Verification failed' }); } // Prevent people from accidentally using the wrong account. This has no other security implications. @@ -57,7 +57,7 @@ export function userMiddleware(opts: UserMiddlewareOpts = {}): DittoMiddleware<{ }]); if (!user || !matchesRole(user, role)) { - throw new HTTPException(403, { message: `Must have ${role} role.` }); + throw new HTTPException(403, { message: `Must have ${role} role` }); } } } From 9c97cc387f68176678935b652b82962ec6fa0744 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 21:15:57 -0600 Subject: [PATCH 59/99] mastoapi: add a test module --- packages/mastoapi/deno.json | 3 +- .../middleware/userMiddleware.test.ts | 27 +--------------- packages/mastoapi/test.ts | 32 +++++++++++++++++++ 3 files changed, 35 insertions(+), 27 deletions(-) create mode 100644 packages/mastoapi/test.ts diff --git a/packages/mastoapi/deno.json b/packages/mastoapi/deno.json index f9abac55..ddeb175f 100644 --- a/packages/mastoapi/deno.json +++ b/packages/mastoapi/deno.json @@ -2,6 +2,7 @@ "name": "@ditto/mastoapi", "version": "1.1.0", "exports": { - "./middleware": "./middleware/mod.ts" + "./middleware": "./middleware/mod.ts", + "./test": "./test.ts" } } diff --git a/packages/mastoapi/middleware/userMiddleware.test.ts b/packages/mastoapi/middleware/userMiddleware.test.ts index a72a5677..2d30b0dc 100644 --- a/packages/mastoapi/middleware/userMiddleware.test.ts +++ b/packages/mastoapi/middleware/userMiddleware.test.ts @@ -1,16 +1,9 @@ -import { DittoConf } from '@ditto/conf'; -import { DummyDB } from '@ditto/db'; -import { DittoApp, type DittoMiddleware } from '@ditto/router'; -import { type NostrSigner, NSecSigner } from '@nostrify/nostrify'; -import { MockRelay } from '@nostrify/nostrify/test'; +import { setUser, testApp } from '@ditto/mastoapi/test'; import { assertEquals } from '@std/assert'; -import { generateSecretKey, nip19 } from 'nostr-tools'; import { userMiddleware } from './userMiddleware.ts'; import { ReadOnlySigner } from '../signers/ReadOnlySigner.ts'; -import type { User } from './User.ts'; - Deno.test('no user 401', async () => { const { app } = testApp(); const response = await app.use(userMiddleware()).request('/'); @@ -79,21 +72,3 @@ Deno.test('admin role 200', async () => { assertEquals(response.status, 200); }); - -function testApp() { - const relay = new MockRelay(); - const signer = new NSecSigner(generateSecretKey()); - const conf = new DittoConf(new Map([['DITTO_NSEC', nip19.nsecEncode(generateSecretKey())]])); - const db = new DummyDB(); - const app = new DittoApp({ conf, relay, db }); - const user = { signer, relay }; - - return { app, relay, conf, db, user }; -} - -function setUser(user: User): DittoMiddleware<{ user: User }> { - return async (c, next) => { - c.set('user', user); - await next(); - }; -} diff --git a/packages/mastoapi/test.ts b/packages/mastoapi/test.ts new file mode 100644 index 00000000..70a5e1af --- /dev/null +++ b/packages/mastoapi/test.ts @@ -0,0 +1,32 @@ +import { DittoConf } from '@ditto/conf'; +import { DummyDB } from '@ditto/db'; +import { DittoApp, type DittoMiddleware } from '@ditto/router'; +import { type NostrSigner, NSecSigner } from '@nostrify/nostrify'; +import { MockRelay } from '@nostrify/nostrify/test'; +import { generateSecretKey, nip19 } from 'nostr-tools'; + +import type { User } from '@ditto/mastoapi/middleware'; + +export function testApp() { + const db = new DummyDB(); + + const nsec = nip19.nsecEncode(generateSecretKey()); + const conf = new DittoConf(new Map([['DITTO_NSEC', nsec]])); + + const relay = new MockRelay(); + const app = new DittoApp({ conf, relay, db }); + + const user = { + signer: new NSecSigner(generateSecretKey()), + relay, + }; + + return { app, relay, conf, db, user }; +} + +export function setUser(user: User): DittoMiddleware<{ user: User }> { + return async (c, next) => { + c.set('user', user); + await next(); + }; +} From 07b68b71d2dc5ad138e95e879ebdbbb6cad0e488 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 21:31:42 -0600 Subject: [PATCH 60/99] Add missing types to testApp --- packages/mastoapi/test.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/mastoapi/test.ts b/packages/mastoapi/test.ts index 70a5e1af..78753511 100644 --- a/packages/mastoapi/test.ts +++ b/packages/mastoapi/test.ts @@ -1,13 +1,22 @@ import { DittoConf } from '@ditto/conf'; -import { DummyDB } from '@ditto/db'; +import { type DittoDB, DummyDB } from '@ditto/db'; import { DittoApp, type DittoMiddleware } from '@ditto/router'; -import { type NostrSigner, NSecSigner } from '@nostrify/nostrify'; +import { type NostrSigner, type NRelay, NSecSigner } from '@nostrify/nostrify'; import { MockRelay } from '@nostrify/nostrify/test'; import { generateSecretKey, nip19 } from 'nostr-tools'; import type { User } from '@ditto/mastoapi/middleware'; -export function testApp() { +export function testApp(): { + app: DittoApp; + relay: NRelay; + conf: DittoConf; + db: DittoDB; + user: { + signer: NostrSigner; + relay: NRelay; + }; +} { const db = new DummyDB(); const nsec = nip19.nsecEncode(generateSecretKey()); From 6c70b4bc4eed2d6542764970a8f5f070820a40ba Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 21:48:10 -0600 Subject: [PATCH 61/99] Make NIP-05 case insensitive --- packages/ditto/controllers/api/admin.ts | 5 +++-- packages/ditto/controllers/api/ditto.ts | 10 ++++++++-- packages/ditto/utils/nip05.ts | 4 +++- packages/ditto/views/mastodon/notifications.ts | 9 ++++++--- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/packages/ditto/controllers/api/admin.ts b/packages/ditto/controllers/api/admin.ts index 9e9ba5d0..720331d3 100644 --- a/packages/ditto/controllers/api/admin.ts +++ b/packages/ditto/controllers/api/admin.ts @@ -193,7 +193,7 @@ const adminApproveController: AppController = async (c) => { } const [existing] = await store.query([ - { kinds: [30360], authors: [await conf.signer.getPublicKey()], '#d': [r], limit: 1 }, + { kinds: [30360], authors: [await conf.signer.getPublicKey()], '#d': [r.toLowerCase()], limit: 1 }, ]); if (existing) { @@ -203,7 +203,8 @@ const adminApproveController: AppController = async (c) => { await createAdminEvent({ kind: 30360, tags: [ - ['d', r], + ['d', r.toLowerCase()], + ['r', r], ['L', 'nip05.domain'], ['l', r.split('@')[1], 'nip05.domain'], ['p', event.pubkey], diff --git a/packages/ditto/controllers/api/ditto.ts b/packages/ditto/controllers/api/ditto.ts index 33db6fc7..deac6f38 100644 --- a/packages/ditto/controllers/api/ditto.ts +++ b/packages/ditto/controllers/api/ditto.ts @@ -92,16 +92,22 @@ export const nameRequestController: AppController = async (c) => { const { name, reason } = result.data; - const [existing] = await store.query([{ kinds: [3036], authors: [pubkey], '#r': [name], limit: 1 }]); + const [existing] = await store.query([{ kinds: [3036], authors: [pubkey], '#r': [name.toLowerCase()], limit: 1 }]); if (existing) { return c.json({ error: 'Name request already exists' }, 400); } + const r: string[][] = [['r', name]]; + + if (name !== name.toLowerCase()) { + r.push(['r', name.toLowerCase()]); + } + const event = await createEvent({ kind: 3036, content: reason, tags: [ - ['r', name], + ...r, ['L', 'nip05.domain'], ['l', name.split('@')[1], 'nip05.domain'], ['p', await conf.signer.getPublicKey()], diff --git a/packages/ditto/utils/nip05.ts b/packages/ditto/utils/nip05.ts index 6c53c18c..7d725ab2 100644 --- a/packages/ditto/utils/nip05.ts +++ b/packages/ditto/utils/nip05.ts @@ -54,9 +54,11 @@ async function getNip05( } export async function localNip05Lookup(store: NStore, localpart: string): Promise { + const name = `${localpart}@${Conf.url.host}`; + const [grant] = await store.query([{ kinds: [30360], - '#d': [`${localpart}@${Conf.url.host}`], + '#d': [name, name.toLowerCase()], authors: [await Conf.signer.getPublicKey()], limit: 1, }]); diff --git a/packages/ditto/views/mastodon/notifications.ts b/packages/ditto/views/mastodon/notifications.ts index 75b0547a..59911606 100644 --- a/packages/ditto/views/mastodon/notifications.ts +++ b/packages/ditto/views/mastodon/notifications.ts @@ -99,15 +99,18 @@ async function renderReaction(event: DittoEvent, opts: RenderNotificationOpts) { } async function renderNameGrant(event: DittoEvent) { + const r = event.tags.find(([name]) => name === 'r')?.[1]; const d = event.tags.find(([name]) => name === 'd')?.[1]; - const account = event.author ? await renderAccount(event.author) : await accountFromPubkey(event.pubkey); + const name = r ?? d; - if (!d) return; + if (name) return; + + const account = event.author ? await renderAccount(event.author) : await accountFromPubkey(event.pubkey); return { id: notificationId(event), type: 'ditto:name_grant' as const, - name: d, + name, created_at: nostrDate(event.created_at).toISOString(), account, }; From 084c6aa94443422f40a91537ae09cde7c354da00 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 21:55:00 -0600 Subject: [PATCH 62/99] Fix DummyDB test --- packages/db/adapters/DummyDB.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/db/adapters/DummyDB.test.ts b/packages/db/adapters/DummyDB.test.ts index c725ab51..9945be45 100644 --- a/packages/db/adapters/DummyDB.test.ts +++ b/packages/db/adapters/DummyDB.test.ts @@ -2,7 +2,7 @@ import { assertEquals } from '@std/assert'; import { DummyDB } from './DummyDB.ts'; Deno.test('DummyDB', async () => { - const db = DummyDB.create(); + const db = new DummyDB(); const rows = await db.kysely.selectFrom('nostr_events').selectAll().execute(); assertEquals(rows, []); From 4ed064076698f19314c0d07797c7fe89a0b851d0 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Feb 2025 23:32:15 -0600 Subject: [PATCH 63/99] @ditto/router -> @ditto/mastoapi/router --- deno.json | 1 - packages/ditto/app.ts | 2 +- packages/ditto/controllers/api/cashu.test.ts | 2 +- packages/ditto/controllers/api/cashu.ts | 2 +- packages/mastoapi/deno.json | 1 + packages/mastoapi/middleware/paginationMiddleware.ts | 2 +- packages/mastoapi/middleware/tokenMiddleware.ts | 2 +- packages/mastoapi/middleware/userMiddleware.ts | 2 +- packages/{ => mastoapi}/router/DittoApp.test.ts | 0 packages/{ => mastoapi}/router/DittoApp.ts | 0 packages/{ => mastoapi}/router/DittoEnv.ts | 0 packages/{ => mastoapi}/router/DittoMiddleware.ts | 0 packages/{ => mastoapi}/router/DittoRoute.test.ts | 0 packages/{ => mastoapi}/router/DittoRoute.ts | 0 packages/{ => mastoapi}/router/mod.ts | 0 packages/mastoapi/test.ts | 2 +- packages/router/deno.json | 7 ------- 17 files changed, 8 insertions(+), 15 deletions(-) rename packages/{ => mastoapi}/router/DittoApp.test.ts (100%) rename packages/{ => mastoapi}/router/DittoApp.ts (100%) rename packages/{ => mastoapi}/router/DittoEnv.ts (100%) rename packages/{ => mastoapi}/router/DittoMiddleware.ts (100%) rename packages/{ => mastoapi}/router/DittoRoute.test.ts (100%) rename packages/{ => mastoapi}/router/DittoRoute.ts (100%) rename packages/{ => mastoapi}/router/mod.ts (100%) delete mode 100644 packages/router/deno.json diff --git a/deno.json b/deno.json index 20d87204..05ecb34a 100644 --- a/deno.json +++ b/deno.json @@ -10,7 +10,6 @@ "./packages/nip98", "./packages/policies", "./packages/ratelimiter", - "./packages/router", "./packages/translators", "./packages/uploaders" ], diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index 2b90f132..eab81b47 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -1,7 +1,7 @@ import { DittoConf } from '@ditto/conf'; import { DittoDB } from '@ditto/db'; import { paginationMiddleware, tokenMiddleware, userMiddleware } from '@ditto/mastoapi/middleware'; -import { DittoApp, type DittoEnv } from '@ditto/router'; +import { DittoApp, type DittoEnv } from '@ditto/mastoapi/router'; import { type DittoTranslator } from '@ditto/translators'; import { type Context, Handler, Input as HonoInput, MiddlewareHandler } from '@hono/hono'; import { every } from '@hono/hono/combine'; diff --git a/packages/ditto/controllers/api/cashu.test.ts b/packages/ditto/controllers/api/cashu.test.ts index 22e9a38f..1b28d099 100644 --- a/packages/ditto/controllers/api/cashu.test.ts +++ b/packages/ditto/controllers/api/cashu.test.ts @@ -1,6 +1,6 @@ import { DittoConf } from '@ditto/conf'; import { type User } from '@ditto/mastoapi/middleware'; -import { DittoApp, DittoMiddleware } from '@ditto/router'; +import { DittoApp, DittoMiddleware } from '@ditto/mastoapi/router'; import { NSecSigner } from '@nostrify/nostrify'; import { genEvent } from '@nostrify/nostrify/test'; import { bytesToString, stringToBytes } from '@scure/base'; diff --git a/packages/ditto/controllers/api/cashu.ts b/packages/ditto/controllers/api/cashu.ts index bb39397c..a98a0309 100644 --- a/packages/ditto/controllers/api/cashu.ts +++ b/packages/ditto/controllers/api/cashu.ts @@ -1,6 +1,6 @@ import { Proof } from '@cashu/cashu-ts'; import { userMiddleware } from '@ditto/mastoapi/middleware'; -import { DittoRoute } from '@ditto/router'; +import { DittoRoute } from '@ditto/mastoapi/router'; import { generateSecretKey, getPublicKey } from 'nostr-tools'; import { bytesToString, stringToBytes } from '@scure/base'; import { z } from 'zod'; diff --git a/packages/mastoapi/deno.json b/packages/mastoapi/deno.json index ddeb175f..d98dbc91 100644 --- a/packages/mastoapi/deno.json +++ b/packages/mastoapi/deno.json @@ -3,6 +3,7 @@ "version": "1.1.0", "exports": { "./middleware": "./middleware/mod.ts", + "./router": "./router/mod.ts", "./test": "./test.ts" } } diff --git a/packages/mastoapi/middleware/paginationMiddleware.ts b/packages/mastoapi/middleware/paginationMiddleware.ts index cca64229..28a7f1a1 100644 --- a/packages/mastoapi/middleware/paginationMiddleware.ts +++ b/packages/mastoapi/middleware/paginationMiddleware.ts @@ -1,7 +1,7 @@ import { paginated, paginatedList } from '../pagination/paginate.ts'; import { paginationSchema } from '../pagination/schema.ts'; -import type { DittoMiddleware } from '@ditto/router'; +import type { DittoMiddleware } from '@ditto/mastoapi/router'; import type { NostrEvent } from '@nostrify/nostrify'; interface Pagination { diff --git a/packages/mastoapi/middleware/tokenMiddleware.ts b/packages/mastoapi/middleware/tokenMiddleware.ts index d4f8a05b..a2241c19 100644 --- a/packages/mastoapi/middleware/tokenMiddleware.ts +++ b/packages/mastoapi/middleware/tokenMiddleware.ts @@ -9,7 +9,7 @@ import { ConnectSigner } from '../signers/ConnectSigner.ts'; import { ReadOnlySigner } from '../signers/ReadOnlySigner.ts'; import { UserStore } from '../storages/UserStore.ts'; -import type { DittoEnv, DittoMiddleware } from '@ditto/router'; +import type { DittoEnv, DittoMiddleware } from '@ditto/mastoapi/router'; import type { Context } from '@hono/hono'; import type { User } from './User.ts'; diff --git a/packages/mastoapi/middleware/userMiddleware.ts b/packages/mastoapi/middleware/userMiddleware.ts index 8308172d..2b964362 100644 --- a/packages/mastoapi/middleware/userMiddleware.ts +++ b/packages/mastoapi/middleware/userMiddleware.ts @@ -1,7 +1,7 @@ import { buildAuthEventTemplate, validateAuthEvent } from '@ditto/nip98'; import { HTTPException } from '@hono/hono/http-exception'; -import type { DittoMiddleware } from '@ditto/router'; +import type { DittoMiddleware } from '@ditto/mastoapi/router'; import type { NostrEvent, NostrSigner } from '@nostrify/nostrify'; import type { SetRequired } from 'type-fest'; import type { User } from './User.ts'; diff --git a/packages/router/DittoApp.test.ts b/packages/mastoapi/router/DittoApp.test.ts similarity index 100% rename from packages/router/DittoApp.test.ts rename to packages/mastoapi/router/DittoApp.test.ts diff --git a/packages/router/DittoApp.ts b/packages/mastoapi/router/DittoApp.ts similarity index 100% rename from packages/router/DittoApp.ts rename to packages/mastoapi/router/DittoApp.ts diff --git a/packages/router/DittoEnv.ts b/packages/mastoapi/router/DittoEnv.ts similarity index 100% rename from packages/router/DittoEnv.ts rename to packages/mastoapi/router/DittoEnv.ts diff --git a/packages/router/DittoMiddleware.ts b/packages/mastoapi/router/DittoMiddleware.ts similarity index 100% rename from packages/router/DittoMiddleware.ts rename to packages/mastoapi/router/DittoMiddleware.ts diff --git a/packages/router/DittoRoute.test.ts b/packages/mastoapi/router/DittoRoute.test.ts similarity index 100% rename from packages/router/DittoRoute.test.ts rename to packages/mastoapi/router/DittoRoute.test.ts diff --git a/packages/router/DittoRoute.ts b/packages/mastoapi/router/DittoRoute.ts similarity index 100% rename from packages/router/DittoRoute.ts rename to packages/mastoapi/router/DittoRoute.ts diff --git a/packages/router/mod.ts b/packages/mastoapi/router/mod.ts similarity index 100% rename from packages/router/mod.ts rename to packages/mastoapi/router/mod.ts diff --git a/packages/mastoapi/test.ts b/packages/mastoapi/test.ts index 78753511..41e35c2c 100644 --- a/packages/mastoapi/test.ts +++ b/packages/mastoapi/test.ts @@ -1,6 +1,6 @@ import { DittoConf } from '@ditto/conf'; import { type DittoDB, DummyDB } from '@ditto/db'; -import { DittoApp, type DittoMiddleware } from '@ditto/router'; +import { DittoApp, type DittoMiddleware } from '@ditto/mastoapi/router'; import { type NostrSigner, type NRelay, NSecSigner } from '@nostrify/nostrify'; import { MockRelay } from '@nostrify/nostrify/test'; import { generateSecretKey, nip19 } from 'nostr-tools'; diff --git a/packages/router/deno.json b/packages/router/deno.json deleted file mode 100644 index 8321baaf..00000000 --- a/packages/router/deno.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "@ditto/router", - "version": "1.1.0", - "exports": { - ".": "./mod.ts" - } -} From 02e284f3aadeec85ec10cdd930a785102986b722 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 01:06:53 -0600 Subject: [PATCH 64/99] Remove unused DittoFilter interface --- packages/ditto/controllers/api/statuses.ts | 2 +- packages/ditto/interfaces/DittoFilter.ts | 5 ----- packages/ditto/queries.ts | 3 --- 3 files changed, 1 insertion(+), 9 deletions(-) delete mode 100644 packages/ditto/interfaces/DittoFilter.ts diff --git a/packages/ditto/controllers/api/statuses.ts b/packages/ditto/controllers/api/statuses.ts index 5b73be9f..252882ff 100644 --- a/packages/ditto/controllers/api/statuses.ts +++ b/packages/ditto/controllers/api/statuses.ts @@ -586,7 +586,7 @@ const zapController: AppController = async (c) => { let lnurl: undefined | string; if (status_id) { - target = await getEvent(status_id, { kind: 1, relations: ['author'], signal }); + target = await getEvent(status_id, { kind: 1, signal }); const author = target?.author; const meta = n.json().pipe(n.metadata()).catch({}).parse(author?.content); lnurl = getLnurl(meta); diff --git a/packages/ditto/interfaces/DittoFilter.ts b/packages/ditto/interfaces/DittoFilter.ts deleted file mode 100644 index f7f1a9ea..00000000 --- a/packages/ditto/interfaces/DittoFilter.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { NostrEvent } from '@nostrify/nostrify'; -import { DittoEvent } from '@/interfaces/DittoEvent.ts'; - -/** Additional properties that may be added by Ditto to events. */ -export type DittoRelation = Exclude; diff --git a/packages/ditto/queries.ts b/packages/ditto/queries.ts index d4f0cb11..a79b2df4 100644 --- a/packages/ditto/queries.ts +++ b/packages/ditto/queries.ts @@ -3,7 +3,6 @@ import { NostrEvent, NostrFilter, NStore } from '@nostrify/nostrify'; import { Conf } from '@/config.ts'; import { Storages } from '@/storages.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { type DittoRelation } from '@/interfaces/DittoFilter.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { fallbackAuthor } from '@/utils.ts'; import { findReplyTag, getTagSet } from '@/utils/tags.ts'; @@ -13,8 +12,6 @@ interface GetEventOpts { signal?: AbortSignal; /** Event kind. */ kind?: number; - /** @deprecated Relations to include on the event. */ - relations?: DittoRelation[]; } /** From 48bd7618f797c4c6c6720e40440e873df1de2e84 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 03:13:05 -0600 Subject: [PATCH 65/99] Start building DittoAPIStore --- packages/ditto/storages/DittoAPIStore.ts | 167 +++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 packages/ditto/storages/DittoAPIStore.ts diff --git a/packages/ditto/storages/DittoAPIStore.ts b/packages/ditto/storages/DittoAPIStore.ts new file mode 100644 index 00000000..8609d1ca --- /dev/null +++ b/packages/ditto/storages/DittoAPIStore.ts @@ -0,0 +1,167 @@ +import { DittoConf } from '@ditto/conf'; +import { pipelineEventsCounter } from '@ditto/metrics'; +import { + NKinds, + NostrEvent, + NostrFilter, + NostrRelayCLOSED, + NostrRelayCOUNT, + NostrRelayEOSE, + NostrRelayEVENT, + NRelay, +} from '@nostrify/nostrify'; +import { logi } from '@soapbox/logi'; +import { LRUCache } from 'lru-cache'; + +import { RelayError } from '@/RelayError.ts'; +import { eventAge, Time } from '@/utils.ts'; +import { purifyEvent } from '@/utils/purify.ts'; +import { getTagSet } from '@/utils/tags.ts'; +import { verifyEventWorker } from '@/workers/verify.ts'; + +interface DittoAPIStoreOpts { + conf: DittoConf; + pool: NRelay; + relay: NRelay; +} + +export class DittoAPIStore implements NRelay { + private encounters = new LRUCache({ max: 5000 }); + + constructor(private opts: DittoAPIStoreOpts) {} + + req( + filters: NostrFilter[], + opts?: { signal?: AbortSignal }, + ): AsyncIterable { + const { relay } = this.opts; + return relay.req(filters, opts); + } + + async event(event: NostrEvent, opts?: { signal?: AbortSignal }): Promise { + const { relay, pool } = this.opts; + + await relay.event(event, opts); + + (async () => { + try { + await pool.event(event, opts); + } catch (e) { + console.error(e); + } + })(); + } + + /** + * Common pipeline function to process (and maybe store) events. + * It is idempotent, so it can be called multiple times for the same event. + */ + async handleEvent(event: NostrEvent, opts?: { signal?: AbortSignal }): Promise { + const { conf, relay } = this.opts; + const { signal } = opts; + + // Skip events that have already been encountered. + if (this.encounters.get(event.id)) { + throw new RelayError('duplicate', 'already have this event'); + } + // Reject events that are too far in the future. + if (eventAge(event) < -Time.minutes(1)) { + throw new RelayError('invalid', 'event too far in the future'); + } + // Integer max value for Postgres. + if (event.kind >= 2_147_483_647) { + throw new RelayError('invalid', 'event kind too large'); + } + // The only point of ephemeral events is to stream them, + // so throw an error if we're not even going to do that. + if (NKinds.ephemeral(event.kind) && !this.isFresh(event)) { + throw new RelayError('invalid', 'event too old'); + } + // Block NIP-70 events, because we have no way to `AUTH`. + if (isProtectedEvent(event)) { + throw new RelayError('invalid', 'protected event'); + } + // Validate the event's signature. + if (!(await verifyEventWorker(event))) { + throw new RelayError('invalid', 'invalid signature'); + } + // Recheck encountered after async ops. + if (this.encounters.has(event.id)) { + throw new RelayError('duplicate', 'already have this event'); + } + // Set the event as encountered after verifying the signature. + this.encounters.set(event.id, true); + + // Log the event. + logi({ level: 'debug', ns: 'ditto.event', source: 'pipeline', id: event.id, kind: event.kind }); + pipelineEventsCounter.inc({ kind: event.kind }); + + // NIP-46 events get special treatment. + // They are exempt from policies and other side-effects, and should be streamed out immediately. + // If streaming fails, an error should be returned. + if (event.kind === 24133) { + await relay.event(event, { signal }); + } + + // Ensure the event doesn't violate the policy. + if (event.pubkey !== await conf.signer.getPublicKey()) { + await this.policyFilter(event, signal); + } + + // Prepare the event for additional checks. + // FIXME: This is kind of hacky. Should be reorganized to fetch only what's needed for each stage. + await hydrateEvent(event, signal); + + // Ensure that the author is not banned. + const n = getTagSet(event.user?.tags ?? [], 'n'); + if (n.has('disabled')) { + throw new RelayError('blocked', 'author is blocked'); + } + + const kysely = await Storages.kysely(); + + try { + await this.storeEvent(purifyEvent(event), signal); + } finally { + // This needs to run in steps, and should not block the API from responding. + Promise.allSettled([ + this.handleZaps(kysely, event), + this.updateAuthorData(event, signal), + this.prewarmLinkPreview(event, signal), + this.generateSetEvents(event), + ]) + .then(() => this.webPush(event)) + .catch(() => {}); + } + } + + /** Determine if the event is being received in a timely manner. */ + private isFresh(event: NostrEvent): boolean { + return eventAge(event) < Time.minutes(1); + } + + query(filters: NostrFilter[], opts?: { signal?: AbortSignal }): Promise { + const { relay } = this.opts; + return relay.query(filters, opts); + } + + count(filters: NostrFilter[], opts?: { signal?: AbortSignal }): Promise { + const { relay } = this.opts; + if (!relay.count) { + return Promise.reject(new Error('Method not implemented.')); + } + return relay.count(filters, opts); + } + + remove(filters: NostrFilter[], opts?: { signal?: AbortSignal }): Promise { + const { relay } = this.opts; + if (!relay.remove) { + return Promise.reject(new Error('Method not implemented.')); + } + return relay.remove(filters, opts); + } + + close(): Promise { + return Promise.reject(new Error('Method not implemented.')); + } +} From 2f0dbc44e4fd7e1f4481a4590296ecc77c2fa856 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 12:01:50 -0600 Subject: [PATCH 66/99] Copy all the pipeline logic into DittoAPIStore (and some into DittoPgStore) --- packages/ditto/storages/DittoAPIStore.ts | 290 ++++++++++++++++++++++- packages/ditto/storages/DittoPgStore.ts | 24 +- 2 files changed, 302 insertions(+), 12 deletions(-) diff --git a/packages/ditto/storages/DittoAPIStore.ts b/packages/ditto/storages/DittoAPIStore.ts index 8609d1ca..89a33eda 100644 --- a/packages/ditto/storages/DittoAPIStore.ts +++ b/packages/ditto/storages/DittoAPIStore.ts @@ -1,5 +1,6 @@ import { DittoConf } from '@ditto/conf'; -import { pipelineEventsCounter } from '@ditto/metrics'; +import { DittoDB, DittoTables } from '@ditto/db'; +import { pipelineEventsCounter, policyEventsCounter, webPushNotificationsCounter } from '@ditto/metrics'; import { NKinds, NostrEvent, @@ -9,17 +10,33 @@ import { NostrRelayEOSE, NostrRelayEVENT, NRelay, + NSchema as n, } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; +import { UpdateObject } from 'kysely'; import { LRUCache } from 'lru-cache'; +import tldts from 'tldts'; +import { z } from 'zod'; +import { DittoPush } from '@/DittoPush.ts'; +import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { RelayError } from '@/RelayError.ts'; +import { hydrateEvents } from '@/storages/hydrate.ts'; import { eventAge, Time } from '@/utils.ts'; +import { getAmount } from '@/utils/bolt11.ts'; +import { errorJson } from '@/utils/log.ts'; import { purifyEvent } from '@/utils/purify.ts'; import { getTagSet } from '@/utils/tags.ts'; +import { policyWorker } from '@/workers/policy.ts'; import { verifyEventWorker } from '@/workers/verify.ts'; +import { faviconCache } from '@/utils/favicon.ts'; +import { parseNoteContent, stripimeta } from '@/utils/note.ts'; +import { unfurlCardCached } from '@/utils/unfurl.ts'; +import { nip05Cache } from '@/utils/nip05.ts'; +import { renderWebPushNotification } from '@/views/mastodon/push.ts'; interface DittoAPIStoreOpts { + db: DittoDB; conf: DittoConf; pool: NRelay; relay: NRelay; @@ -27,8 +44,13 @@ interface DittoAPIStoreOpts { export class DittoAPIStore implements NRelay { private encounters = new LRUCache({ max: 5000 }); + private controller = new AbortController(); - constructor(private opts: DittoAPIStoreOpts) {} + constructor(private opts: DittoAPIStoreOpts) { + this.listen().catch((e: unknown) => { + logi({ level: 'error', ns: 'ditto.apistore', source: 'listen', error: errorJson(e) }); + }); + } req( filters: NostrFilter[], @@ -52,11 +74,24 @@ export class DittoAPIStore implements NRelay { })(); } + /** Open a firehose to the relay. */ + private async listen(): Promise { + const { relay } = this.opts; + const { signal } = this.controller; + + for await (const msg of relay.req([{}], { signal })) { + if (msg[0] === 'EVENT') { + const [, , event] = msg; + await this.handleEvent(event, { signal }); + } + } + } + /** * Common pipeline function to process (and maybe store) events. * It is idempotent, so it can be called multiple times for the same event. */ - async handleEvent(event: NostrEvent, opts?: { signal?: AbortSignal }): Promise { + private async handleEvent(event: DittoEvent, opts: { signal?: AbortSignal } = {}): Promise { const { conf, relay } = this.opts; const { signal } = opts; @@ -78,7 +113,7 @@ export class DittoAPIStore implements NRelay { throw new RelayError('invalid', 'event too old'); } // Block NIP-70 events, because we have no way to `AUTH`. - if (isProtectedEvent(event)) { + if (event.tags.some(([name]) => name === '-')) { throw new RelayError('invalid', 'protected event'); } // Validate the event's signature. @@ -110,7 +145,7 @@ export class DittoAPIStore implements NRelay { // Prepare the event for additional checks. // FIXME: This is kind of hacky. Should be reorganized to fetch only what's needed for each stage. - await hydrateEvent(event, signal); + await this.hydrateEvent(event, signal); // Ensure that the author is not banned. const n = getTagSet(event.user?.tags ?? [], 'n'); @@ -118,14 +153,12 @@ export class DittoAPIStore implements NRelay { throw new RelayError('blocked', 'author is blocked'); } - const kysely = await Storages.kysely(); - try { - await this.storeEvent(purifyEvent(event), signal); + await relay.event(purifyEvent(event), { signal }); } finally { // This needs to run in steps, and should not block the API from responding. Promise.allSettled([ - this.handleZaps(kysely, event), + this.handleZaps(event), this.updateAuthorData(event, signal), this.prewarmLinkPreview(event, signal), this.generateSetEvents(event), @@ -135,6 +168,232 @@ export class DittoAPIStore implements NRelay { } } + private async policyFilter(event: NostrEvent, signal?: AbortSignal): Promise { + try { + const result = await policyWorker.call(event, signal); + const [, , ok, reason] = result; + logi({ level: 'debug', ns: 'ditto.policy', id: event.id, kind: event.kind, ok, reason }); + policyEventsCounter.inc({ ok: String(ok) }); + RelayError.assert(result); + } catch (e) { + if (e instanceof RelayError) { + throw e; + } else { + logi({ level: 'error', ns: 'ditto.policy', id: event.id, kind: event.kind, error: errorJson(e) }); + throw new RelayError('blocked', 'policy error'); + } + } + } + + /** Stores the event in the 'event_zaps' table */ + private async handleZaps(event: NostrEvent) { + if (event.kind !== 9735) return; + + const { db } = this.opts; + + const zapRequestString = event?.tags?.find(([name]) => name === 'description')?.[1]; + if (!zapRequestString) return; + const zapRequest = n.json().pipe(n.event()).optional().catch(undefined).parse(zapRequestString); + if (!zapRequest) return; + + const amountSchema = z.coerce.number().int().nonnegative().catch(0); + const amount_millisats = amountSchema.parse(getAmount(event?.tags.find(([name]) => name === 'bolt11')?.[1])); + if (!amount_millisats || amount_millisats < 1) return; + + const zappedEventId = zapRequest.tags.find(([name]) => name === 'e')?.[1]; + if (!zappedEventId) return; + + try { + await db.kysely.insertInto('event_zaps').values({ + receipt_id: event.id, + target_event_id: zappedEventId, + sender_pubkey: zapRequest.pubkey, + amount_millisats, + comment: zapRequest.content, + }).execute(); + } catch { + // receipt_id is unique, do nothing + } + } + + /** Parse kind 0 metadata and track indexes in the database. */ + private async updateAuthorData(event: NostrEvent, signal?: AbortSignal): Promise { + if (event.kind !== 0) return; + + const { db } = this.opts; + + // Parse metadata. + const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(event.content); + if (!metadata.success) return; + + const { name, nip05 } = metadata.data; + + const updates: UpdateObject = {}; + + const authorStats = await db.kysely + .selectFrom('author_stats') + .selectAll() + .where('pubkey', '=', event.pubkey) + .executeTakeFirst(); + + const lastVerified = authorStats?.nip05_last_verified_at; + const eventNewer = !lastVerified || event.created_at > lastVerified; + + try { + if (nip05 !== authorStats?.nip05 && eventNewer || !lastVerified) { + if (nip05) { + const tld = tldts.parse(nip05); + if (tld.isIcann && !tld.isIp && !tld.isPrivate) { + const pointer = await nip05Cache.fetch(nip05, { signal }); + if (pointer.pubkey === event.pubkey) { + updates.nip05 = nip05; + updates.nip05_domain = tld.domain; + updates.nip05_hostname = tld.hostname; + updates.nip05_last_verified_at = event.created_at; + } + } + } else { + updates.nip05 = null; + updates.nip05_domain = null; + updates.nip05_hostname = null; + updates.nip05_last_verified_at = event.created_at; + } + } + } catch { + // Fallthrough. + } + + // Fetch favicon. + const domain = nip05?.split('@')[1].toLowerCase(); + if (domain) { + try { + await faviconCache.fetch(domain, { signal }); + } catch { + // Fallthrough. + } + } + + const search = [name, nip05].filter(Boolean).join(' ').trim(); + + if (search !== authorStats?.search) { + updates.search = search; + } + + if (Object.keys(updates).length) { + await db.kysely.insertInto('author_stats') + .values({ + pubkey: event.pubkey, + followers_count: 0, + following_count: 0, + notes_count: 0, + search, + ...updates, + }) + .onConflict((oc) => oc.column('pubkey').doUpdateSet(updates)) + .execute(); + } + } + + private async prewarmLinkPreview(event: NostrEvent, signal?: AbortSignal): Promise { + const { firstUrl } = parseNoteContent(stripimeta(event.content, event.tags), []); + if (firstUrl) { + await unfurlCardCached(firstUrl, signal); + } + } + + private async generateSetEvents(event: NostrEvent): Promise { + const { conf } = this.opts; + + const signer = conf.signer; + const pubkey = await signer.getPublicKey(); + + const tagsAdmin = event.tags.some(([name, value]) => ['p', 'P'].includes(name) && value === pubkey); + + if (event.kind === 1984 && tagsAdmin) { + const rel = await signer.signEvent({ + kind: 30383, + content: '', + tags: [ + ['d', event.id], + ['p', event.pubkey], + ['k', '1984'], + ['n', 'open'], + ...[...getTagSet(event.tags, 'p')].map((value) => ['P', value]), + ...[...getTagSet(event.tags, 'e')].map((value) => ['e', value]), + ], + created_at: Math.floor(Date.now() / 1000), + }); + + await this.handleEvent(rel, { signal: AbortSignal.timeout(1000) }); + } + + if (event.kind === 3036 && tagsAdmin) { + const rel = await signer.signEvent({ + kind: 30383, + content: '', + tags: [ + ['d', event.id], + ['p', event.pubkey], + ['k', '3036'], + ['n', 'pending'], + ], + created_at: Math.floor(Date.now() / 1000), + }); + + await this.handleEvent(rel, { signal: AbortSignal.timeout(1000) }); + } + } + + private async webPush(event: NostrEvent): Promise { + if (!this.isFresh(event)) { + throw new RelayError('invalid', 'event too old'); + } + + const { db } = this.opts; + const pubkeys = getTagSet(event.tags, 'p'); + + if (!pubkeys.size) { + return; + } + + const rows = await db.kysely + .selectFrom('push_subscriptions') + .selectAll() + .where('pubkey', 'in', [...pubkeys]) + .execute(); + + for (const row of rows) { + const viewerPubkey = row.pubkey; + + if (viewerPubkey === event.pubkey) { + continue; // Don't notify authors about their own events. + } + + const message = await renderWebPushNotification(event, viewerPubkey); + if (!message) { + continue; + } + + const subscription = { + endpoint: row.endpoint, + keys: { + auth: row.auth, + p256dh: row.p256dh, + }, + }; + + await DittoPush.push(subscription, message); + webPushNotificationsCounter.inc({ type: message.notification_type }); + } + } + + /** Hydrate the event with the user, if applicable. */ + private async hydrateEvent(event: NostrEvent, signal?: AbortSignal): Promise { + const { relay } = this.opts; + const [hydrated] = await hydrateEvents({ events: [event], relay, signal }); + return hydrated; + } + /** Determine if the event is being received in a timely manner. */ private isFresh(event: NostrEvent): boolean { return eventAge(event) < Time.minutes(1); @@ -161,7 +420,16 @@ export class DittoAPIStore implements NRelay { return relay.remove(filters, opts); } - close(): Promise { - return Promise.reject(new Error('Method not implemented.')); + async close(): Promise { + const { relay, pool } = this.opts; + + this.controller.abort(); + + await pool.close(); + await relay.close(); + } + + [Symbol.asyncDispose](): Promise { + return this.close(); } } diff --git a/packages/ditto/storages/DittoPgStore.ts b/packages/ditto/storages/DittoPgStore.ts index 98fad50b..035fd729 100644 --- a/packages/ditto/storages/DittoPgStore.ts +++ b/packages/ditto/storages/DittoPgStore.ts @@ -31,6 +31,7 @@ import { abortError } from '@/utils/abort.ts'; import { purifyEvent } from '@/utils/purify.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { getMediaLinks } from '@/utils/note.ts'; +import { updateStats } from '@/utils/stats.ts'; /** Function to decide whether or not to index a tag. */ type TagCondition = (opts: TagConditionOpts) => boolean; @@ -144,7 +145,7 @@ export class DittoPgStore extends NPostgres { await this.deleteEventsAdmin(event); try { - await super.event(event, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); + await this.storeEvent(event, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); this.fulfill(event); // don't await or catch (should never reject) } catch (e) { if (e instanceof Error && e.message === 'Cannot add a deleted event') { @@ -157,6 +158,27 @@ export class DittoPgStore extends NPostgres { } } + /** Maybe store the event, if eligible. */ + private async storeEvent( + event: NostrEvent, + opts: { signal?: AbortSignal; timeout?: number } = {}, + ): Promise { + try { + await this.transaction(async (store, kysely) => { + await updateStats({ event, store, kysely }); + await super.event(event, opts); + }); + } catch (e) { + // If the failure is only because of updateStats (which runs first), insert the event anyway. + // We can't catch this in the transaction because the error aborts the transaction on the Postgres side. + if (e instanceof Error && e.message.includes('event_stats' satisfies keyof DittoTables)) { + await super.event(event, opts); + } else { + throw e; + } + } + } + /** Fulfill active subscriptions with this event. */ protected async fulfill(event: NostrEvent): Promise { const { maxAge = 60, batchSize = 500 } = this.opts; From 79fc568548f401edadc6273988c4789f246ec461 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 12:59:36 -0600 Subject: [PATCH 67/99] Add DittoPool class --- packages/ditto/controllers/api/ditto.ts | 14 +++- packages/ditto/schema.ts | 11 --- packages/ditto/storages.ts | 53 +------------- packages/ditto/storages/DittoPool.ts | 91 +++++++++++++++++++++++++ packages/ditto/utils/outbox.test.ts | 29 -------- packages/ditto/utils/outbox.ts | 28 -------- 6 files changed, 106 insertions(+), 120 deletions(-) create mode 100644 packages/ditto/storages/DittoPool.ts delete mode 100644 packages/ditto/utils/outbox.test.ts delete mode 100644 packages/ditto/utils/outbox.ts diff --git a/packages/ditto/controllers/api/ditto.ts b/packages/ditto/controllers/api/ditto.ts index 4d470f24..ff1b958f 100644 --- a/packages/ditto/controllers/api/ditto.ts +++ b/packages/ditto/controllers/api/ditto.ts @@ -10,7 +10,7 @@ import { getInstanceMetadata } from '@/utils/instance.ts'; import { deleteTag } from '@/utils/tags.ts'; import { DittoZapSplits, getZapSplits } from '@/utils/zap-split.ts'; import { screenshotsSchema } from '@/schemas/nostr.ts'; -import { booleanParamSchema, percentageSchema, wsUrlSchema } from '@/schema.ts'; +import { booleanParamSchema, percentageSchema } from '@/schema.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { renderNameRequest } from '@/views/ditto.ts'; import { accountFromPubkey } from '@/views/mastodon/accounts.ts'; @@ -20,6 +20,16 @@ import { updateListAdminEvent } from '@/utils/api.ts'; const markerSchema = z.enum(['read', 'write']); +/** WebSocket URL. */ +const wsUrlSchema = z.string().refine((val): val is `wss://${string}` | `ws://${string}` => { + try { + const { protocol } = new URL(val); + return protocol === 'wss:' || protocol === 'ws:'; + } catch { + return false; + } +}, 'Invalid WebSocket URL'); + const relaySchema = z.object({ url: wsUrlSchema, marker: markerSchema.optional(), @@ -62,7 +72,7 @@ function renderRelays(event: NostrEvent): RelayEntity[] { return event.tags.reduce((acc, [name, url, marker]) => { if (name === 'r') { const relay: RelayEntity = { - url, + url: url as `wss://${string}`, marker: markerSchema.safeParse(marker).success ? marker as 'read' | 'write' : undefined, }; acc.push(relay); diff --git a/packages/ditto/schema.ts b/packages/ditto/schema.ts index 56c9b998..c67aa5f6 100644 --- a/packages/ditto/schema.ts +++ b/packages/ditto/schema.ts @@ -22,16 +22,6 @@ const hashtagSchema = z.string().regex(/^\w{1,30}$/); */ const safeUrlSchema = z.string().max(2048).url(); -/** WebSocket URL. */ -const wsUrlSchema = z.string().refine((val) => { - try { - const { protocol } = new URL(val); - return protocol === 'wss:' || protocol === 'ws:'; - } catch { - return false; - } -}, 'Invalid WebSocket URL'); - /** https://github.com/colinhacks/zod/issues/1630#issuecomment-1365983831 */ const booleanParamSchema = z.enum(['true', 'false']).transform((value) => value === 'true'); @@ -93,5 +83,4 @@ export { safeUrlSchema, sizesSchema, walletSchema, - wsUrlSchema, }; diff --git a/packages/ditto/storages.ts b/packages/ditto/storages.ts index d5d0f029..aae165f2 100644 --- a/packages/ditto/storages.ts +++ b/packages/ditto/storages.ts @@ -1,13 +1,11 @@ // deno-lint-ignore-file require-await import { type DittoDB, DittoPolyPg } from '@ditto/db'; import { NPool, NRelay1 } from '@nostrify/nostrify'; -import { logi } from '@soapbox/logi'; import { Conf } from '@/config.ts'; -import { wsUrlSchema } from '@/schema.ts'; import { DittoPgStore } from '@/storages/DittoPgStore.ts'; -import { getRelays } from '@/utils/outbox.ts'; import { seedZapSplits } from '@/utils/zap-split.ts'; +import { DittoPool } from '@/storages/DittoPool.ts'; export class Storages { private static _db: Promise | undefined; @@ -55,53 +53,8 @@ export class Storages { public static async client(): Promise> { if (!this._client) { this._client = (async () => { - const db = await this.db(); - - const [relayList] = await db.query([ - { kinds: [10002], authors: [await Conf.signer.getPublicKey()], limit: 1 }, - ]); - - const tags = relayList?.tags ?? []; - - const activeRelays = tags.reduce((acc, [name, url, marker]) => { - const valid = wsUrlSchema.safeParse(url).success; - - if (valid && name === 'r' && (!marker || marker === 'write')) { - acc.push(url); - } - return acc; - }, []); - - logi({ - level: 'info', - ns: 'ditto.pool', - msg: `connecting to ${activeRelays.length} relays`, - relays: activeRelays, - }); - - return new NPool({ - open(url) { - return new NRelay1(url, { - // Skip event verification (it's done in the pipeline). - verifyEvent: () => true, - log(log) { - logi(log); - }, - }); - }, - reqRouter: async (filters) => { - return new Map(activeRelays.map((relay) => { - return [relay, filters]; - })); - }, - eventRouter: async (event) => { - const relaySet = await getRelays(await Storages.db(), event.pubkey); - relaySet.delete(Conf.relay); - - const relays = [...relaySet].slice(0, 4); - return relays; - }, - }); + const relay = await this.db(); + return new DittoPool({ conf: Conf, relay }); })(); } return this._client; diff --git a/packages/ditto/storages/DittoPool.ts b/packages/ditto/storages/DittoPool.ts new file mode 100644 index 00000000..53545128 --- /dev/null +++ b/packages/ditto/storages/DittoPool.ts @@ -0,0 +1,91 @@ +// deno-lint-ignore-file require-await +import { DittoConf } from '@ditto/conf'; +import { NostrEvent, NostrFilter, NPool, type NRelay, NRelay1 } from '@nostrify/nostrify'; +import { logi } from '@soapbox/logi'; + +interface DittoPoolOpts { + conf: DittoConf; + relay: NRelay; + maxEventRelays?: number; +} + +export class DittoPool extends NPool { + private _opts: DittoPoolOpts; + + constructor(opts: DittoPoolOpts) { + super({ + open(url) { + return new NRelay1(url, { + // Skip event verification (it's done in the pipeline). + verifyEvent: () => true, + log: logi, + }); + }, + reqRouter: (filters) => { + return this.reqRouter(filters); + }, + eventRouter: async (event) => { + return this.eventRouter(event); + }, + }); + + this._opts = opts; + } + + private async reqRouter(filters: NostrFilter[]): Promise> { + const routes = new Map(); + + for (const relayUrl of await this.getRelayUrls({ marker: 'read' })) { + routes.set(relayUrl, filters); + } + + return routes; + } + + private async eventRouter(event: NostrEvent): Promise { + const { conf, maxEventRelays = 4 } = this._opts; + const { pubkey } = event; + + const relaySet = await this.getRelayUrls({ pubkey, marker: 'write' }); + relaySet.delete(conf.relay); + + return [...relaySet].slice(0, maxEventRelays); + } + + private async getRelayUrls(opts: { pubkey?: string; marker?: 'read' | 'write' } = {}): Promise> { + const { conf, relay } = this._opts; + + const relays = new Set<`wss://${string}`>(); + const authors = new Set([await conf.signer.getPublicKey()]); + + if (opts.pubkey) { + authors.add(opts.pubkey); + } + + const events = await relay.query([ + { kinds: [10002], authors: [...authors] }, + ]); + + // Ensure user's own relay list is counted first. + if (opts.pubkey) { + events.sort((a) => a.pubkey === opts.pubkey ? -1 : 1); + } + + for (const event of events) { + for (const [name, relayUrl, marker] of event.tags) { + if (name === 'r' && (!marker || !opts.marker || marker === opts.marker)) { + try { + const url = new URL(relayUrl); + if (url.protocol === 'wss:') { + relays.add(url.toString() as `wss://${string}`); + } + } catch { + // fallthrough + } + } + } + } + + return relays; + } +} diff --git a/packages/ditto/utils/outbox.test.ts b/packages/ditto/utils/outbox.test.ts deleted file mode 100644 index 62dac2d0..00000000 --- a/packages/ditto/utils/outbox.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { MockRelay } from '@nostrify/nostrify/test'; -import { eventFixture } from '@/test.ts'; -import { getRelays } from '@/utils/outbox.ts'; -import { assertEquals } from '@std/assert'; - -Deno.test('Get write relays - kind 10002', async () => { - const db = new MockRelay(); - - const relayListMetadata = await eventFixture('kind-10002-alex'); - - await db.event(relayListMetadata); - - const relays = await getRelays(db, relayListMetadata.pubkey); - - assertEquals(relays.size, 6); -}); - -Deno.test('Get write relays with invalid URL - kind 10002', async () => { - const db = new MockRelay(); - - const relayListMetadata = await eventFixture('kind-10002-alex'); - relayListMetadata.tags[0] = ['r', 'yolo']; - - await db.event(relayListMetadata); - - const relays = await getRelays(db, relayListMetadata.pubkey); - - assertEquals(relays.size, 5); -}); diff --git a/packages/ditto/utils/outbox.ts b/packages/ditto/utils/outbox.ts deleted file mode 100644 index 074518bc..00000000 --- a/packages/ditto/utils/outbox.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { NStore } from '@nostrify/nostrify'; - -import { Conf } from '@/config.ts'; - -export async function getRelays(store: NStore, pubkey: string): Promise> { - const relays = new Set<`wss://${string}`>(); - - const events = await store.query([ - { kinds: [10002], authors: [pubkey, await Conf.signer.getPublicKey()], limit: 2 }, - ]); - - for (const event of events) { - for (const [name, relay, marker] of event.tags) { - if (name === 'r' && (marker === 'write' || !marker)) { - try { - const url = new URL(relay); - if (url.protocol === 'wss:') { - relays.add(url.toString() as `wss://${string}`); - } - } catch (_e) { - // fall through - } - } - } - } - - return relays; -} From 398d79b45edc6b4e0f39acee26d85b685d8d4cba Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 13:05:50 -0600 Subject: [PATCH 68/99] DittoAPIStore: console.error -> logi --- packages/ditto/storages/DittoAPIStore.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/ditto/storages/DittoAPIStore.ts b/packages/ditto/storages/DittoAPIStore.ts index 89a33eda..9e04c6c6 100644 --- a/packages/ditto/storages/DittoAPIStore.ts +++ b/packages/ditto/storages/DittoAPIStore.ts @@ -46,9 +46,11 @@ export class DittoAPIStore implements NRelay { private encounters = new LRUCache({ max: 5000 }); private controller = new AbortController(); + private ns = 'ditto.apistore'; + constructor(private opts: DittoAPIStoreOpts) { this.listen().catch((e: unknown) => { - logi({ level: 'error', ns: 'ditto.apistore', source: 'listen', error: errorJson(e) }); + logi({ level: 'error', ns: this.ns, source: 'listen', error: errorJson(e) }); }); } @@ -62,6 +64,7 @@ export class DittoAPIStore implements NRelay { async event(event: NostrEvent, opts?: { signal?: AbortSignal }): Promise { const { relay, pool } = this.opts; + const { id, kind } = event; await relay.event(event, opts); @@ -69,7 +72,7 @@ export class DittoAPIStore implements NRelay { try { await pool.event(event, opts); } catch (e) { - console.error(e); + logi({ level: 'error', ns: this.ns, source: 'publish', id, kind, error: errorJson(e) }); } })(); } From 63c0f8b0320754005787f8f8ad8a3a062d45bdb6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 15:32:47 -0600 Subject: [PATCH 69/99] ditto/db: make adapters use classes instead of static classes --- packages/db/DittoDB.ts | 1 + packages/db/DittoPgMigrator.ts | 52 ++++++++++++++++ packages/db/adapters/DittoPglite.test.ts | 5 +- packages/db/adapters/DittoPglite.ts | 44 ++++++++------ packages/db/adapters/DittoPolyPg.test.ts | 4 +- packages/db/adapters/DittoPolyPg.ts | 75 +++++++++--------------- packages/db/adapters/DittoPostgres.ts | 71 +++++++++++----------- packages/db/adapters/DummyDB.test.ts | 2 + packages/db/adapters/DummyDB.ts | 4 ++ 9 files changed, 155 insertions(+), 103 deletions(-) create mode 100644 packages/db/DittoPgMigrator.ts diff --git a/packages/db/DittoDB.ts b/packages/db/DittoDB.ts index 99ab4c70..0afbddfd 100644 --- a/packages/db/DittoDB.ts +++ b/packages/db/DittoDB.ts @@ -6,6 +6,7 @@ export interface DittoDB extends AsyncDisposable { readonly kysely: Kysely; readonly poolSize: number; readonly availableConnections: number; + migrate(): Promise; listen(channel: string, callback: (payload: string) => void): void; } diff --git a/packages/db/DittoPgMigrator.ts b/packages/db/DittoPgMigrator.ts new file mode 100644 index 00000000..45407fe4 --- /dev/null +++ b/packages/db/DittoPgMigrator.ts @@ -0,0 +1,52 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; + +import { logi } from '@soapbox/logi'; +import { FileMigrationProvider, type Kysely, Migrator } from 'kysely'; + +import type { JsonValue } from '@std/json'; + +export class DittoPgMigrator { + private migrator: Migrator; + + // deno-lint-ignore no-explicit-any + constructor(private kysely: Kysely) { + this.migrator = new Migrator({ + db: this.kysely, + provider: new FileMigrationProvider({ + fs, + path, + migrationFolder: new URL(import.meta.resolve('./migrations')).pathname, + }), + }); + } + + async migrate(): Promise { + logi({ level: 'info', ns: 'ditto.db.migration', msg: 'Running migrations...', state: 'started' }); + const { results, error } = await this.migrator.migrateToLatest(); + + if (error) { + logi({ + level: 'fatal', + ns: 'ditto.db.migration', + msg: 'Migration failed.', + state: 'failed', + results: results as unknown as JsonValue, + error: error instanceof Error ? error : null, + }); + throw new Error('Migration failed.'); + } else { + if (!results?.length) { + logi({ level: 'info', ns: 'ditto.db.migration', msg: 'Everything up-to-date.', state: 'skipped' }); + } else { + logi({ + level: 'info', + ns: 'ditto.db.migration', + msg: 'Migrations finished!', + state: 'migrated', + results: results as unknown as JsonValue, + }); + } + } + } +} diff --git a/packages/db/adapters/DittoPglite.test.ts b/packages/db/adapters/DittoPglite.test.ts index 449ba02c..b0d9f4d1 100644 --- a/packages/db/adapters/DittoPglite.test.ts +++ b/packages/db/adapters/DittoPglite.test.ts @@ -2,8 +2,9 @@ import { assertEquals } from '@std/assert'; import { DittoPglite } from './DittoPglite.ts'; -Deno.test('DittoPglite.create', async () => { - const db = DittoPglite.create('memory://'); +Deno.test('DittoPglite', async () => { + const db = new DittoPglite('memory://'); + await db.migrate(); assertEquals(db.poolSize, 1); assertEquals(db.availableConnections, 1); diff --git a/packages/db/adapters/DittoPglite.ts b/packages/db/adapters/DittoPglite.ts index 9a4ad657..33516ee2 100644 --- a/packages/db/adapters/DittoPglite.ts +++ b/packages/db/adapters/DittoPglite.ts @@ -4,42 +4,50 @@ import { PgliteDialect } from '@soapbox/kysely-pglite'; import { Kysely } from 'kysely'; import { KyselyLogger } from '../KyselyLogger.ts'; +import { DittoPgMigrator } from '../DittoPgMigrator.ts'; import { isWorker } from '../utils/worker.ts'; import type { DittoDB, DittoDBOpts } from '../DittoDB.ts'; import type { DittoTables } from '../DittoTables.ts'; -export class DittoPglite { - static create(databaseUrl: string, opts?: DittoDBOpts): DittoDB { +export class DittoPglite implements DittoDB { + readonly poolSize = 1; + readonly availableConnections = 1; + readonly kysely: Kysely; + + private pglite: PGlite; + private migrator: DittoPgMigrator; + + constructor(databaseUrl: string, opts?: DittoDBOpts) { const url = new URL(databaseUrl); if (url.protocol === 'file:' && isWorker()) { throw new Error('PGlite is not supported in worker threads.'); } - const pglite = new PGlite(databaseUrl, { + this.pglite = new PGlite(databaseUrl, { extensions: { pg_trgm }, debug: opts?.debug, }); - const kysely = new Kysely({ - dialect: new PgliteDialect({ database: pglite }), + this.kysely = new Kysely({ + dialect: new PgliteDialect({ database: this.pglite }), log: KyselyLogger, }); - const listen = (channel: string, callback: (payload: string) => void): void => { - pglite.listen(channel, callback); - }; + this.migrator = new DittoPgMigrator(this.kysely); + } - return { - kysely, - poolSize: 1, - availableConnections: 1, - listen, - [Symbol.asyncDispose]: async () => { - await pglite.close(); - await kysely.destroy(); - }, - }; + listen(channel: string, callback: (payload: string) => void): void { + this.pglite.listen(channel, callback); + } + + async migrate(): Promise { + await this.migrator.migrate(); + } + + async [Symbol.asyncDispose](): Promise { + await this.pglite.close(); + await this.kysely.destroy(); } } diff --git a/packages/db/adapters/DittoPolyPg.test.ts b/packages/db/adapters/DittoPolyPg.test.ts index 539a6ed0..d38d8eb1 100644 --- a/packages/db/adapters/DittoPolyPg.test.ts +++ b/packages/db/adapters/DittoPolyPg.test.ts @@ -1,6 +1,6 @@ import { DittoPolyPg } from './DittoPolyPg.ts'; Deno.test('DittoPolyPg', async () => { - const db = DittoPolyPg.create('memory://'); - await DittoPolyPg.migrate(db.kysely); + const db = new DittoPolyPg('memory://'); + await db.migrate(); }); diff --git a/packages/db/adapters/DittoPolyPg.ts b/packages/db/adapters/DittoPolyPg.ts index 623ee9fc..2d9358cd 100644 --- a/packages/db/adapters/DittoPolyPg.ts +++ b/packages/db/adapters/DittoPolyPg.ts @@ -1,70 +1,53 @@ -import fs from 'node:fs/promises'; -import path from 'node:path'; - -import { logi } from '@soapbox/logi'; -import { FileMigrationProvider, type Kysely, Migrator } from 'kysely'; - import { DittoPglite } from './DittoPglite.ts'; import { DittoPostgres } from './DittoPostgres.ts'; -import type { JsonValue } from '@std/json'; +import type { Kysely } from 'kysely'; import type { DittoDB, DittoDBOpts } from '../DittoDB.ts'; import type { DittoTables } from '../DittoTables.ts'; /** Creates either a PGlite or Postgres connection depending on the databaseUrl. */ -export class DittoPolyPg { +export class DittoPolyPg implements DittoDB { + private adapter: DittoDB; + /** Open a new database connection. */ - static create(databaseUrl: string, opts?: DittoDBOpts): DittoDB { + constructor(databaseUrl: string, opts?: DittoDBOpts) { const { protocol } = new URL(databaseUrl); switch (protocol) { case 'file:': case 'memory:': - return DittoPglite.create(databaseUrl, opts); + this.adapter = new DittoPglite(databaseUrl, opts); + break; case 'postgres:': case 'postgresql:': - return DittoPostgres.create(databaseUrl, opts); + this.adapter = new DittoPostgres(databaseUrl, opts); + break; default: throw new Error('Unsupported database URL.'); } } - /** Migrate the database to the latest version. */ - static async migrate(kysely: Kysely) { - const migrator = new Migrator({ - db: kysely, - provider: new FileMigrationProvider({ - fs, - path, - migrationFolder: new URL(import.meta.resolve('../migrations')).pathname, - }), - }); + get kysely(): Kysely { + return this.adapter.kysely; + } - logi({ level: 'info', ns: 'ditto.db.migration', msg: 'Running migrations...', state: 'started' }); - const { results, error } = await migrator.migrateToLatest(); + async migrate(): Promise { + await this.adapter.migrate(); + } - if (error) { - logi({ - level: 'fatal', - ns: 'ditto.db.migration', - msg: 'Migration failed.', - state: 'failed', - results: results as unknown as JsonValue, - error: error instanceof Error ? error : null, - }); - throw new Error('Migration failed.'); - } else { - if (!results?.length) { - logi({ level: 'info', ns: 'ditto.db.migration', msg: 'Everything up-to-date.', state: 'skipped' }); - } else { - logi({ - level: 'info', - ns: 'ditto.db.migration', - msg: 'Migrations finished!', - state: 'migrated', - results: results as unknown as JsonValue, - }); - } - } + listen(channel: string, callback: (payload: string) => void): void { + this.adapter.listen(channel, callback); + } + + get poolSize(): number { + return this.adapter.poolSize; + } + + get availableConnections(): number { + return this.adapter.availableConnections; + } + + async [Symbol.asyncDispose](): Promise { + await this.adapter[Symbol.asyncDispose](); } } diff --git a/packages/db/adapters/DittoPostgres.ts b/packages/db/adapters/DittoPostgres.ts index 6657a8d6..ba16b09e 100644 --- a/packages/db/adapters/DittoPostgres.ts +++ b/packages/db/adapters/DittoPostgres.ts @@ -12,53 +12,54 @@ import { import { type PostgresJSDialectConfig, PostgresJSDriver } from 'kysely-postgres-js'; import postgres from 'postgres'; +import { DittoPgMigrator } from '../DittoPgMigrator.ts'; import { KyselyLogger } from '../KyselyLogger.ts'; import type { DittoDB, DittoDBOpts } from '../DittoDB.ts'; import type { DittoTables } from '../DittoTables.ts'; -export class DittoPostgres { - static create(databaseUrl: string, opts?: DittoDBOpts): DittoDB { - const pg = postgres(databaseUrl, { max: opts?.poolSize }); +export class DittoPostgres implements DittoDB { + private pg: ReturnType; + private migrator: DittoPgMigrator; - const kysely = new Kysely({ + readonly kysely: Kysely; + + constructor(databaseUrl: string, opts?: DittoDBOpts) { + this.pg = postgres(databaseUrl, { max: opts?.poolSize }); + + this.kysely = new Kysely({ dialect: { - createAdapter() { - return new PostgresAdapter(); - }, - createDriver() { - return new PostgresJSDriver({ - postgres: pg as unknown as PostgresJSDialectConfig['postgres'], - }); - }, - createIntrospector(db) { - return new PostgresIntrospector(db); - }, - createQueryCompiler() { - return new DittoPostgresQueryCompiler(); - }, + createAdapter: () => new PostgresAdapter(), + createDriver: () => + new PostgresJSDriver({ postgres: this.pg as unknown as PostgresJSDialectConfig['postgres'] }), + createIntrospector: (db) => new PostgresIntrospector(db), + createQueryCompiler: () => new DittoPostgresQueryCompiler(), }, log: KyselyLogger, }); - const listen = (channel: string, callback: (payload: string) => void): void => { - pg.listen(channel, callback); - }; + this.migrator = new DittoPgMigrator(this.kysely); + } - return { - kysely, - get poolSize() { - return pg.connections.open; - }, - get availableConnections() { - return pg.connections.idle; - }, - listen, - [Symbol.asyncDispose]: async () => { - await pg.end(); - await kysely.destroy(); - }, - }; + listen(channel: string, callback: (payload: string) => void): void { + this.pg.listen(channel, callback); + } + + async migrate(): Promise { + await this.migrator.migrate(); + } + + get poolSize(): number { + return this.pg.connections.open; + } + + get availableConnections(): number { + return this.pg.connections.idle; + } + + async [Symbol.asyncDispose](): Promise { + await this.pg.end(); + await this.kysely.destroy(); } } diff --git a/packages/db/adapters/DummyDB.test.ts b/packages/db/adapters/DummyDB.test.ts index 9945be45..a58ddcb0 100644 --- a/packages/db/adapters/DummyDB.test.ts +++ b/packages/db/adapters/DummyDB.test.ts @@ -3,6 +3,8 @@ import { DummyDB } from './DummyDB.ts'; Deno.test('DummyDB', async () => { const db = new DummyDB(); + await db.migrate(); + const rows = await db.kysely.selectFrom('nostr_events').selectAll().execute(); assertEquals(rows, []); diff --git a/packages/db/adapters/DummyDB.ts b/packages/db/adapters/DummyDB.ts index 51c29b10..669b679d 100644 --- a/packages/db/adapters/DummyDB.ts +++ b/packages/db/adapters/DummyDB.ts @@ -23,6 +23,10 @@ export class DummyDB implements DittoDB { // noop } + migrate(): Promise { + return Promise.resolve(); + } + [Symbol.asyncDispose](): Promise { return Promise.resolve(); } From ca5c8877053df83b442a3a0fb147d15a790416eb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 15:54:31 -0600 Subject: [PATCH 70/99] Remove storages.ts from scripts --- scripts/admin-event.ts | 14 +++++++++----- scripts/admin-role.ts | 16 ++++++++++------ scripts/db-export.ts | 12 ++++++++---- scripts/db-import.ts | 13 ++++++++----- scripts/db-migrate.ts | 10 +++++----- scripts/db-policy.ts | 16 +++++++++++----- scripts/db-populate-extensions.ts | 10 ++++++---- scripts/db-populate-nip05.ts | 19 ++++++++++++++----- scripts/db-populate-search.ts | 13 ++++++++----- scripts/db-streak-recompute.ts | 16 +++++++++------- scripts/nostr-pull.ts | 10 +++++++--- scripts/setup-kind0.ts | 18 +++++++++++------- scripts/stats-recompute.ts | 13 ++++++++----- 13 files changed, 114 insertions(+), 66 deletions(-) diff --git a/scripts/admin-event.ts b/scripts/admin-event.ts index aec9e145..bec49460 100644 --- a/scripts/admin-event.ts +++ b/scripts/admin-event.ts @@ -1,13 +1,17 @@ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { JsonParseStream } from '@std/json/json-parse-stream'; import { TextLineStream } from '@std/streams/text-line-stream'; -import { Conf } from '../packages/ditto/config.ts'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; import { type EventStub } from '../packages/ditto/utils/api.ts'; import { nostrNow } from '../packages/ditto/utils.ts'; -const signer = Conf.signer; -const store = await Storages.db(); +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); + +const { signer } = conf; const readable = Deno.stdin.readable .pipeThrough(new TextDecoderStream()) @@ -22,7 +26,7 @@ for await (const t of readable) { ...t as EventStub, }); - await store.event(event); + await relay.event(event); } Deno.exit(0); diff --git a/scripts/admin-role.ts b/scripts/admin-role.ts index 4da9610e..59b95878 100644 --- a/scripts/admin-role.ts +++ b/scripts/admin-role.ts @@ -1,15 +1,20 @@ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { Conf } from '../packages/ditto/config.ts'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; import { nostrNow } from '../packages/ditto/utils.ts'; -const store = await Storages.db(); +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); const [pubkeyOrNpub, role] = Deno.args; const pubkey = pubkeyOrNpub.startsWith('npub1') ? nip19.decode(pubkeyOrNpub as `npub1${string}`).data : pubkeyOrNpub; +const { signer } = conf; + if (!NSchema.id().safeParse(pubkey).success) { console.error('Invalid pubkey'); Deno.exit(1); @@ -20,10 +25,9 @@ if (!['admin', 'user'].includes(role)) { Deno.exit(1); } -const signer = Conf.signer; const admin = await signer.getPublicKey(); -const [existing] = await store.query([{ +const [existing] = await relay.query([{ kinds: [30382], authors: [admin], '#d': [pubkey], @@ -57,6 +61,6 @@ const event = await signer.signEvent({ created_at: nostrNow(), }); -await store.event(event); +await relay.event(event); Deno.exit(0); diff --git a/scripts/db-export.ts b/scripts/db-export.ts index d36d4f3f..d9295420 100644 --- a/scripts/db-export.ts +++ b/scripts/db-export.ts @@ -1,7 +1,13 @@ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { NostrFilter } from '@nostrify/nostrify'; import { Command, InvalidOptionArgumentError } from 'commander'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; + +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); interface ExportFilter { authors?: string[]; @@ -98,8 +104,6 @@ export function buildFilter(args: ExportFilter) { } async function exportEvents(args: ExportFilter) { - const store = await Storages.db(); - let filter: NostrFilter = {}; try { filter = buildFilter(args); @@ -108,7 +112,7 @@ async function exportEvents(args: ExportFilter) { } let count = 0; - for await (const msg of store.req([filter])) { + for await (const msg of relay.req([filter])) { if (msg[0] === 'EOSE') { break; } diff --git a/scripts/db-import.ts b/scripts/db-import.ts index 2f6c1595..4d27e54a 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -1,13 +1,16 @@ import { Semaphore } from '@core/asyncutil'; +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { NostrEvent } from '@nostrify/nostrify'; import { JsonParseStream } from '@std/json/json-parse-stream'; import { TextLineStream } from '@std/streams/text-line-stream'; -import { Conf } from '../packages/ditto/config.ts'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; -const store = await Storages.db(); -const sem = new Semaphore(Conf.pg.poolSize); +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); +const sem = new Semaphore(conf.pg.poolSize); console.warn('Importing events...'); @@ -27,7 +30,7 @@ for await (const line of readable) { sem.lock(async () => { try { - await store.event(event); + await relay.event(event); console.warn(`(${count}) Event<${event.kind}> ${event.id}`); } catch (error) { if (error instanceof Error && error.message.includes('violates unique constraint')) { diff --git a/scripts/db-migrate.ts b/scripts/db-migrate.ts index 21b8db22..23547eea 100644 --- a/scripts/db-migrate.ts +++ b/scripts/db-migrate.ts @@ -1,9 +1,9 @@ -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; -// This migrates kysely internally. -const kysely = await Storages.kysely(); +const conf = new DittoConf(Deno.env); +await using db = new DittoPolyPg(conf.databaseUrl); -// Close the connection before exiting. -await kysely.destroy(); +await db.migrate(); Deno.exit(); diff --git a/scripts/db-policy.ts b/scripts/db-policy.ts index caab55af..80e217c5 100644 --- a/scripts/db-policy.ts +++ b/scripts/db-policy.ts @@ -1,16 +1,22 @@ -import { policyWorker } from '../packages/ditto/workers/policy.ts'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; + +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; +import { policyWorker } from '../packages/ditto/workers/policy.ts'; + +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); -const db = await Storages.db(); let count = 0; -for await (const msg of db.req([{}])) { +for await (const msg of relay.req([{}])) { const [type, , event] = msg; if (type === 'EOSE') console.log('EOSE'); if (type !== 'EVENT') continue; const [, , ok] = await policyWorker.call(event, AbortSignal.timeout(5000)); if (!ok) { - await db.remove([{ ids: [event.id] }]); + await relay.remove([{ ids: [event.id] }]); count += 1; } } diff --git a/scripts/db-populate-extensions.ts b/scripts/db-populate-extensions.ts index 0cb3a49b..9af8be2a 100644 --- a/scripts/db-populate-extensions.ts +++ b/scripts/db-populate-extensions.ts @@ -1,11 +1,13 @@ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { NostrEvent } from '@nostrify/nostrify'; -import { Storages } from '../packages/ditto/storages.ts'; import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; -const kysely = await Storages.kysely(); +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); -const query = kysely +const query = db.kysely .selectFrom('nostr_events') .select(['id', 'kind', 'content', 'pubkey', 'tags', 'created_at', 'sig']); @@ -14,7 +16,7 @@ for await (const row of query.stream()) { const ext = DittoPgStore.indexExtensions(event); try { - await kysely + await db.kysely .updateTable('nostr_events') .set('search_ext', ext) .where('id', '=', event.id) diff --git a/scripts/db-populate-nip05.ts b/scripts/db-populate-nip05.ts index acfe70da..46e0686d 100644 --- a/scripts/db-populate-nip05.ts +++ b/scripts/db-populate-nip05.ts @@ -1,13 +1,22 @@ import { Semaphore } from '@core/asyncutil'; import { NostrEvent } from '@nostrify/nostrify'; +import { MockRelay } from '@nostrify/nostrify/test'; -import { updateAuthorData } from '../packages/ditto/pipeline.ts'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; + +import { DittoAPIStore } from '../packages/ditto/storages/DittoAPIStore.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; + +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); + +const pgstore = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); +const apistore = new DittoAPIStore({ conf, db, relay: pgstore, pool: new MockRelay() }); -const kysely = await Storages.kysely(); const sem = new Semaphore(5); -const query = kysely +const query = db.kysely .selectFrom('nostr_events') .select(['id', 'kind', 'content', 'pubkey', 'tags', 'created_at', 'sig']) .where('kind', '=', 0); @@ -19,7 +28,7 @@ for await (const row of query.stream(100)) { sem.lock(async () => { const event: NostrEvent = { ...row, created_at: Number(row.created_at) }; - await updateAuthorData(event, AbortSignal.timeout(3000)); + await apistore.updateAuthorData(event, AbortSignal.timeout(3000)); }); } diff --git a/scripts/db-populate-search.ts b/scripts/db-populate-search.ts index e73f79ac..7189b30c 100644 --- a/scripts/db-populate-search.ts +++ b/scripts/db-populate-search.ts @@ -1,11 +1,14 @@ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { NSchema as n } from '@nostrify/nostrify'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; -const store = await Storages.db(); -const kysely = await Storages.kysely(); +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); -for await (const msg of store.req([{ kinds: [0] }])) { +for await (const msg of relay.req([{ kinds: [0] }])) { if (msg[0] === 'EVENT') { const { pubkey, content } = msg[2]; @@ -13,7 +16,7 @@ for await (const msg of store.req([{ kinds: [0] }])) { const search = [name, nip05].filter(Boolean).join(' ').trim(); try { - await kysely.insertInto('author_stats').values({ + await db.kysely.insertInto('author_stats').values({ pubkey, search, followers_count: 0, diff --git a/scripts/db-streak-recompute.ts b/scripts/db-streak-recompute.ts index e45d4f64..6a0f313f 100644 --- a/scripts/db-streak-recompute.ts +++ b/scripts/db-streak-recompute.ts @@ -1,12 +1,14 @@ -import { Conf } from '../packages/ditto/config.ts'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; -const kysely = await Storages.kysely(); -const statsQuery = kysely.selectFrom('author_stats').select('pubkey'); -const { streakWindow } = Conf; +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); + +const statsQuery = db.kysely.selectFrom('author_stats').select('pubkey'); +const { streakWindow } = conf; for await (const { pubkey } of statsQuery.stream(10)) { - const eventsQuery = kysely + const eventsQuery = db.kysely .selectFrom('nostr_events') .select('created_at') .where('pubkey', '=', pubkey) @@ -38,7 +40,7 @@ for await (const { pubkey } of statsQuery.stream(10)) { } if (start && end) { - await kysely + await db.kysely .updateTable('author_stats') .set({ streak_end: end, diff --git a/scripts/nostr-pull.ts b/scripts/nostr-pull.ts index 7c21cb80..d8a4513a 100644 --- a/scripts/nostr-pull.ts +++ b/scripts/nostr-pull.ts @@ -3,12 +3,16 @@ * by looking them up on a list of relays. */ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; -const store = await Storages.db(); +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); interface ImportEventsOpts { profilesOnly: boolean; @@ -19,7 +23,7 @@ const importUsers = async ( authors: string[], relays: string[], opts?: Partial, - doEvent: DoEvent = async (event: NostrEvent) => await store.event(event), + doEvent: DoEvent = async (event: NostrEvent) => await relay.event(event), ) => { // Kind 0s + follow lists. const profiles: Record> = {}; diff --git a/scripts/setup-kind0.ts b/scripts/setup-kind0.ts index 85f7a6ca..b3dd0682 100644 --- a/scripts/setup-kind0.ts +++ b/scripts/setup-kind0.ts @@ -1,9 +1,13 @@ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { Command } from 'commander'; import { NostrEvent } from 'nostr-tools'; -import { nostrNow } from '../packages/ditto/utils.ts'; -import { Conf } from '../packages/ditto/config.ts'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; + +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); function die(code: number, ...args: unknown[]) { console.error(...args); @@ -33,19 +37,19 @@ if (import.meta.main) { content.lud16 = lightning; content.name = name; content.picture = image; - content.website = Conf.localDomain; + content.website = conf.localDomain; - const signer = Conf.signer; + const signer = conf.signer; const bare: Omit = { - created_at: nostrNow(), kind: 0, tags: [], content: JSON.stringify(content), + created_at: Math.floor(Date.now() / 1000), }; const signed = await signer.signEvent(bare); console.log({ content, signed }); - await Storages.db().then((store) => store.event(signed)); + await relay.event(signed); }); await kind0.parseAsync(); diff --git a/scripts/stats-recompute.ts b/scripts/stats-recompute.ts index 942d0012..c17e9047 100644 --- a/scripts/stats-recompute.ts +++ b/scripts/stats-recompute.ts @@ -1,8 +1,14 @@ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { nip19 } from 'nostr-tools'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; import { refreshAuthorStats } from '../packages/ditto/utils/stats.ts'; +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); + let pubkey: string; try { const result = nip19.decode(Deno.args[0]); @@ -16,7 +22,4 @@ try { Deno.exit(1); } -const store = await Storages.db(); -const kysely = await Storages.kysely(); - -await refreshAuthorStats({ pubkey, kysely, store }); +await refreshAuthorStats({ pubkey, kysely: db.kysely, store: relay }); From 3b17fd9b45708e225f0e440ed2f45e2ae253d1ca Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 19:27:53 -0600 Subject: [PATCH 71/99] Remove @/storages.ts (jesus christ) --- packages/ditto/DittoPush.ts | 52 +-- packages/ditto/app.ts | 65 +++- packages/ditto/controllers/api/accounts.ts | 75 ++-- packages/ditto/controllers/api/admin.ts | 13 +- packages/ditto/controllers/api/cashu.test.ts | 2 +- packages/ditto/controllers/api/ditto.ts | 19 +- packages/ditto/controllers/api/instance.ts | 13 +- .../ditto/controllers/api/notifications.ts | 10 +- packages/ditto/controllers/api/oauth.ts | 20 +- packages/ditto/controllers/api/pleroma.ts | 8 +- packages/ditto/controllers/api/push.ts | 12 +- packages/ditto/controllers/api/reactions.ts | 8 +- packages/ditto/controllers/api/reports.ts | 23 +- packages/ditto/controllers/api/search.ts | 44 +-- packages/ditto/controllers/api/statuses.ts | 92 ++--- packages/ditto/controllers/api/streaming.ts | 17 +- packages/ditto/controllers/api/suggestions.ts | 6 +- packages/ditto/controllers/api/timelines.ts | 12 +- packages/ditto/controllers/api/translate.ts | 6 +- packages/ditto/controllers/api/trends.ts | 91 ++--- packages/ditto/controllers/frontend.ts | 21 +- packages/ditto/controllers/metrics.ts | 19 +- packages/ditto/controllers/nostr/relay.ts | 3 +- .../ditto/controllers/well-known/nostr.ts | 4 +- packages/ditto/cron.ts | 16 +- packages/ditto/firehose.ts | 24 +- packages/ditto/middleware/cspMiddleware.ts | 10 +- packages/ditto/pipeline.ts | 368 ------------------ packages/ditto/queries.ts | 83 +--- packages/ditto/signers/ConnectSigner.ts | 31 +- packages/ditto/startup.ts | 12 - packages/ditto/storages.ts | 62 --- packages/ditto/storages/DittoAPIStore.ts | 73 +++- packages/ditto/storages/DittoPgStore.ts | 2 +- packages/ditto/storages/hydrate.test.ts | 76 ++-- packages/ditto/storages/hydrate.ts | 38 +- packages/ditto/test.ts | 7 +- packages/ditto/trends.ts | 56 +-- packages/ditto/utils/api.ts | 126 +----- packages/ditto/utils/connect.ts | 28 -- packages/ditto/utils/favicon.ts | 29 +- packages/ditto/utils/lookup.ts | 24 +- packages/ditto/utils/nip05.ts | 45 +-- packages/ditto/views.ts | 10 +- .../ditto/views/mastodon/notifications.ts | 34 +- packages/ditto/views/mastodon/push.ts | 5 +- packages/ditto/views/mastodon/reports.ts | 12 +- packages/ditto/views/mastodon/statuses.ts | 23 +- packages/ditto/workers/policy.worker.ts | 2 +- packages/mastoapi/deno.json | 1 + packages/mastoapi/pagination/mod.ts | 3 + packages/mastoapi/router/DittoApp.test.ts | 2 +- scripts/trends.ts | 18 +- 53 files changed, 639 insertions(+), 1216 deletions(-) delete mode 100644 packages/ditto/pipeline.ts delete mode 100644 packages/ditto/startup.ts delete mode 100644 packages/ditto/storages.ts delete mode 100644 packages/ditto/utils/connect.ts create mode 100644 packages/mastoapi/pagination/mod.ts diff --git a/packages/ditto/DittoPush.ts b/packages/ditto/DittoPush.ts index 7f5dafa0..3a378300 100644 --- a/packages/ditto/DittoPush.ts +++ b/packages/ditto/DittoPush.ts @@ -1,39 +1,41 @@ +import { DittoConf } from '@ditto/conf'; import { ApplicationServer, PushMessageOptions, PushSubscriber, PushSubscription } from '@negrel/webpush'; +import { NStore } from '@nostrify/types'; import { logi } from '@soapbox/logi'; -import { Conf } from '@/config.ts'; -import { Storages } from '@/storages.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; +interface DittoPushOpts { + conf: DittoConf; + relay: NStore; +} + export class DittoPush { - static _server: Promise | undefined; + private server: Promise; - static get server(): Promise { - if (!this._server) { - this._server = (async () => { - const store = await Storages.db(); - const meta = await getInstanceMetadata(store); - const keys = await Conf.vapidKeys; + constructor(opts: DittoPushOpts) { + const { conf, relay } = opts; - if (keys) { - return await ApplicationServer.new({ - contactInformation: `mailto:${meta.email}`, - vapidKeys: keys, - }); - } else { - logi({ - level: 'warn', - ns: 'ditto.push', - msg: 'VAPID keys are not set. Push notifications will be disabled.', - }); - } - })(); - } + this.server = (async () => { + const meta = await getInstanceMetadata(relay); + const keys = await conf.vapidKeys; - return this._server; + if (keys) { + return await ApplicationServer.new({ + contactInformation: `mailto:${meta.email}`, + vapidKeys: keys, + }); + } else { + logi({ + level: 'warn', + ns: 'ditto.push', + msg: 'VAPID keys are not set. Push notifications will be disabled.', + }); + } + })(); } - static async push( + async push( subscription: PushSubscription, json: object, opts: PushMessageOptions = {}, diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index eab81b47..0a9806d6 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -1,7 +1,8 @@ import { DittoConf } from '@ditto/conf'; -import { DittoDB } from '@ditto/db'; +import { DittoDB, DittoPolyPg } from '@ditto/db'; import { paginationMiddleware, tokenMiddleware, userMiddleware } from '@ditto/mastoapi/middleware'; import { DittoApp, type DittoEnv } from '@ditto/mastoapi/router'; +import { relayPoolRelaysSizeGauge, relayPoolSubscriptionsSizeGauge } from '@ditto/metrics'; import { type DittoTranslator } from '@ditto/translators'; import { type Context, Handler, Input as HonoInput, MiddlewareHandler } from '@hono/hono'; import { every } from '@hono/hono/combine'; @@ -9,11 +10,13 @@ import { cors } from '@hono/hono/cors'; import { serveStatic } from '@hono/hono/deno'; import { NostrEvent, NostrSigner, NRelay, NUploader } from '@nostrify/nostrify'; -import '@/startup.ts'; - -import { Conf } from '@/config.ts'; -import { Storages } from '@/storages.ts'; +import { cron } from '@/cron.ts'; +import { startFirehose } from '@/firehose.ts'; +import { DittoAPIStore } from '@/storages/DittoAPIStore.ts'; +import { DittoPgStore } from '@/storages/DittoPgStore.ts'; +import { DittoPool } from '@/storages/DittoPool.ts'; import { Time } from '@/utils/time.ts'; +import { seedZapSplits } from '@/utils/zap-split.ts'; import { accountController, @@ -176,14 +179,42 @@ type AppMiddleware = MiddlewareHandler; // deno-lint-ignore no-explicit-any type AppController

= Handler>; -const app = new DittoApp({ - conf: Conf, - db: await Storages.database(), - relay: await Storages.db(), -}, { - strict: false, +const conf = new DittoConf(Deno.env); + +const db = new DittoPolyPg(conf.databaseUrl, { + poolSize: conf.pg.poolSize, + debug: conf.pgliteDebug, }); +await db.migrate(); + +const store = new DittoPgStore({ + db, + pubkey: await conf.signer.getPublicKey(), + timeout: conf.db.timeouts.default, + notify: conf.notifyEnabled, +}); + +const pool = new DittoPool({ conf, relay: store }); +const relay = new DittoAPIStore({ db, conf, relay: store, pool }); + +await seedZapSplits(relay); + +if (conf.firehoseEnabled) { + startFirehose({ + pool, + store: relay, + concurrency: conf.firehoseConcurrency, + kinds: conf.firehoseKinds, + }); +} + +if (conf.cronEnabled) { + cron({ conf, db, relay }); +} + +const app = new DittoApp({ conf, db, relay }, { strict: false }); + /** User-provided files in the gitignored `public/` directory. */ const publicFiles = serveStatic({ root: './public/' }); /** Static files provided by the Ditto repo, checked into git. */ @@ -218,7 +249,17 @@ app.use( uploaderMiddleware, ); -app.get('/metrics', metricsController); +app.get('/metrics', async (_c, next) => { + relayPoolRelaysSizeGauge.reset(); + relayPoolSubscriptionsSizeGauge.reset(); + + for (const relay of pool.relays.values()) { + relayPoolRelaysSizeGauge.inc({ ready_state: relay.socket.readyState }); + relayPoolSubscriptionsSizeGauge.inc(relay.subscriptions.length); + } + + await next(); +}, metricsController); app.get( '/.well-known/nodeinfo', diff --git a/packages/ditto/controllers/api/accounts.ts b/packages/ditto/controllers/api/accounts.ts index 24f7d5af..685ef70a 100644 --- a/packages/ditto/controllers/api/accounts.ts +++ b/packages/ditto/controllers/api/accounts.ts @@ -1,14 +1,14 @@ -import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; +import { paginated } from '@ditto/mastoapi/pagination'; +import { NostrEvent, NostrFilter, NSchema as n, NStore } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { getAuthor, getFollowedPubkeys } from '@/queries.ts'; import { booleanParamSchema, fileSchema } from '@/schema.ts'; -import { Storages } from '@/storages.ts'; import { uploadFile } from '@/utils/upload.ts'; import { nostrNow } from '@/utils.ts'; -import { assertAuthenticated, createEvent, paginated, parseBody, updateEvent, updateListEvent } from '@/utils/api.ts'; +import { assertAuthenticated, createEvent, parseBody, updateEvent, updateListEvent } from '@/utils/api.ts'; import { extractIdentifier, lookupAccount, lookupPubkey } from '@/utils/lookup.ts'; import { renderAccounts, renderEventAccounts, renderStatuses } from '@/views.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; @@ -54,7 +54,7 @@ const verifyCredentialsController: AppController = async (c) => { const pubkey = await signer.getPublicKey(); const [author, [settingsEvent]] = await Promise.all([ - getAuthor(pubkey, { signal: AbortSignal.timeout(5000) }), + getAuthor(pubkey, c.var), relay.query([{ kinds: [30078], @@ -81,7 +81,7 @@ const verifyCredentialsController: AppController = async (c) => { const accountController: AppController = async (c) => { const pubkey = c.req.param('pubkey'); - const event = await getAuthor(pubkey); + const event = await getAuthor(pubkey, c.var); if (event) { assertAuthenticated(c, event); return c.json(await renderAccount(event)); @@ -97,7 +97,7 @@ const accountLookupController: AppController = async (c) => { return c.json({ error: 'Missing `acct` query parameter.' }, 422); } - const event = await lookupAccount(decodeURIComponent(acct)); + const event = await lookupAccount(decodeURIComponent(acct), c.var); if (event) { assertAuthenticated(c, event); return c.json(await renderAccount(event)); @@ -131,10 +131,10 @@ const accountSearchController: AppController = async (c) => { const query = decodeURIComponent(result.data.q); const lookup = extractIdentifier(query); - const event = await lookupAccount(lookup ?? query); + const event = await lookupAccount(lookup ?? query, c.var); if (!event && lookup) { - const pubkey = await lookupPubkey(lookup); + const pubkey = await lookupPubkey(lookup, c.var); return c.json(pubkey ? [accountFromPubkey(pubkey)] : []); } @@ -143,7 +143,7 @@ const accountSearchController: AppController = async (c) => { if (event) { events.push(event); } else { - const following = viewerPubkey ? await getFollowedPubkeys(viewerPubkey) : new Set(); + const following = viewerPubkey ? await getFollowedPubkeys(relay, viewerPubkey, signal) : new Set(); const authors = [...await getPubkeysBySearch(db.kysely, { q: query, limit, offset: 0, following })]; const profiles = await relay.query([{ kinds: [0], authors, limit }], { signal }); @@ -155,14 +155,14 @@ const accountSearchController: AppController = async (c) => { } } - const accounts = await hydrateEvents({ events, relay, signal }) + const accounts = await hydrateEvents({ ...c.var, events }) .then((events) => events.map((event) => renderAccount(event))); return c.json(accounts); }; const relationshipsController: AppController = async (c) => { - const { user } = c.var; + const { relay, user } = c.var; const pubkey = await user!.signer.getPublicKey(); const ids = z.array(z.string()).safeParse(c.req.queries('id[]')); @@ -171,11 +171,9 @@ const relationshipsController: AppController = async (c) => { return c.json({ error: 'Missing `id[]` query parameters.' }, 422); } - const db = await Storages.db(); - const [sourceEvents, targetEvents] = await Promise.all([ - db.query([{ kinds: [3, 10000], authors: [pubkey] }]), - db.query([{ kinds: [3], authors: ids.data }]), + relay.query([{ kinds: [3, 10000], authors: [pubkey] }]), + relay.query([{ kinds: [3], authors: ids.data }]), ]); const event3 = sourceEvents.find((event) => event.kind === 3 && event.pubkey === pubkey); @@ -267,7 +265,7 @@ const accountStatusesController: AppController = async (c) => { const opts = { signal, limit, timeout: conf.db.timeouts.timelines }; const events = await relay.query([filter], opts) - .then((events) => hydrateEvents({ events, relay, signal })) + .then((events) => hydrateEvents({ ...c.var, events })) .then((events) => { if (exclude_replies) { return events.filter((event) => { @@ -282,8 +280,8 @@ const accountStatusesController: AppController = async (c) => { const statuses = await Promise.all( events.map((event) => { - if (event.kind === 6) return renderReblog(event, { viewerPubkey }); - return renderStatus(event, { viewerPubkey }); + if (event.kind === 6) return renderReblog(relay, event, { viewerPubkey }); + return renderStatus(relay, event, { viewerPubkey }); }), ); return paginated(c, events, statuses); @@ -305,7 +303,7 @@ const updateCredentialsSchema = z.object({ }); const updateCredentialsController: AppController = async (c) => { - const { relay, user, signal } = c.var; + const { relay, user } = c.var; const pubkey = await user!.signer.getPublicKey(); const body = await parseBody(c.req.raw); @@ -375,7 +373,7 @@ const updateCredentialsController: AppController = async (c) => { let account: MastodonAccount; if (event) { - await hydrateEvents({ events: [event], relay, signal }); + await hydrateEvents({ ...c.var, events: [event] }); account = await renderAccount(event, { withSource: true, settingsStore }); } else { account = await accountFromPubkey(pubkey, { withSource: true, settingsStore }); @@ -394,7 +392,7 @@ const updateCredentialsController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/accounts/#follow */ const followController: AppController = async (c) => { - const { user } = c.var; + const { relay, user } = c.var; const sourcePubkey = await user!.signer.getPublicKey(); const targetPubkey = c.req.param('pubkey'); @@ -405,7 +403,7 @@ const followController: AppController = async (c) => { c, ); - const relationship = await getRelationship(sourcePubkey, targetPubkey); + const relationship = await getRelationship(relay, sourcePubkey, targetPubkey); relationship.following = true; return c.json(relationship); @@ -413,7 +411,7 @@ const followController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/accounts/#unfollow */ const unfollowController: AppController = async (c) => { - const { user } = c.var; + const { relay, user } = c.var; const sourcePubkey = await user!.signer.getPublicKey(); const targetPubkey = c.req.param('pubkey'); @@ -424,7 +422,7 @@ const unfollowController: AppController = async (c) => { c, ); - const relationship = await getRelationship(sourcePubkey, targetPubkey); + const relationship = await getRelationship(relay, sourcePubkey, targetPubkey); return c.json(relationship); }; @@ -435,8 +433,9 @@ const followersController: AppController = (c) => { }; const followingController: AppController = async (c) => { + const { relay, signal } = c.var; const pubkey = c.req.param('pubkey'); - const pubkeys = await getFollowedPubkeys(pubkey); + const pubkeys = await getFollowedPubkeys(relay, pubkey, signal); return renderAccounts(c, [...pubkeys]); }; @@ -452,7 +451,7 @@ const unblockController: AppController = (c) => { /** https://docs.joinmastodon.org/methods/accounts/#mute */ const muteController: AppController = async (c) => { - const { user } = c.var; + const { relay, user } = c.var; const sourcePubkey = await user!.signer.getPublicKey(); const targetPubkey = c.req.param('pubkey'); @@ -463,13 +462,13 @@ const muteController: AppController = async (c) => { c, ); - const relationship = await getRelationship(sourcePubkey, targetPubkey); + const relationship = await getRelationship(relay, sourcePubkey, targetPubkey); return c.json(relationship); }; /** https://docs.joinmastodon.org/methods/accounts/#unmute */ const unmuteController: AppController = async (c) => { - const { user } = c.var; + const { relay, user } = c.var; const sourcePubkey = await user!.signer.getPublicKey(); const targetPubkey = c.req.param('pubkey'); @@ -480,7 +479,7 @@ const unmuteController: AppController = async (c) => { c, ); - const relationship = await getRelationship(sourcePubkey, targetPubkey); + const relationship = await getRelationship(relay, sourcePubkey, targetPubkey); return c.json(relationship); }; @@ -499,26 +498,26 @@ const favouritesController: AppController = async (c) => { .filter((id): id is string => !!id); const events1 = await relay.query([{ kinds: [1, 20], ids }], { signal }) - .then((events) => hydrateEvents({ events, relay, signal })); + .then((events) => hydrateEvents({ ...c.var, events })); const viewerPubkey = await user?.signer.getPublicKey(); const statuses = await Promise.all( - events1.map((event) => renderStatus(event, { viewerPubkey })), + events1.map((event) => renderStatus(relay, event, { viewerPubkey })), ); return paginated(c, events1, statuses); }; const familiarFollowersController: AppController = async (c) => { - const { relay, user } = c.var; + const { relay, user, signal } = c.var; const pubkey = await user!.signer.getPublicKey(); const ids = z.array(z.string()).parse(c.req.queries('id[]')); - const follows = await getFollowedPubkeys(pubkey); + const follows = await getFollowedPubkeys(relay, pubkey, signal); const results = await Promise.all(ids.map(async (id) => { const followLists = await relay.query([{ kinds: [3], authors: [...follows], '#p': [id] }]) - .then((events) => hydrateEvents({ events, relay })); + .then((events) => hydrateEvents({ ...c.var, events })); const accounts = await Promise.all( followLists.map((event) => event.author ? renderAccount(event.author) : accountFromPubkey(event.pubkey)), @@ -530,12 +529,10 @@ const familiarFollowersController: AppController = async (c) => { return c.json(results); }; -async function getRelationship(sourcePubkey: string, targetPubkey: string) { - const db = await Storages.db(); - +async function getRelationship(relay: NStore, sourcePubkey: string, targetPubkey: string) { const [sourceEvents, targetEvents] = await Promise.all([ - db.query([{ kinds: [3, 10000], authors: [sourcePubkey] }]), - db.query([{ kinds: [3], authors: [targetPubkey] }]), + relay.query([{ kinds: [3, 10000], authors: [sourcePubkey] }]), + relay.query([{ kinds: [3], authors: [targetPubkey] }]), ]); return renderRelationship({ diff --git a/packages/ditto/controllers/api/admin.ts b/packages/ditto/controllers/api/admin.ts index b4e18f0d..411aa841 100644 --- a/packages/ditto/controllers/api/admin.ts +++ b/packages/ditto/controllers/api/admin.ts @@ -1,3 +1,4 @@ +import { paginated } from '@ditto/mastoapi/pagination'; import { NostrFilter } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import { z } from 'zod'; @@ -5,7 +6,7 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { booleanParamSchema } from '@/schema.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { createAdminEvent, paginated, parseBody, updateEventInfo, updateUser } from '@/utils/api.ts'; +import { createAdminEvent, parseBody, updateEventInfo, updateUser } from '@/utils/api.ts'; import { renderNameRequest } from '@/views/ditto.ts'; import { renderAdminAccount, renderAdminAccountFromPubkey } from '@/views/mastodon/admin-accounts.ts'; import { errorJson } from '@/utils/log.ts'; @@ -59,7 +60,7 @@ const adminAccountsController: AppController = async (c) => { ); const events = await relay.query([{ kinds: [3036], ids: [...ids] }]) - .then((events) => hydrateEvents({ relay, events, signal })); + .then((events) => hydrateEvents({ ...c.var, events })); const nameRequests = await Promise.all(events.map(renderNameRequest)); return paginated(c, orig, nameRequests); @@ -97,7 +98,7 @@ const adminAccountsController: AppController = async (c) => { ); const authors = await relay.query([{ kinds: [0], authors: [...pubkeys] }]) - .then((events) => hydrateEvents({ relay, events, signal })); + .then((events) => hydrateEvents({ ...c.var, events })); const accounts = await Promise.all( [...pubkeys].map((pubkey) => { @@ -116,7 +117,7 @@ const adminAccountsController: AppController = async (c) => { } const events = await relay.query([filter], { signal }) - .then((events) => hydrateEvents({ relay, events, signal })); + .then((events) => hydrateEvents({ ...c.var, events })); const accounts = await Promise.all(events.map(renderAdminAccount)); return paginated(c, events, accounts); @@ -210,7 +211,7 @@ const adminApproveController: AppController = async (c) => { }, c); await updateEventInfo(eventId, { pending: false, approved: true, rejected: false }, c); - await hydrateEvents({ events: [event], relay }); + await hydrateEvents({ ...c.var, events: [event] }); const nameRequest = await renderNameRequest(event); return c.json(nameRequest); @@ -226,7 +227,7 @@ const adminRejectController: AppController = async (c) => { } await updateEventInfo(eventId, { pending: false, approved: false, rejected: true }, c); - await hydrateEvents({ events: [event], relay }); + await hydrateEvents({ ...c.var, events: [event] }); const nameRequest = await renderNameRequest(event); return c.json(nameRequest); diff --git a/packages/ditto/controllers/api/cashu.test.ts b/packages/ditto/controllers/api/cashu.test.ts index 1b28d099..85803f18 100644 --- a/packages/ditto/controllers/api/cashu.test.ts +++ b/packages/ditto/controllers/api/cashu.test.ts @@ -252,7 +252,7 @@ async function createTestRoute() { const sk = generateSecretKey(); const signer = new NSecSigner(sk); - const route = new DittoApp({ db, relay, conf }); + const route = new DittoApp({ db: db.db, relay, conf }); route.use(testUserMiddleware({ signer, relay })); route.route('/', cashuRoute); diff --git a/packages/ditto/controllers/api/ditto.ts b/packages/ditto/controllers/api/ditto.ts index ff1b958f..2aa8da2b 100644 --- a/packages/ditto/controllers/api/ditto.ts +++ b/packages/ditto/controllers/api/ditto.ts @@ -1,3 +1,4 @@ +import { paginated, paginatedList } from '@ditto/mastoapi/pagination'; import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { z } from 'zod'; @@ -5,7 +6,7 @@ import { AppController } from '@/app.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { getAuthor } from '@/queries.ts'; import { addTag } from '@/utils/tags.ts'; -import { createEvent, paginated, parseBody, updateAdminEvent } from '@/utils/api.ts'; +import { createEvent, parseBody, updateAdminEvent } from '@/utils/api.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; import { deleteTag } from '@/utils/tags.ts'; import { DittoZapSplits, getZapSplits } from '@/utils/zap-split.ts'; @@ -15,7 +16,6 @@ import { hydrateEvents } from '@/storages/hydrate.ts'; import { renderNameRequest } from '@/views/ditto.ts'; import { accountFromPubkey } from '@/views/mastodon/accounts.ts'; import { renderAccount } from '@/views/mastodon/accounts.ts'; -import { Storages } from '@/storages.ts'; import { updateListAdminEvent } from '@/utils/api.ts'; const markerSchema = z.enum(['read', 'write']); @@ -120,7 +120,7 @@ export const nameRequestController: AppController = async (c) => { ], }, c); - await hydrateEvents({ events: [event], relay }); + await hydrateEvents({ ...c.var, events: [event] }); const nameRequest = await renderNameRequest(event); return c.json(nameRequest); @@ -132,7 +132,7 @@ const nameRequestsSchema = z.object({ }); export const nameRequestsController: AppController = async (c) => { - const { conf, relay, user, signal } = c.var; + const { conf, relay, user } = c.var; const pubkey = await user!.signer.getPublicKey(); const params = c.get('pagination'); @@ -168,7 +168,7 @@ export const nameRequestsController: AppController = async (c) => { } const events = await relay.query([{ kinds: [3036], ids: [...ids], authors: [pubkey] }]) - .then((events) => hydrateEvents({ relay, events: events, signal })); + .then((events) => hydrateEvents({ ...c.var, events })); const nameRequests = await Promise.all( events.map((event) => renderNameRequest(event)), @@ -263,7 +263,7 @@ export const getZapSplitsController: AppController = async (c) => { const pubkeys = Object.keys(dittoZapSplit); const zapSplits = await Promise.all(pubkeys.map(async (pubkey) => { - const author = await getAuthor(pubkey); + const author = await getAuthor(pubkey, c.var); const account = author ? renderAccount(author) : accountFromPubkey(pubkey); @@ -292,7 +292,7 @@ export const statusZapSplitsController: AppController = async (c) => { const pubkeys = zapsTag.map((name) => name[1]); const users = await relay.query([{ authors: pubkeys, kinds: [0], limit: pubkeys.length }], { signal }); - await hydrateEvents({ events: users, relay, signal }); + await hydrateEvents({ ...c.var, events: users }); const zapSplits = (await Promise.all(pubkeys.map((pubkey) => { const author = (users.find((event) => event.pubkey === pubkey) as DittoEvent | undefined)?.author; @@ -325,7 +325,8 @@ const updateInstanceSchema = z.object({ }); export const updateInstanceController: AppController = async (c) => { - const { conf } = c.var; + const { conf, relay, signal } = c.var; + const body = await parseBody(c.req.raw); const result = updateInstanceSchema.safeParse(body); const pubkey = await conf.signer.getPublicKey(); @@ -334,7 +335,7 @@ export const updateInstanceController: AppController = async (c) => { return c.json(result.error, 422); } - const meta = await getInstanceMetadata(await Storages.db(), c.req.raw.signal); + const meta = await getInstanceMetadata(relay, signal); await updateAdminEvent( { kinds: [0], authors: [pubkey], limit: 1 }, diff --git a/packages/ditto/controllers/api/instance.ts b/packages/ditto/controllers/api/instance.ts index 8c3c6e4c..1fb742e5 100644 --- a/packages/ditto/controllers/api/instance.ts +++ b/packages/ditto/controllers/api/instance.ts @@ -1,7 +1,6 @@ import denoJson from 'deno.json' with { type: 'json' }; import { AppController } from '@/app.ts'; -import { Storages } from '@/storages.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; const version = `3.0.0 (compatible; Ditto ${denoJson.version})`; @@ -16,9 +15,9 @@ const features = [ ]; const instanceV1Controller: AppController = async (c) => { - const { conf } = c.var; + const { conf, relay, signal } = c.var; const { host, protocol } = conf.url; - const meta = await getInstanceMetadata(await Storages.db(), c.req.raw.signal); + const meta = await getInstanceMetadata(relay, signal); /** Protocol to use for WebSocket URLs, depending on the protocol of the `LOCAL_DOMAIN`. */ const wsProtocol = protocol === 'http:' ? 'ws:' : 'wss:'; @@ -76,9 +75,9 @@ const instanceV1Controller: AppController = async (c) => { }; const instanceV2Controller: AppController = async (c) => { - const { conf } = c.var; + const { conf, relay, signal } = c.var; const { host, protocol } = conf.url; - const meta = await getInstanceMetadata(await Storages.db(), c.req.raw.signal); + const meta = await getInstanceMetadata(relay, signal); /** Protocol to use for WebSocket URLs, depending on the protocol of the `LOCAL_DOMAIN`. */ const wsProtocol = protocol === 'http:' ? 'ws:' : 'wss:'; @@ -165,7 +164,9 @@ const instanceV2Controller: AppController = async (c) => { }; const instanceDescriptionController: AppController = async (c) => { - const meta = await getInstanceMetadata(await Storages.db(), c.req.raw.signal); + const { relay, signal } = c.var; + + const meta = await getInstanceMetadata(relay, signal); return c.json({ content: meta.about, diff --git a/packages/ditto/controllers/api/notifications.ts b/packages/ditto/controllers/api/notifications.ts index f0435bc4..53edf354 100644 --- a/packages/ditto/controllers/api/notifications.ts +++ b/packages/ditto/controllers/api/notifications.ts @@ -1,10 +1,10 @@ +import { paginated } from '@ditto/mastoapi/pagination'; import { NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { z } from 'zod'; import { AppContext, AppController } from '@/app.ts'; import { DittoPagination } from '@/interfaces/DittoPagination.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { paginated } from '@/utils/api.ts'; import { renderNotification } from '@/views/mastodon/notifications.ts'; /** Set of known notification types across backends. */ @@ -90,9 +90,9 @@ const notificationController: AppController = async (c) => { return c.json({ error: 'Event not found' }, { status: 404 }); } - await hydrateEvents({ events: [event], relay }); + await hydrateEvents({ ...c.var, events: [event] }); - const notification = await renderNotification(event, { viewerPubkey: pubkey }); + const notification = await renderNotification(relay, event, { viewerPubkey: pubkey }); if (!notification) { return c.json({ error: 'Notification not found' }, { status: 404 }); @@ -116,14 +116,14 @@ async function renderNotifications( const events = await relay .query(filters, opts) .then((events) => events.filter((event) => event.pubkey !== pubkey)) - .then((events) => hydrateEvents({ events, relay, signal })); + .then((events) => hydrateEvents({ ...c.var, events })); if (!events.length) { return c.json([]); } const notifications = (await Promise.all(events.map((event) => { - return renderNotification(event, { viewerPubkey: pubkey }); + return renderNotification(relay, event, { viewerPubkey: pubkey }); }))) .filter((notification) => notification && types.has(notification.type)); diff --git a/packages/ditto/controllers/api/oauth.ts b/packages/ditto/controllers/api/oauth.ts index c48963a9..aa4ed125 100644 --- a/packages/ditto/controllers/api/oauth.ts +++ b/packages/ditto/controllers/api/oauth.ts @@ -3,8 +3,7 @@ import { escape } from 'entities'; import { generateSecretKey } from 'nostr-tools'; import { z } from 'zod'; -import { AppController } from '@/app.ts'; -import { Storages } from '@/storages.ts'; +import { AppContext, AppController } from '@/app.ts'; import { nostrNow } from '@/utils.ts'; import { parseBody } from '@/utils/api.ts'; import { aesEncrypt } from '@/utils/aes.ts'; @@ -40,6 +39,7 @@ const createTokenSchema = z.discriminatedUnion('grant_type', [ const createTokenController: AppController = async (c) => { const { conf } = c.var; + const body = await parseBody(c.req.raw); const result = createTokenSchema.safeParse(body); @@ -50,7 +50,7 @@ const createTokenController: AppController = async (c) => { switch (result.data.grant_type) { case 'nostr_bunker': return c.json({ - access_token: await getToken(result.data, conf.seckey), + access_token: await getToken(c, result.data, conf.seckey), token_type: 'Bearer', scope: 'read write follow push', created_at: nostrNow(), @@ -90,6 +90,8 @@ const revokeTokenSchema = z.object({ * https://docs.joinmastodon.org/methods/oauth/#revoke */ const revokeTokenController: AppController = async (c) => { + const { db } = c.var; + const body = await parseBody(c.req.raw); const result = revokeTokenSchema.safeParse(body); @@ -99,10 +101,9 @@ const revokeTokenController: AppController = async (c) => { const { token } = result.data; - const kysely = await Storages.kysely(); const tokenHash = await getTokenHash(token as `token1${string}`); - await kysely + await db.kysely .deleteFrom('auth_tokens') .where('token_hash', '=', tokenHash) .execute(); @@ -111,10 +112,11 @@ const revokeTokenController: AppController = async (c) => { }; async function getToken( + c: AppContext, { pubkey: bunkerPubkey, secret, relays = [] }: { pubkey: string; secret?: string; relays?: string[] }, dittoSeckey: Uint8Array, ): Promise<`token1${string}`> { - const kysely = await Storages.kysely(); + const { db, relay } = c.var; const { token, hash } = await generateToken(); const nip46Seckey = generateSecretKey(); @@ -123,14 +125,14 @@ async function getToken( encryption: 'nip44', pubkey: bunkerPubkey, signer: new NSecSigner(nip46Seckey), - relay: await Storages.db(), // TODO: Use the relays from the request. + relay, timeout: 60_000, }); await signer.connect(secret); const userPubkey = await signer.getPublicKey(); - await kysely.insertInto('auth_tokens').values({ + await db.kysely.insertInto('auth_tokens').values({ token_hash: hash, pubkey: userPubkey, bunker_pubkey: bunkerPubkey, @@ -236,7 +238,7 @@ const oauthAuthorizeController: AppController = async (c) => { const bunker = new URL(bunker_uri); - const token = await getToken({ + const token = await getToken(c, { pubkey: bunker.hostname, secret: bunker.searchParams.get('secret') || undefined, relays: bunker.searchParams.getAll('relay'), diff --git a/packages/ditto/controllers/api/pleroma.ts b/packages/ditto/controllers/api/pleroma.ts index dc4b0c68..ef27696d 100644 --- a/packages/ditto/controllers/api/pleroma.ts +++ b/packages/ditto/controllers/api/pleroma.ts @@ -71,7 +71,7 @@ const pleromaAdminTagController: AppController = async (c) => { const params = pleromaAdminTagSchema.parse(await c.req.json()); for (const nickname of params.nicknames) { - const pubkey = await lookupPubkey(nickname); + const pubkey = await lookupPubkey(nickname, c.var); if (!pubkey) continue; await updateAdminEvent( @@ -104,7 +104,7 @@ const pleromaAdminUntagController: AppController = async (c) => { const params = pleromaAdminTagSchema.parse(await c.req.json()); for (const nickname of params.nicknames) { - const pubkey = await lookupPubkey(nickname); + const pubkey = await lookupPubkey(nickname, c.var); if (!pubkey) continue; await updateAdminEvent( @@ -130,7 +130,7 @@ const pleromaAdminSuggestController: AppController = async (c) => { const { nicknames } = pleromaAdminSuggestSchema.parse(await c.req.json()); for (const nickname of nicknames) { - const pubkey = await lookupPubkey(nickname); + const pubkey = await lookupPubkey(nickname, c.var); if (!pubkey) continue; await updateUser(pubkey, { suggested: true }, c); } @@ -142,7 +142,7 @@ const pleromaAdminUnsuggestController: AppController = async (c) => { const { nicknames } = pleromaAdminSuggestSchema.parse(await c.req.json()); for (const nickname of nicknames) { - const pubkey = await lookupPubkey(nickname); + const pubkey = await lookupPubkey(nickname, c.var); if (!pubkey) continue; await updateUser(pubkey, { suggested: false }, c); } diff --git a/packages/ditto/controllers/api/push.ts b/packages/ditto/controllers/api/push.ts index e613c5f8..c99963aa 100644 --- a/packages/ditto/controllers/api/push.ts +++ b/packages/ditto/controllers/api/push.ts @@ -3,7 +3,6 @@ import { nip19 } from 'nostr-tools'; import { z } from 'zod'; import { AppController } from '@/app.ts'; -import { Storages } from '@/storages.ts'; import { parseBody } from '@/utils/api.ts'; import { getTokenHash } from '@/utils/auth.ts'; @@ -42,7 +41,7 @@ const pushSubscribeSchema = z.object({ }); export const pushSubscribeController: AppController = async (c) => { - const { conf, user } = c.var; + const { conf, db, user } = c.var; const vapidPublicKey = await conf.vapidPublicKey; if (!vapidPublicKey) { @@ -50,8 +49,6 @@ export const pushSubscribeController: AppController = async (c) => { } const accessToken = getAccessToken(c.req.raw); - - const kysely = await Storages.kysely(); const signer = user!.signer; const result = pushSubscribeSchema.safeParse(await parseBody(c.req.raw)); @@ -65,7 +62,7 @@ export const pushSubscribeController: AppController = async (c) => { const pubkey = await signer.getPublicKey(); const tokenHash = await getTokenHash(accessToken); - const { id } = await kysely.transaction().execute(async (trx) => { + const { id } = await db.kysely.transaction().execute(async (trx) => { await trx .deleteFrom('push_subscriptions') .where('token_hash', '=', tokenHash) @@ -97,7 +94,7 @@ export const pushSubscribeController: AppController = async (c) => { }; export const getSubscriptionController: AppController = async (c) => { - const { conf } = c.var; + const { conf, db } = c.var; const vapidPublicKey = await conf.vapidPublicKey; if (!vapidPublicKey) { @@ -106,10 +103,9 @@ export const getSubscriptionController: AppController = async (c) => { const accessToken = getAccessToken(c.req.raw); - const kysely = await Storages.kysely(); const tokenHash = await getTokenHash(accessToken); - const row = await kysely + const row = await db.kysely .selectFrom('push_subscriptions') .selectAll() .where('token_hash', '=', tokenHash) diff --git a/packages/ditto/controllers/api/reactions.ts b/packages/ditto/controllers/api/reactions.ts index a69ba363..74e499d4 100644 --- a/packages/ditto/controllers/api/reactions.ts +++ b/packages/ditto/controllers/api/reactions.ts @@ -31,9 +31,9 @@ const reactionController: AppController = async (c) => { tags: [['e', id], ['p', event.pubkey]], }, c); - await hydrateEvents({ events: [event], relay }); + await hydrateEvents({ ...c.var, events: [event] }); - const status = await renderStatus(event, { viewerPubkey: await user!.signer.getPublicKey() }); + const status = await renderStatus(relay, event, { viewerPubkey: await user!.signer.getPublicKey() }); return c.json(status); }; @@ -76,7 +76,7 @@ const deleteReactionController: AppController = async (c) => { tags, }, c); - const status = renderStatus(event, { viewerPubkey: pubkey }); + const status = renderStatus(relay, event, { viewerPubkey: pubkey }); return c.json(status); }; @@ -99,7 +99,7 @@ const reactionsController: AppController = async (c) => { const events = await relay.query([{ kinds: [7], '#e': [id], limit: 100 }]) .then((events) => events.filter(({ content }) => /^\p{RGI_Emoji}$/v.test(content))) .then((events) => events.filter((event) => !emoji || event.content === emoji)) - .then((events) => hydrateEvents({ events, relay })); + .then((events) => hydrateEvents({ ...c.var, events })); /** Events grouped by emoji. */ const byEmoji = events.reduce((acc, event) => { diff --git a/packages/ditto/controllers/api/reports.ts b/packages/ditto/controllers/api/reports.ts index 7c98ce4e..66dde2e2 100644 --- a/packages/ditto/controllers/api/reports.ts +++ b/packages/ditto/controllers/api/reports.ts @@ -1,8 +1,9 @@ +import { paginated } from '@ditto/mastoapi/pagination'; import { NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; -import { createEvent, paginated, parseBody, updateEventInfo } from '@/utils/api.ts'; +import { createEvent, parseBody, updateEventInfo } from '@/utils/api.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { renderAdminReport } from '@/views/mastodon/reports.ts'; import { renderReport } from '@/views/mastodon/reports.ts'; @@ -18,7 +19,7 @@ const reportSchema = z.object({ /** https://docs.joinmastodon.org/methods/reports/#post */ const reportController: AppController = async (c) => { - const { conf, relay } = c.var; + const { conf } = c.var; const body = await parseBody(c.req.raw); const result = reportSchema.safeParse(body); @@ -49,7 +50,7 @@ const reportController: AppController = async (c) => { tags, }, c); - await hydrateEvents({ events: [event], relay }); + await hydrateEvents({ ...c.var, events: [event] }); return c.json(await renderReport(event)); }; @@ -94,10 +95,10 @@ const adminReportsController: AppController = async (c) => { } const events = await relay.query([{ kinds: [1984], ids: [...ids] }]) - .then((events) => hydrateEvents({ relay, events: events, signal: c.req.raw.signal })); + .then((events) => hydrateEvents({ ...c.var, events })); const reports = await Promise.all( - events.map((event) => renderAdminReport(event, { viewerPubkey })), + events.map((event) => renderAdminReport(relay, event, { viewerPubkey })), ); return paginated(c, orig, reports); @@ -120,9 +121,9 @@ const adminReportController: AppController = async (c) => { return c.json({ error: 'Not found' }, 404); } - await hydrateEvents({ events: [event], relay, signal }); + await hydrateEvents({ ...c.var, events: [event] }); - const report = await renderAdminReport(event, { viewerPubkey: pubkey }); + const report = await renderAdminReport(relay, event, { viewerPubkey: pubkey }); return c.json(report); }; @@ -144,9 +145,9 @@ const adminReportResolveController: AppController = async (c) => { } await updateEventInfo(eventId, { open: false, closed: true }, c); - await hydrateEvents({ events: [event], relay, signal }); + await hydrateEvents({ ...c.var, events: [event] }); - const report = await renderAdminReport(event, { viewerPubkey: pubkey }); + const report = await renderAdminReport(relay, event, { viewerPubkey: pubkey }); return c.json(report); }; @@ -167,9 +168,9 @@ const adminReportReopenController: AppController = async (c) => { } await updateEventInfo(eventId, { open: true, closed: false }, c); - await hydrateEvents({ events: [event], relay, signal }); + await hydrateEvents({ ...c.var, events: [event] }); - const report = await renderAdminReport(event, { viewerPubkey: pubkey }); + const report = await renderAdminReport(relay, event, { viewerPubkey: pubkey }); return c.json(report); }; diff --git a/packages/ditto/controllers/api/search.ts b/packages/ditto/controllers/api/search.ts index 3ce9e0ac..964f0729 100644 --- a/packages/ditto/controllers/api/search.ts +++ b/packages/ditto/controllers/api/search.ts @@ -1,18 +1,17 @@ +import { paginated, paginatedList } from '@ditto/mastoapi/pagination'; import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; import { z } from 'zod'; -import { AppController } from '@/app.ts'; +import { AppContext, AppController } from '@/app.ts'; import { booleanParamSchema } from '@/schema.ts'; -import { Storages } from '@/storages.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { extractIdentifier, lookupPubkey } from '@/utils/lookup.ts'; -import { nip05Cache } from '@/utils/nip05.ts'; +import { lookupNip05 } from '@/utils/nip05.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; import { getFollowedPubkeys } from '@/queries.ts'; import { getPubkeysBySearch } from '@/utils/search.ts'; -import { paginated, paginatedList } from '@/utils/api.ts'; const searchQuerySchema = z.object({ q: z.string().transform(decodeURIComponent), @@ -26,7 +25,7 @@ const searchQuerySchema = z.object({ type SearchQuery = z.infer & { since?: number; until?: number; limit: number }; const searchController: AppController = async (c) => { - const { user, pagination, signal } = c.var; + const { relay, user, pagination, signal } = c.var; const result = searchQuerySchema.safeParse(c.req.query()); const viewerPubkey = await user?.signer.getPublicKey(); @@ -35,12 +34,12 @@ const searchController: AppController = async (c) => { return c.json({ error: 'Bad request', schema: result.error }, 422); } - const event = await lookupEvent({ ...result.data, ...pagination }, signal); + const event = await lookupEvent(c, { ...result.data, ...pagination }); const lookup = extractIdentifier(result.data.q); // Render account from pubkey. if (!event && lookup) { - const pubkey = await lookupPubkey(lookup); + const pubkey = await lookupPubkey(lookup, c.var); return c.json({ accounts: pubkey ? [accountFromPubkey(pubkey)] : [], statuses: [], @@ -54,7 +53,7 @@ const searchController: AppController = async (c) => { events = [event]; } - events.push(...(await searchEvents({ ...result.data, ...pagination, viewerPubkey }, signal))); + events.push(...(await searchEvents(c, { ...result.data, ...pagination, viewerPubkey }, signal))); const [accounts, statuses] = await Promise.all([ Promise.all( @@ -66,7 +65,7 @@ const searchController: AppController = async (c) => { Promise.all( events .filter((event) => event.kind === 1) - .map((event) => renderStatus(event, { viewerPubkey })) + .map((event) => renderStatus(relay, event, { viewerPubkey })) .filter(Boolean), ), ]); @@ -86,16 +85,17 @@ const searchController: AppController = async (c) => { /** Get events for the search params. */ async function searchEvents( + c: AppContext, { q, type, since, until, limit, offset, account_id, viewerPubkey }: SearchQuery & { viewerPubkey?: string }, signal: AbortSignal, ): Promise { + const { relay, db } = c.var; + // Hashtag search is not supported. if (type === 'hashtags') { return Promise.resolve([]); } - const relay = await Storages.db(); - const filter: NostrFilter = { kinds: typeToKinds(type), search: q, @@ -104,12 +104,10 @@ async function searchEvents( limit, }; - const kysely = await Storages.kysely(); - // For account search, use a special index, and prioritize followed accounts. if (type === 'accounts') { - const following = viewerPubkey ? await getFollowedPubkeys(viewerPubkey) : new Set(); - const searchPubkeys = await getPubkeysBySearch(kysely, { q, limit, offset, following }); + const following = viewerPubkey ? await getFollowedPubkeys(relay, viewerPubkey) : new Set(); + const searchPubkeys = await getPubkeysBySearch(db.kysely, { q, limit, offset, following }); filter.authors = [...searchPubkeys]; filter.search = undefined; @@ -123,7 +121,7 @@ async function searchEvents( // Query the events. let events = await relay .query([filter], { signal }) - .then((events) => hydrateEvents({ events, relay, signal })); + .then((events) => hydrateEvents({ ...c.var, events })); // When using an authors filter, return the events in the same order as the filter. if (filter.authors) { @@ -148,17 +146,17 @@ function typeToKinds(type: SearchQuery['type']): number[] { } /** Resolve a searched value into an event, if applicable. */ -async function lookupEvent(query: SearchQuery, signal: AbortSignal): Promise { - const filters = await getLookupFilters(query, signal); - const relay = await Storages.db(); +async function lookupEvent(c: AppContext, query: SearchQuery): Promise { + const { relay, signal } = c.var; + const filters = await getLookupFilters(c, query); - return relay.query(filters, { limit: 1, signal }) - .then((events) => hydrateEvents({ events, relay, signal })) + return relay.query(filters, { signal }) + .then((events) => hydrateEvents({ ...c.var, events })) .then(([event]) => event); } /** Get filters to lookup the input value. */ -async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: AbortSignal): Promise { +async function getLookupFilters(c: AppContext, { q, type, resolve }: SearchQuery): Promise { const accounts = !type || type === 'accounts'; const statuses = !type || type === 'statuses'; @@ -199,7 +197,7 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort } try { - const { pubkey } = await nip05Cache.fetch(lookup, { signal }); + const { pubkey } = await lookupNip05(lookup, c.var); if (pubkey) { return [{ kinds: [0], authors: [pubkey] }]; } diff --git a/packages/ditto/controllers/api/statuses.ts b/packages/ditto/controllers/api/statuses.ts index 252882ff..4bf2ed23 100644 --- a/packages/ditto/controllers/api/statuses.ts +++ b/packages/ditto/controllers/api/statuses.ts @@ -1,4 +1,5 @@ import { HTTPException } from '@hono/hono/http-exception'; +import { paginated, paginatedList } from '@ditto/mastoapi/pagination'; import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; import 'linkify-plugin-hashtag'; import linkify from 'linkifyjs'; @@ -15,7 +16,7 @@ import { asyncReplaceAll } from '@/utils/text.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; import { languageSchema } from '@/schema.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { assertAuthenticated, createEvent, paginated, paginatedList, parseBody, updateListEvent } from '@/utils/api.ts'; +import { assertAuthenticated, createEvent, parseBody, updateListEvent } from '@/utils/api.ts'; import { getInvoice, getLnurl } from '@/utils/lnurl.ts'; import { purifyEvent } from '@/utils/purify.ts'; import { getZapSplits } from '@/utils/zap-split.ts'; @@ -46,10 +47,10 @@ const createStatusSchema = z.object({ ); const statusController: AppController = async (c) => { - const { user, signal } = c.var; + const { relay, user } = c.var; const id = c.req.param('id'); - const event = await getEvent(id, { signal }); + const event = await getEvent(id, c.var); if (event?.author) { assertAuthenticated(c, event.author); @@ -57,7 +58,7 @@ const statusController: AppController = async (c) => { if (event) { const viewerPubkey = await user?.signer.getPublicKey(); - const status = await renderStatus(event, { viewerPubkey }); + const status = await renderStatus(relay, event, { viewerPubkey }); return c.json(status); } @@ -65,7 +66,7 @@ const statusController: AppController = async (c) => { }; const createStatusController: AppController = async (c) => { - const { conf, relay, user, signal } = c.var; + const { conf, relay, user } = c.var; const body = await parseBody(c.req.raw); const result = createStatusSchema.safeParse(body); @@ -153,7 +154,7 @@ const createStatusController: AppController = async (c) => { data.status ?? '', /(? { - const pubkey = await lookupPubkey(username); + const pubkey = await lookupPubkey(username, c.var); if (!pubkey) return match; // Content addressing (default) @@ -171,7 +172,7 @@ const createStatusController: AppController = async (c) => { // Explicit addressing for (const to of data.to ?? []) { - const pubkey = await lookupPubkey(to); + const pubkey = await lookupPubkey(to, c.var); if (pubkey) { pubkeys.add(pubkey); } @@ -191,7 +192,7 @@ const createStatusController: AppController = async (c) => { } const pubkey = await user!.signer.getPublicKey(); - const author = pubkey ? await getAuthor(pubkey) : undefined; + const author = pubkey ? await getAuthor(pubkey, c.var) : undefined; if (conf.zapSplitsEnabled) { const meta = n.json().pipe(n.metadata()).catch({}).parse(author?.content); @@ -254,22 +255,18 @@ const createStatusController: AppController = async (c) => { }, c); if (data.quote_id) { - await hydrateEvents({ - events: [event], - relay, - signal, - }); + await hydrateEvents({ ...c.var, events: [event] }); } - return c.json(await renderStatus({ ...event, author }, { viewerPubkey: author?.pubkey })); + return c.json(await renderStatus(relay, { ...event, author }, { viewerPubkey: author?.pubkey })); }; const deleteStatusController: AppController = async (c) => { - const { conf, user, signal } = c.var; + const { conf, relay, user } = c.var; const id = c.req.param('id'); const pubkey = await user?.signer.getPublicKey(); - const event = await getEvent(id, { signal }); + const event = await getEvent(id, c.var); if (event) { if (event.pubkey === pubkey) { @@ -278,8 +275,8 @@ const deleteStatusController: AppController = async (c) => { tags: [['e', id, conf.relay, '', pubkey]], }, c); - const author = await getAuthor(event.pubkey); - return c.json(await renderStatus({ ...event, author }, { viewerPubkey: pubkey })); + const author = await getAuthor(event.pubkey, c.var); + return c.json(await renderStatus(relay, { ...event, author }, { viewerPubkey: pubkey })); } else { return c.json({ error: 'Unauthorized' }, 403); } @@ -297,7 +294,7 @@ const contextController: AppController = async (c) => { async function renderStatuses(events: NostrEvent[]) { const statuses = await Promise.all( - events.map((event) => renderStatus(event, { viewerPubkey })), + events.map((event) => renderStatus(relay, event, { viewerPubkey })), ); return statuses.filter(Boolean); } @@ -308,11 +305,7 @@ const contextController: AppController = async (c) => { getDescendants(relay, event), ]); - await hydrateEvents({ - events: [...ancestorEvents, ...descendantEvents], - signal: c.req.raw.signal, - relay, - }); + await hydrateEvents({ ...c.var, events: [...ancestorEvents, ...descendantEvents] }); const [ancestors, descendants] = await Promise.all([ renderStatuses(ancestorEvents), @@ -341,9 +334,9 @@ const favouriteController: AppController = async (c) => { ], }, c); - await hydrateEvents({ events: [target], relay }); + await hydrateEvents({ ...c.var, events: [target] }); - const status = await renderStatus(target, { viewerPubkey: await user?.signer.getPublicKey() }); + const status = await renderStatus(relay, target, { viewerPubkey: await user?.signer.getPublicKey() }); if (status) { status.favourited = true; @@ -367,10 +360,10 @@ const favouritedByController: AppController = (c) => { /** https://docs.joinmastodon.org/methods/statuses/#boost */ const reblogStatusController: AppController = async (c) => { - const { conf, relay, user, signal } = c.var; + const { conf, relay, user } = c.var; const eventId = c.req.param('id'); - const event = await getEvent(eventId); + const event = await getEvent(eventId, c.var); if (!event) { return c.json({ error: 'Event not found.' }, 404); @@ -384,13 +377,9 @@ const reblogStatusController: AppController = async (c) => { ], }, c); - await hydrateEvents({ - events: [reblogEvent], - relay, - signal: signal, - }); + await hydrateEvents({ ...c.var, events: [reblogEvent] }); - const status = await renderReblog(reblogEvent, { viewerPubkey: await user?.signer.getPublicKey() }); + const status = await renderReblog(relay, reblogEvent, { viewerPubkey: await user?.signer.getPublicKey() }); return c.json(status); }; @@ -420,7 +409,7 @@ const unreblogStatusController: AppController = async (c) => { tags: [['e', repostEvent.id, conf.relay, '', repostEvent.pubkey]], }, c); - return c.json(await renderStatus(event, { viewerPubkey: pubkey })); + return c.json(await renderStatus(relay, event, { viewerPubkey: pubkey })); }; const rebloggedByController: AppController = (c) => { @@ -441,12 +430,12 @@ const quotesController: AppController = async (c) => { const quotes = await relay .query([{ kinds: [1, 20], '#q': [event.id], ...pagination }]) - .then((events) => hydrateEvents({ events, relay })); + .then((events) => hydrateEvents({ ...c.var, events })); const viewerPubkey = await user?.signer.getPublicKey(); const statuses = await Promise.all( - quotes.map((event) => renderStatus(event, { viewerPubkey })), + quotes.map((event) => renderStatus(relay, event, { viewerPubkey })), ); if (!statuses.length) { @@ -458,11 +447,11 @@ const quotesController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/statuses/#bookmark */ const bookmarkController: AppController = async (c) => { - const { conf, user } = c.var; + const { conf, relay, user } = c.var; const pubkey = await user!.signer.getPublicKey(); const eventId = c.req.param('id'); - const event = await getEvent(eventId); + const event = await getEvent(eventId, c.var); if (event) { await updateListEvent( @@ -471,7 +460,7 @@ const bookmarkController: AppController = async (c) => { c, ); - const status = await renderStatus(event, { viewerPubkey: pubkey }); + const status = await renderStatus(relay, event, { viewerPubkey: pubkey }); if (status) { status.bookmarked = true; } @@ -483,12 +472,12 @@ const bookmarkController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/statuses/#unbookmark */ const unbookmarkController: AppController = async (c) => { - const { conf, user } = c.var; + const { conf, relay, user } = c.var; const pubkey = await user!.signer.getPublicKey(); const eventId = c.req.param('id'); - const event = await getEvent(eventId); + const event = await getEvent(eventId, c.var); if (event) { await updateListEvent( @@ -497,7 +486,7 @@ const unbookmarkController: AppController = async (c) => { c, ); - const status = await renderStatus(event, { viewerPubkey: pubkey }); + const status = await renderStatus(relay, event, { viewerPubkey: pubkey }); if (status) { status.bookmarked = false; } @@ -509,12 +498,12 @@ const unbookmarkController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/statuses/#pin */ const pinController: AppController = async (c) => { - const { conf, user } = c.var; + const { conf, relay, user } = c.var; const pubkey = await user!.signer.getPublicKey(); const eventId = c.req.param('id'); - const event = await getEvent(eventId); + const event = await getEvent(eventId, c.var); if (event) { await updateListEvent( @@ -523,7 +512,7 @@ const pinController: AppController = async (c) => { c, ); - const status = await renderStatus(event, { viewerPubkey: pubkey }); + const status = await renderStatus(relay, event, { viewerPubkey: pubkey }); if (status) { status.pinned = true; } @@ -535,15 +524,12 @@ const pinController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/statuses/#unpin */ const unpinController: AppController = async (c) => { - const { conf, user, signal } = c.var; + const { conf, relay, user } = c.var; const pubkey = await user!.signer.getPublicKey(); const eventId = c.req.param('id'); - const event = await getEvent(eventId, { - kind: 1, - signal, - }); + const event = await getEvent(eventId, c.var); if (event) { await updateListEvent( @@ -552,7 +538,7 @@ const unpinController: AppController = async (c) => { c, ); - const status = await renderStatus(event, { viewerPubkey: pubkey }); + const status = await renderStatus(relay, event, { viewerPubkey: pubkey }); if (status) { status.pinned = false; } @@ -586,7 +572,7 @@ const zapController: AppController = async (c) => { let lnurl: undefined | string; if (status_id) { - target = await getEvent(status_id, { kind: 1, signal }); + target = await getEvent(status_id, c.var); const author = target?.author; const meta = n.json().pipe(n.metadata()).catch({}).parse(author?.content); lnurl = getLnurl(meta); diff --git a/packages/ditto/controllers/api/streaming.ts b/packages/ditto/controllers/api/streaming.ts index cdd8dae3..01a829df 100644 --- a/packages/ditto/controllers/api/streaming.ts +++ b/packages/ditto/controllers/api/streaming.ts @@ -5,7 +5,7 @@ import { streamingServerMessagesCounter, } from '@ditto/metrics'; import TTLCache from '@isaacs/ttlcache'; -import { NostrEvent, NostrFilter } from '@nostrify/nostrify'; +import { NostrEvent, NostrFilter, NStore } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import { z } from 'zod'; @@ -111,7 +111,7 @@ const streamingController: AppController = async (c) => { } } - await hydrateEvents({ events: [event], relay, signal: AbortSignal.timeout(1000) }); + await hydrateEvents({ ...c.var, events: [event] }); const result = await render(event); @@ -130,17 +130,17 @@ const streamingController: AppController = async (c) => { streamingConnectionsGauge.set(connections.size); if (!stream) return; - const topicFilter = await topicToFilter(stream, c.req.query(), pubkey, conf.url.host); + const topicFilter = await topicToFilter(relay, stream, c.req.query(), pubkey, conf.url.host); if (topicFilter) { sub(topicFilter, async (event) => { let payload: object | undefined; if (event.kind === 1) { - payload = await renderStatus(event, { viewerPubkey: pubkey }); + payload = await renderStatus(relay, event, { viewerPubkey: pubkey }); } if (event.kind === 6) { - payload = await renderReblog(event, { viewerPubkey: pubkey }); + payload = await renderReblog(relay, event, { viewerPubkey: pubkey }); } if (payload) { @@ -156,13 +156,13 @@ const streamingController: AppController = async (c) => { if (['user', 'user:notification'].includes(stream) && pubkey) { sub({ '#p': [pubkey], limit: 0 }, async (event) => { if (event.pubkey === pubkey) return; // skip own events - const payload = await renderNotification(event, { viewerPubkey: pubkey }); + const payload = await renderNotification(relay, event, { viewerPubkey: pubkey }); if (payload) { return { event: 'notification', payload: JSON.stringify(payload), stream: [stream], - }; + } satisfies StreamingEvent; } }); return; @@ -198,6 +198,7 @@ const streamingController: AppController = async (c) => { }; async function topicToFilter( + relay: NStore, topic: Stream, query: Record, pubkey: string | undefined, @@ -218,7 +219,7 @@ async function topicToFilter( // HACK: this puts the user's entire contacts list into RAM, // and then calls `matchFilters` over it. Refreshing the page // is required after following a new user. - return pubkey ? { kinds: [1, 6, 20], authors: [...await getFeedPubkeys(pubkey)], limit: 0 } : undefined; + return pubkey ? { kinds: [1, 6, 20], authors: [...await getFeedPubkeys(relay, pubkey)], limit: 0 } : undefined; } } diff --git a/packages/ditto/controllers/api/suggestions.ts b/packages/ditto/controllers/api/suggestions.ts index 3af4f678..39cbd235 100644 --- a/packages/ditto/controllers/api/suggestions.ts +++ b/packages/ditto/controllers/api/suggestions.ts @@ -1,10 +1,10 @@ +import { paginated, paginatedList } from '@ditto/mastoapi/pagination'; import { NostrFilter } from '@nostrify/nostrify'; import { matchFilter } from 'nostr-tools'; import { AppContext, AppController } from '@/app.ts'; import { paginationSchema } from '@/schemas/pagination.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { paginated, paginatedList } from '@/utils/api.ts'; import { getTagSet } from '@/utils/tags.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; @@ -82,7 +82,7 @@ async function renderV2Suggestions(c: AppContext, params: { offset: number; limi [{ kinds: [0], authors, limit: authors.length }], { signal }, ) - .then((events) => hydrateEvents({ events, relay, signal })); + .then((events) => hydrateEvents({ ...c.var, events })); return Promise.all(authors.map(async (pubkey) => { const profile = profiles.find((event) => event.pubkey === pubkey); @@ -115,7 +115,7 @@ export const localSuggestionsController: AppController = async (c) => { [{ kinds: [0], authors: [...pubkeys], search: `domain:${conf.url.host}`, ...pagination }], { signal }, ) - .then((events) => hydrateEvents({ relay, events, signal })); + .then((events) => hydrateEvents({ ...c.var, events })); const suggestions = [...pubkeys].map((pubkey) => { const profile = profiles.find((event) => event.pubkey === pubkey); diff --git a/packages/ditto/controllers/api/timelines.ts b/packages/ditto/controllers/api/timelines.ts index 5ef83856..820ebd75 100644 --- a/packages/ditto/controllers/api/timelines.ts +++ b/packages/ditto/controllers/api/timelines.ts @@ -1,3 +1,4 @@ +import { paginated } from '@ditto/mastoapi/pagination'; import { NostrFilter } from '@nostrify/nostrify'; import { z } from 'zod'; @@ -5,7 +6,6 @@ import { type AppContext, type AppController } from '@/app.ts'; import { getFeedPubkeys } from '@/queries.ts'; import { booleanParamSchema, languageSchema } from '@/schema.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { paginated } from '@/utils/api.ts'; import { getTagSet } from '@/utils/tags.ts'; import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts'; @@ -15,7 +15,7 @@ const homeQuerySchema = z.object({ }); const homeTimelineController: AppController = async (c) => { - const { user, pagination } = c.var; + const { relay, user, pagination } = c.var; const pubkey = await user?.signer.getPublicKey()!; const result = homeQuerySchema.safeParse(c.req.query()); @@ -25,7 +25,7 @@ const homeTimelineController: AppController = async (c) => { const { exclude_replies, only_media } = result.data; - const authors = [...await getFeedPubkeys(pubkey)]; + const authors = [...await getFeedPubkeys(relay, pubkey)]; const filter: NostrFilter = { authors, kinds: [1, 6, 20], ...pagination }; const search: string[] = []; @@ -110,7 +110,7 @@ async function renderStatuses(c: AppContext, filters: NostrFilter[]) { const events = await relay .query(filters, opts) - .then((events) => hydrateEvents({ events, relay, signal })); + .then((events) => hydrateEvents({ ...c.var, events })); if (!events.length) { return c.json([]); @@ -120,9 +120,9 @@ async function renderStatuses(c: AppContext, filters: NostrFilter[]) { const statuses = (await Promise.all(events.map((event) => { if (event.kind === 6) { - return renderReblog(event, { viewerPubkey }); + return renderReblog(relay, event, { viewerPubkey }); } - return renderStatus(event, { viewerPubkey }); + return renderStatus(relay, event, { viewerPubkey }); }))).filter(Boolean); if (!statuses.length) { diff --git a/packages/ditto/controllers/api/translate.ts b/packages/ditto/controllers/api/translate.ts index f9ff4dcd..7a0f7731 100644 --- a/packages/ditto/controllers/api/translate.ts +++ b/packages/ditto/controllers/api/translate.ts @@ -17,7 +17,7 @@ const translateSchema = z.object({ }); const translateController: AppController = async (c) => { - const { user, signal } = c.var; + const { relay, user, signal } = c.var; const result = translateSchema.safeParse(await parseBody(c.req.raw)); @@ -34,7 +34,7 @@ const translateController: AppController = async (c) => { const id = c.req.param('id'); - const event = await getEvent(id, { signal }); + const event = await getEvent(id, c.var); if (!event) { return c.json({ error: 'Record not found' }, 400); } @@ -45,7 +45,7 @@ const translateController: AppController = async (c) => { return c.json({ error: 'Source and target languages are the same. No translation needed.' }, 400); } - const status = await renderStatus(event, { viewerPubkey }); + const status = await renderStatus(relay, event, { viewerPubkey }); if (!status?.content) { return c.json({ error: 'Bad request.', schema: result.error }, 400); } diff --git a/packages/ditto/controllers/api/trends.ts b/packages/ditto/controllers/api/trends.ts index 5af88557..ce35601f 100644 --- a/packages/ditto/controllers/api/trends.ts +++ b/packages/ditto/controllers/api/trends.ts @@ -1,34 +1,45 @@ import { type DittoConf } from '@ditto/conf'; +import { paginated } from '@ditto/mastoapi/pagination'; import { NostrEvent, NostrFilter, NStore } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import { z } from 'zod'; import { AppController } from '@/app.ts'; -import { Conf } from '@/config.ts'; import { paginationSchema } from '@/schemas/pagination.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { Storages } from '@/storages.ts'; import { generateDateRange, Time } from '@/utils/time.ts'; -import { unfurlCardCached } from '@/utils/unfurl.ts'; -import { paginated } from '@/utils/api.ts'; +import { PreviewCard, unfurlCardCached } from '@/utils/unfurl.ts'; import { errorJson } from '@/utils/log.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; -let trendingHashtagsCache = getTrendingHashtags(Conf).catch((e: unknown) => { - logi({ - level: 'error', - ns: 'ditto.trends.api', - type: 'tags', - msg: 'Failed to get trending hashtags', - error: errorJson(e), - }); - return Promise.resolve([]); +interface TrendHistory { + day: string; + accounts: string; + uses: string; +} + +interface TrendingHashtag { + name: string; + url: string; + history: TrendHistory[]; +} + +interface TrendingLink extends PreviewCard { + history: TrendHistory[]; +} + +const trendingTagsQuerySchema = z.object({ + limit: z.coerce.number().catch(10).transform((value) => Math.min(Math.max(value, 0), 20)), + offset: z.number().nonnegative().catch(0), }); -Deno.cron('update trending hashtags cache', '35 * * * *', async () => { +const trendingTagsController: AppController = async (c) => { + const { conf, relay } = c.var; + const { limit, offset } = trendingTagsQuerySchema.parse(c.req.query()); + try { - const trends = await getTrendingHashtags(Conf); - trendingHashtagsCache = Promise.resolve(trends); + const trends = await getTrendingHashtags(conf, relay); + return c.json(trends.slice(offset, offset + limit)); } catch (e) { logi({ level: 'error', @@ -37,22 +48,11 @@ Deno.cron('update trending hashtags cache', '35 * * * *', async () => { msg: 'Failed to get trending hashtags', error: errorJson(e), }); + return c.json([]); } -}); - -const trendingTagsQuerySchema = z.object({ - limit: z.coerce.number().catch(10).transform((value) => Math.min(Math.max(value, 0), 20)), - offset: z.number().nonnegative().catch(0), -}); - -const trendingTagsController: AppController = async (c) => { - const { limit, offset } = trendingTagsQuerySchema.parse(c.req.query()); - const trends = await trendingHashtagsCache; - return c.json(trends.slice(offset, offset + limit)); }; -async function getTrendingHashtags(conf: DittoConf) { - const relay = await Storages.db(); +async function getTrendingHashtags(conf: DittoConf, relay: NStore): Promise { const trends = await getTrendingTags(relay, 't', await conf.signer.getPublicKey()); return trends.map((trend) => { @@ -72,21 +72,12 @@ async function getTrendingHashtags(conf: DittoConf) { }); } -let trendingLinksCache = getTrendingLinks(Conf).catch((e: unknown) => { - logi({ - level: 'error', - ns: 'ditto.trends.api', - type: 'links', - msg: 'Failed to get trending links', - error: errorJson(e), - }); - return Promise.resolve([]); -}); - -Deno.cron('update trending links cache', '50 * * * *', async () => { +const trendingLinksController: AppController = async (c) => { + const { conf, relay } = c.var; + const { limit, offset } = trendingTagsQuerySchema.parse(c.req.query()); try { - const trends = await getTrendingLinks(Conf); - trendingLinksCache = Promise.resolve(trends); + const trends = await getTrendingLinks(conf, relay); + return c.json(trends.slice(offset, offset + limit)); } catch (e) { logi({ level: 'error', @@ -95,17 +86,11 @@ Deno.cron('update trending links cache', '50 * * * *', async () => { msg: 'Failed to get trending links', error: errorJson(e), }); + return c.json([]); } -}); - -const trendingLinksController: AppController = async (c) => { - const { limit, offset } = trendingTagsQuerySchema.parse(c.req.query()); - const trends = await trendingLinksCache; - return c.json(trends.slice(offset, offset + limit)); }; -async function getTrendingLinks(conf: DittoConf) { - const relay = await Storages.db(); +async function getTrendingLinks(conf: DittoConf, relay: NStore): Promise { const trends = await getTrendingTags(relay, 'r', await conf.signer.getPublicKey()); return Promise.all(trends.map(async (trend) => { @@ -162,7 +147,7 @@ const trendingStatusesController: AppController = async (c) => { } const results = await relay.query([{ kinds: [1, 20], ids }]) - .then((events) => hydrateEvents({ events, relay })); + .then((events) => hydrateEvents({ ...c.var, events })); // Sort events in the order they appear in the label. const events = ids @@ -170,7 +155,7 @@ const trendingStatusesController: AppController = async (c) => { .filter((event): event is NostrEvent => !!event); const statuses = await Promise.all( - events.map((event) => renderStatus(event, {})), + events.map((event) => renderStatus(relay, event, {})), ); return paginated(c, results, statuses); diff --git a/packages/ditto/controllers/frontend.ts b/packages/ditto/controllers/frontend.ts index d19a20cb..ad98a9aa 100644 --- a/packages/ditto/controllers/frontend.ts +++ b/packages/ditto/controllers/frontend.ts @@ -1,6 +1,6 @@ import { logi } from '@soapbox/logi'; -import { AppMiddleware } from '@/app.ts'; +import { AppContext, AppMiddleware } from '@/app.ts'; import { getPathParams, MetadataEntities } from '@/utils/og-metadata.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; import { errorJson } from '@/utils/log.ts'; @@ -9,14 +9,11 @@ import { renderMetadata } from '@/views/meta.ts'; import { getAuthor, getEvent } from '@/queries.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; import { renderAccount } from '@/views/mastodon/accounts.ts'; -import { NStore } from '@nostrify/nostrify'; /** Placeholder to find & replace with metadata. */ const META_PLACEHOLDER = '' as const; export const frontendController: AppMiddleware = async (c) => { - const { relay } = c.var; - c.header('Cache-Control', 'max-age=86400, s-maxage=30, public, stale-if-error=604800'); try { @@ -25,7 +22,7 @@ export const frontendController: AppMiddleware = async (c) => { if (content.includes(META_PLACEHOLDER)) { const params = getPathParams(c.req.path); try { - const entities = await getEntities(relay, params ?? {}); + const entities = await getEntities(c, params ?? {}); const meta = renderMetadata(c.req.url, entities); return c.html(content.replace(META_PLACEHOLDER, meta)); } catch (e) { @@ -39,25 +36,27 @@ export const frontendController: AppMiddleware = async (c) => { } }; -async function getEntities(relay: NStore, params: { acct?: string; statusId?: string }): Promise { +async function getEntities(c: AppContext, params: { acct?: string; statusId?: string }): Promise { + const { relay } = c.var; + const entities: MetadataEntities = { instance: await getInstanceMetadata(relay), }; if (params.statusId) { - const event = await getEvent(params.statusId, { kind: 1 }); + const event = await getEvent(params.statusId, c.var); if (event) { - entities.status = await renderStatus(event, {}); + entities.status = await renderStatus(relay, event, {}); entities.account = entities.status?.account; } return entities; } if (params.acct) { - const pubkey = await lookupPubkey(params.acct.replace(/^@/, '')); - const event = pubkey ? await getAuthor(pubkey) : undefined; + const pubkey = await lookupPubkey(params.acct.replace(/^@/, ''), c.var); + const event = pubkey ? await getAuthor(pubkey, c.var) : undefined; if (event) { - entities.account = await renderAccount(event); + entities.account = renderAccount(event); } } diff --git a/packages/ditto/controllers/metrics.ts b/packages/ditto/controllers/metrics.ts index 32a8783d..be3ef624 100644 --- a/packages/ditto/controllers/metrics.ts +++ b/packages/ditto/controllers/metrics.ts @@ -1,31 +1,16 @@ -import { - dbAvailableConnectionsGauge, - dbPoolSizeGauge, - relayPoolRelaysSizeGauge, - relayPoolSubscriptionsSizeGauge, -} from '@ditto/metrics'; +import { dbAvailableConnectionsGauge, dbPoolSizeGauge } from '@ditto/metrics'; import { register } from 'prom-client'; import { AppController } from '@/app.ts'; -import { Storages } from '@/storages.ts'; /** Prometheus/OpenMetrics controller. */ export const metricsController: AppController = async (c) => { - const db = await Storages.database(); - const pool = await Storages.client(); + const { db } = c.var; // Update some metrics at request time. dbPoolSizeGauge.set(db.poolSize); dbAvailableConnectionsGauge.set(db.availableConnections); - relayPoolRelaysSizeGauge.reset(); - relayPoolSubscriptionsSizeGauge.reset(); - - for (const relay of pool.relays.values()) { - relayPoolRelaysSizeGauge.inc({ ready_state: relay.socket.readyState }); - relayPoolSubscriptionsSizeGauge.inc(relay.subscriptions.length); - } - // Serve the metrics. const metrics = await register.metrics(); diff --git a/packages/ditto/controllers/nostr/relay.ts b/packages/ditto/controllers/nostr/relay.ts index 191aed36..6b56743c 100644 --- a/packages/ditto/controllers/nostr/relay.ts +++ b/packages/ditto/controllers/nostr/relay.ts @@ -16,7 +16,6 @@ import { import { AppController } from '@/app.ts'; import { relayInfoController } from '@/controllers/nostr/relay-info.ts'; -import * as pipeline from '@/pipeline.ts'; import { RelayError } from '@/RelayError.ts'; import { type DittoPgStore } from '@/storages/DittoPgStore.ts'; import { errorJson } from '@/utils/log.ts'; @@ -159,7 +158,7 @@ function connectStream(conf: DittoConf, relay: DittoPgStore, socket: WebSocket, try { // This will store it (if eligible) and run other side-effects. - await pipeline.handleEvent(purifyEvent(event), { source: 'relay', signal: AbortSignal.timeout(1000) }); + await relay.event(purifyEvent(event), { signal: AbortSignal.timeout(1000) }); send(['OK', event.id, true, '']); } catch (e) { if (e instanceof RelayError) { diff --git a/packages/ditto/controllers/well-known/nostr.ts b/packages/ditto/controllers/well-known/nostr.ts index ee442788..7c27aa70 100644 --- a/packages/ditto/controllers/well-known/nostr.ts +++ b/packages/ditto/controllers/well-known/nostr.ts @@ -12,8 +12,6 @@ const emptyResult: NostrJson = { names: {}, relays: {} }; * https://github.com/nostr-protocol/nips/blob/master/05.md */ const nostrController: AppController = async (c) => { - const { relay } = c.var; - // If there are no query parameters, this will always return an empty result. if (!Object.entries(c.req.queries()).length) { c.header('Cache-Control', 'max-age=31536000, public, immutable, stale-while-revalidate=86400'); @@ -22,7 +20,7 @@ const nostrController: AppController = async (c) => { const result = nameSchema.safeParse(c.req.query('name')); const name = result.success ? result.data : undefined; - const pointer = name ? await localNip05Lookup(relay, name) : undefined; + const pointer = name ? await localNip05Lookup(name, c.var) : undefined; if (!name || !pointer) { // Not found, cache for 5 minutes. diff --git a/packages/ditto/cron.ts b/packages/ditto/cron.ts index ba8a18d5..bcbbffb0 100644 --- a/packages/ditto/cron.ts +++ b/packages/ditto/cron.ts @@ -1,7 +1,7 @@ import { sql } from 'kysely'; -import { Storages } from '@/storages.ts'; import { + type TrendsCtx, updateTrendingEvents, updateTrendingHashtags, updateTrendingLinks, @@ -10,15 +10,15 @@ import { } from '@/trends.ts'; /** Start cron jobs for the application. */ -export function cron() { - Deno.cron('update trending pubkeys', '0 * * * *', updateTrendingPubkeys); - Deno.cron('update trending zapped events', '7 * * * *', updateTrendingZappedEvents); - Deno.cron('update trending events', '15 * * * *', updateTrendingEvents); - Deno.cron('update trending hashtags', '30 * * * *', updateTrendingHashtags); - Deno.cron('update trending links', '45 * * * *', updateTrendingLinks); +export function cron(ctx: TrendsCtx) { + Deno.cron('update trending pubkeys', '0 * * * *', () => updateTrendingPubkeys(ctx)); + Deno.cron('update trending zapped events', '7 * * * *', () => updateTrendingZappedEvents(ctx)); + Deno.cron('update trending events', '15 * * * *', () => updateTrendingEvents(ctx)); + Deno.cron('update trending hashtags', '30 * * * *', () => updateTrendingHashtags(ctx)); + Deno.cron('update trending links', '45 * * * *', () => updateTrendingLinks(ctx)); Deno.cron('refresh top authors', '20 * * * *', async () => { - const kysely = await Storages.kysely(); + const { kysely } = ctx.db; await sql`refresh materialized view top_authors`.execute(kysely); }); } diff --git a/packages/ditto/firehose.ts b/packages/ditto/firehose.ts index e967e1f2..f6f3d27f 100644 --- a/packages/ditto/firehose.ts +++ b/packages/ditto/firehose.ts @@ -1,32 +1,38 @@ import { firehoseEventsCounter } from '@ditto/metrics'; import { Semaphore } from '@core/asyncutil'; +import { NRelay, NStore } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; -import { Conf } from '@/config.ts'; -import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; -import * as pipeline from '@/pipeline.ts'; - -const sem = new Semaphore(Conf.firehoseConcurrency); +interface FirehoseOpts { + pool: NRelay; + store: NStore; + concurrency: number; + kinds: number[]; + timeout?: number; +} /** * This function watches events on all known relays and performs * side-effects based on them, such as trending hashtag tracking * and storing events for notifications and the home feed. */ -export async function startFirehose(): Promise { - const store = await Storages.client(); +export async function startFirehose(opts: FirehoseOpts): Promise { + const { pool, store, kinds, concurrency, timeout = 5000 } = opts; - for await (const msg of store.req([{ kinds: Conf.firehoseKinds, limit: 0, since: nostrNow() }])) { + const sem = new Semaphore(concurrency); + + for await (const msg of pool.req([{ kinds, limit: 0, since: nostrNow() }])) { if (msg[0] === 'EVENT') { const event = msg[2]; + logi({ level: 'debug', ns: 'ditto.event', source: 'firehose', id: event.id, kind: event.kind }); firehoseEventsCounter.inc({ kind: event.kind }); sem.lock(async () => { try { - await pipeline.handleEvent(event, { source: 'firehose', signal: AbortSignal.timeout(5000) }); + await store.event(event, { signal: AbortSignal.timeout(timeout) }); } catch { // Ignore } diff --git a/packages/ditto/middleware/cspMiddleware.ts b/packages/ditto/middleware/cspMiddleware.ts index e16829cc..8e890101 100644 --- a/packages/ditto/middleware/cspMiddleware.ts +++ b/packages/ditto/middleware/cspMiddleware.ts @@ -1,17 +1,15 @@ import { AppMiddleware } from '@/app.ts'; import { PleromaConfigDB } from '@/utils/PleromaConfigDB.ts'; -import { Storages } from '@/storages.ts'; import { getPleromaConfigs } from '@/utils/pleroma.ts'; -let configDBCache: Promise | undefined; - export const cspMiddleware = (): AppMiddleware => { + let configDBCache: Promise | undefined; + return async (c, next) => { - const { conf } = c.var; - const store = await Storages.db(); + const { conf, relay } = c.var; if (!configDBCache) { - configDBCache = getPleromaConfigs(store); + configDBCache = getPleromaConfigs(relay); } const { host, protocol, origin } = conf.url; diff --git a/packages/ditto/pipeline.ts b/packages/ditto/pipeline.ts deleted file mode 100644 index 2ae55b96..00000000 --- a/packages/ditto/pipeline.ts +++ /dev/null @@ -1,368 +0,0 @@ -import { DittoTables } from '@ditto/db'; -import { pipelineEventsCounter, policyEventsCounter, webPushNotificationsCounter } from '@ditto/metrics'; -import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify'; -import { logi } from '@soapbox/logi'; -import { Kysely, UpdateObject } from 'kysely'; -import tldts from 'tldts'; -import { z } from 'zod'; - -import { pipelineEncounters } from '@/caches/pipelineEncounters.ts'; -import { Conf } from '@/config.ts'; -import { DittoPush } from '@/DittoPush.ts'; -import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { RelayError } from '@/RelayError.ts'; -import { hydrateEvents } from '@/storages/hydrate.ts'; -import { Storages } from '@/storages.ts'; -import { eventAge, Time } from '@/utils.ts'; -import { getAmount } from '@/utils/bolt11.ts'; -import { faviconCache } from '@/utils/favicon.ts'; -import { errorJson } from '@/utils/log.ts'; -import { nip05Cache } from '@/utils/nip05.ts'; -import { parseNoteContent, stripimeta } from '@/utils/note.ts'; -import { purifyEvent } from '@/utils/purify.ts'; -import { updateStats } from '@/utils/stats.ts'; -import { getTagSet } from '@/utils/tags.ts'; -import { unfurlCardCached } from '@/utils/unfurl.ts'; -import { renderWebPushNotification } from '@/views/mastodon/push.ts'; -import { policyWorker } from '@/workers/policy.ts'; -import { verifyEventWorker } from '@/workers/verify.ts'; - -interface PipelineOpts { - signal: AbortSignal; - source: 'relay' | 'api' | 'firehose' | 'pipeline' | 'notify' | 'internal'; -} - -/** - * Common pipeline function to process (and maybe store) events. - * It is idempotent, so it can be called multiple times for the same event. - */ -async function handleEvent(event: DittoEvent, opts: PipelineOpts): Promise { - // Skip events that have already been encountered. - if (pipelineEncounters.get(event.id)) { - throw new RelayError('duplicate', 'already have this event'); - } - // Reject events that are too far in the future. - if (eventAge(event) < -Time.minutes(1)) { - throw new RelayError('invalid', 'event too far in the future'); - } - // Integer max value for Postgres. - if (event.kind >= 2_147_483_647) { - throw new RelayError('invalid', 'event kind too large'); - } - // The only point of ephemeral events is to stream them, - // so throw an error if we're not even going to do that. - if (NKinds.ephemeral(event.kind) && !isFresh(event)) { - throw new RelayError('invalid', 'event too old'); - } - // Block NIP-70 events, because we have no way to `AUTH`. - if (isProtectedEvent(event)) { - throw new RelayError('invalid', 'protected event'); - } - // Validate the event's signature. - if (!(await verifyEventWorker(event))) { - throw new RelayError('invalid', 'invalid signature'); - } - // Recheck encountered after async ops. - if (pipelineEncounters.has(event.id)) { - throw new RelayError('duplicate', 'already have this event'); - } - // Set the event as encountered after verifying the signature. - pipelineEncounters.set(event.id, true); - - // Log the event. - logi({ level: 'debug', ns: 'ditto.event', source: 'pipeline', id: event.id, kind: event.kind }); - pipelineEventsCounter.inc({ kind: event.kind }); - - // NIP-46 events get special treatment. - // They are exempt from policies and other side-effects, and should be streamed out immediately. - // If streaming fails, an error should be returned. - if (event.kind === 24133) { - const store = await Storages.db(); - await store.event(event, { signal: opts.signal }); - } - - // Ensure the event doesn't violate the policy. - if (event.pubkey !== await Conf.signer.getPublicKey()) { - await policyFilter(event, opts.signal); - } - - // Prepare the event for additional checks. - // FIXME: This is kind of hacky. Should be reorganized to fetch only what's needed for each stage. - await hydrateEvent(event, opts.signal); - - // Ensure that the author is not banned. - const n = getTagSet(event.user?.tags ?? [], 'n'); - if (n.has('disabled')) { - throw new RelayError('blocked', 'author is blocked'); - } - - const kysely = await Storages.kysely(); - - try { - await storeEvent(purifyEvent(event), opts.signal); - } finally { - // This needs to run in steps, and should not block the API from responding. - Promise.allSettled([ - handleZaps(kysely, event), - updateAuthorData(event, opts.signal), - prewarmLinkPreview(event, opts.signal), - generateSetEvents(event), - ]) - .then(() => webPush(event)) - .catch(() => {}); - } -} - -async function policyFilter(event: NostrEvent, signal: AbortSignal): Promise { - try { - const result = await policyWorker.call(event, signal); - const [, , ok, reason] = result; - logi({ level: 'debug', ns: 'ditto.policy', id: event.id, kind: event.kind, ok, reason }); - policyEventsCounter.inc({ ok: String(ok) }); - RelayError.assert(result); - } catch (e) { - if (e instanceof RelayError) { - throw e; - } else { - logi({ level: 'error', ns: 'ditto.policy', id: event.id, kind: event.kind, error: errorJson(e) }); - throw new RelayError('blocked', 'policy error'); - } - } -} - -/** Check whether the event has a NIP-70 `-` tag. */ -function isProtectedEvent(event: NostrEvent): boolean { - return event.tags.some(([name]) => name === '-'); -} - -/** Hydrate the event with the user, if applicable. */ -async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { - await hydrateEvents({ events: [event], relay: await Storages.db(), signal }); -} - -/** Maybe store the event, if eligible. */ -async function storeEvent(event: NostrEvent, signal?: AbortSignal): Promise { - const store = await Storages.db(); - - try { - await store.transaction(async (store, kysely) => { - if (!NKinds.ephemeral(event.kind)) { - await updateStats({ event, store, kysely }); - } - await store.event(event, { signal }); - }); - } catch (e) { - // If the failure is only because of updateStats (which runs first), insert the event anyway. - // We can't catch this in the transaction because the error aborts the transaction on the Postgres side. - if (e instanceof Error && e.message.includes('event_stats' satisfies keyof DittoTables)) { - await store.event(event, { signal }); - } else { - throw e; - } - } -} - -/** Parse kind 0 metadata and track indexes in the database. */ -async function updateAuthorData(event: NostrEvent, signal: AbortSignal): Promise { - if (event.kind !== 0) return; - - // Parse metadata. - const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(event.content); - if (!metadata.success) return; - - const { name, nip05 } = metadata.data; - - const kysely = await Storages.kysely(); - - const updates: UpdateObject = {}; - - const authorStats = await kysely - .selectFrom('author_stats') - .selectAll() - .where('pubkey', '=', event.pubkey) - .executeTakeFirst(); - - const lastVerified = authorStats?.nip05_last_verified_at; - const eventNewer = !lastVerified || event.created_at > lastVerified; - - try { - if (nip05 !== authorStats?.nip05 && eventNewer || !lastVerified) { - if (nip05) { - const tld = tldts.parse(nip05); - if (tld.isIcann && !tld.isIp && !tld.isPrivate) { - const pointer = await nip05Cache.fetch(nip05, { signal }); - if (pointer.pubkey === event.pubkey) { - updates.nip05 = nip05; - updates.nip05_domain = tld.domain; - updates.nip05_hostname = tld.hostname; - updates.nip05_last_verified_at = event.created_at; - } - } - } else { - updates.nip05 = null; - updates.nip05_domain = null; - updates.nip05_hostname = null; - updates.nip05_last_verified_at = event.created_at; - } - } - } catch { - // Fallthrough. - } - - // Fetch favicon. - const domain = nip05?.split('@')[1].toLowerCase(); - if (domain) { - try { - await faviconCache.fetch(domain, { signal }); - } catch { - // Fallthrough. - } - } - - const search = [name, nip05].filter(Boolean).join(' ').trim(); - - if (search !== authorStats?.search) { - updates.search = search; - } - - if (Object.keys(updates).length) { - await kysely.insertInto('author_stats') - .values({ - pubkey: event.pubkey, - followers_count: 0, - following_count: 0, - notes_count: 0, - search, - ...updates, - }) - .onConflict((oc) => oc.column('pubkey').doUpdateSet(updates)) - .execute(); - } -} - -async function prewarmLinkPreview(event: NostrEvent, signal: AbortSignal): Promise { - const { firstUrl } = parseNoteContent(stripimeta(event.content, event.tags), []); - if (firstUrl) { - await unfurlCardCached(firstUrl, signal); - } -} - -/** Determine if the event is being received in a timely manner. */ -function isFresh(event: NostrEvent): boolean { - return eventAge(event) < Time.minutes(1); -} - -async function webPush(event: NostrEvent): Promise { - if (!isFresh(event)) { - throw new RelayError('invalid', 'event too old'); - } - - const kysely = await Storages.kysely(); - const pubkeys = getTagSet(event.tags, 'p'); - - if (!pubkeys.size) { - return; - } - - const rows = await kysely - .selectFrom('push_subscriptions') - .selectAll() - .where('pubkey', 'in', [...pubkeys]) - .execute(); - - for (const row of rows) { - const viewerPubkey = row.pubkey; - - if (viewerPubkey === event.pubkey) { - continue; // Don't notify authors about their own events. - } - - const message = await renderWebPushNotification(event, viewerPubkey); - if (!message) { - continue; - } - - const subscription = { - endpoint: row.endpoint, - keys: { - auth: row.auth, - p256dh: row.p256dh, - }, - }; - - await DittoPush.push(subscription, message); - webPushNotificationsCounter.inc({ type: message.notification_type }); - } -} - -async function generateSetEvents(event: NostrEvent): Promise { - const signer = Conf.signer; - const pubkey = await signer.getPublicKey(); - - const tagsAdmin = event.tags.some(([name, value]) => ['p', 'P'].includes(name) && value === pubkey); - - if (event.kind === 1984 && tagsAdmin) { - const rel = await signer.signEvent({ - kind: 30383, - content: '', - tags: [ - ['d', event.id], - ['p', event.pubkey], - ['k', '1984'], - ['n', 'open'], - ...[...getTagSet(event.tags, 'p')].map((value) => ['P', value]), - ...[...getTagSet(event.tags, 'e')].map((value) => ['e', value]), - ], - created_at: Math.floor(Date.now() / 1000), - }); - - await handleEvent(rel, { source: 'pipeline', signal: AbortSignal.timeout(1000) }); - } - - if (event.kind === 3036 && tagsAdmin) { - const rel = await signer.signEvent({ - kind: 30383, - content: '', - tags: [ - ['d', event.id], - ['p', event.pubkey], - ['k', '3036'], - ['n', 'pending'], - ], - created_at: Math.floor(Date.now() / 1000), - }); - - await handleEvent(rel, { source: 'pipeline', signal: AbortSignal.timeout(1000) }); - } -} - -/** Stores the event in the 'event_zaps' table */ -async function handleZaps(kysely: Kysely, event: NostrEvent) { - if (event.kind !== 9735) return; - - const zapRequestString = event?.tags?.find(([name]) => name === 'description')?.[1]; - if (!zapRequestString) return; - const zapRequest = n.json().pipe(n.event()).optional().catch(undefined).parse(zapRequestString); - if (!zapRequest) return; - - const amountSchema = z.coerce.number().int().nonnegative().catch(0); - const amount_millisats = amountSchema.parse(getAmount(event?.tags.find(([name]) => name === 'bolt11')?.[1])); - if (!amount_millisats || amount_millisats < 1) return; - - const zappedEventId = zapRequest.tags.find(([name]) => name === 'e')?.[1]; - if (!zappedEventId) return; - - try { - await kysely.insertInto('event_zaps').values({ - receipt_id: event.id, - target_event_id: zappedEventId, - sender_pubkey: zapRequest.pubkey, - amount_millisats, - comment: zapRequest.content, - }).execute(); - } catch { - // receipt_id is unique, do nothing - } -} - -export { handleEvent, handleZaps, updateAuthorData }; diff --git a/packages/ditto/queries.ts b/packages/ditto/queries.ts index a79b2df4..e14b4f28 100644 --- a/packages/ditto/queries.ts +++ b/packages/ditto/queries.ts @@ -1,73 +1,55 @@ +import { DittoDB } from '@ditto/db'; +import { DittoConf } from '@ditto/conf'; import { NostrEvent, NostrFilter, NStore } from '@nostrify/nostrify'; -import { Conf } from '@/config.ts'; -import { Storages } from '@/storages.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { fallbackAuthor } from '@/utils.ts'; import { findReplyTag, getTagSet } from '@/utils/tags.ts'; interface GetEventOpts { - /** Signal to abort the request. */ + db: DittoDB; + conf: DittoConf; + relay: NStore; signal?: AbortSignal; - /** Event kind. */ - kind?: number; } /** * Get a Nostr event by its ID. * @deprecated Use `relay.query` directly. */ -const getEvent = async ( - id: string, - opts: GetEventOpts = {}, -): Promise => { - const relay = await Storages.db(); - const { kind, signal = AbortSignal.timeout(1000) } = opts; - +async function getEvent(id: string, opts: GetEventOpts): Promise { const filter: NostrFilter = { ids: [id], limit: 1 }; - if (kind) { - filter.kinds = [kind]; - } - - return await relay.query([filter], { limit: 1, signal }) - .then((events) => hydrateEvents({ events, relay, signal })) - .then(([event]) => event); -}; + const [event] = await opts.relay.query([filter], opts); + hydrateEvents({ ...opts, events: [event] }); + return event; +} /** * Get a Nostr `set_medatadata` event for a user's pubkey. * @deprecated Use `relay.query` directly. */ -async function getAuthor(pubkey: string, opts: GetEventOpts = {}): Promise { - const relay = await Storages.db(); - const { signal = AbortSignal.timeout(1000) } = opts; - - const events = await relay.query([{ authors: [pubkey], kinds: [0], limit: 1 }], { limit: 1, signal }); - const event = events[0] ?? fallbackAuthor(pubkey); - - await hydrateEvents({ events: [event], relay, signal }); - +async function getAuthor(pubkey: string, opts: GetEventOpts): Promise { + const [event] = await opts.relay.query([{ authors: [pubkey], kinds: [0], limit: 1 }], opts); + hydrateEvents({ ...opts, events: [event] }); return event; } /** Get users the given pubkey follows. */ -const getFollows = async (pubkey: string, signal?: AbortSignal): Promise => { - const store = await Storages.db(); - const [event] = await store.query([{ authors: [pubkey], kinds: [3], limit: 1 }], { limit: 1, signal }); +const getFollows = async (relay: NStore, pubkey: string, signal?: AbortSignal): Promise => { + const [event] = await relay.query([{ authors: [pubkey], kinds: [3], limit: 1 }], { signal }); return event; }; /** Get pubkeys the user follows. */ -async function getFollowedPubkeys(pubkey: string, signal?: AbortSignal): Promise> { - const event = await getFollows(pubkey, signal); +async function getFollowedPubkeys(relay: NStore, pubkey: string, signal?: AbortSignal): Promise> { + const event = await getFollows(relay, pubkey, signal); if (!event) return new Set(); return getTagSet(event.tags, 'p'); } /** Get pubkeys the user follows, including the user's own pubkey. */ -async function getFeedPubkeys(pubkey: string): Promise> { - const authors = await getFollowedPubkeys(pubkey); +async function getFeedPubkeys(relay: NStore, pubkey: string): Promise> { + const authors = await getFollowedPubkeys(relay, pubkey); return authors.add(pubkey); } @@ -92,34 +74,11 @@ async function getAncestors(store: NStore, event: NostrEvent, result: NostrEvent async function getDescendants( store: NStore, event: NostrEvent, - signal = AbortSignal.timeout(2000), + signal?: AbortSignal, ): Promise { return await store .query([{ kinds: [1], '#e': [event.id], since: event.created_at, limit: 200 }], { signal }) .then((events) => events.filter(({ tags }) => findReplyTag(tags)?.[1] === event.id)); } -/** Returns whether the pubkey is followed by a local user. */ -async function isLocallyFollowed(pubkey: string): Promise { - const { host } = Conf.url; - - const store = await Storages.db(); - - const [event] = await store.query( - [{ kinds: [3], '#p': [pubkey], search: `domain:${host}`, limit: 1 }], - { limit: 1 }, - ); - - return Boolean(event); -} - -export { - getAncestors, - getAuthor, - getDescendants, - getEvent, - getFeedPubkeys, - getFollowedPubkeys, - getFollows, - isLocallyFollowed, -}; +export { getAncestors, getAuthor, getDescendants, getEvent, getFeedPubkeys, getFollowedPubkeys, getFollows }; diff --git a/packages/ditto/signers/ConnectSigner.ts b/packages/ditto/signers/ConnectSigner.ts index c6d23d37..4f5a6f3e 100644 --- a/packages/ditto/signers/ConnectSigner.ts +++ b/packages/ditto/signers/ConnectSigner.ts @@ -1,13 +1,12 @@ // deno-lint-ignore-file require-await import { HTTPException } from '@hono/hono/http-exception'; -import { NConnectSigner, NostrEvent, NostrSigner } from '@nostrify/nostrify'; - -import { Storages } from '@/storages.ts'; +import { NConnectSigner, NostrEvent, NostrSigner, NRelay } from '@nostrify/nostrify'; interface ConnectSignerOpts { bunkerPubkey: string; userPubkey: string; signer: NostrSigner; + relay: NRelay; relays?: string[]; } @@ -17,27 +16,23 @@ interface ConnectSignerOpts { * Simple extension of nostrify's `NConnectSigner`, with our options to keep it DRY. */ export class ConnectSigner implements NostrSigner { - private signer: Promise; + private signer: NConnectSigner; constructor(private opts: ConnectSignerOpts) { - this.signer = this.init(opts.signer); - } + const { relay, signer } = this.opts; - async init(signer: NostrSigner): Promise { - return new NConnectSigner({ + this.signer = new NConnectSigner({ encryption: 'nip44', pubkey: this.opts.bunkerPubkey, - // TODO: use a remote relay for `nprofile` signing (if present and `Conf.relay` isn't already in the list) - relay: await Storages.db(), + relay, signer, timeout: 60_000, }); } async signEvent(event: Omit): Promise { - const signer = await this.signer; try { - return await signer.signEvent(event); + return await this.signer.signEvent(event); } catch (e) { if (e instanceof Error && e.name === 'AbortError') { throw new HTTPException(408, { message: 'The event was not signed quickly enough' }); @@ -49,9 +44,8 @@ export class ConnectSigner implements NostrSigner { readonly nip04 = { encrypt: async (pubkey: string, plaintext: string): Promise => { - const signer = await this.signer; try { - return await signer.nip04.encrypt(pubkey, plaintext); + return await this.signer.nip04.encrypt(pubkey, plaintext); } catch (e) { if (e instanceof Error && e.name === 'AbortError') { throw new HTTPException(408, { @@ -64,9 +58,8 @@ export class ConnectSigner implements NostrSigner { }, decrypt: async (pubkey: string, ciphertext: string): Promise => { - const signer = await this.signer; try { - return await signer.nip04.decrypt(pubkey, ciphertext); + return await this.signer.nip04.decrypt(pubkey, ciphertext); } catch (e) { if (e instanceof Error && e.name === 'AbortError') { throw new HTTPException(408, { @@ -81,9 +74,8 @@ export class ConnectSigner implements NostrSigner { readonly nip44 = { encrypt: async (pubkey: string, plaintext: string): Promise => { - const signer = await this.signer; try { - return await signer.nip44.encrypt(pubkey, plaintext); + return await this.signer.nip44.encrypt(pubkey, plaintext); } catch (e) { if (e instanceof Error && e.name === 'AbortError') { throw new HTTPException(408, { @@ -96,9 +88,8 @@ export class ConnectSigner implements NostrSigner { }, decrypt: async (pubkey: string, ciphertext: string): Promise => { - const signer = await this.signer; try { - return await signer.nip44.decrypt(pubkey, ciphertext); + return await this.signer.nip44.decrypt(pubkey, ciphertext); } catch (e) { if (e instanceof Error && e.name === 'AbortError') { throw new HTTPException(408, { diff --git a/packages/ditto/startup.ts b/packages/ditto/startup.ts deleted file mode 100644 index 0372a1d1..00000000 --- a/packages/ditto/startup.ts +++ /dev/null @@ -1,12 +0,0 @@ -// Starts up applications required to run before the HTTP server is on. -import { Conf } from '@/config.ts'; -import { cron } from '@/cron.ts'; -import { startFirehose } from '@/firehose.ts'; - -if (Conf.firehoseEnabled) { - startFirehose(); -} - -if (Conf.cronEnabled) { - cron(); -} diff --git a/packages/ditto/storages.ts b/packages/ditto/storages.ts deleted file mode 100644 index aae165f2..00000000 --- a/packages/ditto/storages.ts +++ /dev/null @@ -1,62 +0,0 @@ -// deno-lint-ignore-file require-await -import { type DittoDB, DittoPolyPg } from '@ditto/db'; -import { NPool, NRelay1 } from '@nostrify/nostrify'; - -import { Conf } from '@/config.ts'; -import { DittoPgStore } from '@/storages/DittoPgStore.ts'; -import { seedZapSplits } from '@/utils/zap-split.ts'; -import { DittoPool } from '@/storages/DittoPool.ts'; - -export class Storages { - private static _db: Promise | undefined; - private static _database: Promise | undefined; - private static _client: Promise> | undefined; - - public static async database(): Promise { - if (!this._database) { - this._database = (async () => { - const db = DittoPolyPg.create(Conf.databaseUrl, { - poolSize: Conf.pg.poolSize, - debug: Conf.pgliteDebug, - }); - await DittoPolyPg.migrate(db.kysely); - return db; - })(); - } - return this._database; - } - - public static async kysely(): Promise { - const { kysely } = await this.database(); - return kysely; - } - - /** SQL database to store events this Ditto server cares about. */ - public static async db(): Promise { - if (!this._db) { - this._db = (async () => { - const db = await this.database(); - const store = new DittoPgStore({ - db, - pubkey: await Conf.signer.getPublicKey(), - timeout: Conf.db.timeouts.default, - notify: Conf.notifyEnabled, - }); - await seedZapSplits(store); - return store; - })(); - } - return this._db; - } - - /** Relay pool storage. */ - public static async client(): Promise> { - if (!this._client) { - this._client = (async () => { - const relay = await this.db(); - return new DittoPool({ conf: Conf, relay }); - })(); - } - return this._client; - } -} diff --git a/packages/ditto/storages/DittoAPIStore.ts b/packages/ditto/storages/DittoAPIStore.ts index 9e04c6c6..7a479899 100644 --- a/packages/ditto/storages/DittoAPIStore.ts +++ b/packages/ditto/storages/DittoAPIStore.ts @@ -1,6 +1,12 @@ import { DittoConf } from '@ditto/conf'; import { DittoDB, DittoTables } from '@ditto/db'; -import { pipelineEventsCounter, policyEventsCounter, webPushNotificationsCounter } from '@ditto/metrics'; +import { + cachedFaviconsSizeGauge, + cachedNip05sSizeGauge, + pipelineEventsCounter, + policyEventsCounter, + webPushNotificationsCounter, +} from '@ditto/metrics'; import { NKinds, NostrEvent, @@ -22,18 +28,20 @@ import { DittoPush } from '@/DittoPush.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { RelayError } from '@/RelayError.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { eventAge, Time } from '@/utils.ts'; +import { eventAge, nostrNow, Time } from '@/utils.ts'; import { getAmount } from '@/utils/bolt11.ts'; import { errorJson } from '@/utils/log.ts'; import { purifyEvent } from '@/utils/purify.ts'; import { getTagSet } from '@/utils/tags.ts'; import { policyWorker } from '@/workers/policy.ts'; import { verifyEventWorker } from '@/workers/verify.ts'; -import { faviconCache } from '@/utils/favicon.ts'; +import { fetchFavicon, insertFavicon, queryFavicon } from '@/utils/favicon.ts'; +import { lookupNip05 } from '@/utils/nip05.ts'; import { parseNoteContent, stripimeta } from '@/utils/note.ts'; +import { SimpleLRU } from '@/utils/SimpleLRU.ts'; import { unfurlCardCached } from '@/utils/unfurl.ts'; -import { nip05Cache } from '@/utils/nip05.ts'; import { renderWebPushNotification } from '@/views/mastodon/push.ts'; +import { nip19 } from 'nostr-tools'; interface DittoAPIStoreOpts { db: DittoDB; @@ -43,15 +51,45 @@ interface DittoAPIStoreOpts { } export class DittoAPIStore implements NRelay { + private push: DittoPush; private encounters = new LRUCache({ max: 5000 }); private controller = new AbortController(); + private faviconCache: SimpleLRU; + private nip05Cache: SimpleLRU; + private ns = 'ditto.apistore'; constructor(private opts: DittoAPIStoreOpts) { + const { conf, db } = this.opts; + + this.push = new DittoPush(opts); + this.listen().catch((e: unknown) => { logi({ level: 'error', ns: this.ns, source: 'listen', error: errorJson(e) }); }); + + this.faviconCache = new SimpleLRU( + async (domain, { signal }) => { + const row = await queryFavicon(db.kysely, domain); + + if (row && (nostrNow() - row.last_updated_at) < (conf.caches.favicon.ttl / 1000)) { + return new URL(row.favicon); + } + + const url = await fetchFavicon(domain, signal); + await insertFavicon(db.kysely, domain, url.href); + return url; + }, + { ...conf.caches.favicon, gauge: cachedFaviconsSizeGauge }, + ); + + this.nip05Cache = new SimpleLRU( + (nip05, { signal }) => { + return lookupNip05(nip05, { ...this.opts, signal }); + }, + { ...conf.caches.nip05, gauge: cachedNip05sSizeGauge }, + ); } req( @@ -220,7 +258,7 @@ export class DittoAPIStore implements NRelay { } /** Parse kind 0 metadata and track indexes in the database. */ - private async updateAuthorData(event: NostrEvent, signal?: AbortSignal): Promise { + async updateAuthorData(event: NostrEvent, signal?: AbortSignal): Promise { if (event.kind !== 0) return; const { db } = this.opts; @@ -247,7 +285,7 @@ export class DittoAPIStore implements NRelay { if (nip05) { const tld = tldts.parse(nip05); if (tld.isIcann && !tld.isIp && !tld.isPrivate) { - const pointer = await nip05Cache.fetch(nip05, { signal }); + const pointer = await this.nip05Cache.fetch(nip05, { signal }); if (pointer.pubkey === event.pubkey) { updates.nip05 = nip05; updates.nip05_domain = tld.domain; @@ -270,7 +308,7 @@ export class DittoAPIStore implements NRelay { const domain = nip05?.split('@')[1].toLowerCase(); if (domain) { try { - await faviconCache.fetch(domain, { signal }); + await this.faviconCache.fetch(domain, { signal }); } catch { // Fallthrough. } @@ -352,7 +390,7 @@ export class DittoAPIStore implements NRelay { throw new RelayError('invalid', 'event too old'); } - const { db } = this.opts; + const { db, relay } = this.opts; const pubkeys = getTagSet(event.tags, 'p'); if (!pubkeys.size) { @@ -372,7 +410,7 @@ export class DittoAPIStore implements NRelay { continue; // Don't notify authors about their own events. } - const message = await renderWebPushNotification(event, viewerPubkey); + const message = await renderWebPushNotification(relay, event, viewerPubkey); if (!message) { continue; } @@ -385,15 +423,14 @@ export class DittoAPIStore implements NRelay { }, }; - await DittoPush.push(subscription, message); + await this.push.push(subscription, message); webPushNotificationsCounter.inc({ type: message.notification_type }); } } /** Hydrate the event with the user, if applicable. */ private async hydrateEvent(event: NostrEvent, signal?: AbortSignal): Promise { - const { relay } = this.opts; - const [hydrated] = await hydrateEvents({ events: [event], relay, signal }); + const [hydrated] = await hydrateEvents({ ...this.opts, events: [event], signal }); return hydrated; } @@ -402,9 +439,17 @@ export class DittoAPIStore implements NRelay { return eventAge(event) < Time.minutes(1); } - query(filters: NostrFilter[], opts?: { signal?: AbortSignal }): Promise { + async query(filters: NostrFilter[], opts: { pure?: boolean; signal?: AbortSignal } = {}): Promise { const { relay } = this.opts; - return relay.query(filters, opts); + const { pure = true, signal } = opts; // TODO: make pure `false` by default + + const events = await relay.query(filters, opts); + + if (!pure) { + return hydrateEvents({ ...this.opts, events, signal }); + } + + return events; } count(filters: NostrFilter[], opts?: { signal?: AbortSignal }): Promise { diff --git a/packages/ditto/storages/DittoPgStore.ts b/packages/ditto/storages/DittoPgStore.ts index 035fd729..bf6babb5 100644 --- a/packages/ditto/storages/DittoPgStore.ts +++ b/packages/ditto/storages/DittoPgStore.ts @@ -55,7 +55,7 @@ interface DittoPgStoreOpts { /** Pubkey of the admin account. */ pubkey: string; /** Timeout in milliseconds for database queries. */ - timeout: number; + timeout?: number; /** Whether the event returned should be a Nostr event or a Ditto event. Defaults to false. */ pure?: boolean; /** Chunk size for streaming events. Defaults to 20. */ diff --git a/packages/ditto/storages/hydrate.test.ts b/packages/ditto/storages/hydrate.test.ts index ebafa6af..6ba4870b 100644 --- a/packages/ditto/storages/hydrate.test.ts +++ b/packages/ditto/storages/hydrate.test.ts @@ -1,13 +1,15 @@ +import { DittoConf } from '@ditto/conf'; +import { DummyDB } from '@ditto/db'; import { MockRelay } from '@nostrify/nostrify/test'; import { assertEquals } from '@std/assert'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { createTestDB, eventFixture } from '@/test.ts'; +import { eventFixture } from '@/test.ts'; Deno.test('hydrateEvents(): author --- WITHOUT stats', async () => { - const relay = new MockRelay(); - await using db = await createTestDB(); + const opts = setupTest(); + const { relay } = opts; const event0 = await eventFixture('event-0'); const event1 = await eventFixture('event-1'); @@ -16,19 +18,15 @@ Deno.test('hydrateEvents(): author --- WITHOUT stats', async () => { await relay.event(event0); await relay.event(event1); - await hydrateEvents({ - events: [event1], - relay, - kysely: db.kysely, - }); + await hydrateEvents({ ...opts, events: [event1] }); const expectedEvent = { ...event1, author: event0 }; assertEquals(event1, expectedEvent); }); Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { - const relay = new MockRelay(); - await using db = await createTestDB(); + const opts = setupTest(); + const { relay } = opts; const event0madePost = await eventFixture('event-0-the-one-who-post-and-users-repost'); const event0madeRepost = await eventFixture('event-0-the-one-who-repost'); @@ -41,23 +39,20 @@ Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { await relay.event(event1reposted); await relay.event(event6); - await hydrateEvents({ - events: [event6], - relay, - kysely: db.kysely, - }); + await hydrateEvents({ ...opts, events: [event6] }); const expectedEvent6 = { ...event6, author: event0madeRepost, repost: { ...event1reposted, author: event0madePost }, }; + assertEquals(event6, expectedEvent6); }); Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { - const relay = new MockRelay(); - await using db = await createTestDB(); + const opts = setupTest(); + const { relay } = opts; const event0madeQuoteRepost = await eventFixture('event-0-the-one-who-quote-repost'); const event0 = await eventFixture('event-0'); @@ -70,11 +65,7 @@ Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { await relay.event(event1quoteRepost); await relay.event(event1willBeQuoteReposted); - await hydrateEvents({ - events: [event1quoteRepost], - relay, - kysely: db.kysely, - }); + await hydrateEvents({ ...opts, events: [event1quoteRepost] }); const expectedEvent1quoteRepost = { ...event1quoteRepost, @@ -86,8 +77,8 @@ Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { }); Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () => { - const relay = new MockRelay(); - await using db = await createTestDB(); + const opts = setupTest(); + const { relay } = opts; const author = await eventFixture('event-0-makes-repost-with-quote-repost'); const event1 = await eventFixture('event-1-will-be-reposted-with-quote-repost'); @@ -100,23 +91,20 @@ Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () await relay.event(event1quote); await relay.event(event6); - await hydrateEvents({ - events: [event6], - relay, - kysely: db.kysely, - }); + await hydrateEvents({ ...opts, events: [event6] }); const expectedEvent6 = { ...event6, author, repost: { ...event1quote, author, quote: { author, ...event1 } }, }; + assertEquals(event6, expectedEvent6); }); Deno.test('hydrateEvents(): report pubkey and post // kind 1984 --- WITHOUT stats', async () => { - const relay = new MockRelay(); - await using db = await createTestDB(); + const opts = setupTest(); + const { relay } = opts; const authorDictator = await eventFixture('kind-0-dictator'); const authorVictim = await eventFixture('kind-0-george-orwell'); @@ -129,11 +117,7 @@ Deno.test('hydrateEvents(): report pubkey and post // kind 1984 --- WITHOUT stat await relay.event(reportEvent); await relay.event(event1); - await hydrateEvents({ - events: [reportEvent], - relay, - kysely: db.kysely, - }); + await hydrateEvents({ ...opts, events: [reportEvent] }); const expectedEvent: DittoEvent = { ...reportEvent, @@ -141,12 +125,13 @@ Deno.test('hydrateEvents(): report pubkey and post // kind 1984 --- WITHOUT stat reported_notes: [event1], reported_profile: authorVictim, }; + assertEquals(reportEvent, expectedEvent); }); Deno.test('hydrateEvents(): zap sender, zap amount, zapped post // kind 9735 --- WITHOUT stats', async () => { - const relay = new MockRelay(); - await using db = await createTestDB(); + const opts = setupTest(); + const { relay } = opts; const zapSender = await eventFixture('kind-0-jack'); const zapReceipt = await eventFixture('kind-9735-jack-zap-patrick'); @@ -159,11 +144,7 @@ Deno.test('hydrateEvents(): zap sender, zap amount, zapped post // kind 9735 --- await relay.event(zappedPost); await relay.event(zapReceiver); - await hydrateEvents({ - events: [zapReceipt], - relay, - kysely: db.kysely, - }); + await hydrateEvents({ ...opts, events: [zapReceipt] }); const expectedEvent: DittoEvent = { ...zapReceipt, @@ -175,5 +156,14 @@ Deno.test('hydrateEvents(): zap sender, zap amount, zapped post // kind 9735 --- zap_amount: 5225000, // millisats zap_message: '🫂', }; + assertEquals(zapReceipt, expectedEvent); }); + +function setupTest() { + const db = new DummyDB(); + const conf = new DittoConf(new Map()); + const relay = new MockRelay(); + + return { conf, db, relay }; +} diff --git a/packages/ditto/storages/hydrate.ts b/packages/ditto/storages/hydrate.ts index 5bf51f96..5fdb691f 100644 --- a/packages/ditto/storages/hydrate.ts +++ b/packages/ditto/storages/hydrate.ts @@ -1,28 +1,28 @@ -import { DittoTables } from '@ditto/db'; +import { DittoDB, DittoTables } from '@ditto/db'; +import { DittoConf } from '@ditto/conf'; import { NStore } from '@nostrify/nostrify'; import { Kysely } from 'kysely'; import { matchFilter } from 'nostr-tools'; import { NSchema as n } from '@nostrify/nostrify'; import { z } from 'zod'; -import { Conf } from '@/config.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { fallbackAuthor } from '@/utils.ts'; import { findQuoteTag } from '@/utils/tags.ts'; import { findQuoteInContent } from '@/utils/note.ts'; import { getAmount } from '@/utils/bolt11.ts'; -import { Storages } from '@/storages.ts'; interface HydrateOpts { - events: DittoEvent[]; + db: DittoDB; + conf: DittoConf; relay: NStore; + events: DittoEvent[]; signal?: AbortSignal; - kysely?: Kysely; } /** Hydrate events using the provided storage. */ async function hydrateEvents(opts: HydrateOpts): Promise { - const { events, relay, signal, kysely = await Storages.kysely() } = opts; + const { conf, db, events } = opts; if (!events.length) { return events; @@ -30,28 +30,28 @@ async function hydrateEvents(opts: HydrateOpts): Promise { const cache = [...events]; - for (const event of await gatherRelatedEvents({ events: cache, relay, signal })) { + for (const event of await gatherRelatedEvents({ ...opts, events: cache })) { cache.push(event); } - for (const event of await gatherQuotes({ events: cache, relay, signal })) { + for (const event of await gatherQuotes({ ...opts, events: cache })) { cache.push(event); } - for (const event of await gatherProfiles({ events: cache, relay, signal })) { + for (const event of await gatherProfiles({ ...opts, events: cache })) { cache.push(event); } - for (const event of await gatherUsers({ events: cache, relay, signal })) { + for (const event of await gatherUsers({ ...opts, events: cache })) { cache.push(event); } - for (const event of await gatherInfo({ events: cache, relay, signal })) { + for (const event of await gatherInfo({ ...opts, events: cache })) { cache.push(event); } - const authorStats = await gatherAuthorStats(cache, kysely as Kysely); - const eventStats = await gatherEventStats(cache, kysely as Kysely); + const authorStats = await gatherAuthorStats(cache, db.kysely); + const eventStats = await gatherEventStats(cache, db.kysely); const domains = authorStats.reduce((result, { nip05_hostname }) => { if (nip05_hostname) result.add(nip05_hostname); @@ -59,7 +59,7 @@ async function hydrateEvents(opts: HydrateOpts): Promise { }, new Set()); const favicons = ( - await kysely + await db.kysely .selectFrom('domain_favicons') .select(['domain', 'favicon']) .where('domain', 'in', [...domains]) @@ -79,7 +79,7 @@ async function hydrateEvents(opts: HydrateOpts): Promise { // Dedupe events. const results = [...new Map(cache.map((event) => [event.id, event])).values()]; - const admin = await Conf.signer.getPublicKey(); + const admin = await conf.signer.getPublicKey(); // First connect all the events to each-other, then connect the connected events to the original list. assembleEvents(admin, results, results, stats); @@ -317,7 +317,7 @@ async function gatherProfiles({ events, relay, signal }: HydrateOpts): Promise { +async function gatherUsers({ conf, events, relay, signal }: HydrateOpts): Promise { const pubkeys = new Set(events.map((event) => event.pubkey)); if (!pubkeys.size) { @@ -325,13 +325,13 @@ async function gatherUsers({ events, relay, signal }: HydrateOpts): Promise { +async function gatherInfo({ conf, events, relay, signal }: HydrateOpts): Promise { const ids = new Set(); for (const event of events) { @@ -345,7 +345,7 @@ async function gatherInfo({ events, relay, signal }: HydrateOpts): Promise { /** Create a database for testing. It uses `DATABASE_URL`, or creates an in-memory database by default. */ export async function createTestDB(opts?: { pure?: boolean }) { - const db = DittoPolyPg.create(Conf.databaseUrl, { poolSize: 1 }); - - await DittoPolyPg.migrate(db.kysely); + const db = new DittoPolyPg(Conf.databaseUrl, { poolSize: 1 }); + await db.migrate(); const store = new DittoPgStore({ db, @@ -26,8 +25,10 @@ export async function createTestDB(opts?: { pure?: boolean }) { }); return { + db, ...db, store, + kysely: db.kysely, [Symbol.asyncDispose]: async () => { const { rows } = await sql< { tablename: string } diff --git a/packages/ditto/trends.ts b/packages/ditto/trends.ts index 4cec1712..47afdb9a 100644 --- a/packages/ditto/trends.ts +++ b/packages/ditto/trends.ts @@ -1,11 +1,9 @@ -import { DittoTables } from '@ditto/db'; -import { NostrFilter } from '@nostrify/nostrify'; +import { DittoConf } from '@ditto/conf'; +import { DittoDB, DittoTables } from '@ditto/db'; +import { NostrFilter, NStore } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import { Kysely, sql } from 'kysely'; -import { Conf } from '@/config.ts'; -import { handleEvent } from '@/pipeline.ts'; -import { Storages } from '@/storages.ts'; import { errorJson } from '@/utils/log.ts'; import { Time } from '@/utils/time.ts'; @@ -63,8 +61,15 @@ export async function getTrendingTagValues( })); } +export interface TrendsCtx { + conf: DittoConf; + db: DittoDB; + relay: NStore; +} + /** Get trending tags and publish an event with them. */ export async function updateTrendingTags( + ctx: TrendsCtx, l: string, tagName: string, kinds: number[], @@ -73,10 +78,11 @@ export async function updateTrendingTags( aliases?: string[], values?: string[], ) { + const { conf, db, relay } = ctx; const params = { l, tagName, kinds, limit, extra, aliases, values }; + logi({ level: 'info', ns: 'ditto.trends', msg: 'Updating trending', ...params }); - const kysely = await Storages.kysely(); const signal = AbortSignal.timeout(1000); const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); @@ -85,7 +91,7 @@ export async function updateTrendingTags( const tagNames = aliases ? [tagName, ...aliases] : [tagName]; try { - const trends = await getTrendingTagValues(kysely, tagNames, { + const trends = await getTrendingTagValues(db.kysely, tagNames, { kinds, since: yesterday, until: now, @@ -99,7 +105,7 @@ export async function updateTrendingTags( return; } - const signer = Conf.signer; + const signer = conf.signer; const label = await signer.signEvent({ kind: 1985, @@ -112,7 +118,7 @@ export async function updateTrendingTags( created_at: Math.floor(Date.now() / 1000), }); - await handleEvent(label, { source: 'internal', signal }); + await relay.event(label, { signal }); logi({ level: 'info', ns: 'ditto.trends', msg: 'Trends updated', ...params }); } catch (e) { logi({ level: 'error', ns: 'ditto.trends', msg: 'Error updating trends', ...params, error: errorJson(e) }); @@ -120,28 +126,28 @@ export async function updateTrendingTags( } /** Update trending pubkeys. */ -export function updateTrendingPubkeys(): Promise { - return updateTrendingTags('#p', 'p', [1, 3, 6, 7, 9735], 40, Conf.relay); +export function updateTrendingPubkeys(ctx: TrendsCtx): Promise { + return updateTrendingTags(ctx, '#p', 'p', [1, 3, 6, 7, 9735], 40, ctx.conf.relay); } /** Update trending zapped events. */ -export function updateTrendingZappedEvents(): Promise { - return updateTrendingTags('zapped', 'e', [9735], 40, Conf.relay, ['q']); +export function updateTrendingZappedEvents(ctx: TrendsCtx): Promise { + return updateTrendingTags(ctx, 'zapped', 'e', [9735], 40, ctx.conf.relay, ['q']); } /** Update trending events. */ -export async function updateTrendingEvents(): Promise { +export async function updateTrendingEvents(ctx: TrendsCtx): Promise { + const { conf, db } = ctx; + const results: Promise[] = [ - updateTrendingTags('#e', 'e', [1, 6, 7, 9735], 40, Conf.relay, ['q']), + updateTrendingTags(ctx, '#e', 'e', [1, 6, 7, 9735], 40, ctx.conf.relay, ['q']), ]; - const kysely = await Storages.kysely(); - - for (const language of Conf.preferredLanguages ?? []) { + for (const language of conf.preferredLanguages ?? []) { const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); const now = Math.floor(Date.now() / 1000); - const rows = await kysely + const rows = await db.kysely .selectFrom('nostr_events') .select('nostr_events.id') .where(sql`nostr_events.search_ext->>'language'`, '=', language) @@ -151,18 +157,20 @@ export async function updateTrendingEvents(): Promise { const ids = rows.map((row) => row.id); - results.push(updateTrendingTags(`#e.${language}`, 'e', [1, 6, 7, 9735], 40, Conf.relay, ['q'], ids)); + results.push( + updateTrendingTags(ctx, `#e.${language}`, 'e', [1, 6, 7, 9735], 40, conf.relay, ['q'], ids), + ); } await Promise.allSettled(results); } /** Update trending hashtags. */ -export function updateTrendingHashtags(): Promise { - return updateTrendingTags('#t', 't', [1], 20); +export function updateTrendingHashtags(ctx: TrendsCtx): Promise { + return updateTrendingTags(ctx, '#t', 't', [1], 20); } /** Update trending links. */ -export function updateTrendingLinks(): Promise { - return updateTrendingTags('#r', 'r', [1], 20); +export function updateTrendingLinks(ctx: TrendsCtx): Promise { + return updateTrendingTags(ctx, '#r', 'r', [1], 20); } diff --git a/packages/ditto/utils/api.ts b/packages/ditto/utils/api.ts index 7605e138..80dc4e57 100644 --- a/packages/ditto/utils/api.ts +++ b/packages/ditto/utils/api.ts @@ -1,25 +1,18 @@ import { HTTPException } from '@hono/hono/http-exception'; import { NostrEvent, NostrFilter } from '@nostrify/nostrify'; -import { logi } from '@soapbox/logi'; import { EventTemplate } from 'nostr-tools'; import * as TypeFest from 'type-fest'; import { type AppContext } from '@/app.ts'; -import { Conf } from '@/config.ts'; -import * as pipeline from '@/pipeline.ts'; -import { RelayError } from '@/RelayError.ts'; -import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; import { parseFormData } from '@/utils/formdata.ts'; -import { errorJson } from '@/utils/log.ts'; -import { purifyEvent } from '@/utils/purify.ts'; /** EventTemplate with defaults. */ type EventStub = TypeFest.SetOptional; /** Publish an event through the pipeline. */ async function createEvent(t: EventStub, c: AppContext): Promise { - const { user } = c.var; + const { user, relay, signal } = c.var; if (!user) { throw new HTTPException(401, { @@ -34,7 +27,8 @@ async function createEvent(t: EventStub, c: AppContext): Promise { ...t, }); - return publishEvent(event, c); + await relay.event(event, { signal }); + return event; } /** Filter for fetching an existing event to update. */ @@ -49,9 +43,9 @@ async function updateEvent( fn: (prev: NostrEvent) => E | Promise, c: AppContext, ): Promise { - const store = await Storages.db(); + const { relay } = c.var; - const [prev] = await store.query( + const [prev] = await relay.query( [filter], { signal: c.req.raw.signal }, ); @@ -80,16 +74,17 @@ function updateListEvent( /** Publish an admin event through the pipeline. */ async function createAdminEvent(t: EventStub, c: AppContext): Promise { - const signer = Conf.signer; + const { conf, relay, signal } = c.var; - const event = await signer.signEvent({ + const event = await conf.signer.signEvent({ content: '', created_at: nostrNow(), tags: [], ...t, }); - return publishEvent(event, c); + await relay.event(event, { signal }); + return event; } /** Fetch existing event, update its tags, then publish the new admin event. */ @@ -111,8 +106,8 @@ async function updateAdminEvent( fn: (prev: NostrEvent | undefined) => E, c: AppContext, ): Promise { - const store = await Storages.db(); - const [prev] = await store.query([filter], { limit: 1, signal: c.req.raw.signal }); + const { relay, signal } = c.var; + const [prev] = await relay.query([filter], { signal }); return createAdminEvent(fn(prev), c); } @@ -125,8 +120,8 @@ function updateEventInfo(id: string, n: Record, c: AppContext): } async function updateNames(k: number, d: string, n: Record, c: AppContext): Promise { - const signer = Conf.signer; - const admin = await signer.getPublicKey(); + const { conf } = c.var; + const admin = await conf.signer.getPublicKey(); return updateAdminEvent( { kinds: [k], authors: [admin], '#d': [d], limit: 1 }, @@ -154,33 +149,6 @@ async function updateNames(k: number, d: string, n: Record, c: ); } -/** Push the event through the pipeline, rethrowing any RelayError. */ -async function publishEvent(event: NostrEvent, c: AppContext): Promise { - logi({ level: 'info', ns: 'ditto.event', source: 'api', id: event.id, kind: event.kind }); - try { - const promise = pipeline.handleEvent(event, { source: 'api', signal: c.req.raw.signal }); - - promise.then(async () => { - const client = await Storages.client(); - await client.event(purifyEvent(event)); - }).catch((e: unknown) => { - logi({ level: 'error', ns: 'ditto.pool', id: event.id, kind: event.kind, error: errorJson(e) }); - }); - - await promise; - } catch (e) { - if (e instanceof RelayError) { - throw new HTTPException(422, { - res: c.json({ error: e.message }, 422), - }); - } else { - throw e; - } - } - - return event; -} - /** Parse request body to JSON, depending on the content-type of the request. */ async function parseBody(req: Request): Promise { switch (req.headers.get('content-type')?.split(';')[0]) { @@ -196,74 +164,8 @@ async function parseBody(req: Request): Promise { } } -/** Build HTTP Link header for Mastodon API pagination. */ -function buildLinkHeader(url: string, events: NostrEvent[]): string | undefined { - if (events.length <= 1) return; - const firstEvent = events[0]; - const lastEvent = events[events.length - 1]; - - const { origin } = Conf.url; - const { pathname, search } = new URL(url); - const next = new URL(pathname + search, origin); - const prev = new URL(pathname + search, origin); - - next.searchParams.set('until', String(lastEvent.created_at)); - prev.searchParams.set('since', String(firstEvent.created_at)); - - return `<${next}>; rel="next", <${prev}>; rel="prev"`; -} - type HeaderRecord = Record; -/** Return results with pagination headers. Assumes chronological sorting of events. */ -function paginated(c: AppContext, events: NostrEvent[], body: object | unknown[], headers: HeaderRecord = {}) { - const link = buildLinkHeader(c.req.url, events); - - if (link) { - headers.link = link; - } - - // Filter out undefined entities. - const results = Array.isArray(body) ? body.filter(Boolean) : body; - return c.json(results, 200, headers); -} - -/** Build HTTP Link header for paginating Nostr lists. */ -function buildListLinkHeader(url: string, params: { offset: number; limit: number }): string | undefined { - const { origin } = Conf.url; - const { pathname, search } = new URL(url); - const { offset, limit } = params; - const next = new URL(pathname + search, origin); - const prev = new URL(pathname + search, origin); - - next.searchParams.set('offset', String(offset + limit)); - prev.searchParams.set('offset', String(Math.max(offset - limit, 0))); - - next.searchParams.set('limit', String(limit)); - prev.searchParams.set('limit', String(limit)); - - return `<${next}>; rel="next", <${prev}>; rel="prev"`; -} - -/** paginate a list of tags. */ -function paginatedList( - c: AppContext, - params: { offset: number; limit: number }, - body: object | unknown[], - headers: HeaderRecord = {}, -) { - const link = buildListLinkHeader(c.req.url, params); - const hasMore = Array.isArray(body) ? body.length > 0 : true; - - if (link) { - headers.link = hasMore ? link : link.split(', ').find((link) => link.endsWith('; rel="prev"'))!; - } - - // Filter out undefined entities. - const results = Array.isArray(body) ? body.filter(Boolean) : body; - return c.json(results, 200, headers); -} - /** Actors with Bluesky's `!no-unauthenticated` self-label should require authorization to view. */ function assertAuthenticated(c: AppContext, author: NostrEvent): void { if ( @@ -282,8 +184,6 @@ export { createAdminEvent, createEvent, type EventStub, - paginated, - paginatedList, parseBody, updateAdminEvent, updateEvent, diff --git a/packages/ditto/utils/connect.ts b/packages/ditto/utils/connect.ts deleted file mode 100644 index 095b93c4..00000000 --- a/packages/ditto/utils/connect.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Conf } from '@/config.ts'; -import { Storages } from '@/storages.ts'; -import { getInstanceMetadata } from '@/utils/instance.ts'; - -/** NIP-46 client-connect metadata. */ -interface ConnectMetadata { - name: string; - description: string; - url: string; -} - -/** Get NIP-46 `nostrconnect://` URI for the Ditto server. */ -export async function getClientConnectUri(signal?: AbortSignal): Promise { - const uri = new URL('nostrconnect://'); - const { name, tagline } = await getInstanceMetadata(await Storages.db(), signal); - - const metadata: ConnectMetadata = { - name, - description: tagline, - url: Conf.localDomain, - }; - - uri.host = await Conf.signer.getPublicKey(); - uri.searchParams.set('relay', Conf.relay); - uri.searchParams.set('metadata', JSON.stringify(metadata)); - - return uri.toString(); -} diff --git a/packages/ditto/utils/favicon.ts b/packages/ditto/utils/favicon.ts index ed218cfa..448dfe0d 100644 --- a/packages/ditto/utils/favicon.ts +++ b/packages/ditto/utils/favicon.ts @@ -1,36 +1,13 @@ import { DOMParser } from '@b-fuze/deno-dom'; import { DittoTables } from '@ditto/db'; -import { cachedFaviconsSizeGauge } from '@ditto/metrics'; import { logi } from '@soapbox/logi'; import { safeFetch } from '@soapbox/safe-fetch'; import { Kysely } from 'kysely'; import tldts from 'tldts'; -import { Conf } from '@/config.ts'; -import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; -import { SimpleLRU } from '@/utils/SimpleLRU.ts'; -export const faviconCache = new SimpleLRU( - async (domain, { signal }) => { - const kysely = await Storages.kysely(); - - const row = await queryFavicon(kysely, domain); - - if (row && (nostrNow() - row.last_updated_at) < (Conf.caches.favicon.ttl / 1000)) { - return new URL(row.favicon); - } - - const url = await fetchFavicon(domain, signal); - - await insertFavicon(kysely, domain, url.href); - - return url; - }, - { ...Conf.caches.favicon, gauge: cachedFaviconsSizeGauge }, -); - -async function queryFavicon( +export async function queryFavicon( kysely: Kysely, domain: string, ): Promise { @@ -41,7 +18,7 @@ async function queryFavicon( .executeTakeFirst(); } -async function insertFavicon(kysely: Kysely, domain: string, favicon: string): Promise { +export async function insertFavicon(kysely: Kysely, domain: string, favicon: string): Promise { await kysely .insertInto('domain_favicons') .values({ domain, favicon, last_updated_at: nostrNow() }) @@ -49,7 +26,7 @@ async function insertFavicon(kysely: Kysely, domain: string, favico .execute(); } -async function fetchFavicon(domain: string, signal?: AbortSignal): Promise { +export async function fetchFavicon(domain: string, signal?: AbortSignal): Promise { logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'started' }); const tld = tldts.parse(domain); diff --git a/packages/ditto/utils/lookup.ts b/packages/ditto/utils/lookup.ts index 9afd8a08..e0f10a0e 100644 --- a/packages/ditto/utils/lookup.ts +++ b/packages/ditto/utils/lookup.ts @@ -1,32 +1,42 @@ -import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; +import { NostrEvent, NSchema as n, NStore } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; import { match } from 'path-to-regexp'; import tldts from 'tldts'; import { getAuthor } from '@/queries.ts'; import { bech32ToPubkey } from '@/utils.ts'; -import { nip05Cache } from '@/utils/nip05.ts'; +import { lookupNip05 } from '@/utils/nip05.ts'; + +import type { DittoConf } from '@ditto/conf'; +import type { DittoDB } from '@ditto/db'; + +interface LookupAccountOpts { + db: DittoDB; + conf: DittoConf; + relay: NStore; + signal?: AbortSignal; +} /** Resolve a bech32 or NIP-05 identifier to an account. */ export async function lookupAccount( value: string, - signal = AbortSignal.timeout(3000), + opts: LookupAccountOpts, ): Promise { - const pubkey = await lookupPubkey(value, signal); + const pubkey = await lookupPubkey(value, opts); if (pubkey) { - return getAuthor(pubkey); + return getAuthor(pubkey, opts); } } /** Resolve a bech32 or NIP-05 identifier to a pubkey. */ -export async function lookupPubkey(value: string, signal?: AbortSignal): Promise { +export async function lookupPubkey(value: string, opts: LookupAccountOpts): Promise { if (n.bech32().safeParse(value).success) { return bech32ToPubkey(value); } try { - const { pubkey } = await nip05Cache.fetch(value, { signal }); + const { pubkey } = await lookupNip05(value, opts); return pubkey; } catch { return; diff --git a/packages/ditto/utils/nip05.ts b/packages/ditto/utils/nip05.ts index 7d725ab2..60eb8c32 100644 --- a/packages/ditto/utils/nip05.ts +++ b/packages/ditto/utils/nip05.ts @@ -1,28 +1,20 @@ -import { cachedNip05sSizeGauge } from '@ditto/metrics'; +import { DittoConf } from '@ditto/conf'; import { NIP05, NStore } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import { safeFetch } from '@soapbox/safe-fetch'; import { nip19 } from 'nostr-tools'; import tldts from 'tldts'; -import { Conf } from '@/config.ts'; -import { Storages } from '@/storages.ts'; import { errorJson } from '@/utils/log.ts'; -import { SimpleLRU } from '@/utils/SimpleLRU.ts'; -export const nip05Cache = new SimpleLRU( - async (nip05, { signal }) => { - const store = await Storages.db(); - return getNip05(store, nip05, signal); - }, - { ...Conf.caches.nip05, gauge: cachedNip05sSizeGauge }, -); +interface GetNip05Opts { + conf: DittoConf; + relay: NStore; + signal?: AbortSignal; +} -async function getNip05( - store: NStore, - nip05: string, - signal?: AbortSignal, -): Promise { +export async function lookupNip05(nip05: string, opts: GetNip05Opts): Promise { + const { conf, signal } = opts; const tld = tldts.parse(nip05); if (!tld.isIcann || tld.isIp || tld.isPrivate) { @@ -34,8 +26,8 @@ async function getNip05( const [name, domain] = nip05.split('@'); try { - if (domain === Conf.url.host) { - const pointer = await localNip05Lookup(store, name); + if (domain === conf.url.host) { + const pointer = await localNip05Lookup(name, opts); if (pointer) { logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'local', pubkey: pointer.pubkey }); return pointer; @@ -53,19 +45,24 @@ async function getNip05( } } -export async function localNip05Lookup(store: NStore, localpart: string): Promise { - const name = `${localpart}@${Conf.url.host}`; +export async function localNip05Lookup( + localpart: string, + opts: GetNip05Opts, +): Promise { + const { conf, relay, signal } = opts; - const [grant] = await store.query([{ + const name = `${localpart}@${conf.url.host}`; + + const [grant] = await relay.query([{ kinds: [30360], '#d': [name, name.toLowerCase()], - authors: [await Conf.signer.getPublicKey()], + authors: [await conf.signer.getPublicKey()], limit: 1, - }]); + }], { signal }); const pubkey = grant?.tags.find(([name]) => name === 'p')?.[1]; if (pubkey) { - return { pubkey, relays: [Conf.relay] }; + return { pubkey, relays: [conf.relay] }; } } diff --git a/packages/ditto/views.ts b/packages/ditto/views.ts index 879c3196..ae708360 100644 --- a/packages/ditto/views.ts +++ b/packages/ditto/views.ts @@ -1,10 +1,10 @@ +import { paginated, paginatedList } from '@ditto/mastoapi/pagination'; import { NostrEvent, NostrFilter } from '@nostrify/nostrify'; import { AppContext } from '@/app.ts'; import { paginationSchema } from '@/schemas/pagination.ts'; import { renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; -import { paginated, paginatedList } from '@/utils/api.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { accountFromPubkey } from '@/views/mastodon/accounts.ts'; @@ -25,7 +25,7 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], opts?: const events = await relay.query(filters, { signal }) // Deduplicate by author. .then((events) => Array.from(new Map(events.map((event) => [event.pubkey, event])).values())) - .then((events) => hydrateEvents({ events, relay, signal })) + .then((events) => hydrateEvents({ ...c.var, events, relay, signal })) .then((events) => filterFn ? events.filter(filterFn) : events); const accounts = await Promise.all( @@ -48,7 +48,7 @@ async function renderAccounts(c: AppContext, pubkeys: string[]) { const { relay, signal } = c.var; const events = await relay.query([{ kinds: [0], authors }], { signal }) - .then((events) => hydrateEvents({ events, relay, signal })); + .then((events) => hydrateEvents({ ...c.var, events })); const accounts = await Promise.all( authors.map((pubkey) => { @@ -74,7 +74,7 @@ async function renderStatuses(c: AppContext, ids: string[], signal = AbortSignal const { limit } = pagination; const events = await relay.query([{ kinds: [1, 20], ids, limit }], { signal }) - .then((events) => hydrateEvents({ events, relay, signal })); + .then((events) => hydrateEvents({ ...c.var, events })); if (!events.length) { return c.json([]); @@ -85,7 +85,7 @@ async function renderStatuses(c: AppContext, ids: string[], signal = AbortSignal const viewerPubkey = await user?.signer.getPublicKey(); const statuses = await Promise.all( - sortedEvents.map((event) => renderStatus(event, { viewerPubkey })), + sortedEvents.map((event) => renderStatus(relay, event, { viewerPubkey })), ); // TODO: pagination with min_id and max_id based on the order of `ids`. diff --git a/packages/ditto/views/mastodon/notifications.ts b/packages/ditto/views/mastodon/notifications.ts index 59911606..7f71c1ea 100644 --- a/packages/ditto/views/mastodon/notifications.ts +++ b/packages/ditto/views/mastodon/notifications.ts @@ -1,4 +1,4 @@ -import { NostrEvent } from '@nostrify/nostrify'; +import { NostrEvent, NStore } from '@nostrify/nostrify'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { Conf } from '@/config.ts'; @@ -10,23 +10,23 @@ interface RenderNotificationOpts { viewerPubkey: string; } -async function renderNotification(event: DittoEvent, opts: RenderNotificationOpts) { +async function renderNotification(store: NStore, event: DittoEvent, opts: RenderNotificationOpts) { const mentioned = !!event.tags.find(([name, value]) => name === 'p' && value === opts.viewerPubkey); if (event.kind === 1 && mentioned) { - return renderMention(event, opts); + return renderMention(store, event, opts); } if (event.kind === 6) { - return renderReblog(event, opts); + return renderReblog(store, event, opts); } if (event.kind === 7 && event.content === '+') { - return renderFavourite(event, opts); + return renderFavourite(store, event, opts); } if (event.kind === 7) { - return renderReaction(event, opts); + return renderReaction(store, event, opts); } if (event.kind === 30360 && event.pubkey === await Conf.signer.getPublicKey()) { @@ -34,12 +34,12 @@ async function renderNotification(event: DittoEvent, opts: RenderNotificationOpt } if (event.kind === 9735) { - return renderZap(event, opts); + return renderZap(store, event, opts); } } -async function renderMention(event: DittoEvent, opts: RenderNotificationOpts) { - const status = await renderStatus(event, opts); +async function renderMention(store: NStore, event: DittoEvent, opts: RenderNotificationOpts) { + const status = await renderStatus(store, event, opts); if (!status) return; return { @@ -51,9 +51,9 @@ async function renderMention(event: DittoEvent, opts: RenderNotificationOpts) { }; } -async function renderReblog(event: DittoEvent, opts: RenderNotificationOpts) { +async function renderReblog(store: NStore, event: DittoEvent, opts: RenderNotificationOpts) { if (event.repost?.kind !== 1) return; - const status = await renderStatus(event.repost, opts); + const status = await renderStatus(store, event.repost, opts); if (!status) return; const account = event.author ? await renderAccount(event.author) : await accountFromPubkey(event.pubkey); @@ -66,9 +66,9 @@ async function renderReblog(event: DittoEvent, opts: RenderNotificationOpts) { }; } -async function renderFavourite(event: DittoEvent, opts: RenderNotificationOpts) { +async function renderFavourite(store: NStore, event: DittoEvent, opts: RenderNotificationOpts) { if (event.reacted?.kind !== 1) return; - const status = await renderStatus(event.reacted, opts); + const status = await renderStatus(store, event.reacted, opts); if (!status) return; const account = event.author ? await renderAccount(event.author) : await accountFromPubkey(event.pubkey); @@ -81,9 +81,9 @@ async function renderFavourite(event: DittoEvent, opts: RenderNotificationOpts) }; } -async function renderReaction(event: DittoEvent, opts: RenderNotificationOpts) { +async function renderReaction(store: NStore, event: DittoEvent, opts: RenderNotificationOpts) { if (event.reacted?.kind !== 1) return; - const status = await renderStatus(event.reacted, opts); + const status = await renderStatus(store, event.reacted, opts); if (!status) return; const account = event.author ? await renderAccount(event.author) : await accountFromPubkey(event.pubkey); @@ -116,7 +116,7 @@ async function renderNameGrant(event: DittoEvent) { }; } -async function renderZap(event: DittoEvent, opts: RenderNotificationOpts) { +async function renderZap(store: NStore, event: DittoEvent, opts: RenderNotificationOpts) { if (!event.zap_sender) return; const { zap_amount = 0, zap_message = '' } = event; @@ -133,7 +133,7 @@ async function renderZap(event: DittoEvent, opts: RenderNotificationOpts) { message: zap_message, created_at: nostrDate(event.created_at).toISOString(), account, - ...(event.zapped ? { status: await renderStatus(event.zapped, opts) } : {}), + ...(event.zapped ? { status: await renderStatus(store, event.zapped, opts) } : {}), }; } diff --git a/packages/ditto/views/mastodon/push.ts b/packages/ditto/views/mastodon/push.ts index 0a13179b..eb2e064c 100644 --- a/packages/ditto/views/mastodon/push.ts +++ b/packages/ditto/views/mastodon/push.ts @@ -1,4 +1,4 @@ -import type { NostrEvent } from '@nostrify/nostrify'; +import type { NostrEvent, NStore } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; import { MastodonPush } from '@/types/MastodonPush.ts'; @@ -9,10 +9,11 @@ import { renderNotification } from '@/views/mastodon/notifications.ts'; * Unlike other views, only one will be rendered at a time, so making use of async calls is okay. */ export async function renderWebPushNotification( + store: NStore, event: NostrEvent, viewerPubkey: string, ): Promise { - const notification = await renderNotification(event, { viewerPubkey }); + const notification = await renderNotification(store, event, { viewerPubkey }); if (!notification) { return; } diff --git a/packages/ditto/views/mastodon/reports.ts b/packages/ditto/views/mastodon/reports.ts index 48baa42f..a2ad8d62 100644 --- a/packages/ditto/views/mastodon/reports.ts +++ b/packages/ditto/views/mastodon/reports.ts @@ -1,3 +1,5 @@ +import { NStore } from '@nostrify/nostrify'; + import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { nostrDate } from '@/utils.ts'; @@ -6,7 +8,7 @@ import { renderStatus } from '@/views/mastodon/statuses.ts'; import { getTagSet } from '@/utils/tags.ts'; /** Expects a `reportEvent` of kind 1984 and a `profile` of kind 0 of the person being reported */ -async function renderReport(event: DittoEvent) { +function renderReport(event: DittoEvent) { // The category is present in both the 'e' and 'p' tag, however, it is possible to report a user without reporting a note, so it's better to get the category from the 'p' tag const category = event.tags.find(([name]) => name === 'p')?.[2]; const statusIds = event.tags.filter(([name]) => name === 'e').map((tag) => tag[1]) ?? []; @@ -23,9 +25,7 @@ async function renderReport(event: DittoEvent) { created_at: nostrDate(event.created_at).toISOString(), status_ids: statusIds, rules_ids: null, - target_account: event.reported_profile - ? await renderAccount(event.reported_profile) - : await accountFromPubkey(reportedPubkey), + target_account: event.reported_profile ? renderAccount(event.reported_profile) : accountFromPubkey(reportedPubkey), }; } @@ -36,7 +36,7 @@ interface RenderAdminReportOpts { /** Admin-level information about a filed report. * Expects an event of kind 1984 fully hydrated. * https://docs.joinmastodon.org/entities/Admin_Report */ -async function renderAdminReport(event: DittoEvent, opts: RenderAdminReportOpts) { +async function renderAdminReport(store: NStore, event: DittoEvent, opts: RenderAdminReportOpts) { const { viewerPubkey } = opts; // The category is present in both the 'e' and 'p' tag, however, it is possible to report a user without reporting a note, so it's better to get the category from the 'p' tag @@ -45,7 +45,7 @@ async function renderAdminReport(event: DittoEvent, opts: RenderAdminReportOpts) const statuses = []; if (event.reported_notes) { for (const status of event.reported_notes) { - statuses.push(await renderStatus(status, { viewerPubkey })); + statuses.push(await renderStatus(store, status, { viewerPubkey })); } } diff --git a/packages/ditto/views/mastodon/statuses.ts b/packages/ditto/views/mastodon/statuses.ts index 00f7dd55..5957356e 100644 --- a/packages/ditto/views/mastodon/statuses.ts +++ b/packages/ditto/views/mastodon/statuses.ts @@ -1,4 +1,4 @@ -import { NostrEvent } from '@nostrify/nostrify'; +import { NostrEvent, NStore } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; import { Conf } from '@/config.ts'; @@ -6,7 +6,6 @@ import { MastodonAttachment } from '@/entities/MastodonAttachment.ts'; import { MastodonMention } from '@/entities/MastodonMention.ts'; import { MastodonStatus } from '@/entities/MastodonStatus.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { Storages } from '@/storages.ts'; import { nostrDate } from '@/utils.ts'; import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts'; import { findReplyTag } from '@/utils/tags.ts'; @@ -20,7 +19,11 @@ interface RenderStatusOpts { depth?: number; } -async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise { +async function renderStatus( + store: NStore, + event: DittoEvent, + opts: RenderStatusOpts, +): Promise { const { viewerPubkey, depth = 1 } = opts; if (depth > 2 || depth < 0) return; @@ -38,8 +41,6 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< const replyId = findReplyTag(event.tags)?.[1]; - const store = await Storages.db(); - const mentions = event.mentions?.map((event) => renderMention(event)) ?? []; const { html, links, firstUrl } = parseNoteContent(stripimeta(event.content, event.tags), mentions); @@ -123,7 +124,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< tags: [], emojis: renderEmojis(event), poll: null, - quote: !event.quote ? null : await renderStatus(event.quote, { depth: depth + 1 }), + quote: !event.quote ? null : await renderStatus(store, event.quote, { depth: depth + 1 }), quote_id: event.quote?.id ?? null, uri: Conf.local(`/users/${account.acct}/statuses/${event.id}`), url: Conf.local(`/@${account.acct}/${event.id}`), @@ -139,14 +140,18 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< }; } -async function renderReblog(event: DittoEvent, opts: RenderStatusOpts): Promise { +async function renderReblog( + store: NStore, + event: DittoEvent, + opts: RenderStatusOpts, +): Promise { const { viewerPubkey } = opts; if (!event.repost) return; - const status = await renderStatus(event, {}); // omit viewerPubkey intentionally + const status = await renderStatus(store, event, {}); // omit viewerPubkey intentionally if (!status) return; - const reblog = await renderStatus(event.repost, { viewerPubkey }) ?? null; + const reblog = await renderStatus(store, event.repost, { viewerPubkey }) ?? null; return { ...status, diff --git a/packages/ditto/workers/policy.worker.ts b/packages/ditto/workers/policy.worker.ts index 49fc75ef..539830a5 100644 --- a/packages/ditto/workers/policy.worker.ts +++ b/packages/ditto/workers/policy.worker.ts @@ -30,7 +30,7 @@ export class CustomPolicy implements NPolicy { async init({ path, databaseUrl, pubkey }: PolicyInit): Promise { const Policy = (await import(path)).default; - const db = DittoPolyPg.create(databaseUrl, { poolSize: 1 }); + const db = new DittoPolyPg(databaseUrl, { poolSize: 1 }); const store = new DittoPgStore({ db, diff --git a/packages/mastoapi/deno.json b/packages/mastoapi/deno.json index d98dbc91..b9626b3e 100644 --- a/packages/mastoapi/deno.json +++ b/packages/mastoapi/deno.json @@ -3,6 +3,7 @@ "version": "1.1.0", "exports": { "./middleware": "./middleware/mod.ts", + "./pagination": "./pagination/mod.ts", "./router": "./router/mod.ts", "./test": "./test.ts" } diff --git a/packages/mastoapi/pagination/mod.ts b/packages/mastoapi/pagination/mod.ts new file mode 100644 index 00000000..18998a36 --- /dev/null +++ b/packages/mastoapi/pagination/mod.ts @@ -0,0 +1,3 @@ +export { buildLinkHeader, buildListLinkHeader } from './link-header.ts'; +export { paginated, paginatedList } from './paginate.ts'; +export { paginationSchema } from './schema.ts'; diff --git a/packages/mastoapi/router/DittoApp.test.ts b/packages/mastoapi/router/DittoApp.test.ts index 329b9dbc..c828d68a 100644 --- a/packages/mastoapi/router/DittoApp.test.ts +++ b/packages/mastoapi/router/DittoApp.test.ts @@ -7,7 +7,7 @@ import { DittoApp } from './DittoApp.ts'; import { DittoRoute } from './DittoRoute.ts'; Deno.test('DittoApp', async () => { - await using db = DittoPolyPg.create('memory://'); + await using db = new DittoPolyPg('memory://'); const conf = new DittoConf(new Map()); const relay = new MockRelay(); diff --git a/scripts/trends.ts b/scripts/trends.ts index bb9708ab..2a878a12 100644 --- a/scripts/trends.ts +++ b/scripts/trends.ts @@ -1,5 +1,8 @@ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { z } from 'zod'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; import { updateTrendingEvents, updateTrendingHashtags, @@ -8,6 +11,11 @@ import { updateTrendingZappedEvents, } from '../packages/ditto/trends.ts'; +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); +const ctx = { conf, db, relay }; + const trendSchema = z.enum(['pubkeys', 'zapped_events', 'events', 'hashtags', 'links']); const trends = trendSchema.array().parse(Deno.args); @@ -19,23 +27,23 @@ for (const trend of trends) { switch (trend) { case 'pubkeys': console.log('Updating trending pubkeys...'); - await updateTrendingPubkeys(); + await updateTrendingPubkeys(ctx); break; case 'zapped_events': console.log('Updating trending zapped events...'); - await updateTrendingZappedEvents(); + await updateTrendingZappedEvents(ctx); break; case 'events': console.log('Updating trending events...'); - await updateTrendingEvents(); + await updateTrendingEvents(ctx); break; case 'hashtags': console.log('Updating trending hashtags...'); - await updateTrendingHashtags(); + await updateTrendingHashtags(ctx); break; case 'links': console.log('Updating trending links...'); - await updateTrendingLinks(); + await updateTrendingLinks(ctx); break; } } From f2e2072184320fd0bcf3e694f662391d893484bd Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 19:46:38 -0600 Subject: [PATCH 72/99] Export PolicyWorker as a regular class --- packages/ditto/storages/DittoAPIStore.ts | 6 ++++-- packages/ditto/workers/policy.ts | 26 ++++++++++++------------ scripts/db-policy.ts | 3 ++- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/packages/ditto/storages/DittoAPIStore.ts b/packages/ditto/storages/DittoAPIStore.ts index 7a479899..2df28da5 100644 --- a/packages/ditto/storages/DittoAPIStore.ts +++ b/packages/ditto/storages/DittoAPIStore.ts @@ -33,7 +33,7 @@ import { getAmount } from '@/utils/bolt11.ts'; import { errorJson } from '@/utils/log.ts'; import { purifyEvent } from '@/utils/purify.ts'; import { getTagSet } from '@/utils/tags.ts'; -import { policyWorker } from '@/workers/policy.ts'; +import { PolicyWorker } from '@/workers/policy.ts'; import { verifyEventWorker } from '@/workers/verify.ts'; import { fetchFavicon, insertFavicon, queryFavicon } from '@/utils/favicon.ts'; import { lookupNip05 } from '@/utils/nip05.ts'; @@ -54,6 +54,7 @@ export class DittoAPIStore implements NRelay { private push: DittoPush; private encounters = new LRUCache({ max: 5000 }); private controller = new AbortController(); + private policyWorker: PolicyWorker; private faviconCache: SimpleLRU; private nip05Cache: SimpleLRU; @@ -64,6 +65,7 @@ export class DittoAPIStore implements NRelay { const { conf, db } = this.opts; this.push = new DittoPush(opts); + this.policyWorker = new PolicyWorker(conf); this.listen().catch((e: unknown) => { logi({ level: 'error', ns: this.ns, source: 'listen', error: errorJson(e) }); @@ -211,7 +213,7 @@ export class DittoAPIStore implements NRelay { private async policyFilter(event: NostrEvent, signal?: AbortSignal): Promise { try { - const result = await policyWorker.call(event, signal); + const result = await this.policyWorker.call(event, signal); const [, , ok, reason] = result; logi({ level: 'debug', ns: 'ditto.policy', id: event.id, kind: event.kind, ok, reason }); policyEventsCounter.inc({ ok: String(ok) }); diff --git a/packages/ditto/workers/policy.ts b/packages/ditto/workers/policy.ts index 02de539c..e2617f72 100644 --- a/packages/ditto/workers/policy.ts +++ b/packages/ditto/workers/policy.ts @@ -1,16 +1,16 @@ +import { DittoConf } from '@ditto/conf'; import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import * as Comlink from 'comlink'; -import { Conf } from '@/config.ts'; import type { CustomPolicy } from '@/workers/policy.worker.ts'; -class PolicyWorker implements NPolicy { +export class PolicyWorker implements NPolicy { private worker: Comlink.Remote; private ready: Promise; private enabled = true; - constructor() { + constructor(private conf: DittoConf) { this.worker = Comlink.wrap( new Worker( new URL('./policy.worker.ts', import.meta.url), @@ -19,8 +19,8 @@ class PolicyWorker implements NPolicy { name: 'PolicyWorker', deno: { permissions: { - read: [Conf.denoDir, Conf.policy, Conf.dataDir], - write: [Conf.dataDir], + read: [conf.denoDir, conf.policy, conf.dataDir], + write: [conf.dataDir], net: 'inherit', env: false, import: true, @@ -44,18 +44,20 @@ class PolicyWorker implements NPolicy { } private async init(): Promise { + const conf = this.conf; + try { await this.worker.init({ - path: Conf.policy, - databaseUrl: Conf.databaseUrl, - pubkey: await Conf.signer.getPublicKey(), + path: conf.policy, + databaseUrl: conf.databaseUrl, + pubkey: await conf.signer.getPublicKey(), }); logi({ level: 'info', ns: 'ditto.system.policy', msg: 'Using custom policy', - path: Conf.policy, + path: conf.policy, enabled: true, }); } catch (e) { @@ -76,16 +78,14 @@ class PolicyWorker implements NPolicy { level: 'warn', ns: 'ditto.system.policy', msg: 'Custom policies are not supported with PGlite. The policy is disabled.', - path: Conf.policy, + path: conf.policy, enabled: false, }); this.enabled = false; return; } - throw new Error(`DITTO_POLICY (error importing policy): ${Conf.policy}`); + throw new Error(`DITTO_POLICY (error importing policy): ${conf.policy}`); } } } - -export const policyWorker = new PolicyWorker(); diff --git a/scripts/db-policy.ts b/scripts/db-policy.ts index 80e217c5..b7ceee96 100644 --- a/scripts/db-policy.ts +++ b/scripts/db-policy.ts @@ -2,11 +2,12 @@ import { DittoConf } from '@ditto/conf'; import { DittoPolyPg } from '@ditto/db'; import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; -import { policyWorker } from '../packages/ditto/workers/policy.ts'; +import { PolicyWorker } from '../packages/ditto/workers/policy.ts'; const conf = new DittoConf(Deno.env); const db = new DittoPolyPg(conf.databaseUrl); const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); +const policyWorker = new PolicyWorker(conf); let count = 0; From 70f0eb3b0337e97224161c07b881561c52843eaa Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 19:49:43 -0600 Subject: [PATCH 73/99] Fix pagination lint errors --- packages/ditto/controllers/api/ditto.ts | 2 +- packages/mastoapi/pagination/schema.ts | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/ditto/controllers/api/ditto.ts b/packages/ditto/controllers/api/ditto.ts index 2aa8da2b..38c72eb4 100644 --- a/packages/ditto/controllers/api/ditto.ts +++ b/packages/ditto/controllers/api/ditto.ts @@ -1,4 +1,4 @@ -import { paginated, paginatedList } from '@ditto/mastoapi/pagination'; +import { paginated } from '@ditto/mastoapi/pagination'; import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { z } from 'zod'; diff --git a/packages/mastoapi/pagination/schema.ts b/packages/mastoapi/pagination/schema.ts index 89e3c5f6..5647246d 100644 --- a/packages/mastoapi/pagination/schema.ts +++ b/packages/mastoapi/pagination/schema.ts @@ -1,7 +1,16 @@ import { z } from 'zod'; +export interface Pagination { + max_id?: string; + min_id?: string; + since?: number; + until?: number; + limit: number; + offset: number; +} + /** Schema to parse pagination query params. */ -export const paginationSchema = z.object({ +export const paginationSchema: z.ZodType = z.object({ max_id: z.string().transform((val) => { if (!val.includes('-')) return val; return val.split('-')[1]; @@ -11,4 +20,4 @@ export const paginationSchema = z.object({ until: z.coerce.number().nonnegative().optional().catch(undefined), limit: z.coerce.number().catch(20).transform((value) => Math.min(Math.max(value, 0), 40)), offset: z.coerce.number().nonnegative().catch(0), -}); +}) as z.ZodType; From f893a8146473bc53f5e718ee42637eb0e85890e9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 19:55:58 -0600 Subject: [PATCH 74/99] DittoAPIStore: add limit 0 --- packages/ditto/storages/DittoAPIStore.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ditto/storages/DittoAPIStore.ts b/packages/ditto/storages/DittoAPIStore.ts index 2df28da5..ad7c6028 100644 --- a/packages/ditto/storages/DittoAPIStore.ts +++ b/packages/ditto/storages/DittoAPIStore.ts @@ -122,7 +122,7 @@ export class DittoAPIStore implements NRelay { const { relay } = this.opts; const { signal } = this.controller; - for await (const msg of relay.req([{}], { signal })) { + for await (const msg of relay.req([{ limit: 0 }], { signal })) { if (msg[0] === 'EVENT') { const [, , event] = msg; await this.handleEvent(event, { signal }); From 4f46a69131d72f9860eaa0e9f5c4e0569947fc0c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 20:56:53 -0600 Subject: [PATCH 75/99] I did a fucked up polymorphism --- packages/ditto/controllers/api/cashu.test.ts | 15 +++------------ packages/ditto/storages/DittoPgStore.ts | 6 +++--- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/packages/ditto/controllers/api/cashu.test.ts b/packages/ditto/controllers/api/cashu.test.ts index 85803f18..75017b11 100644 --- a/packages/ditto/controllers/api/cashu.test.ts +++ b/packages/ditto/controllers/api/cashu.test.ts @@ -13,10 +13,7 @@ import { createTestDB } from '@/test.ts'; import cashuRoute from './cashu.ts'; import { walletSchema } from '@/schema.ts'; -Deno.test('PUT /wallet must be successful', { - sanitizeOps: false, - sanitizeResources: false, -}, async () => { +Deno.test('PUT /wallet must be successful', async () => { await using test = await createTestRoute(); const { route, signer, sk, relay } = test; @@ -101,10 +98,7 @@ Deno.test('PUT /wallet must NOT be successful: wrong request body/schema', async assertObjectMatch(body, { error: 'Bad schema' }); }); -Deno.test('PUT /wallet must NOT be successful: wallet already exists', { - sanitizeOps: false, - sanitizeResources: false, -}, async () => { +Deno.test('PUT /wallet must NOT be successful: wallet already exists', async () => { await using test = await createTestRoute(); const { route, sk, relay } = test; @@ -127,10 +121,7 @@ Deno.test('PUT /wallet must NOT be successful: wallet already exists', { assertEquals(body2, { error: 'You already have a wallet 😏' }); }); -Deno.test('GET /wallet must be successful', { - sanitizeOps: false, - sanitizeResources: false, -}, async () => { +Deno.test('GET /wallet must be successful', async () => { await using test = await createTestRoute(); const { route, sk, relay, signer } = test; diff --git a/packages/ditto/storages/DittoPgStore.ts b/packages/ditto/storages/DittoPgStore.ts index bf6babb5..619495c0 100644 --- a/packages/ditto/storages/DittoPgStore.ts +++ b/packages/ditto/storages/DittoPgStore.ts @@ -164,9 +164,9 @@ export class DittoPgStore extends NPostgres { opts: { signal?: AbortSignal; timeout?: number } = {}, ): Promise { try { - await this.transaction(async (store, kysely) => { - await updateStats({ event, store, kysely }); - await super.event(event, opts); + await super.transaction(async (store, kysely) => { + await updateStats({ event, store, kysely: kysely as unknown as Kysely }); + await store.event(event, opts); }); } catch (e) { // If the failure is only because of updateStats (which runs first), insert the event anyway. From 6cd64500ce4fa18edfc4d99d86fd747129dd1e52 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 21:24:17 -0600 Subject: [PATCH 76/99] Fix stats test --- packages/db/adapters/DittoPglite.ts | 1 - packages/ditto/storages/DittoPgStore.ts | 6 +- packages/ditto/utils/stats.test.ts | 154 ++++++++++++++---------- packages/ditto/utils/stats.ts | 34 +++--- 4 files changed, 113 insertions(+), 82 deletions(-) diff --git a/packages/db/adapters/DittoPglite.ts b/packages/db/adapters/DittoPglite.ts index 33516ee2..7fcd5bab 100644 --- a/packages/db/adapters/DittoPglite.ts +++ b/packages/db/adapters/DittoPglite.ts @@ -47,7 +47,6 @@ export class DittoPglite implements DittoDB { } async [Symbol.asyncDispose](): Promise { - await this.pglite.close(); await this.kysely.destroy(); } } diff --git a/packages/ditto/storages/DittoPgStore.ts b/packages/ditto/storages/DittoPgStore.ts index 619495c0..f473a791 100644 --- a/packages/ditto/storages/DittoPgStore.ts +++ b/packages/ditto/storages/DittoPgStore.ts @@ -164,9 +164,9 @@ export class DittoPgStore extends NPostgres { opts: { signal?: AbortSignal; timeout?: number } = {}, ): Promise { try { - await super.transaction(async (store, kysely) => { - await updateStats({ event, store, kysely: kysely as unknown as Kysely }); - await store.event(event, opts); + await super.transaction(async (relay, kysely) => { + await updateStats({ event, relay, kysely: kysely as unknown as Kysely }); + await relay.event(event, opts); }); } catch (e) { // If the failure is only because of updateStats (which runs first), insert the event anyway. diff --git a/packages/ditto/utils/stats.test.ts b/packages/ditto/utils/stats.test.ts index 762db37c..043e6f13 100644 --- a/packages/ditto/utils/stats.test.ts +++ b/packages/ditto/utils/stats.test.ts @@ -1,43 +1,48 @@ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; +import { NPostgres } from '@nostrify/db'; import { genEvent } from '@nostrify/nostrify/test'; import { assertEquals } from '@std/assert'; +import { sql } from 'kysely'; import { generateSecretKey, getPublicKey } from 'nostr-tools'; -import { createTestDB } from '@/test.ts'; import { countAuthorStats, getAuthorStats, getEventStats, getFollowDiff, updateStats } from '@/utils/stats.ts'; Deno.test('updateStats with kind 1 increments notes count', async () => { - await using db = await createTestDB(); + await using test = await setupTest(); const sk = generateSecretKey(); const pubkey = getPublicKey(sk); - await updateStats({ ...db, event: genEvent({ kind: 1 }, sk) }); + await updateStats({ ...test, event: genEvent({ kind: 1 }, sk) }); - const stats = await getAuthorStats(db.kysely, pubkey); + const stats = await getAuthorStats(test.kysely, pubkey); assertEquals(stats!.notes_count, 1); }); Deno.test('updateStats with kind 1 increments replies count', async () => { - await using db = await createTestDB(); + await using test = await setupTest(); + const { relay, kysely } = test; const sk = generateSecretKey(); const note = genEvent({ kind: 1 }, sk); - await updateStats({ ...db, event: note }); - await db.store.event(note); + await updateStats({ ...test, event: note }); + await relay.event(note); const reply = genEvent({ kind: 1, tags: [['e', note.id]] }, sk); - await updateStats({ ...db, event: reply }); - await db.store.event(reply); + await updateStats({ ...test, event: reply }); + await relay.event(reply); - const stats = await getEventStats(db.kysely, note.id); + const stats = await getEventStats(kysely, note.id); assertEquals(stats!.replies_count, 1); }); Deno.test('updateStats with kind 5 decrements notes count', async () => { - await using db = await createTestDB(); + await using test = await setupTest(); + const { relay, kysely } = test; const sk = generateSecretKey(); const pubkey = getPublicKey(sk); @@ -45,41 +50,43 @@ Deno.test('updateStats with kind 5 decrements notes count', async () => { const create = genEvent({ kind: 1 }, sk); const remove = genEvent({ kind: 5, tags: [['e', create.id]] }, sk); - await updateStats({ ...db, event: create }); - assertEquals((await getAuthorStats(db.kysely, pubkey))!.notes_count, 1); - await db.store.event(create); + await updateStats({ ...test, event: create }); + assertEquals((await getAuthorStats(kysely, pubkey))!.notes_count, 1); + await relay.event(create); - await updateStats({ ...db, event: remove }); - assertEquals((await getAuthorStats(db.kysely, pubkey))!.notes_count, 0); - await db.store.event(remove); + await updateStats({ ...test, event: remove }); + assertEquals((await getAuthorStats(kysely, pubkey))!.notes_count, 0); + await relay.event(remove); }); Deno.test('updateStats with kind 3 increments followers count', async () => { - await using db = await createTestDB(); + await using test = await setupTest(); + const { kysely } = test; - await updateStats({ ...db, event: genEvent({ kind: 3, tags: [['p', 'alex']] }) }); - await updateStats({ ...db, event: genEvent({ kind: 3, tags: [['p', 'alex']] }) }); - await updateStats({ ...db, event: genEvent({ kind: 3, tags: [['p', 'alex']] }) }); + await updateStats({ ...test, event: genEvent({ kind: 3, tags: [['p', 'alex']] }) }); + await updateStats({ ...test, event: genEvent({ kind: 3, tags: [['p', 'alex']] }) }); + await updateStats({ ...test, event: genEvent({ kind: 3, tags: [['p', 'alex']] }) }); - const stats = await getAuthorStats(db.kysely, 'alex'); + const stats = await getAuthorStats(kysely, 'alex'); assertEquals(stats!.followers_count, 3); }); Deno.test('updateStats with kind 3 decrements followers count', async () => { - await using db = await createTestDB(); + await using test = await setupTest(); + const { relay, kysely } = test; const sk = generateSecretKey(); const follow = genEvent({ kind: 3, tags: [['p', 'alex']], created_at: 0 }, sk); const remove = genEvent({ kind: 3, tags: [], created_at: 1 }, sk); - await updateStats({ ...db, event: follow }); - assertEquals((await getAuthorStats(db.kysely, 'alex'))!.followers_count, 1); - await db.store.event(follow); + await updateStats({ ...test, event: follow }); + assertEquals((await getAuthorStats(kysely, 'alex'))!.followers_count, 1); + await relay.event(follow); - await updateStats({ ...db, event: remove }); - assertEquals((await getAuthorStats(db.kysely, 'alex'))!.followers_count, 0); - await db.store.event(remove); + await updateStats({ ...test, event: remove }); + assertEquals((await getAuthorStats(kysely, 'alex'))!.followers_count, 0); + await relay.event(remove); }); Deno.test('getFollowDiff returns added and removed followers', () => { @@ -93,86 +100,91 @@ Deno.test('getFollowDiff returns added and removed followers', () => { }); Deno.test('updateStats with kind 6 increments reposts count', async () => { - await using db = await createTestDB(); + await using test = await setupTest(); + const { relay, kysely } = test; const note = genEvent({ kind: 1 }); - await updateStats({ ...db, event: note }); - await db.store.event(note); + await updateStats({ ...test, event: note }); + await relay.event(note); const repost = genEvent({ kind: 6, tags: [['e', note.id]] }); - await updateStats({ ...db, event: repost }); - await db.store.event(repost); + await updateStats({ ...test, event: repost }); + await relay.event(repost); - const stats = await getEventStats(db.kysely, note.id); + const stats = await getEventStats(kysely, note.id); assertEquals(stats!.reposts_count, 1); }); Deno.test('updateStats with kind 5 decrements reposts count', async () => { - await using db = await createTestDB(); + await using test = await setupTest(); + const { relay, kysely } = test; const note = genEvent({ kind: 1 }); - await updateStats({ ...db, event: note }); - await db.store.event(note); + await updateStats({ ...test, event: note }); + await relay.event(note); const sk = generateSecretKey(); const repost = genEvent({ kind: 6, tags: [['e', note.id]] }, sk); - await updateStats({ ...db, event: repost }); - await db.store.event(repost); + await updateStats({ ...test, event: repost }); + await relay.event(repost); - await updateStats({ ...db, event: genEvent({ kind: 5, tags: [['e', repost.id]] }, sk) }); + await updateStats({ ...test, event: genEvent({ kind: 5, tags: [['e', repost.id]] }, sk) }); - const stats = await getEventStats(db.kysely, note.id); + const stats = await getEventStats(kysely, note.id); assertEquals(stats!.reposts_count, 0); }); Deno.test('updateStats with kind 7 increments reactions count', async () => { - await using db = await createTestDB(); + await using test = await setupTest(); + const { relay, kysely } = test; const note = genEvent({ kind: 1 }); - await updateStats({ ...db, event: note }); - await db.store.event(note); + await updateStats({ ...test, event: note }); + await relay.event(note); - await updateStats({ ...db, event: genEvent({ kind: 7, content: '+', tags: [['e', note.id]] }) }); - await updateStats({ ...db, event: genEvent({ kind: 7, content: '😂', tags: [['e', note.id]] }) }); + await updateStats({ ...test, event: genEvent({ kind: 7, content: '+', tags: [['e', note.id]] }) }); + await updateStats({ ...test, event: genEvent({ kind: 7, content: '😂', tags: [['e', note.id]] }) }); - const stats = await getEventStats(db.kysely, note.id); + const stats = await getEventStats(kysely, note.id); assertEquals(stats!.reactions, JSON.stringify({ '+': 1, '😂': 1 })); assertEquals(stats!.reactions_count, 2); }); Deno.test('updateStats with kind 5 decrements reactions count', async () => { - await using db = await createTestDB(); + await using test = await setupTest(); + const { relay, kysely } = test; const note = genEvent({ kind: 1 }); - await updateStats({ ...db, event: note }); - await db.store.event(note); + await updateStats({ ...test, event: note }); + await relay.event(note); const sk = generateSecretKey(); const reaction = genEvent({ kind: 7, content: '+', tags: [['e', note.id]] }, sk); - await updateStats({ ...db, event: reaction }); - await db.store.event(reaction); + await updateStats({ ...test, event: reaction }); + await relay.event(reaction); - await updateStats({ ...db, event: genEvent({ kind: 5, tags: [['e', reaction.id]] }, sk) }); + await updateStats({ ...test, event: genEvent({ kind: 5, tags: [['e', reaction.id]] }, sk) }); - const stats = await getEventStats(db.kysely, note.id); + const stats = await getEventStats(kysely, note.id); assertEquals(stats!.reactions, JSON.stringify({})); }); Deno.test('countAuthorStats counts author stats from the database', async () => { - await using db = await createTestDB(); + await using test = await setupTest(); + const { relay } = test; const sk = generateSecretKey(); const pubkey = getPublicKey(sk); - await db.store.event(genEvent({ kind: 1, content: 'hello' }, sk)); - await db.store.event(genEvent({ kind: 1, content: 'yolo' }, sk)); - await db.store.event(genEvent({ kind: 3, tags: [['p', pubkey]] })); + await relay.event(genEvent({ kind: 1, content: 'hello' }, sk)); + await relay.event(genEvent({ kind: 1, content: 'yolo' }, sk)); + await relay.event(genEvent({ kind: 3, tags: [['p', pubkey]] })); - await db.kysely.insertInto('author_stats').values({ + await test.kysely.insertInto('author_stats').values({ pubkey, search: 'Yolo Lolo', notes_count: 0, @@ -181,8 +193,28 @@ Deno.test('countAuthorStats counts author stats from the database', async () => }).onConflict((oc) => oc.column('pubkey').doUpdateSet({ 'search': 'baka' })) .execute(); - const stats = await countAuthorStats({ store: db.store, pubkey, kysely: db.kysely }); + const stats = await countAuthorStats({ ...test, pubkey }); assertEquals(stats!.notes_count, 2); assertEquals(stats!.followers_count, 1); }); + +async function setupTest() { + const conf = new DittoConf(Deno.env); + + const db = new DittoPolyPg(conf.databaseUrl); + await db.migrate(); + + const { kysely } = db; + const relay = new NPostgres(kysely); + + return { + relay, + kysely, + [Symbol.asyncDispose]: async () => { + await sql`truncate table event_stats cascade`.execute(kysely); + await sql`truncate table author_stats cascade`.execute(kysely); + await db[Symbol.asyncDispose](); + }, + }; +} diff --git a/packages/ditto/utils/stats.ts b/packages/ditto/utils/stats.ts index 01ec80d9..448ba241 100644 --- a/packages/ditto/utils/stats.ts +++ b/packages/ditto/utils/stats.ts @@ -9,14 +9,14 @@ import { findQuoteTag, findReplyTag, getTagSet } from '@/utils/tags.ts'; interface UpdateStatsOpts { kysely: Kysely; - store: NStore; + relay: NStore; event: NostrEvent; x?: 1 | -1; } /** Handle one event at a time and update relevant stats for it. */ // deno-lint-ignore require-await -export async function updateStats({ event, kysely, store, x = 1 }: UpdateStatsOpts): Promise { +export async function updateStats({ event, kysely, relay, x = 1 }: UpdateStatsOpts): Promise { switch (event.kind) { case 1: case 20: @@ -24,9 +24,9 @@ export async function updateStats({ event, kysely, store, x = 1 }: UpdateStatsOp case 30023: return handleEvent1(kysely, event, x); case 3: - return handleEvent3(kysely, event, x, store); + return handleEvent3(kysely, event, x, relay); case 5: - return handleEvent5(kysely, event, -1, store); + return handleEvent5(kysely, event, -1, relay); case 6: return handleEvent6(kysely, event, x); case 7: @@ -88,12 +88,12 @@ async function handleEvent1(kysely: Kysely, event: NostrEvent, x: n } /** Update stats for kind 3 event. */ -async function handleEvent3(kysely: Kysely, event: NostrEvent, x: number, store: NStore): Promise { +async function handleEvent3(kysely: Kysely, event: NostrEvent, x: number, relay: NStore): Promise { const following = getTagSet(event.tags, 'p'); await updateAuthorStats(kysely, event.pubkey, () => ({ following_count: following.size })); - const [prev] = await store.query([ + const [prev] = await relay.query([ { kinds: [3], authors: [event.pubkey], limit: 1 }, ]); @@ -117,12 +117,12 @@ async function handleEvent3(kysely: Kysely, event: NostrEvent, x: n } /** Update stats for kind 5 event. */ -async function handleEvent5(kysely: Kysely, event: NostrEvent, x: -1, store: NStore): Promise { +async function handleEvent5(kysely: Kysely, event: NostrEvent, x: -1, relay: NStore): Promise { const id = event.tags.find(([name]) => name === 'e')?.[1]; if (id) { - const [target] = await store.query([{ ids: [id], authors: [event.pubkey], limit: 1 }]); + const [target] = await relay.query([{ ids: [id], authors: [event.pubkey], limit: 1 }]); if (target) { - await updateStats({ event: target, kysely, store, x }); + await updateStats({ event: target, kysely, relay, x }); } } } @@ -300,13 +300,13 @@ export async function updateEventStats( /** Calculate author stats from the database. */ export async function countAuthorStats( - { pubkey, store }: RefreshAuthorStatsOpts, + { pubkey, relay }: RefreshAuthorStatsOpts, ): Promise { const [{ count: followers_count }, { count: notes_count }, [followList], [kind0]] = await Promise.all([ - store.count([{ kinds: [3], '#p': [pubkey] }]), - store.count([{ kinds: [1, 20], authors: [pubkey] }]), - store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]), - store.query([{ kinds: [0], authors: [pubkey], limit: 1 }]), + relay.count([{ kinds: [3], '#p': [pubkey] }]), + relay.count([{ kinds: [1, 20], authors: [pubkey] }]), + relay.query([{ kinds: [3], authors: [pubkey], limit: 1 }]), + relay.query([{ kinds: [0], authors: [pubkey], limit: 1 }]), ]); let search: string = ''; const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(kind0?.content); @@ -333,14 +333,14 @@ export async function countAuthorStats( export interface RefreshAuthorStatsOpts { pubkey: string; kysely: Kysely; - store: SetRequired; + relay: SetRequired; } /** Refresh the author's stats in the database. */ export async function refreshAuthorStats( - { pubkey, kysely, store }: RefreshAuthorStatsOpts, + { pubkey, kysely, relay }: RefreshAuthorStatsOpts, ): Promise { - const stats = await countAuthorStats({ store, pubkey, kysely }); + const stats = await countAuthorStats({ relay, pubkey, kysely }); await kysely.insertInto('author_stats') .values(stats) From 1ae9da5793449441edec838a0cb4dbe01c0d4930 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 21:26:23 -0600 Subject: [PATCH 77/99] Fix hydrate tests --- packages/ditto/storages/hydrate.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/ditto/storages/hydrate.test.ts b/packages/ditto/storages/hydrate.test.ts index 6ba4870b..fa14d50d 100644 --- a/packages/ditto/storages/hydrate.test.ts +++ b/packages/ditto/storages/hydrate.test.ts @@ -2,6 +2,7 @@ import { DittoConf } from '@ditto/conf'; import { DummyDB } from '@ditto/db'; import { MockRelay } from '@nostrify/nostrify/test'; import { assertEquals } from '@std/assert'; +import { generateSecretKey, nip19 } from 'nostr-tools'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; @@ -162,7 +163,7 @@ Deno.test('hydrateEvents(): zap sender, zap amount, zapped post // kind 9735 --- function setupTest() { const db = new DummyDB(); - const conf = new DittoConf(new Map()); + const conf = new DittoConf(new Map([['DITTO_NSEC', nip19.nsecEncode(generateSecretKey())]])); const relay = new MockRelay(); return { conf, db, relay }; From 979f2cffb487a429515d55e4739a4f99a0fff6f5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 21:30:57 -0600 Subject: [PATCH 78/99] Fix stats:recompute script --- scripts/stats-recompute.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/stats-recompute.ts b/scripts/stats-recompute.ts index c17e9047..16614e45 100644 --- a/scripts/stats-recompute.ts +++ b/scripts/stats-recompute.ts @@ -9,6 +9,8 @@ const conf = new DittoConf(Deno.env); const db = new DittoPolyPg(conf.databaseUrl); const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); +const { kysely } = db; + let pubkey: string; try { const result = nip19.decode(Deno.args[0]); @@ -22,4 +24,4 @@ try { Deno.exit(1); } -await refreshAuthorStats({ pubkey, kysely: db.kysely, store: relay }); +await refreshAuthorStats({ pubkey, kysely, relay }); From 6f1312b67fb6a6498de0f7e535e9aacec1e084f9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 21:38:48 -0600 Subject: [PATCH 79/99] Remove old paginationSchema --- packages/ditto/controllers/api/accounts.ts | 4 ++-- packages/ditto/controllers/api/statuses.ts | 3 +-- packages/ditto/controllers/api/suggestions.ts | 3 +-- packages/ditto/controllers/api/trends.ts | 3 +-- packages/ditto/schemas/pagination.ts | 14 -------------- packages/ditto/views.ts | 3 +-- 6 files changed, 6 insertions(+), 24 deletions(-) delete mode 100644 packages/ditto/schemas/pagination.ts diff --git a/packages/ditto/controllers/api/accounts.ts b/packages/ditto/controllers/api/accounts.ts index 685ef70a..495e79b5 100644 --- a/packages/ditto/controllers/api/accounts.ts +++ b/packages/ditto/controllers/api/accounts.ts @@ -72,8 +72,8 @@ const verifyCredentialsController: AppController = async (c) => { } const account = author - ? await renderAccount(author, { withSource: true, settingsStore }) - : await accountFromPubkey(pubkey, { withSource: true, settingsStore }); + ? renderAccount(author, { withSource: true, settingsStore }) + : accountFromPubkey(pubkey, { withSource: true, settingsStore }); return c.json(account); }; diff --git a/packages/ditto/controllers/api/statuses.ts b/packages/ditto/controllers/api/statuses.ts index 4bf2ed23..8bc04151 100644 --- a/packages/ditto/controllers/api/statuses.ts +++ b/packages/ditto/controllers/api/statuses.ts @@ -1,5 +1,5 @@ import { HTTPException } from '@hono/hono/http-exception'; -import { paginated, paginatedList } from '@ditto/mastoapi/pagination'; +import { paginated, paginatedList, paginationSchema } from '@ditto/mastoapi/pagination'; import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; import 'linkify-plugin-hashtag'; import linkify from 'linkifyjs'; @@ -10,7 +10,6 @@ import { type AppController } from '@/app.ts'; import { DittoUpload, dittoUploads } from '@/DittoUploads.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { getAncestors, getAuthor, getDescendants, getEvent } from '@/queries.ts'; -import { paginationSchema } from '@/schemas/pagination.ts'; import { addTag, deleteTag } from '@/utils/tags.ts'; import { asyncReplaceAll } from '@/utils/text.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; diff --git a/packages/ditto/controllers/api/suggestions.ts b/packages/ditto/controllers/api/suggestions.ts index 39cbd235..cb6a8206 100644 --- a/packages/ditto/controllers/api/suggestions.ts +++ b/packages/ditto/controllers/api/suggestions.ts @@ -1,9 +1,8 @@ -import { paginated, paginatedList } from '@ditto/mastoapi/pagination'; +import { paginated, paginatedList, paginationSchema } from '@ditto/mastoapi/pagination'; import { NostrFilter } from '@nostrify/nostrify'; import { matchFilter } from 'nostr-tools'; import { AppContext, AppController } from '@/app.ts'; -import { paginationSchema } from '@/schemas/pagination.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { getTagSet } from '@/utils/tags.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; diff --git a/packages/ditto/controllers/api/trends.ts b/packages/ditto/controllers/api/trends.ts index ce35601f..a687c2cc 100644 --- a/packages/ditto/controllers/api/trends.ts +++ b/packages/ditto/controllers/api/trends.ts @@ -1,11 +1,10 @@ import { type DittoConf } from '@ditto/conf'; -import { paginated } from '@ditto/mastoapi/pagination'; +import { paginated, paginationSchema } from '@ditto/mastoapi/pagination'; import { NostrEvent, NostrFilter, NStore } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import { z } from 'zod'; import { AppController } from '@/app.ts'; -import { paginationSchema } from '@/schemas/pagination.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { generateDateRange, Time } from '@/utils/time.ts'; import { PreviewCard, unfurlCardCached } from '@/utils/unfurl.ts'; diff --git a/packages/ditto/schemas/pagination.ts b/packages/ditto/schemas/pagination.ts deleted file mode 100644 index 89e3c5f6..00000000 --- a/packages/ditto/schemas/pagination.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { z } from 'zod'; - -/** Schema to parse pagination query params. */ -export const paginationSchema = z.object({ - max_id: z.string().transform((val) => { - if (!val.includes('-')) return val; - return val.split('-')[1]; - }).optional().catch(undefined), - min_id: z.string().optional().catch(undefined), - since: z.coerce.number().nonnegative().optional().catch(undefined), - until: z.coerce.number().nonnegative().optional().catch(undefined), - limit: z.coerce.number().catch(20).transform((value) => Math.min(Math.max(value, 0), 40)), - offset: z.coerce.number().nonnegative().catch(0), -}); diff --git a/packages/ditto/views.ts b/packages/ditto/views.ts index ae708360..79379e6c 100644 --- a/packages/ditto/views.ts +++ b/packages/ditto/views.ts @@ -1,8 +1,7 @@ -import { paginated, paginatedList } from '@ditto/mastoapi/pagination'; +import { paginated, paginatedList, paginationSchema } from '@ditto/mastoapi/pagination'; import { NostrEvent, NostrFilter } from '@nostrify/nostrify'; import { AppContext } from '@/app.ts'; -import { paginationSchema } from '@/schemas/pagination.ts'; import { renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; From 7f059b4daceb61708c44b7f92b8b44f54fa6d25b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 21:40:53 -0600 Subject: [PATCH 80/99] Fix event hydration with getEvent/getAuthor --- packages/ditto/queries.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/ditto/queries.ts b/packages/ditto/queries.ts index e14b4f28..dd1e54e1 100644 --- a/packages/ditto/queries.ts +++ b/packages/ditto/queries.ts @@ -19,8 +19,8 @@ interface GetEventOpts { */ async function getEvent(id: string, opts: GetEventOpts): Promise { const filter: NostrFilter = { ids: [id], limit: 1 }; - const [event] = await opts.relay.query([filter], opts); - hydrateEvents({ ...opts, events: [event] }); + const events = await opts.relay.query([filter], opts); + const [event] = await hydrateEvents({ ...opts, events }); return event; } @@ -29,8 +29,8 @@ async function getEvent(id: string, opts: GetEventOpts): Promise { - const [event] = await opts.relay.query([{ authors: [pubkey], kinds: [0], limit: 1 }], opts); - hydrateEvents({ ...opts, events: [event] }); + const events = await opts.relay.query([{ authors: [pubkey], kinds: [0], limit: 1 }], opts); + const [event] = await hydrateEvents({ ...opts, events }); return event; } From f1cb8c778a7245bdc82c9850723a1e13908666a3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 21:45:04 -0600 Subject: [PATCH 81/99] Normalize Link header URLs --- packages/mastoapi/pagination/paginate.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/mastoapi/pagination/paginate.ts b/packages/mastoapi/pagination/paginate.ts index 2da2e478..26be72bd 100644 --- a/packages/mastoapi/pagination/paginate.ts +++ b/packages/mastoapi/pagination/paginate.ts @@ -1,5 +1,6 @@ import { buildLinkHeader, buildListLinkHeader } from './link-header.ts'; +import type { DittoEnv } from '@ditto/mastoapi/router'; import type { Context } from '@hono/hono'; import type { NostrEvent } from '@nostrify/nostrify'; @@ -7,12 +8,15 @@ type HeaderRecord = Record; /** Return results with pagination headers. Assumes chronological sorting of events. */ export function paginated( - c: Context, + c: Context, events: NostrEvent[], body: object | unknown[], headers: HeaderRecord = {}, ): Response { - const link = buildLinkHeader(c.req.url, events); + const { conf } = c.var; + + const url = conf.local(c.req.url); + const link = buildLinkHeader(url, events); if (link) { headers.link = link; @@ -25,12 +29,15 @@ export function paginated( /** paginate a list of tags. */ export function paginatedList( - c: Context, + c: Context, params: { offset: number; limit: number }, body: object | unknown[], headers: HeaderRecord = {}, ): Response { - const link = buildListLinkHeader(c.req.url, params); + const { conf } = c.var; + + const url = conf.local(c.req.url); + const link = buildListLinkHeader(url, params); const hasMore = Array.isArray(body) ? body.length > 0 : true; if (link) { From 237f6e55ad3312b50a93f32f91233f90b0879cf7 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 21:52:26 -0600 Subject: [PATCH 82/99] Fix DittoEnv type check --- packages/ditto/controllers/api/cashu.ts | 2 -- packages/ditto/utils/api.ts | 8 +++++++- packages/mastoapi/pagination/paginate.ts | 8 ++++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/ditto/controllers/api/cashu.ts b/packages/ditto/controllers/api/cashu.ts index a98a0309..4546dda3 100644 --- a/packages/ditto/controllers/api/cashu.ts +++ b/packages/ditto/controllers/api/cashu.ts @@ -78,7 +78,6 @@ route.put('/wallet', userMiddleware({ enc: 'nip44' }), async (c) => { await createEvent({ kind: 17375, content: encryptedWalletContentTags, - // @ts-ignore kill me }, c); // Nutzap information @@ -89,7 +88,6 @@ route.put('/wallet', userMiddleware({ enc: 'nip44' }), async (c) => { ['relay', conf.relay], // TODO: add more relays once things get more stable ['pubkey', p2pk], ], - // @ts-ignore kill me }, c); // TODO: hydrate wallet and add a 'balance' field when a 'renderWallet' view function is created diff --git a/packages/ditto/utils/api.ts b/packages/ditto/utils/api.ts index 80dc4e57..58740917 100644 --- a/packages/ditto/utils/api.ts +++ b/packages/ditto/utils/api.ts @@ -1,3 +1,5 @@ +import { User } from '@ditto/mastoapi/middleware'; +import { DittoEnv } from '@ditto/mastoapi/router'; import { HTTPException } from '@hono/hono/http-exception'; import { NostrEvent, NostrFilter } from '@nostrify/nostrify'; import { EventTemplate } from 'nostr-tools'; @@ -6,12 +8,16 @@ import * as TypeFest from 'type-fest'; import { type AppContext } from '@/app.ts'; import { nostrNow } from '@/utils.ts'; import { parseFormData } from '@/utils/formdata.ts'; +import { Context } from '@hono/hono'; /** EventTemplate with defaults. */ type EventStub = TypeFest.SetOptional; /** Publish an event through the pipeline. */ -async function createEvent(t: EventStub, c: AppContext): Promise { +async function createEvent( + t: EventStub, + c: Context, +): Promise { const { user, relay, signal } = c.var; if (!user) { diff --git a/packages/mastoapi/pagination/paginate.ts b/packages/mastoapi/pagination/paginate.ts index 26be72bd..aab93a47 100644 --- a/packages/mastoapi/pagination/paginate.ts +++ b/packages/mastoapi/pagination/paginate.ts @@ -7,8 +7,8 @@ import type { NostrEvent } from '@nostrify/nostrify'; type HeaderRecord = Record; /** Return results with pagination headers. Assumes chronological sorting of events. */ -export function paginated( - c: Context, +export function paginated( + c: Context, events: NostrEvent[], body: object | unknown[], headers: HeaderRecord = {}, @@ -28,8 +28,8 @@ export function paginated( } /** paginate a list of tags. */ -export function paginatedList( - c: Context, +export function paginatedList( + c: Context, params: { offset: number; limit: number }, body: object | unknown[], headers: HeaderRecord = {}, From a9c696936b66cddde5e7d38fa837718ff212d945 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Feb 2025 00:11:42 -0600 Subject: [PATCH 83/99] Upgrade Nostrify --- deno.json | 4 ++-- deno.lock | 42 +++++++++++++++++++++++------------------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/deno.json b/deno.json index 05ecb34a..fccea26b 100644 --- a/deno.json +++ b/deno.json @@ -62,8 +62,8 @@ "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@negrel/webpush": "jsr:@negrel/webpush@^0.3.0", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/db": "jsr:@nostrify/db@^0.39.3", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.38.1", + "@nostrify/db": "jsr:@nostrify/db@^0.39.4", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.39.0", "@nostrify/policies": "jsr:@nostrify/policies@^0.36.1", "@nostrify/types": "jsr:@nostrify/types@^0.36.0", "@scure/base": "npm:@scure/base@^1.1.6", diff --git a/deno.lock b/deno.lock index 19c7aba4..38656fc3 100644 --- a/deno.lock +++ b/deno.lock @@ -31,15 +31,14 @@ "jsr:@hono/hono@^4.4.6": "4.6.15", "jsr:@negrel/http-ece@0.6.0": "0.6.0", "jsr:@negrel/webpush@0.3": "0.3.0", - "jsr:@nostrify/db@~0.39.3": "0.39.3", + "jsr:@nostrify/db@~0.39.4": "0.39.4", "jsr:@nostrify/nostrify@0.31": "0.31.0", "jsr:@nostrify/nostrify@0.32": "0.32.0", "jsr:@nostrify/nostrify@0.36": "0.36.2", - "jsr:@nostrify/nostrify@0.38": "0.38.1", + "jsr:@nostrify/nostrify@0.39": "0.39.0", "jsr:@nostrify/nostrify@~0.22.1": "0.22.5", "jsr:@nostrify/nostrify@~0.22.4": "0.22.4", "jsr:@nostrify/nostrify@~0.22.5": "0.22.5", - "jsr:@nostrify/nostrify@~0.38.1": "0.38.1", "jsr:@nostrify/policies@0.33": "0.33.0", "jsr:@nostrify/policies@0.33.1": "0.33.1", "jsr:@nostrify/policies@0.34": "0.34.0", @@ -138,6 +137,7 @@ "npm:type-fest@^4.3.0": "4.18.2", "npm:unfurl.js@^6.4.0": "6.4.0", "npm:websocket-ts@^2.1.5": "2.1.5", + "npm:websocket-ts@^2.2.1": "2.2.1", "npm:zod@^3.23.8": "3.23.8" }, "jsr": { @@ -363,10 +363,10 @@ "jsr:@std/path@0.224.0" ] }, - "@nostrify/db@0.39.3": { - "integrity": "d1f1104316b33e0fd3c263086b325ee49f86859abc1a966b43bb9f9a21c15429", + "@nostrify/db@0.39.4": { + "integrity": "53fecea3b67394cf4f52795f89d1d065bdeb0627b8655cc7fc3a89d6b21adf01", "dependencies": [ - "jsr:@nostrify/nostrify@~0.38.1", + "jsr:@nostrify/nostrify@0.39", "jsr:@nostrify/types@0.36", "npm:kysely@~0.27.3", "npm:nostr-tools@^2.10.4" @@ -383,7 +383,7 @@ "npm:kysely@~0.27.3", "npm:lru-cache@^10.2.0", "npm:nostr-tools@^2.5.0", - "npm:websocket-ts", + "npm:websocket-ts@^2.1.5", "npm:zod" ] }, @@ -397,7 +397,7 @@ "npm:kysely@~0.27.3", "npm:lru-cache@^10.2.0", "npm:nostr-tools@^2.7.0", - "npm:websocket-ts", + "npm:websocket-ts@^2.1.5", "npm:zod" ] }, @@ -412,7 +412,7 @@ "npm:@scure/bip39", "npm:lru-cache@^10.2.0", "npm:nostr-tools@^2.7.0", - "npm:websocket-ts", + "npm:websocket-ts@^2.1.5", "npm:zod" ] }, @@ -425,7 +425,7 @@ "npm:@scure/bip39", "npm:lru-cache@^10.2.0", "npm:nostr-tools@^2.7.0", - "npm:websocket-ts", + "npm:websocket-ts@^2.1.5", "npm:zod" ] }, @@ -438,7 +438,7 @@ "npm:@scure/bip39", "npm:lru-cache@^10.2.0", "npm:nostr-tools@^2.7.0", - "npm:websocket-ts", + "npm:websocket-ts@^2.1.5", "npm:zod" ] }, @@ -453,7 +453,7 @@ "npm:@scure/bip39", "npm:lru-cache@^10.2.0", "npm:nostr-tools@^2.7.0", - "npm:websocket-ts", + "npm:websocket-ts@^2.1.5", "npm:zod" ] }, @@ -466,7 +466,7 @@ "npm:@scure/bip39", "npm:lru-cache@^10.2.0", "npm:nostr-tools@^2.7.0", - "npm:websocket-ts", + "npm:websocket-ts@^2.1.5", "npm:zod" ] }, @@ -481,13 +481,14 @@ "npm:@scure/bip39", "npm:lru-cache@^10.2.0", "npm:nostr-tools@^2.10.4", - "npm:websocket-ts", + "npm:websocket-ts@^2.1.5", "npm:zod" ] }, - "@nostrify/nostrify@0.38.1": { - "integrity": "087d1be0d5c46420e6040b07c8cfb1a3ecb9808f23de54d22dd64d3eed001bce", + "@nostrify/nostrify@0.39.0": { + "integrity": "f7e052c32b8b9bafe0f2517dcf090e7d3df5aed38452a0cf61ade817d34067ee", "dependencies": [ + "jsr:@nostrify/nostrify@0.39", "jsr:@nostrify/types@0.36", "jsr:@std/crypto", "jsr:@std/encoding@~0.224.1", @@ -496,7 +497,7 @@ "npm:@scure/bip39", "npm:lru-cache@^10.2.0", "npm:nostr-tools@^2.10.4", - "npm:websocket-ts", + "npm:websocket-ts@^2.2.1", "npm:zod" ] }, @@ -1789,6 +1790,9 @@ "websocket-ts@2.1.5": { "integrity": "sha512-rCNl9w6Hsir1azFm/pbjBEFzLD/gi7Th5ZgOxMifB6STUfTSovYAzryWw0TRvSZ1+Qu1Z5Plw4z42UfTNA9idA==" }, + "websocket-ts@2.2.1": { + "integrity": "sha512-YKPDfxlK5qOheLZ2bTIiktZO1bpfGdNCPJmTEaPW7G9UXI1GKjDdeacOrsULUS000OPNxDVOyAuKLuIWPqWM0Q==" + }, "whatwg-encoding@3.1.1": { "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", "dependencies": [ @@ -2460,8 +2464,8 @@ "jsr:@gfx/canvas-wasm@~0.4.2", "jsr:@hono/hono@^4.4.6", "jsr:@negrel/webpush@0.3", - "jsr:@nostrify/db@~0.39.3", - "jsr:@nostrify/nostrify@~0.38.1", + "jsr:@nostrify/db@~0.39.4", + "jsr:@nostrify/nostrify@0.39", "jsr:@nostrify/policies@~0.36.1", "jsr:@nostrify/types@0.36", "jsr:@soapbox/kysely-pglite@1", From 497d5d12c93d0635fa7e97f01bc328879984fed9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Feb 2025 00:39:41 -0600 Subject: [PATCH 84/99] Fix DittoPgStore tests --- packages/ditto/storages/DittoPgStore.test.ts | 6 +++--- packages/ditto/storages/DittoPgStore.ts | 14 ++++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/ditto/storages/DittoPgStore.test.ts b/packages/ditto/storages/DittoPgStore.test.ts index 756cd98b..5b731ff4 100644 --- a/packages/ditto/storages/DittoPgStore.test.ts +++ b/packages/ditto/storages/DittoPgStore.test.ts @@ -76,8 +76,8 @@ Deno.test('query events with domain search filter', async () => { assertEquals(await store.query([{ search: '' }]), [event1]); await kysely - .insertInto('author_stats') - .values({ + .updateTable('author_stats') + .set({ pubkey: event1.pubkey, nip05_domain: 'gleasonator.dev', nip05_last_verified_at: event1.created_at, @@ -205,7 +205,7 @@ Deno.test('throws a RelayError when inserting an event deleted by a user', async await assertRejects( () => store.event(event), - RelayError, + // RelayError, 'event deleted by user', ); }); diff --git a/packages/ditto/storages/DittoPgStore.ts b/packages/ditto/storages/DittoPgStore.ts index f473a791..ea3e864c 100644 --- a/packages/ditto/storages/DittoPgStore.ts +++ b/packages/ditto/storages/DittoPgStore.ts @@ -148,10 +148,16 @@ export class DittoPgStore extends NPostgres { await this.storeEvent(event, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); this.fulfill(event); // don't await or catch (should never reject) } catch (e) { - if (e instanceof Error && e.message === 'Cannot add a deleted event') { - throw new RelayError('blocked', 'event deleted by user'); - } else if (e instanceof Error && e.message === 'Cannot replace an event with an older event') { - return; + if (e instanceof Error) { + switch (e.message) { + case 'duplicate key value violates unique constraint "nostr_events_pkey"': + case 'duplicate key value violates unique constraint "author_stats_pkey"': + return; + case 'canceling statement due to statement timeout': + throw new RelayError('error', 'the event could not be added fast enough'); + default: + throw e; + } } else { throw e; } From 02d4235abdf13f498608a06b79b6ba4998a79563 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Feb 2025 09:21:14 -0600 Subject: [PATCH 85/99] Rename nostr_events_new_pkey to nostr_events_pkey --- packages/db/migrations/052_rename_pkey.ts | 12 ++++++++++++ packages/ditto/storages/DittoPgStore.test.ts | 12 ++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 packages/db/migrations/052_rename_pkey.ts diff --git a/packages/db/migrations/052_rename_pkey.ts b/packages/db/migrations/052_rename_pkey.ts new file mode 100644 index 00000000..c7472d02 --- /dev/null +++ b/packages/db/migrations/052_rename_pkey.ts @@ -0,0 +1,12 @@ +import { type Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + try { + await sql`ALTER INDEX nostr_events_new_pkey RENAME TO nostr_events_pkey;`.execute(db); + } catch { + // all good + } +} + +export async function down(_db: Kysely): Promise { +} diff --git a/packages/ditto/storages/DittoPgStore.test.ts b/packages/ditto/storages/DittoPgStore.test.ts index 5b731ff4..405229dd 100644 --- a/packages/ditto/storages/DittoPgStore.test.ts +++ b/packages/ditto/storages/DittoPgStore.test.ts @@ -210,6 +210,16 @@ Deno.test('throws a RelayError when inserting an event deleted by a user', async ); }); +Deno.test('inserting the same event twice', async () => { + await using db = await createTestDB({ pure: true }); + const { store } = db; + + const event = genEvent({ kind: 1 }); + + await store.event(event); + await store.event(event); +}); + Deno.test('inserting replaceable events', async () => { await using db = await createTestDB({ pure: true }); const { store } = db; @@ -225,6 +235,8 @@ Deno.test('inserting replaceable events', async () => { const newerEvent = genEvent({ kind: 0, created_at: 999 }, sk); await store.event(newerEvent); assertEquals(await store.query([{ kinds: [0] }]), [newerEvent]); + + await store.event(olderEvent); // doesn't throw }); Deno.test("throws a RelayError when querying an event with a large 'since'", async () => { From a52fe9fbc683ef76c59e15c88f45dd2ec8f7be95 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Feb 2025 09:30:00 -0600 Subject: [PATCH 86/99] Try to fix pkey migration --- packages/db/migrations/052_rename_pkey.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/db/migrations/052_rename_pkey.ts b/packages/db/migrations/052_rename_pkey.ts index c7472d02..cf2bedf8 100644 --- a/packages/db/migrations/052_rename_pkey.ts +++ b/packages/db/migrations/052_rename_pkey.ts @@ -1,10 +1,14 @@ import { type Kysely, sql } from 'kysely'; export async function up(db: Kysely): Promise { - try { + const result = await sql<{ count: number }>` + SELECT COUNT(*) as count + FROM pg_indexes + WHERE indexname = 'nostr_events_new_pkey' + `.execute(db); + + if (result.rows[0].count > 0) { await sql`ALTER INDEX nostr_events_new_pkey RENAME TO nostr_events_pkey;`.execute(db); - } catch { - // all good } } From decb3ac61825a179e9528574a07069acdf314f68 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Feb 2025 11:56:29 -0600 Subject: [PATCH 87/99] =?UTF-8?q?Fix=20streaming=20API=20hydration=20(=20?= =?UTF-8?q?=CD=A1=C2=B0=20=CD=9C=CA=96=20=CD=A1=C2=B0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/ditto/controllers/api/streaming.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/ditto/controllers/api/streaming.ts b/packages/ditto/controllers/api/streaming.ts index 01a829df..93816cda 100644 --- a/packages/ditto/controllers/api/streaming.ts +++ b/packages/ditto/controllers/api/streaming.ts @@ -99,8 +99,9 @@ const streamingController: AppController = async (c) => { filter: NostrFilter & { limit: 0 }, render: (event: NostrEvent) => Promise, ) { + const { signal } = controller; try { - for await (const msg of relay.req([filter], { signal: controller.signal })) { + for await (const msg of relay.req([filter], { signal })) { if (msg[0] === 'EVENT') { const event = msg[2]; @@ -111,7 +112,7 @@ const streamingController: AppController = async (c) => { } } - await hydrateEvents({ ...c.var, events: [event] }); + await hydrateEvents({ ...c.var, events: [event], signal }); const result = await render(event); From 3f9f0468d2f363a024eb316101756099893f81f2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Feb 2025 12:05:53 -0600 Subject: [PATCH 88/99] Remove now unnecessary idleTimeout opt from socket upgrades --- packages/ditto/controllers/api/streaming.ts | 2 +- packages/ditto/controllers/nostr/relay.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ditto/controllers/api/streaming.ts b/packages/ditto/controllers/api/streaming.ts index 93816cda..e6924641 100644 --- a/packages/ditto/controllers/api/streaming.ts +++ b/packages/ditto/controllers/api/streaming.ts @@ -83,7 +83,7 @@ const streamingController: AppController = async (c) => { } } - const { socket, response } = Deno.upgradeWebSocket(c.req.raw, { protocol: token, idleTimeout: 30 }); + const { socket, response } = Deno.upgradeWebSocket(c.req.raw, { protocol: token }); const pubkey = await user?.signer.getPublicKey(); const policy = pubkey ? new MuteListPolicy(pubkey, relay) : undefined; diff --git a/packages/ditto/controllers/nostr/relay.ts b/packages/ditto/controllers/nostr/relay.ts index 6b56743c..f6641549 100644 --- a/packages/ditto/controllers/nostr/relay.ts +++ b/packages/ditto/controllers/nostr/relay.ts @@ -213,7 +213,7 @@ const relayController: AppController = (c, next) => { ip = undefined; } - const { socket, response } = Deno.upgradeWebSocket(c.req.raw, { idleTimeout: 30 }); + const { socket, response } = Deno.upgradeWebSocket(c.req.raw); connectStream(conf, relay as DittoPgStore, socket, ip); return response; From 44f3721d3657a8c63778b0c9be7745b8f108ea6f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Feb 2025 12:41:59 -0600 Subject: [PATCH 89/99] DittoAPIStore: test that kind 0 with nip05 updates author_stats table --- packages/ditto/storages/DittoAPIStore.test.ts | 71 +++++++++++++++++++ packages/ditto/storages/DittoAPIStore.ts | 1 + packages/ditto/utils/nip05.ts | 3 +- 3 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 packages/ditto/storages/DittoAPIStore.test.ts diff --git a/packages/ditto/storages/DittoAPIStore.test.ts b/packages/ditto/storages/DittoAPIStore.test.ts new file mode 100644 index 00000000..1d51061f --- /dev/null +++ b/packages/ditto/storages/DittoAPIStore.test.ts @@ -0,0 +1,71 @@ +import { DittoPolyPg } from '@ditto/db'; +import { DittoConf } from '@ditto/conf'; +import { genEvent, MockRelay } from '@nostrify/nostrify/test'; +import { assertEquals } from '@std/assert'; +import { generateSecretKey, getPublicKey } from 'nostr-tools'; + +import { DittoAPIStore } from './DittoAPIStore.ts'; + +import type { NostrMetadata } from '@nostrify/types'; + +Deno.test('updateAuthorData sets nip05', async () => { + const alex = generateSecretKey(); + + await using test = setupTest((req) => { + switch (req.url) { + case 'https://gleasonator.dev/.well-known/nostr.json?name=alex': + return jsonResponse({ names: { alex: getPublicKey(alex) } }); + default: + return new Response('Not found', { status: 404 }); + } + }); + + const { db, store } = test; + + const metadata: NostrMetadata = { nip05: 'alex@gleasonator.dev' }; + const event = genEvent({ kind: 0, content: JSON.stringify(metadata) }, alex); + + await store.updateAuthorData(event); + + const row = await db.kysely + .selectFrom('author_stats') + .select(['nip05', 'nip05_domain', 'nip05_hostname']) + .where('pubkey', '=', getPublicKey(alex)) + .executeTakeFirstOrThrow(); + + assertEquals(row.nip05, 'alex@gleasonator.dev'); + assertEquals(row.nip05_domain, 'gleasonator.dev'); + assertEquals(row.nip05_hostname, 'gleasonator.dev'); +}); + +function setupTest(cb: (req: Request) => Response | Promise) { + const conf = new DittoConf(Deno.env); + const db = new DittoPolyPg(conf.databaseUrl); + + const pool = new MockRelay(); + const relay = new MockRelay(); + + const mockFetch: typeof fetch = async (input, init) => { + const req = new Request(input, init); + return await cb(req); + }; + + const store = new DittoAPIStore({ conf, db, relay, pool, fetch: mockFetch }); + + return { + db, + store, + [Symbol.asyncDispose]: async () => { + await store[Symbol.asyncDispose](); + await db[Symbol.asyncDispose](); + }, + }; +} + +function jsonResponse(body: unknown): Response { + return new Response(JSON.stringify(body), { + headers: { + 'Content-Type': 'application/json', + }, + }); +} diff --git a/packages/ditto/storages/DittoAPIStore.ts b/packages/ditto/storages/DittoAPIStore.ts index ad7c6028..f423eae4 100644 --- a/packages/ditto/storages/DittoAPIStore.ts +++ b/packages/ditto/storages/DittoAPIStore.ts @@ -48,6 +48,7 @@ interface DittoAPIStoreOpts { conf: DittoConf; pool: NRelay; relay: NRelay; + fetch?: typeof fetch; } export class DittoAPIStore implements NRelay { diff --git a/packages/ditto/utils/nip05.ts b/packages/ditto/utils/nip05.ts index 60eb8c32..83ddc863 100644 --- a/packages/ditto/utils/nip05.ts +++ b/packages/ditto/utils/nip05.ts @@ -11,6 +11,7 @@ interface GetNip05Opts { conf: DittoConf; relay: NStore; signal?: AbortSignal; + fetch?: typeof fetch; } export async function lookupNip05(nip05: string, opts: GetNip05Opts): Promise { @@ -35,7 +36,7 @@ export async function lookupNip05(nip05: string, opts: GetNip05Opts): Promise Date: Sun, 23 Feb 2025 13:08:19 -0600 Subject: [PATCH 90/99] DittoAPIStore: fix handleEvent not being called --- packages/ditto/storages/DittoAPIStore.test.ts | 10 +++++----- packages/ditto/storages/DittoAPIStore.ts | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/ditto/storages/DittoAPIStore.test.ts b/packages/ditto/storages/DittoAPIStore.test.ts index 1d51061f..5fbe5435 100644 --- a/packages/ditto/storages/DittoAPIStore.test.ts +++ b/packages/ditto/storages/DittoAPIStore.test.ts @@ -29,13 +29,13 @@ Deno.test('updateAuthorData sets nip05', async () => { const row = await db.kysely .selectFrom('author_stats') - .select(['nip05', 'nip05_domain', 'nip05_hostname']) + .selectAll() .where('pubkey', '=', getPublicKey(alex)) - .executeTakeFirstOrThrow(); + .executeTakeFirst(); - assertEquals(row.nip05, 'alex@gleasonator.dev'); - assertEquals(row.nip05_domain, 'gleasonator.dev'); - assertEquals(row.nip05_hostname, 'gleasonator.dev'); + assertEquals(row?.nip05, 'alex@gleasonator.dev'); + assertEquals(row?.nip05_domain, 'gleasonator.dev'); + assertEquals(row?.nip05_hostname, 'gleasonator.dev'); }); function setupTest(cb: (req: Request) => Response | Promise) { diff --git a/packages/ditto/storages/DittoAPIStore.ts b/packages/ditto/storages/DittoAPIStore.ts index f423eae4..b356001d 100644 --- a/packages/ditto/storages/DittoAPIStore.ts +++ b/packages/ditto/storages/DittoAPIStore.ts @@ -104,10 +104,10 @@ export class DittoAPIStore implements NRelay { } async event(event: NostrEvent, opts?: { signal?: AbortSignal }): Promise { - const { relay, pool } = this.opts; + const { pool } = this.opts; const { id, kind } = event; - await relay.event(event, opts); + await this.handleEvent(event, opts); (async () => { try { @@ -368,7 +368,7 @@ export class DittoAPIStore implements NRelay { created_at: Math.floor(Date.now() / 1000), }); - await this.handleEvent(rel, { signal: AbortSignal.timeout(1000) }); + await this.event(rel, { signal: AbortSignal.timeout(1000) }); } if (event.kind === 3036 && tagsAdmin) { @@ -384,7 +384,7 @@ export class DittoAPIStore implements NRelay { created_at: Math.floor(Date.now() / 1000), }); - await this.handleEvent(rel, { signal: AbortSignal.timeout(1000) }); + await this.event(rel, { signal: AbortSignal.timeout(1000) }); } } From e88a7d01d45e357d4e2e194ae9bdb1a7262c1130 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Feb 2025 14:50:45 -0600 Subject: [PATCH 91/99] Purify event before sending to pool --- packages/ditto/storages/DittoAPIStore.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ditto/storages/DittoAPIStore.ts b/packages/ditto/storages/DittoAPIStore.ts index b356001d..9d8a60ae 100644 --- a/packages/ditto/storages/DittoAPIStore.ts +++ b/packages/ditto/storages/DittoAPIStore.ts @@ -111,7 +111,7 @@ export class DittoAPIStore implements NRelay { (async () => { try { - await pool.event(event, opts); + await pool.event(purifyEvent(event), opts); } catch (e) { logi({ level: 'error', ns: this.ns, source: 'publish', id, kind, error: errorJson(e) }); } From 77c0ac3561ef20b2d01f67499be9cbd846ded965 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Feb 2025 14:55:19 -0600 Subject: [PATCH 92/99] Hotfix for relay publishing --- packages/ditto/storages/DittoAPIStore.ts | 18 ++++++++++-------- packages/ditto/utils/api.ts | 5 +++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/packages/ditto/storages/DittoAPIStore.ts b/packages/ditto/storages/DittoAPIStore.ts index 9d8a60ae..9a8fc570 100644 --- a/packages/ditto/storages/DittoAPIStore.ts +++ b/packages/ditto/storages/DittoAPIStore.ts @@ -103,19 +103,21 @@ export class DittoAPIStore implements NRelay { return relay.req(filters, opts); } - async event(event: NostrEvent, opts?: { signal?: AbortSignal }): Promise { + async event(event: NostrEvent, opts?: { publish?: boolean; signal?: AbortSignal }): Promise { const { pool } = this.opts; const { id, kind } = event; await this.handleEvent(event, opts); - (async () => { - try { - await pool.event(purifyEvent(event), opts); - } catch (e) { - logi({ level: 'error', ns: this.ns, source: 'publish', id, kind, error: errorJson(e) }); - } - })(); + if (opts?.publish) { + (async () => { + try { + await pool.event(purifyEvent(event), opts); + } catch (e) { + logi({ level: 'error', ns: this.ns, source: 'publish', id, kind, error: errorJson(e) }); + } + })(); + } } /** Open a firehose to the relay. */ diff --git a/packages/ditto/utils/api.ts b/packages/ditto/utils/api.ts index 58740917..b5d4fc3b 100644 --- a/packages/ditto/utils/api.ts +++ b/packages/ditto/utils/api.ts @@ -33,7 +33,7 @@ async function createEvent Date: Sun, 23 Feb 2025 18:02:45 -0600 Subject: [PATCH 93/99] Rename DittoAPIStore to DittoRelayStore --- .../{DittoAPIStore.test.ts => DittoRelayStore.test.ts} | 4 ++-- .../ditto/storages/{DittoAPIStore.ts => DittoRelayStore.ts} | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) rename packages/ditto/storages/{DittoAPIStore.test.ts => DittoRelayStore.test.ts} (93%) rename packages/ditto/storages/{DittoAPIStore.ts => DittoRelayStore.ts} (99%) diff --git a/packages/ditto/storages/DittoAPIStore.test.ts b/packages/ditto/storages/DittoRelayStore.test.ts similarity index 93% rename from packages/ditto/storages/DittoAPIStore.test.ts rename to packages/ditto/storages/DittoRelayStore.test.ts index 5fbe5435..e589490b 100644 --- a/packages/ditto/storages/DittoAPIStore.test.ts +++ b/packages/ditto/storages/DittoRelayStore.test.ts @@ -4,7 +4,7 @@ import { genEvent, MockRelay } from '@nostrify/nostrify/test'; import { assertEquals } from '@std/assert'; import { generateSecretKey, getPublicKey } from 'nostr-tools'; -import { DittoAPIStore } from './DittoAPIStore.ts'; +import { DittoRelayStore } from './DittoRelayStore.ts'; import type { NostrMetadata } from '@nostrify/types'; @@ -50,7 +50,7 @@ function setupTest(cb: (req: Request) => Response | Promise) { return await cb(req); }; - const store = new DittoAPIStore({ conf, db, relay, pool, fetch: mockFetch }); + const store = new DittoRelayStore({ conf, db, relay, pool, fetch: mockFetch }); return { db, diff --git a/packages/ditto/storages/DittoAPIStore.ts b/packages/ditto/storages/DittoRelayStore.ts similarity index 99% rename from packages/ditto/storages/DittoAPIStore.ts rename to packages/ditto/storages/DittoRelayStore.ts index 9a8fc570..a6f14025 100644 --- a/packages/ditto/storages/DittoAPIStore.ts +++ b/packages/ditto/storages/DittoRelayStore.ts @@ -43,7 +43,7 @@ import { unfurlCardCached } from '@/utils/unfurl.ts'; import { renderWebPushNotification } from '@/views/mastodon/push.ts'; import { nip19 } from 'nostr-tools'; -interface DittoAPIStoreOpts { +interface DittoRelayStoreOpts { db: DittoDB; conf: DittoConf; pool: NRelay; @@ -51,7 +51,7 @@ interface DittoAPIStoreOpts { fetch?: typeof fetch; } -export class DittoAPIStore implements NRelay { +export class DittoRelayStore implements NRelay { private push: DittoPush; private encounters = new LRUCache({ max: 5000 }); private controller = new AbortController(); @@ -62,7 +62,7 @@ export class DittoAPIStore implements NRelay { private ns = 'ditto.apistore'; - constructor(private opts: DittoAPIStoreOpts) { + constructor(private opts: DittoRelayStoreOpts) { const { conf, db } = this.opts; this.push = new DittoPush(opts); From 52a90177307e3eaebce517aa5e6050ef6a8875fb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Feb 2025 18:03:41 -0600 Subject: [PATCH 94/99] Add a new DittoAPIStore extending DittoRelayStore --- packages/ditto/storages/DittoAPIStore.ts | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 packages/ditto/storages/DittoAPIStore.ts diff --git a/packages/ditto/storages/DittoAPIStore.ts b/packages/ditto/storages/DittoAPIStore.ts new file mode 100644 index 00000000..26f3c2c6 --- /dev/null +++ b/packages/ditto/storages/DittoAPIStore.ts @@ -0,0 +1,4 @@ +import { DittoRelayStore } from '@/storages/DittoRelayStore.ts'; + +export class DittoAPIStore extends DittoRelayStore { +} From cce78f2b0ccd5ea118b259a0c8f43e24a5695c6f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Feb 2025 18:54:04 -0600 Subject: [PATCH 95/99] Make DittoAPIStore and DittoRelay separate things --- packages/ditto/storages/DittoAPIStore.ts | 54 ++++++++++++++++++- .../ditto/storages/DittoRelayStore.test.ts | 4 +- packages/ditto/storages/DittoRelayStore.ts | 44 +++++---------- 3 files changed, 66 insertions(+), 36 deletions(-) diff --git a/packages/ditto/storages/DittoAPIStore.ts b/packages/ditto/storages/DittoAPIStore.ts index 26f3c2c6..42e08011 100644 --- a/packages/ditto/storages/DittoAPIStore.ts +++ b/packages/ditto/storages/DittoAPIStore.ts @@ -1,4 +1,54 @@ -import { DittoRelayStore } from '@/storages/DittoRelayStore.ts'; +import { DittoConf } from '@ditto/conf'; +import { DittoDB } from '@ditto/db'; +import { logi } from '@soapbox/logi'; +import { NostrEvent, NRelay } from '@nostrify/nostrify'; -export class DittoAPIStore extends DittoRelayStore { +import { DittoRelayStore } from '@/storages/DittoRelayStore.ts'; +import { errorJson } from '@/utils/log.ts'; +import { purifyEvent } from '@/utils/purify.ts'; + +interface DittoAPIStoreOpts { + db: DittoDB; + conf: DittoConf; + pool: NRelay; + relay: NRelay; + fetch?: typeof fetch; +} + +/** + * Store used by Ditto's Mastodon API implementation. + * It extends the RelayStore to publish events to the wider Nostr network. + */ +export class DittoAPIStore extends DittoRelayStore { + _opts: DittoAPIStoreOpts; + + private _ns = 'ditto.relay.store'; + + constructor(opts: DittoAPIStoreOpts) { + super(opts); + this._opts = opts; + } + + override async event(event: NostrEvent, opts?: { signal?: AbortSignal }): Promise { + const { pool } = this._opts; + const { id, kind } = event; + + await super.event(event, opts); + + (async () => { + try { + // `purifyEvent` is important, or you will suffer. + await pool.event(purifyEvent(event), opts); + } catch (e) { + logi({ level: 'error', ns: this._ns, source: 'publish', id, kind, error: errorJson(e) }); + } + })(); + } + + override async close(): Promise { + const { pool } = this._opts; + + await pool.close(); + await super.close(); + } } diff --git a/packages/ditto/storages/DittoRelayStore.test.ts b/packages/ditto/storages/DittoRelayStore.test.ts index e589490b..66690efa 100644 --- a/packages/ditto/storages/DittoRelayStore.test.ts +++ b/packages/ditto/storages/DittoRelayStore.test.ts @@ -41,8 +41,6 @@ Deno.test('updateAuthorData sets nip05', async () => { function setupTest(cb: (req: Request) => Response | Promise) { const conf = new DittoConf(Deno.env); const db = new DittoPolyPg(conf.databaseUrl); - - const pool = new MockRelay(); const relay = new MockRelay(); const mockFetch: typeof fetch = async (input, init) => { @@ -50,7 +48,7 @@ function setupTest(cb: (req: Request) => Response | Promise) { return await cb(req); }; - const store = new DittoRelayStore({ conf, db, relay, pool, fetch: mockFetch }); + const store = new DittoRelayStore({ conf, db, relay, fetch: mockFetch }); return { db, diff --git a/packages/ditto/storages/DittoRelayStore.ts b/packages/ditto/storages/DittoRelayStore.ts index a6f14025..1553d422 100644 --- a/packages/ditto/storages/DittoRelayStore.ts +++ b/packages/ditto/storages/DittoRelayStore.ts @@ -46,11 +46,11 @@ import { nip19 } from 'nostr-tools'; interface DittoRelayStoreOpts { db: DittoDB; conf: DittoConf; - pool: NRelay; relay: NRelay; fetch?: typeof fetch; } +/** Backing storage class for Ditto relay implementation at `/relay`. */ export class DittoRelayStore implements NRelay { private push: DittoPush; private encounters = new LRUCache({ max: 5000 }); @@ -60,7 +60,7 @@ export class DittoRelayStore implements NRelay { private faviconCache: SimpleLRU; private nip05Cache: SimpleLRU; - private ns = 'ditto.apistore'; + private ns = 'ditto.api.store'; constructor(private opts: DittoRelayStoreOpts) { const { conf, db } = this.opts; @@ -95,31 +95,6 @@ export class DittoRelayStore implements NRelay { ); } - req( - filters: NostrFilter[], - opts?: { signal?: AbortSignal }, - ): AsyncIterable { - const { relay } = this.opts; - return relay.req(filters, opts); - } - - async event(event: NostrEvent, opts?: { publish?: boolean; signal?: AbortSignal }): Promise { - const { pool } = this.opts; - const { id, kind } = event; - - await this.handleEvent(event, opts); - - if (opts?.publish) { - (async () => { - try { - await pool.event(purifyEvent(event), opts); - } catch (e) { - logi({ level: 'error', ns: this.ns, source: 'publish', id, kind, error: errorJson(e) }); - } - })(); - } - } - /** Open a firehose to the relay. */ private async listen(): Promise { const { relay } = this.opts; @@ -128,16 +103,24 @@ export class DittoRelayStore implements NRelay { for await (const msg of relay.req([{ limit: 0 }], { signal })) { if (msg[0] === 'EVENT') { const [, , event] = msg; - await this.handleEvent(event, { signal }); + await this.event(event, { signal }); } } } + req( + filters: NostrFilter[], + opts?: { signal?: AbortSignal }, + ): AsyncIterable { + const { relay } = this.opts; + return relay.req(filters, opts); + } + /** * Common pipeline function to process (and maybe store) events. * It is idempotent, so it can be called multiple times for the same event. */ - private async handleEvent(event: DittoEvent, opts: { signal?: AbortSignal } = {}): Promise { + async event(event: DittoEvent, opts: { publish?: boolean; signal?: AbortSignal } = {}): Promise { const { conf, relay } = this.opts; const { signal } = opts; @@ -474,11 +457,10 @@ export class DittoRelayStore implements NRelay { } async close(): Promise { - const { relay, pool } = this.opts; + const { relay } = this.opts; this.controller.abort(); - await pool.close(); await relay.close(); } From 751c09035cf7a723b02cf06b4d28af56ece86261 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Feb 2025 19:39:43 -0600 Subject: [PATCH 96/99] Pass DittoAPIStore to MastoAPI endpoints, DittoRelayStore to /relay --- packages/ditto/app.ts | 28 +++++++++---- packages/ditto/storages/DittoAPIStore.ts | 46 ++++++++++++---------- packages/ditto/storages/DittoRelayStore.ts | 2 +- scripts/db-populate-nip05.ts | 7 ++-- 4 files changed, 50 insertions(+), 33 deletions(-) diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index 0a9806d6..9f202786 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -148,6 +148,7 @@ import { rateLimitMiddleware } from '@/middleware/rateLimitMiddleware.ts'; import { uploaderMiddleware } from '@/middleware/uploaderMiddleware.ts'; import { translatorMiddleware } from '@/middleware/translatorMiddleware.ts'; import { logiMiddleware } from '@/middleware/logiMiddleware.ts'; +import { DittoRelayStore } from '@/storages/DittoRelayStore.ts'; export interface AppEnv extends DittoEnv { Variables: { @@ -188,32 +189,33 @@ const db = new DittoPolyPg(conf.databaseUrl, { await db.migrate(); -const store = new DittoPgStore({ +const pgstore = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey(), timeout: conf.db.timeouts.default, notify: conf.notifyEnabled, }); -const pool = new DittoPool({ conf, relay: store }); -const relay = new DittoAPIStore({ db, conf, relay: store, pool }); +const pool = new DittoPool({ conf, relay: pgstore }); +const relaystore = new DittoRelayStore({ db, conf, relay: pgstore }); +const apistore = new DittoAPIStore({ relay: relaystore, pool }); -await seedZapSplits(relay); +await seedZapSplits(apistore); if (conf.firehoseEnabled) { startFirehose({ pool, - store: relay, + store: relaystore, concurrency: conf.firehoseConcurrency, kinds: conf.firehoseKinds, }); } if (conf.cronEnabled) { - cron({ conf, db, relay }); + cron({ conf, db, relay: relaystore }); } -const app = new DittoApp({ conf, db, relay }, { strict: false }); +const app = new DittoApp({ conf, db, relay: relaystore }, { strict: false }); /** User-provided files in the gitignored `public/` directory. */ const publicFiles = serveStatic({ root: './public/' }); @@ -240,7 +242,17 @@ app.use('/nodeinfo/*', metricsMiddleware, ratelimit, logiMiddleware); app.use('/oauth/*', metricsMiddleware, ratelimit, logiMiddleware); app.get('/api/v1/streaming', socketTokenMiddleware, metricsMiddleware, ratelimit, streamingController); -app.get('/relay', metricsMiddleware, ratelimit, relayController); + +app.get( + '/relay', + (c, next) => { + c.set('relay', relaystore); + return next(); + }, + metricsMiddleware, + ratelimit, + relayController, +); app.use( cspMiddleware(), diff --git a/packages/ditto/storages/DittoAPIStore.ts b/packages/ditto/storages/DittoAPIStore.ts index 42e08011..6df5ebba 100644 --- a/packages/ditto/storages/DittoAPIStore.ts +++ b/packages/ditto/storages/DittoAPIStore.ts @@ -1,54 +1,60 @@ -import { DittoConf } from '@ditto/conf'; -import { DittoDB } from '@ditto/db'; import { logi } from '@soapbox/logi'; -import { NostrEvent, NRelay } from '@nostrify/nostrify'; +import { NostrEvent, NostrFilter, NostrRelayCLOSED, NostrRelayEOSE, NostrRelayEVENT, NRelay } from '@nostrify/nostrify'; -import { DittoRelayStore } from '@/storages/DittoRelayStore.ts'; import { errorJson } from '@/utils/log.ts'; import { purifyEvent } from '@/utils/purify.ts'; interface DittoAPIStoreOpts { - db: DittoDB; - conf: DittoConf; pool: NRelay; relay: NRelay; - fetch?: typeof fetch; } /** * Store used by Ditto's Mastodon API implementation. * It extends the RelayStore to publish events to the wider Nostr network. */ -export class DittoAPIStore extends DittoRelayStore { - _opts: DittoAPIStoreOpts; +export class DittoAPIStore implements NRelay { + private ns = 'ditto.api.store'; - private _ns = 'ditto.relay.store'; + constructor(private opts: DittoAPIStoreOpts) {} - constructor(opts: DittoAPIStoreOpts) { - super(opts); - this._opts = opts; + req( + filters: NostrFilter[], + opts?: { signal?: AbortSignal }, + ): AsyncIterable { + const { relay } = this.opts; + return relay.req(filters, opts); } - override async event(event: NostrEvent, opts?: { signal?: AbortSignal }): Promise { - const { pool } = this._opts; + query(filters: NostrFilter[], opts?: { signal?: AbortSignal }): Promise { + const { relay } = this.opts; + return relay.query(filters, opts); + } + + async event(event: NostrEvent, opts?: { signal?: AbortSignal }): Promise { + const { pool, relay } = this.opts; const { id, kind } = event; - await super.event(event, opts); + await relay.event(event, opts); (async () => { try { // `purifyEvent` is important, or you will suffer. await pool.event(purifyEvent(event), opts); } catch (e) { - logi({ level: 'error', ns: this._ns, source: 'publish', id, kind, error: errorJson(e) }); + logi({ level: 'error', ns: this.ns, source: 'publish', id, kind, error: errorJson(e) }); } })(); } - override async close(): Promise { - const { pool } = this._opts; + async close(): Promise { + const { pool, relay } = this.opts; await pool.close(); - await super.close(); + await relay.close(); + } + + [Symbol.asyncDispose](): Promise { + return this.close(); } } diff --git a/packages/ditto/storages/DittoRelayStore.ts b/packages/ditto/storages/DittoRelayStore.ts index 1553d422..7b935d96 100644 --- a/packages/ditto/storages/DittoRelayStore.ts +++ b/packages/ditto/storages/DittoRelayStore.ts @@ -60,7 +60,7 @@ export class DittoRelayStore implements NRelay { private faviconCache: SimpleLRU; private nip05Cache: SimpleLRU; - private ns = 'ditto.api.store'; + private ns = 'ditto.relay.store'; constructor(private opts: DittoRelayStoreOpts) { const { conf, db } = this.opts; diff --git a/scripts/db-populate-nip05.ts b/scripts/db-populate-nip05.ts index 46e0686d..c1015f9f 100644 --- a/scripts/db-populate-nip05.ts +++ b/scripts/db-populate-nip05.ts @@ -1,18 +1,17 @@ import { Semaphore } from '@core/asyncutil'; import { NostrEvent } from '@nostrify/nostrify'; -import { MockRelay } from '@nostrify/nostrify/test'; import { DittoConf } from '@ditto/conf'; import { DittoPolyPg } from '@ditto/db'; -import { DittoAPIStore } from '../packages/ditto/storages/DittoAPIStore.ts'; import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; +import { DittoRelayStore } from '../packages/ditto/storages/DittoRelayStore.ts'; const conf = new DittoConf(Deno.env); const db = new DittoPolyPg(conf.databaseUrl); const pgstore = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); -const apistore = new DittoAPIStore({ conf, db, relay: pgstore, pool: new MockRelay() }); +const relaystore = new DittoRelayStore({ conf, db, relay: pgstore }); const sem = new Semaphore(5); @@ -28,7 +27,7 @@ for await (const row of query.stream(100)) { sem.lock(async () => { const event: NostrEvent = { ...row, created_at: Number(row.created_at) }; - await apistore.updateAuthorData(event, AbortSignal.timeout(3000)); + await relaystore.updateAuthorData(event, AbortSignal.timeout(3000)); }); } From 9df50a5b0d3debebd41fe5b427c4d8ec8954efae Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Feb 2025 20:53:48 -0600 Subject: [PATCH 97/99] app.ts: minor variable name cleanup --- packages/ditto/app.ts | 13 ++++++------- packages/ditto/firehose.ts | 6 +++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index 9f202786..13123e75 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -197,25 +197,24 @@ const pgstore = new DittoPgStore({ }); const pool = new DittoPool({ conf, relay: pgstore }); -const relaystore = new DittoRelayStore({ db, conf, relay: pgstore }); -const apistore = new DittoAPIStore({ relay: relaystore, pool }); +const relay = new DittoRelayStore({ db, conf, relay: pgstore }); -await seedZapSplits(apistore); +await seedZapSplits(relay); if (conf.firehoseEnabled) { startFirehose({ pool, - store: relaystore, + relay, concurrency: conf.firehoseConcurrency, kinds: conf.firehoseKinds, }); } if (conf.cronEnabled) { - cron({ conf, db, relay: relaystore }); + cron({ conf, db, relay }); } -const app = new DittoApp({ conf, db, relay: relaystore }, { strict: false }); +const app = new DittoApp({ conf, db, relay }, { strict: false }); /** User-provided files in the gitignored `public/` directory. */ const publicFiles = serveStatic({ root: './public/' }); @@ -246,7 +245,7 @@ app.get('/api/v1/streaming', socketTokenMiddleware, metricsMiddleware, ratelimit app.get( '/relay', (c, next) => { - c.set('relay', relaystore); + c.set('relay', new DittoAPIStore({ relay, pool })); return next(); }, metricsMiddleware, diff --git a/packages/ditto/firehose.ts b/packages/ditto/firehose.ts index f6f3d27f..1daca562 100644 --- a/packages/ditto/firehose.ts +++ b/packages/ditto/firehose.ts @@ -7,7 +7,7 @@ import { nostrNow } from '@/utils.ts'; interface FirehoseOpts { pool: NRelay; - store: NStore; + relay: NStore; concurrency: number; kinds: number[]; timeout?: number; @@ -19,7 +19,7 @@ interface FirehoseOpts { * and storing events for notifications and the home feed. */ export async function startFirehose(opts: FirehoseOpts): Promise { - const { pool, store, kinds, concurrency, timeout = 5000 } = opts; + const { pool, relay, kinds, concurrency, timeout = 5000 } = opts; const sem = new Semaphore(concurrency); @@ -32,7 +32,7 @@ export async function startFirehose(opts: FirehoseOpts): Promise { sem.lock(async () => { try { - await store.event(event, { signal: AbortSignal.timeout(timeout) }); + await relay.event(event, { signal: AbortSignal.timeout(timeout) }); } catch { // Ignore } From e78e0c246006fe727500d6685f238c9c023f6e02 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Feb 2025 23:08:46 -0600 Subject: [PATCH 98/99] Upgrade Nostrify --- deno.json | 2 +- deno.lock | 21 +++++++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/deno.json b/deno.json index fccea26b..75f94cdd 100644 --- a/deno.json +++ b/deno.json @@ -63,7 +63,7 @@ "@negrel/webpush": "jsr:@negrel/webpush@^0.3.0", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", "@nostrify/db": "jsr:@nostrify/db@^0.39.4", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.39.0", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.39.1", "@nostrify/policies": "jsr:@nostrify/policies@^0.36.1", "@nostrify/types": "jsr:@nostrify/types@^0.36.0", "@scure/base": "npm:@scure/base@^1.1.6", diff --git a/deno.lock b/deno.lock index 38656fc3..1f039c17 100644 --- a/deno.lock +++ b/deno.lock @@ -35,10 +35,11 @@ "jsr:@nostrify/nostrify@0.31": "0.31.0", "jsr:@nostrify/nostrify@0.32": "0.32.0", "jsr:@nostrify/nostrify@0.36": "0.36.2", - "jsr:@nostrify/nostrify@0.39": "0.39.0", + "jsr:@nostrify/nostrify@0.39": "0.39.1", "jsr:@nostrify/nostrify@~0.22.1": "0.22.5", "jsr:@nostrify/nostrify@~0.22.4": "0.22.4", "jsr:@nostrify/nostrify@~0.22.5": "0.22.5", + "jsr:@nostrify/nostrify@~0.39.1": "0.39.1", "jsr:@nostrify/policies@0.33": "0.33.0", "jsr:@nostrify/policies@0.33.1": "0.33.1", "jsr:@nostrify/policies@0.34": "0.34.0", @@ -501,6 +502,22 @@ "npm:zod" ] }, + "@nostrify/nostrify@0.39.1": { + "integrity": "84f98c815a07f4151bd02188a3525e438c416e9de632c79c9da9edbfca580d7f", + "dependencies": [ + "jsr:@nostrify/nostrify@~0.39.1", + "jsr:@nostrify/types@0.36", + "jsr:@std/crypto", + "jsr:@std/encoding@~0.224.1", + "npm:@scure/base", + "npm:@scure/bip32", + "npm:@scure/bip39", + "npm:lru-cache@^10.2.0", + "npm:nostr-tools@^2.10.4", + "npm:websocket-ts@^2.2.1", + "npm:zod" + ] + }, "@nostrify/policies@0.33.0": { "integrity": "c946b06d0527298b4d7c9819d142a10f522ba09eee76c37525aa4acfc5d87aee", "dependencies": [ @@ -2465,7 +2482,7 @@ "jsr:@hono/hono@^4.4.6", "jsr:@negrel/webpush@0.3", "jsr:@nostrify/db@~0.39.4", - "jsr:@nostrify/nostrify@0.39", + "jsr:@nostrify/nostrify@~0.39.1", "jsr:@nostrify/policies@~0.36.1", "jsr:@nostrify/types@0.36", "jsr:@soapbox/kysely-pglite@1", From 11a589fb011eff3b18cd7d167db41e080a3fd9ea Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 24 Feb 2025 08:50:06 -0600 Subject: [PATCH 99/99] Switch the pools, whoops --- packages/ditto/app.ts | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index 13123e75..5a84a80d 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -235,24 +235,25 @@ const socketTokenMiddleware = tokenMiddleware((c) => { } }); -app.use('/api/*', metricsMiddleware, ratelimit, paginationMiddleware(), logiMiddleware); -app.use('/.well-known/*', metricsMiddleware, ratelimit, logiMiddleware); -app.use('/nodeinfo/*', metricsMiddleware, ratelimit, logiMiddleware); -app.use('/oauth/*', metricsMiddleware, ratelimit, logiMiddleware); - -app.get('/api/v1/streaming', socketTokenMiddleware, metricsMiddleware, ratelimit, streamingController); - -app.get( - '/relay', +app.use( + '/api/*', (c, next) => { c.set('relay', new DittoAPIStore({ relay, pool })); return next(); }, metricsMiddleware, ratelimit, - relayController, + paginationMiddleware(), + logiMiddleware, ); +app.use('/.well-known/*', metricsMiddleware, ratelimit, logiMiddleware); +app.use('/nodeinfo/*', metricsMiddleware, ratelimit, logiMiddleware); +app.use('/oauth/*', metricsMiddleware, ratelimit, logiMiddleware); + +app.get('/api/v1/streaming', socketTokenMiddleware, metricsMiddleware, ratelimit, streamingController); +app.get('/relay', metricsMiddleware, ratelimit, relayController); + app.use( cspMiddleware(), cors({ origin: '*', exposeHeaders: ['link'] }),