From ddb93af09fbc740602da1b1e87ae18682bd4dc87 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 20 Jun 2024 14:33:25 -0300 Subject: [PATCH 01/75] feat(DittoTables): create EventZapRow --- src/db/DittoTables.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index aed8c8c2..e56a062e 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -7,6 +7,7 @@ export interface DittoTables { author_stats: AuthorStatsRow; event_stats: EventStatsRow; pubkey_domains: PubkeyDomainRow; + event_zaps: EventZapRow; } interface AuthorStatsRow { @@ -69,3 +70,11 @@ interface PubkeyDomainRow { domain: string; last_updated_at: number; } + +interface EventZapRow { + receipt_id: string; + target_event_id: string; + sender_pubkey: string; + amount: number; + comment: string; +} From bac0b488010740becae4d81fbcd2bafddfceccc9 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 20 Jun 2024 14:36:19 -0300 Subject: [PATCH 02/75] feat: add migration for event_zaps;create idx_event_zaps_id_amount --- src/db/migrations/027_add_zap_events.ts | 26 +++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/db/migrations/027_add_zap_events.ts diff --git a/src/db/migrations/027_add_zap_events.ts b/src/db/migrations/027_add_zap_events.ts new file mode 100644 index 00000000..058e6baf --- /dev/null +++ b/src/db/migrations/027_add_zap_events.ts @@ -0,0 +1,26 @@ +import { Kysely } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('event_zaps') + .ifNotExists() + .addColumn('receipt_id', 'text', (col) => col.primaryKey()) + .addColumn('target_event_id', 'text', (col) => col.notNull()) + .addColumn('sender_pubkey', 'text', (col) => col.notNull()) + .addColumn('amount', 'integer', (col) => col.notNull()) + .addColumn('comment', 'text', (col) => col.notNull()) + .execute(); + + await db.schema + .createIndex('idx_event_zaps_id_amount') + .on('event_zaps') + .column('amount') + .column('target_event_id') + .ifNotExists() + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropIndex('idx_event_zaps_id_amount').execute(); + await db.schema.dropTable('event_zaps').execute(); +} From 1b4ebaccd82f5c8eb1763bc0b6891b178719bcba Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 20 Jun 2024 15:26:08 -0300 Subject: [PATCH 03/75] refactor: resolve import specifier via the active import map --- src/firehose.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/firehose.ts b/src/firehose.ts index 2c776fe4..f715c686 100644 --- a/src/firehose.ts +++ b/src/firehose.ts @@ -3,7 +3,7 @@ import { Stickynotes } from '@soapbox/stickynotes'; import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; -import * as pipeline from './pipeline.ts'; +import * as pipeline from '@/pipeline.ts'; const console = new Stickynotes('ditto:firehose'); From 2d937a7378b4b3e5d4d7899d02fd9d4d721c9974 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 21 Jun 2024 18:20:10 -0300 Subject: [PATCH 04/75] refactor(event_zaps): rename amount to amount_millisats --- src/db/DittoTables.ts | 2 +- src/db/migrations/027_add_zap_events.ts | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index e56a062e..863ca61e 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -75,6 +75,6 @@ interface EventZapRow { receipt_id: string; target_event_id: string; sender_pubkey: string; - amount: number; + amount_millisats: number; comment: string; } diff --git a/src/db/migrations/027_add_zap_events.ts b/src/db/migrations/027_add_zap_events.ts index 058e6baf..fe2e1d20 100644 --- a/src/db/migrations/027_add_zap_events.ts +++ b/src/db/migrations/027_add_zap_events.ts @@ -3,11 +3,10 @@ import { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema .createTable('event_zaps') - .ifNotExists() .addColumn('receipt_id', 'text', (col) => col.primaryKey()) .addColumn('target_event_id', 'text', (col) => col.notNull()) .addColumn('sender_pubkey', 'text', (col) => col.notNull()) - .addColumn('amount', 'integer', (col) => col.notNull()) + .addColumn('amount_millisats', 'integer', (col) => col.notNull()) .addColumn('comment', 'text', (col) => col.notNull()) .execute(); @@ -21,6 +20,6 @@ export async function up(db: Kysely): Promise { } export async function down(db: Kysely): Promise { - await db.schema.dropIndex('idx_event_zaps_id_amount').execute(); + await db.schema.dropIndex('idx_event_zaps_id_amount').ifExists().execute(); await db.schema.dropTable('event_zaps').execute(); } From ec82e14410c57019ac1567a6c56aea8e44a85018 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 21 Jun 2024 18:35:26 -0300 Subject: [PATCH 05/75] fix: add amount_millisats in event_zaps index --- src/db/migrations/027_add_zap_events.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db/migrations/027_add_zap_events.ts b/src/db/migrations/027_add_zap_events.ts index fe2e1d20..58b231fa 100644 --- a/src/db/migrations/027_add_zap_events.ts +++ b/src/db/migrations/027_add_zap_events.ts @@ -13,7 +13,7 @@ export async function up(db: Kysely): Promise { await db.schema .createIndex('idx_event_zaps_id_amount') .on('event_zaps') - .column('amount') + .column('amount_millisats') .column('target_event_id') .ifNotExists() .execute(); From 2c08b9a2f0a16a3cf275cd49bc8bcfefec890f55 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 21 Jun 2024 20:36:59 -0300 Subject: [PATCH 06/75] feat: create scavenger and handle kind 9735 --- src/utils/scavenger.ts | 48 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 src/utils/scavenger.ts diff --git a/src/utils/scavenger.ts b/src/utils/scavenger.ts new file mode 100644 index 00000000..0ffaa669 --- /dev/null +++ b/src/utils/scavenger.ts @@ -0,0 +1,48 @@ +import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; +import { Kysely } from 'kysely'; +import { z } from 'zod'; + +import { DittoTables } from '@/db/DittoTables.ts'; +import { getAmount } from '@/utils/bolt11.ts'; + +interface ScavengerEventOpts { + savedEvent: Promise; + kysely: Kysely; +} + +/** Consumes the event already stored in the database and uses it to insert into a new custom table, if eligible. + * Scavenger is organism that eats dead or rotting biomass, such as animal flesh or plant material. */ +async function scavengerEvent({ savedEvent, kysely }: ScavengerEventOpts): Promise { + const event = await savedEvent; + if (!event) return; + + switch (event.kind) { + case 9735: + await handleEvent9735(kysely, event); + break; + } +} + +async function handleEvent9735(kysely: Kysely, event: NostrEvent) { + const zapRequestString = event?.tags?.find(([name]) => name === 'description')?.[1]; + if (!zapRequestString) return; + const zapRequest = n.json().pipe(n.event()).optional().catch(undefined).parse(zapRequestString); + if (!zapRequest) return; + + const amountSchema = z.coerce.number().int().nonnegative().catch(0); + const amount_millisats = amountSchema.parse(getAmount(event?.tags.find(([name]) => name === 'bolt11')?.[1])); + if (!amount_millisats || amount_millisats < 1) return; + + const zappedEventId = zapRequest.tags.find(([name]) => name === 'e')?.[1]; + if (!zappedEventId) return; + + await kysely.insertInto('event_zaps').values({ + receipt_id: event.id, + target_event_id: zappedEventId, + sender_pubkey: zapRequest.pubkey, + amount_millisats, + comment: zapRequest.content, + }).execute(); +} + +export { scavengerEvent }; From 1b30f10a9fec1f554d488a88d3825dd8d8f9f6e2 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 21 Jun 2024 21:39:18 -0300 Subject: [PATCH 07/75] test(scavenger): store valid data into event_zaps table --- src/utils/scavenger.test.ts | 55 +++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 src/utils/scavenger.test.ts diff --git a/src/utils/scavenger.test.ts b/src/utils/scavenger.test.ts new file mode 100644 index 00000000..a557c5b1 --- /dev/null +++ b/src/utils/scavenger.test.ts @@ -0,0 +1,55 @@ +import { assertEquals } from '@std/assert'; +import { generateSecretKey } from 'nostr-tools'; + +import { genEvent, getTestDB } from '@/test.ts'; +import { scavengerEvent } from '@/utils/scavenger.ts'; + +Deno.test('store one zap receipt in nostr_events; convert it into event_zaps table format and store it', async () => { + await using db = await getTestDB(); + const kysely = db.kysely; + + const sk = generateSecretKey(); + + const event = genEvent({ + 'id': '67b48a14fb66c60c8f9070bdeb37afdfcc3d08ad01989460448e4081eddda446', + 'pubkey': '9630f464cca6a5147aa8a35f0bcdd3ce485324e732fd39e09233b1d848238f31', + 'created_at': 1674164545, + 'kind': 9735, + 'tags': [ + ['p', '32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245'], + ['P', '97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322'], + ['e', '3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8'], + [ + 'bolt11', + 'lnbc10u1p3unwfusp5t9r3yymhpfqculx78u027lxspgxcr2n2987mx2j55nnfs95nxnzqpp5jmrh92pfld78spqs78v9euf2385t83uvpwk9ldrlvf6ch7tpascqhp5zvkrmemgth3tufcvflmzjzfvjt023nazlhljz2n9hattj4f8jq8qxqyjw5qcqpjrzjqtc4fc44feggv7065fqe5m4ytjarg3repr5j9el35xhmtfexc42yczarjuqqfzqqqqqqqqlgqqqqqqgq9q9qxpqysgq079nkq507a5tw7xgttmj4u990j7wfggtrasah5gd4ywfr2pjcn29383tphp4t48gquelz9z78p4cq7ml3nrrphw5w6eckhjwmhezhnqpy6gyf0', + ], + [ + 'description', + '{"pubkey":"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322","content":"","id":"d9cc14d50fcb8c27539aacf776882942c1a11ea4472f8cdec1dea82fab66279d","created_at":1674164539,"sig":"77127f636577e9029276be060332ea565deaf89ff215a494ccff16ae3f757065e2bc59b2e8c113dd407917a010b3abd36c8d7ad84c0e3ab7dab3a0b0caa9835d","kind":9734,"tags":[["e","3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8"],["p","32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245"],["relays","wss://relay.damus.io","wss://nostr-relay.wlvs.space","wss://nostr.fmt.wiz.biz","wss://relay.nostr.bg","wss://nostr.oxtr.dev","wss://nostr.v0l.io","wss://brb.io","wss://nostr.bitcoiner.social","ws://monad.jb55.com:8080","wss://relay.snort.social"]]}', + ], + ['preimage', '5d006d2cf1e73c7148e7519a4c68adc81642ce0e25a432b2434c99f97344c15f'], + ], + 'content': '', + }, sk); + + await db.store.event(event); + + // deno-lint-ignore require-await + await scavengerEvent({ savedEvent: (async () => event)(), kysely: kysely }); + + const zapReceipts = await kysely.selectFrom('nostr_events').selectAll().execute(); + const customEventZaps = await kysely.selectFrom('event_zaps').selectAll().execute(); + + assertEquals(zapReceipts.length, 1); // basic check + assertEquals(customEventZaps.length, 1); // basic check + + const expected = { + receipt_id: event.id, + target_event_id: '3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8', + sender_pubkey: '97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322', + amount_millisats: 1000000, + comment: '', + }; + + assertEquals(customEventZaps[0], expected); +}); From 771d7f79db09ae8f302b2302e094734c1860c6be Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 21 Jun 2024 21:45:55 -0300 Subject: [PATCH 08/75] refactor(scavenger): put SQL insert into try-catch block --- src/utils/scavenger.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/utils/scavenger.ts b/src/utils/scavenger.ts index 0ffaa669..d9874cb7 100644 --- a/src/utils/scavenger.ts +++ b/src/utils/scavenger.ts @@ -36,13 +36,17 @@ async function handleEvent9735(kysely: Kysely, event: NostrEvent) { const zappedEventId = zapRequest.tags.find(([name]) => name === 'e')?.[1]; if (!zappedEventId) return; - await kysely.insertInto('event_zaps').values({ - receipt_id: event.id, - target_event_id: zappedEventId, - sender_pubkey: zapRequest.pubkey, - amount_millisats, - comment: zapRequest.content, - }).execute(); + try { + await kysely.insertInto('event_zaps').values({ + receipt_id: event.id, + target_event_id: zappedEventId, + sender_pubkey: zapRequest.pubkey, + amount_millisats, + comment: zapRequest.content, + }).execute(); + } catch { + // receipt_id is unique, do nothing + } } export { scavengerEvent }; From 89b56539d177bcd1816b8928917fbf437b458a3e Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 21 Jun 2024 22:23:34 -0300 Subject: [PATCH 09/75] test(scavenger): code coverage 100.00% --- src/utils/scavenger.test.ts | 86 +++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/src/utils/scavenger.test.ts b/src/utils/scavenger.test.ts index a557c5b1..9d768999 100644 --- a/src/utils/scavenger.test.ts +++ b/src/utils/scavenger.test.ts @@ -36,6 +36,8 @@ Deno.test('store one zap receipt in nostr_events; convert it into event_zaps tab // deno-lint-ignore require-await await scavengerEvent({ savedEvent: (async () => event)(), kysely: kysely }); + // deno-lint-ignore require-await + await scavengerEvent({ savedEvent: (async () => event)(), kysely: kysely }); // just to trigger "Error: FOREIGN KEY constraint failed" const zapReceipts = await kysely.selectFrom('nostr_events').selectAll().execute(); const customEventZaps = await kysely.selectFrom('event_zaps').selectAll().execute(); @@ -53,3 +55,87 @@ Deno.test('store one zap receipt in nostr_events; convert it into event_zaps tab assertEquals(customEventZaps[0], expected); }); + +// The function tests below only handle the edge cases and don't assert anything +// If no error happens = ok + +Deno.test('savedEvent is undefined', async () => { + await using db = await getTestDB(); + const kysely = db.kysely; + + // deno-lint-ignore require-await + await scavengerEvent({ savedEvent: (async () => undefined)(), kysely: kysely }); + + // no error happened = ok +}); + +Deno.test('zap receipt does not have a "description" tag', async () => { + await using db = await getTestDB(); + const kysely = db.kysely; + + const sk = generateSecretKey(); + + const event = genEvent({ kind: 9735 }, sk); + + // deno-lint-ignore require-await + await scavengerEvent({ savedEvent: (async () => event)(), kysely: kysely }); + + // no error happened = ok +}); + +Deno.test('zap receipt does not have a zap request stringified value in the "description" tag', async () => { + await using db = await getTestDB(); + const kysely = db.kysely; + + const sk = generateSecretKey(); + + const event = genEvent({ kind: 9735, tags: [['description', 'yolo']] }, sk); + + // deno-lint-ignore require-await + await scavengerEvent({ savedEvent: (async () => event)(), kysely: kysely }); + + // no error happened = ok +}); + +Deno.test('zap receipt does not have a "bolt11" tag', async () => { + await using db = await getTestDB(); + const kysely = db.kysely; + + const sk = generateSecretKey(); + + const event = genEvent({ + kind: 9735, + tags: [[ + 'description', + '{"pubkey":"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322","content":"","id":"d9cc14d50fcb8c27539aacf776882942c1a11ea4472f8cdec1dea82fab66279d","created_at":1674164539,"sig":"77127f636577e9029276be060332ea565deaf89ff215a494ccff16ae3f757065e2bc59b2e8c113dd407917a010b3abd36c8d7ad84c0e3ab7dab3a0b0caa9835d","kind":9734,"tags":[["e","3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8"],["p","32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245"],["relays","wss://relay.damus.io","wss://nostr-relay.wlvs.space","wss://nostr.fmt.wiz.biz","wss://relay.nostr.bg","wss://nostr.oxtr.dev","wss://nostr.v0l.io","wss://brb.io","wss://nostr.bitcoiner.social","ws://monad.jb55.com:8080","wss://relay.snort.social"]]}', + ]], + }, sk); + + // deno-lint-ignore require-await + await scavengerEvent({ savedEvent: (async () => event)(), kysely: kysely }); + + // no error happened = ok +}); + +Deno.test('zap request inside zap receipt does not have an "e" tag', async () => { + await using db = await getTestDB(); + const kysely = db.kysely; + + const sk = generateSecretKey(); + + const event = genEvent({ + kind: 9735, + tags: [[ + 'bolt11', + 'lnbc10u1p3unwfusp5t9r3yymhpfqculx78u027lxspgxcr2n2987mx2j55nnfs95nxnzqpp5jmrh92pfld78spqs78v9euf2385t83uvpwk9ldrlvf6ch7tpascqhp5zvkrmemgth3tufcvflmzjzfvjt023nazlhljz2n9hattj4f8jq8qxqyjw5qcqpjrzjqtc4fc44feggv7065fqe5m4ytjarg3repr5j9el35xhmtfexc42yczarjuqqfzqqqqqqqqlgqqqqqqgq9q9qxpqysgq079nkq507a5tw7xgttmj4u990j7wfggtrasah5gd4ywfr2pjcn29383tphp4t48gquelz9z78p4cq7ml3nrrphw5w6eckhjwmhezhnqpy6gyf0', + ], [ + 'description', + '{"pubkey":"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322","content":"","id":"d9cc14d50fcb8c27539aacf776882942c1a11ea4472f8cdec1dea82fab66279d","created_at":1674164539,"sig":"77127f636577e9029276be060332ea565deaf89ff215a494ccff16ae3f757065e2bc59b2e8c113dd407917a010b3abd36c8d7ad84c0e3ab7dab3a0b0caa9835d","kind":9734,"tags":[["p","32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245"],["relays","wss://relay.damus.io","wss://nostr-relay.wlvs.space","wss://nostr.fmt.wiz.biz","wss://relay.nostr.bg","wss://nostr.oxtr.dev","wss://nostr.v0l.io","wss://brb.io","wss://nostr.bitcoiner.social","ws://monad.jb55.com:8080","wss://relay.snort.social"]]}', + ]], + }, sk); + + // deno-lint-ignore require-await + await scavengerEvent({ savedEvent: (async () => event)(), kysely: kysely }); + + // no error happened = ok +}); From 9731fc257299efe7a5f3663a3a4f4e5a88874f33 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 21 Jun 2024 22:25:34 -0300 Subject: [PATCH 10/75] feat: add scavenger to the pipeline --- src/pipeline.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index 40ba9c12..bd56f09e 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -17,6 +17,7 @@ import { verifyEventWorker } from '@/workers/verify.ts'; import { nip05Cache } from '@/utils/nip05.ts'; import { updateStats } from '@/utils/stats.ts'; import { getTagSet } from '@/utils/tags.ts'; +import { scavengerEvent } from '@/utils/scavenger.ts'; const debug = Debug('ditto:pipeline'); @@ -50,7 +51,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { +async function storeEvent(event: DittoEvent, signal?: AbortSignal): Promise { if (NKinds.ephemeral(event.kind)) return; const store = await Storages.db(); const kysely = await DittoDB.getInstance(); await updateStats({ event, store, kysely }).catch(debug); await store.event(event, { signal }); + + return event; } /** Parse kind 0 metadata and track indexes in the database. */ From 31a5533fd71bd1245962bedfe487ca751addca7a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Jun 2024 22:38:25 -0500 Subject: [PATCH 11/75] Add Prometheus metrics --- deno.json | 1 + installation/ditto.conf | 6 +++ src/controllers/metrics.ts | 14 +++++++ src/controllers/nostr/relay.ts | 5 +++ src/firehose.ts | 4 +- src/metrics.ts | 58 +++++++++++++++++++++++++++++ src/middleware/metricsMiddleware.ts | 10 +++++ src/pipeline.ts | 2 + src/storages/EventsDB.ts | 3 ++ src/workers/fetch.ts | 4 +- 10 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 src/controllers/metrics.ts create mode 100644 src/metrics.ts create mode 100644 src/middleware/metricsMiddleware.ts diff --git a/deno.json b/deno.json index bc5d1e29..240b6c3a 100644 --- a/deno.json +++ b/deno.json @@ -59,6 +59,7 @@ "nostr-relaypool": "npm:nostr-relaypool2@0.6.34", "nostr-tools": "npm:nostr-tools@2.5.1", "nostr-wasm": "npm:nostr-wasm@^0.1.0", + "prom-client": "npm:prom-client@^15.1.2", "question-deno": "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/mod.ts", "tldts": "npm:tldts@^6.0.14", "tseep": "npm:tseep@^1.2.1", diff --git a/installation/ditto.conf b/installation/ditto.conf index d74a8865..256498f4 100644 --- a/installation/ditto.conf +++ b/installation/ditto.conf @@ -50,6 +50,12 @@ server { root /opt/ditto/public; } + location /metrics { + allow 127.0.0.1; + deny all; + proxy_pass http://ditto; + } + location ~ ^/(instance|sw\.js$|sw\.js\.map$) { root /opt/ditto/public; try_files $uri =404; diff --git a/src/controllers/metrics.ts b/src/controllers/metrics.ts new file mode 100644 index 00000000..419931da --- /dev/null +++ b/src/controllers/metrics.ts @@ -0,0 +1,14 @@ +import { register } from 'prom-client'; + +import { AppController } from '@/app.ts'; + +/** Prometheus/OpenMetrics controller. */ +export const metricsController: AppController = async (c) => { + const metrics = await register.metrics(); + + const headers: HeadersInit = { + 'Content-Type': register.contentType, + }; + + return c.text(metrics, 200, headers); +}; diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 5d08e02c..c19395dd 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -10,6 +10,7 @@ import { import { AppController } from '@/app.ts'; import { relayInfoController } from '@/controllers/nostr/relay-info.ts'; +import { relayEventCounter, relayMessageCounter } from '@/metrics.ts'; import * as pipeline from '@/pipeline.ts'; import { RelayError } from '@/RelayError.ts'; import { Storages } from '@/storages.ts'; @@ -22,6 +23,7 @@ function connectStream(socket: WebSocket) { const controllers = new Map(); socket.onmessage = (e) => { + relayMessageCounter.inc(); const result = n.json().pipe(n.clientMsg()).safeParse(e.data); if (result.success) { handleMsg(result.data); @@ -40,15 +42,18 @@ function connectStream(socket: WebSocket) { function handleMsg(msg: NostrClientMsg) { switch (msg[0]) { case 'REQ': + relayEventCounter.inc(); handleReq(msg); return; case 'EVENT': + relayEventCounter.inc({ kind: msg[1].kind.toString() }); handleEvent(msg); return; case 'CLOSE': handleClose(msg); return; case 'COUNT': + relayEventCounter.inc(); handleCount(msg); return; } diff --git a/src/firehose.ts b/src/firehose.ts index 2c776fe4..3e6c8fc0 100644 --- a/src/firehose.ts +++ b/src/firehose.ts @@ -1,5 +1,6 @@ import { Stickynotes } from '@soapbox/stickynotes'; +import { firehoseEventCounter } from '@/metrics.ts'; import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; @@ -12,13 +13,14 @@ const console = new Stickynotes('ditto:firehose'); * side-effects based on them, such as trending hashtag tracking * and storing events for notifications and the home feed. */ -export async function startFirehose() { +export async function startFirehose(): Promise { const store = await Storages.client(); for await (const msg of store.req([{ kinds: [0, 1, 3, 5, 6, 7, 9735, 10002], limit: 0, since: nostrNow() }])) { if (msg[0] === 'EVENT') { const event = msg[2]; console.debug(`NostrEvent<${event.kind}> ${event.id}`); + firehoseEventCounter.inc({ kind: event.kind }); pipeline .handleEvent(event, AbortSignal.timeout(5000)) diff --git a/src/metrics.ts b/src/metrics.ts new file mode 100644 index 00000000..efe1a895 --- /dev/null +++ b/src/metrics.ts @@ -0,0 +1,58 @@ +import { Counter } from 'prom-client'; + +export const httpRequestCounter = new Counter({ + name: 'http_requests_total', + help: 'Total number of HTTP requests', + labelNames: ['method', 'path'], +}); + +export const fetchCounter = new Counter({ + name: 'fetch_total', + help: 'Total number of fetch requests', + labelNames: ['method', 'path'], +}); + +export const firehoseEventCounter = new Counter({ + name: 'firehose_events_total', + help: 'Total number of Nostr events processed by the firehose', + labelNames: ['kind'], +}); + +export const pipelineEventCounter = new Counter({ + name: 'pipeline_events_total', + help: 'Total number of Nostr events processed by the pipeline', + labelNames: ['kind'], +}); + +export const relayReqCounter = new Counter({ + name: 'relay_reqs_total', + help: 'Total number of REQ messages processed by the relay', +}); + +export const relayEventCounter = new Counter({ + name: 'relay_events_total', + help: 'Total number of EVENT messages processed by the relay', + labelNames: ['kind'], +}); + +export const relayCountCounter = new Counter({ + name: 'relay_counts_total', + help: 'Total number of COUNT messages processed by the relay', +}); + +export const relayMessageCounter = new Counter({ + name: 'relay_messages_total', + help: 'Total number of Nostr messages processed by the relay', +}); + +export const dbQueryCounter = new Counter({ + name: 'db_query_total', + help: 'Total number of database queries', + labelNames: ['kind'], +}); + +export const dbEventCounter = new Counter({ + name: 'db_events_total', + help: 'Total number of database inserts', + labelNames: ['kind'], +}); diff --git a/src/middleware/metricsMiddleware.ts b/src/middleware/metricsMiddleware.ts new file mode 100644 index 00000000..1e88ff3f --- /dev/null +++ b/src/middleware/metricsMiddleware.ts @@ -0,0 +1,10 @@ +import { MiddlewareHandler } from '@hono/hono'; + +import { httpRequestCounter } from '@/metrics.ts'; + +export const metricsMiddleware: MiddlewareHandler = async (c, next) => { + const { method, path } = c.req; + httpRequestCounter.inc({ method, path }); + + await next(); +}; diff --git a/src/pipeline.ts b/src/pipeline.ts index 40ba9c12..695a027e 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -7,6 +7,7 @@ import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; import { deleteAttachedMedia } from '@/db/unattached-media.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; +import { pipelineEventCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; @@ -36,6 +37,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise ${event.id}`); + pipelineEventCounter.inc({ kind: event.kind }); if (event.kind !== 24133) { await policyFilter(event); diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 085a4270..bd350173 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -7,6 +7,7 @@ import { nip27 } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { DittoTables } from '@/db/DittoTables.ts'; +import { dbEventCounter, dbQueryCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; import { isNostrId, isURL } from '@/utils.ts'; @@ -53,6 +54,7 @@ class EventsDB implements NStore { async event(event: NostrEvent, _opts?: { signal?: AbortSignal }): Promise { event = purifyEvent(event); this.console.debug('EVENT', JSON.stringify(event)); + dbEventCounter.inc({ kind: event.kind }); if (await this.isDeletedAdmin(event)) { throw new RelayError('blocked', 'event deleted by admin'); @@ -137,6 +139,7 @@ class EventsDB implements NStore { /** Get events for filters from the database. */ async query(filters: NostrFilter[], opts: { signal?: AbortSignal; limit?: number } = {}): Promise { filters = await this.expandFilters(filters); + dbQueryCounter.inc(); for (const filter of filters) { if (filter.since && filter.since >= 2_147_483_647) { diff --git a/src/workers/fetch.ts b/src/workers/fetch.ts index f0bece58..ad0c834e 100644 --- a/src/workers/fetch.ts +++ b/src/workers/fetch.ts @@ -1,8 +1,9 @@ import * as Comlink from 'comlink'; +import { FetchWorker } from './fetch.worker.ts'; import './handlers/abortsignal.ts'; -import type { FetchWorker } from './fetch.worker.ts'; +import { fetchCounter } from '@/metrics.ts'; const worker = new Worker(new URL('./fetch.worker.ts', import.meta.url), { type: 'module' }); const client = Comlink.wrap(worker); @@ -24,6 +25,7 @@ const fetchWorker: typeof fetch = async (...args) => { await ready; const [url, init] = serializeFetchArgs(args); const { body, signal, ...rest } = init; + fetchCounter.inc({ method: init.method, path: new URL(url).pathname }); const result = await client.fetch(url, { ...rest, body: await prepareBodyForWorker(body) }, signal); return new Response(...result); }; From 88df19494ae5099600c21602573091847c0ad3a9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 21 Jun 2024 22:46:37 -0500 Subject: [PATCH 12/75] Expose /metrics in hono --- deno.lock | 1 + src/app.ts | 3 +++ 2 files changed, 4 insertions(+) diff --git a/deno.lock b/deno.lock index f59adb73..a43c7cea 100644 --- a/deno.lock +++ b/deno.lock @@ -1418,6 +1418,7 @@ "npm:nostr-relaypool2@0.6.34", "npm:nostr-tools@2.5.1", "npm:nostr-wasm@^0.1.0", + "npm:prom-client@^15.1.2", "npm:tldts@^6.0.14", "npm:tseep@^1.2.1", "npm:type-fest@^4.3.0", diff --git a/src/app.ts b/src/app.ts index 0c21ecf0..8bcaa244 100644 --- a/src/app.ts +++ b/src/app.ts @@ -108,6 +108,7 @@ import { trendingStatusesController, trendingTagsController, } from '@/controllers/api/trends.ts'; +import { metricsController } from '@/controllers/metrics.ts'; import { indexController } from '@/controllers/site.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; import { nostrController } from '@/controllers/well-known/nostr.ts'; @@ -168,6 +169,8 @@ app.use( storeMiddleware, ); +app.get('/metrics', metricsController); + app.get('/.well-known/nodeinfo', nodeInfoController); app.get('/.well-known/nostr.json', nostrController); From e50ba819b9485939ca22a0068744c70c404f6d04 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Jun 2024 09:08:32 -0500 Subject: [PATCH 13/75] metrics: remove path from fetch and request metrics --- src/metrics.ts | 4 ++-- src/middleware/metricsMiddleware.ts | 4 ++-- src/workers/fetch.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/metrics.ts b/src/metrics.ts index efe1a895..2d74bd45 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -3,13 +3,13 @@ import { Counter } from 'prom-client'; export const httpRequestCounter = new Counter({ name: 'http_requests_total', help: 'Total number of HTTP requests', - labelNames: ['method', 'path'], + labelNames: ['method'], }); export const fetchCounter = new Counter({ name: 'fetch_total', help: 'Total number of fetch requests', - labelNames: ['method', 'path'], + labelNames: ['method'], }); export const firehoseEventCounter = new Counter({ diff --git a/src/middleware/metricsMiddleware.ts b/src/middleware/metricsMiddleware.ts index 1e88ff3f..1a491186 100644 --- a/src/middleware/metricsMiddleware.ts +++ b/src/middleware/metricsMiddleware.ts @@ -3,8 +3,8 @@ import { MiddlewareHandler } from '@hono/hono'; import { httpRequestCounter } from '@/metrics.ts'; export const metricsMiddleware: MiddlewareHandler = async (c, next) => { - const { method, path } = c.req; - httpRequestCounter.inc({ method, path }); + const { method } = c.req; + httpRequestCounter.inc({ method }); await next(); }; diff --git a/src/workers/fetch.ts b/src/workers/fetch.ts index ad0c834e..3ed98fbb 100644 --- a/src/workers/fetch.ts +++ b/src/workers/fetch.ts @@ -25,7 +25,7 @@ const fetchWorker: typeof fetch = async (...args) => { await ready; const [url, init] = serializeFetchArgs(args); const { body, signal, ...rest } = init; - fetchCounter.inc({ method: init.method, path: new URL(url).pathname }); + fetchCounter.inc({ method: init.method }); const result = await client.fetch(url, { ...rest, body: await prepareBodyForWorker(body) }, signal); return new Response(...result); }; From c44347e9d118d461987de27b11c3f5caf3a83131 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Jun 2024 10:28:42 -0500 Subject: [PATCH 14/75] Fix relay metrics --- src/controllers/nostr/relay.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index c19395dd..730c1ff9 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -10,7 +10,7 @@ import { import { AppController } from '@/app.ts'; import { relayInfoController } from '@/controllers/nostr/relay-info.ts'; -import { relayEventCounter, relayMessageCounter } from '@/metrics.ts'; +import { relayCountCounter, relayEventCounter, relayMessageCounter, relayReqCounter } from '@/metrics.ts'; import * as pipeline from '@/pipeline.ts'; import { RelayError } from '@/RelayError.ts'; import { Storages } from '@/storages.ts'; @@ -42,7 +42,7 @@ function connectStream(socket: WebSocket) { function handleMsg(msg: NostrClientMsg) { switch (msg[0]) { case 'REQ': - relayEventCounter.inc(); + relayReqCounter.inc(); handleReq(msg); return; case 'EVENT': @@ -53,7 +53,7 @@ function connectStream(socket: WebSocket) { handleClose(msg); return; case 'COUNT': - relayEventCounter.inc(); + relayCountCounter.inc(); handleCount(msg); return; } From a1810219f1dd959cb3da2ea9c9c55313968333e2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Jun 2024 10:38:55 -0500 Subject: [PATCH 15/75] Update deno.lock --- deno.lock | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/deno.lock b/deno.lock index a43c7cea..e69798dc 100644 --- a/deno.lock +++ b/deno.lock @@ -7,6 +7,7 @@ "jsr:@denosaurs/plug@1": "jsr:@denosaurs/plug@1.0.6", "jsr:@gleasonator/policy": "jsr:@gleasonator/policy@0.2.0", "jsr:@gleasonator/policy@0.2.0": "jsr:@gleasonator/policy@0.2.0", + "jsr:@gleasonator/policy@0.4.0": "jsr:@gleasonator/policy@0.4.0", "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.4.6", "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", @@ -61,6 +62,7 @@ "npm:nostr-tools@^2.5.0": "npm:nostr-tools@2.5.1", "npm:nostr-tools@^2.7.0": "npm:nostr-tools@2.7.0", "npm:nostr-wasm@^0.1.0": "npm:nostr-wasm@0.1.0", + "npm:prom-client@^15.1.2": "npm:prom-client@15.1.2", "npm:tldts@^6.0.14": "npm:tldts@6.1.18", "npm:type-fest@^4.3.0": "npm:type-fest@4.18.2", "npm:unfurl.js@^6.4.0": "npm:unfurl.js@6.4.0", @@ -96,6 +98,12 @@ "jsr:@nostrify/nostrify@^0.22.1" ] }, + "@gleasonator/policy@0.4.0": { + "integrity": "59c2f3ab1dc663e99a3e10b7eb69bf9fe581ce5d428fe56653e38f7f961da5ea", + "dependencies": [ + "jsr:@nostrify/nostrify@^0.22.1" + ] + }, "@hono/hono@4.4.6": { "integrity": "aa557ca9930787ee86b9ca1730691f1ce1c379174c2cb244d5934db2b6314453" }, @@ -275,6 +283,10 @@ "integrity": "sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg==", "dependencies": {} }, + "@opentelemetry/api@1.9.0": { + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "dependencies": {} + }, "@scure/base@1.1.1": { "integrity": "sha512-ZxOhsSyxYwLJj3pLZCefNitxsj093tb2vq90mp2txoYeBqbcjDjqFhyM8eUjq/uFm6zJ+mUuqxlS2FkuSY1MTA==", "dependencies": {} @@ -351,6 +363,10 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", "dependencies": {} }, + "bintrees@1.0.2": { + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==", + "dependencies": {} + }, "braces@3.0.2": { "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", "dependencies": { @@ -852,6 +868,13 @@ "integrity": "sha512-eG2dWTVw5bzqGRztnHExczNxt5VGsE6OwTeCG3fdUf9KBsZzO3R5OIIIzWR+iZA0NtZ+RDVdaoE2dK1cn6jH4g==", "dependencies": {} }, + "prom-client@15.1.2": { + "integrity": "sha512-on3h1iXb04QFLLThrmVYg1SChBQ9N1c+nKAjebBjokBqipddH3uxmOUcEkTnzmJ8Jh/5TSUnUqS40i2QB2dJHQ==", + "dependencies": { + "@opentelemetry/api": "@opentelemetry/api@1.9.0", + "tdigest": "tdigest@0.1.2" + } + }, "psl@1.9.0": { "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==", "dependencies": {} @@ -955,6 +978,12 @@ "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", "dependencies": {} }, + "tdigest@0.1.2": { + "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "dependencies": { + "bintrees": "bintrees@1.0.2" + } + }, "tldts-core@6.1.18": { "integrity": "sha512-e4wx32F/7dMBSZyKAx825Yte3U0PQtZZ0bkWxYQiwLteRVnQ5zM40fEbi0IyNtwQssgJAk3GCr7Q+w39hX0VKA==", "dependencies": {} From 0d7ef68353817977e8acc3a81091b0f9e0cdcaaa Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 23 Jun 2024 13:38:40 -0300 Subject: [PATCH 16/75] feat: add pagination and sort by amount - zapped_by endpoint --- src/controllers/api/statuses.ts | 50 ++++++++++++++++----------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 28e0778a..30550bbb 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -8,14 +8,21 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; -import { getAmount } from '@/utils/bolt11.ts'; import { getAncestors, getAuthor, getDescendants, getEvent } from '@/queries.ts'; import { getUnattachedMediaByIds } from '@/db/unattached-media.ts'; import { renderEventAccounts } from '@/views.ts'; import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts'; import { Storages } from '@/storages.ts'; import { hydrateEvents, purifyEvent } from '@/storages/hydrate.ts'; -import { createEvent, paginated, paginationSchema, parseBody, updateListEvent } from '@/utils/api.ts'; +import { + createEvent, + listPaginationSchema, + paginated, + paginatedList, + paginationSchema, + parseBody, + updateListEvent, +} from '@/utils/api.ts'; import { getInvoice, getLnurl } from '@/utils/lnurl.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; import { addTag, deleteTag } from '@/utils/tags.ts'; @@ -545,33 +552,26 @@ const zapController: AppController = async (c) => { const zappedByController: AppController = async (c) => { const id = c.req.param('id'); + const params = listPaginationSchema.parse(c.req.query()); const store = await Storages.db(); - const amountSchema = z.coerce.number().int().nonnegative().catch(0); + const db = await DittoDB.getInstance(); - const events = (await store.query([{ kinds: [9735], '#e': [id], limit: 100 }])).map((event) => { - const zapRequestString = event.tags.find(([name]) => name === 'description')?.[1]; - if (!zapRequestString) return; - try { - const zapRequest = n.json().pipe(n.event()).parse(zapRequestString); - const amount = zapRequest?.tags.find(([name]: any) => name === 'amount')?.[1]; - if (!amount) { - const amount = getAmount(event?.tags.find(([name]) => name === 'bolt11')?.[1]); - if (!amount) return; - zapRequest.tags.push(['amount', amount]); - } - return zapRequest; - } catch { - return; - } - }).filter(Boolean) as DittoEvent[]; + const zaps = await db.selectFrom('event_zaps') + .selectAll() + .where('target_event_id', '=', id) + .orderBy('amount_millisats', 'desc') + .limit(params.limit) + .offset(params.offset).execute(); - await hydrateEvents({ events, store }); + const authors = await store.query([{ kinds: [0], authors: zaps.map((zap) => zap.sender_pubkey) }]); const results = (await Promise.all( - events.map(async (event) => { - const amount = amountSchema.parse(event.tags.find(([name]) => name === 'amount')?.[1]); - const comment = event?.content ?? ''; - const account = event?.author ? await renderAccount(event.author) : await accountFromPubkey(event.pubkey); + zaps.map(async (zap) => { + const amount = zap.amount_millisats; + const comment = zap.comment; + + const sender = authors.find((author) => author.pubkey === zap.sender_pubkey); + const account = sender ? await renderAccount(sender) : await accountFromPubkey(zap.sender_pubkey); return { comment, @@ -581,7 +581,7 @@ const zappedByController: AppController = async (c) => { }), )).filter(Boolean); - return c.json(results); + return paginatedList(c, params, results); }; export { From 39fb1eee5fd740e7d02d20ad70c35552e10e2ecd Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Jun 2024 11:59:08 -0500 Subject: [PATCH 17/75] Import Sentry into workers --- src/workers/fetch.worker.ts | 1 + src/workers/verify.worker.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/src/workers/fetch.worker.ts b/src/workers/fetch.worker.ts index d44e043c..0012088b 100644 --- a/src/workers/fetch.worker.ts +++ b/src/workers/fetch.worker.ts @@ -2,6 +2,7 @@ import Debug from '@soapbox/stickynotes/debug'; import * as Comlink from 'comlink'; import './handlers/abortsignal.ts'; +import '@/sentry.ts'; const debug = Debug('ditto:fetch.worker'); diff --git a/src/workers/verify.worker.ts b/src/workers/verify.worker.ts index e218474e..3e71215d 100644 --- a/src/workers/verify.worker.ts +++ b/src/workers/verify.worker.ts @@ -3,6 +3,7 @@ import * as Comlink from 'comlink'; import { VerifiedEvent, verifyEvent } from 'nostr-tools'; import '@/nostr-wasm.ts'; +import '@/sentry.ts'; export const VerifyWorker = { verifyEvent(event: NostrEvent): event is VerifiedEvent { From 60b4ea4643216d50f5455ac1207a253dbc3de2c1 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 24 Jun 2024 02:02:58 +0530 Subject: [PATCH 18/75] add config values for external nostr viewer back to source code --- src/config.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/config.ts b/src/config.ts index 502544d8..f80e5eed 100644 --- a/src/config.ts +++ b/src/config.ts @@ -63,6 +63,14 @@ class Conf { static get localDomain(): string { return Deno.env.get('LOCAL_DOMAIN') || `http://localhost:${Conf.port}`; } + /** Link to an external nostr viewer. */ + static get externalDomain(): string { + return Deno.env.get('NOSTR_EXTERNAL') || 'https://njump.me'; + } + /** Get a link to a nip19-encoded entity in the configured external viewer. */ + static external(path: string) { + return new URL(path, Conf.externalDomain); + } /** * Heroku-style database URL. This is used in production to connect to the * database. From f19afa449adb98b11b355661ae5028b2cd02fe87 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 24 Jun 2024 02:03:31 +0530 Subject: [PATCH 19/75] add the ditto.external field to statuses and accounts --- src/entities/MastodonAccount.ts | 1 + src/entities/MastodonStatus.ts | 3 +++ src/views/mastodon/accounts.ts | 1 + src/views/mastodon/statuses.ts | 3 +++ 4 files changed, 8 insertions(+) diff --git a/src/entities/MastodonAccount.ts b/src/entities/MastodonAccount.ts index 27cad244..4873ed4c 100644 --- a/src/entities/MastodonAccount.ts +++ b/src/entities/MastodonAccount.ts @@ -40,6 +40,7 @@ export interface MastodonAccount { username: string; ditto: { accepts_zaps: boolean; + external?: URL; }; pleroma: { deactivated: boolean; diff --git a/src/entities/MastodonStatus.ts b/src/entities/MastodonStatus.ts index 1fcbcacb..430f8e7f 100644 --- a/src/entities/MastodonStatus.ts +++ b/src/entities/MastodonStatus.ts @@ -39,4 +39,7 @@ export interface MastodonStatus { expires_at?: string; quotes_count: number; }; + ditto: { + external?: URL; + }; } diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index f9ed1cdc..4217f699 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -82,6 +82,7 @@ async function renderAccount( username: parsed05?.nickname || npub.substring(0, 8), ditto: { accepts_zaps: Boolean(getLnurl({ lud06, lud16 })), + external: Conf.externalDomain === Conf.localDomain ? undefined : Conf.external(npub), }, pleroma: { deactivated: names.has('disabled'), diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index d440b65c..7ac9c07a 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -124,6 +124,9 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< uri: Conf.local(`/${note}`), url: Conf.local(`/${note}`), zapped: Boolean(zapEvent), + ditto: { + external: Conf.externalDomain === Conf.localDomain ? undefined : Conf.external(note), + }, pleroma: { emoji_reactions: reactions, expires_at: !isNaN(expiresAt.getTime()) ? expiresAt.toISOString() : undefined, From 3f7687d59fc1a9b9bb5c71ed482426d791b41acf Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 24 Jun 2024 02:34:49 +0530 Subject: [PATCH 20/75] make changes according to 24-06-24 review --- src/config.ts | 2 +- src/entities/MastodonAccount.ts | 2 +- src/entities/MastodonStatus.ts | 2 +- src/views/mastodon/accounts.ts | 2 +- src/views/mastodon/statuses.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/config.ts b/src/config.ts index f80e5eed..1dd688b7 100644 --- a/src/config.ts +++ b/src/config.ts @@ -69,7 +69,7 @@ class Conf { } /** Get a link to a nip19-encoded entity in the configured external viewer. */ static external(path: string) { - return new URL(path, Conf.externalDomain); + return new URL(path, Conf.externalDomain).toString(); } /** * Heroku-style database URL. This is used in production to connect to the diff --git a/src/entities/MastodonAccount.ts b/src/entities/MastodonAccount.ts index 4873ed4c..a7fef5de 100644 --- a/src/entities/MastodonAccount.ts +++ b/src/entities/MastodonAccount.ts @@ -40,7 +40,7 @@ export interface MastodonAccount { username: string; ditto: { accepts_zaps: boolean; - external?: URL; + external_url: string; }; pleroma: { deactivated: boolean; diff --git a/src/entities/MastodonStatus.ts b/src/entities/MastodonStatus.ts index 430f8e7f..20c52438 100644 --- a/src/entities/MastodonStatus.ts +++ b/src/entities/MastodonStatus.ts @@ -40,6 +40,6 @@ export interface MastodonStatus { quotes_count: number; }; ditto: { - external?: URL; + external_url: string; }; } diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index 4217f699..5abb1aca 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -82,7 +82,7 @@ async function renderAccount( username: parsed05?.nickname || npub.substring(0, 8), ditto: { accepts_zaps: Boolean(getLnurl({ lud06, lud16 })), - external: Conf.externalDomain === Conf.localDomain ? undefined : Conf.external(npub), + external_url: Conf.external(npub), }, pleroma: { deactivated: names.has('disabled'), diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 7ac9c07a..2fa8f313 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -125,7 +125,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< url: Conf.local(`/${note}`), zapped: Boolean(zapEvent), ditto: { - external: Conf.externalDomain === Conf.localDomain ? undefined : Conf.external(note), + external_url: Conf.external(note), }, pleroma: { emoji_reactions: reactions, From 05bf417bcccb0f75e6cc229d9c2879cbb74bf590 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 23 Jun 2024 22:49:15 -0300 Subject: [PATCH 21/75] perf(event_zaps): make two separate indexes instead of a compound index --- src/db/migrations/027_add_zap_events.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/db/migrations/027_add_zap_events.ts b/src/db/migrations/027_add_zap_events.ts index 58b231fa..2fcc101c 100644 --- a/src/db/migrations/027_add_zap_events.ts +++ b/src/db/migrations/027_add_zap_events.ts @@ -11,15 +11,22 @@ export async function up(db: Kysely): Promise { .execute(); await db.schema - .createIndex('idx_event_zaps_id_amount') + .createIndex('idx_event_zaps_amount_millisats') .on('event_zaps') .column('amount_millisats') + .ifNotExists() + .execute(); + + await db.schema + .createIndex('idx_event_zaps_target_event_id') + .on('event_zaps') .column('target_event_id') .ifNotExists() .execute(); } export async function down(db: Kysely): Promise { - await db.schema.dropIndex('idx_event_zaps_id_amount').ifExists().execute(); + await db.schema.dropIndex('idx_event_zaps_amount_millisats').ifExists().execute(); + await db.schema.dropIndex('idx_event_zaps_target_event_id').ifExists().execute(); await db.schema.dropTable('event_zaps').execute(); } From e1ee3bd8e97665826426aada49f2867501dcd50d Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 23 Jun 2024 23:45:32 -0300 Subject: [PATCH 22/75] refactor: remove scavenger, put logic directly into pipeline --- src/pipeline.test.ts | 125 ++++++++++++++++++++++++++++++++ src/pipeline.ts | 38 +++++++++- src/utils/scavenger.test.ts | 141 ------------------------------------ src/utils/scavenger.ts | 52 ------------- 4 files changed, 159 insertions(+), 197 deletions(-) create mode 100644 src/pipeline.test.ts delete mode 100644 src/utils/scavenger.ts diff --git a/src/pipeline.test.ts b/src/pipeline.test.ts new file mode 100644 index 00000000..64cb523b --- /dev/null +++ b/src/pipeline.test.ts @@ -0,0 +1,125 @@ +import { assertEquals } from '@std/assert'; +import { generateSecretKey } from 'nostr-tools'; + +import { genEvent, getTestDB } from '@/test.ts'; +import { handleZaps } from '@/pipeline.ts'; + +Deno.test('store one zap receipt in nostr_events; convert it into event_zaps table format and store it', async () => { + await using db = await getTestDB(); + const kysely = db.kysely; + + const sk = generateSecretKey(); + + const event = genEvent({ + 'id': '67b48a14fb66c60c8f9070bdeb37afdfcc3d08ad01989460448e4081eddda446', + 'pubkey': '9630f464cca6a5147aa8a35f0bcdd3ce485324e732fd39e09233b1d848238f31', + 'created_at': 1674164545, + 'kind': 9735, + 'tags': [ + ['p', '32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245'], + ['P', '97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322'], + ['e', '3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8'], + [ + 'bolt11', + 'lnbc10u1p3unwfusp5t9r3yymhpfqculx78u027lxspgxcr2n2987mx2j55nnfs95nxnzqpp5jmrh92pfld78spqs78v9euf2385t83uvpwk9ldrlvf6ch7tpascqhp5zvkrmemgth3tufcvflmzjzfvjt023nazlhljz2n9hattj4f8jq8qxqyjw5qcqpjrzjqtc4fc44feggv7065fqe5m4ytjarg3repr5j9el35xhmtfexc42yczarjuqqfzqqqqqqqqlgqqqqqqgq9q9qxpqysgq079nkq507a5tw7xgttmj4u990j7wfggtrasah5gd4ywfr2pjcn29383tphp4t48gquelz9z78p4cq7ml3nrrphw5w6eckhjwmhezhnqpy6gyf0', + ], + [ + 'description', + '{"pubkey":"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322","content":"","id":"d9cc14d50fcb8c27539aacf776882942c1a11ea4472f8cdec1dea82fab66279d","created_at":1674164539,"sig":"77127f636577e9029276be060332ea565deaf89ff215a494ccff16ae3f757065e2bc59b2e8c113dd407917a010b3abd36c8d7ad84c0e3ab7dab3a0b0caa9835d","kind":9734,"tags":[["e","3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8"],["p","32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245"],["relays","wss://relay.damus.io","wss://nostr-relay.wlvs.space","wss://nostr.fmt.wiz.biz","wss://relay.nostr.bg","wss://nostr.oxtr.dev","wss://nostr.v0l.io","wss://brb.io","wss://nostr.bitcoiner.social","ws://monad.jb55.com:8080","wss://relay.snort.social"]]}', + ], + ['preimage', '5d006d2cf1e73c7148e7519a4c68adc81642ce0e25a432b2434c99f97344c15f'], + ], + 'content': '', + }, sk); + + await db.store.event(event); + + await handleZaps(kysely, event); + await handleZaps(kysely, event); + + const zapReceipts = await kysely.selectFrom('nostr_events').selectAll().execute(); + const customEventZaps = await kysely.selectFrom('event_zaps').selectAll().execute(); + + assertEquals(zapReceipts.length, 1); // basic check + assertEquals(customEventZaps.length, 1); // basic check + + const expected = { + receipt_id: event.id, + target_event_id: '3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8', + sender_pubkey: '97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322', + amount_millisats: 1000000, + comment: '', + }; + + assertEquals(customEventZaps[0], expected); +}); + +// The function tests below only handle the edge cases and don't assert anything +// If no error happens = ok + +Deno.test('zap receipt does not have a "description" tag', async () => { + await using db = await getTestDB(); + const kysely = db.kysely; + + const sk = generateSecretKey(); + + const event = genEvent({ kind: 9735 }, sk); + + await handleZaps(kysely, event); + + // no error happened = ok +}); + +Deno.test('zap receipt does not have a zap request stringified value in the "description" tag', async () => { + await using db = await getTestDB(); + const kysely = db.kysely; + + const sk = generateSecretKey(); + + const event = genEvent({ kind: 9735, tags: [['description', 'yolo']] }, sk); + + await handleZaps(kysely, event); + + // no error happened = ok +}); + +Deno.test('zap receipt does not have a "bolt11" tag', async () => { + await using db = await getTestDB(); + const kysely = db.kysely; + + const sk = generateSecretKey(); + + const event = genEvent({ + kind: 9735, + tags: [[ + 'description', + '{"pubkey":"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322","content":"","id":"d9cc14d50fcb8c27539aacf776882942c1a11ea4472f8cdec1dea82fab66279d","created_at":1674164539,"sig":"77127f636577e9029276be060332ea565deaf89ff215a494ccff16ae3f757065e2bc59b2e8c113dd407917a010b3abd36c8d7ad84c0e3ab7dab3a0b0caa9835d","kind":9734,"tags":[["e","3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8"],["p","32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245"],["relays","wss://relay.damus.io","wss://nostr-relay.wlvs.space","wss://nostr.fmt.wiz.biz","wss://relay.nostr.bg","wss://nostr.oxtr.dev","wss://nostr.v0l.io","wss://brb.io","wss://nostr.bitcoiner.social","ws://monad.jb55.com:8080","wss://relay.snort.social"]]}', + ]], + }, sk); + + await handleZaps(kysely, event); + + // no error happened = ok +}); + +Deno.test('zap request inside zap receipt does not have an "e" tag', async () => { + await using db = await getTestDB(); + const kysely = db.kysely; + + const sk = generateSecretKey(); + + const event = genEvent({ + kind: 9735, + tags: [[ + 'bolt11', + 'lnbc10u1p3unwfusp5t9r3yymhpfqculx78u027lxspgxcr2n2987mx2j55nnfs95nxnzqpp5jmrh92pfld78spqs78v9euf2385t83uvpwk9ldrlvf6ch7tpascqhp5zvkrmemgth3tufcvflmzjzfvjt023nazlhljz2n9hattj4f8jq8qxqyjw5qcqpjrzjqtc4fc44feggv7065fqe5m4ytjarg3repr5j9el35xhmtfexc42yczarjuqqfzqqqqqqqqlgqqqqqqgq9q9qxpqysgq079nkq507a5tw7xgttmj4u990j7wfggtrasah5gd4ywfr2pjcn29383tphp4t48gquelz9z78p4cq7ml3nrrphw5w6eckhjwmhezhnqpy6gyf0', + ], [ + 'description', + '{"pubkey":"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322","content":"","id":"d9cc14d50fcb8c27539aacf776882942c1a11ea4472f8cdec1dea82fab66279d","created_at":1674164539,"sig":"77127f636577e9029276be060332ea565deaf89ff215a494ccff16ae3f757065e2bc59b2e8c113dd407917a010b3abd36c8d7ad84c0e3ab7dab3a0b0caa9835d","kind":9734,"tags":[["p","32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245"],["relays","wss://relay.damus.io","wss://nostr-relay.wlvs.space","wss://nostr.fmt.wiz.biz","wss://relay.nostr.bg","wss://nostr.oxtr.dev","wss://nostr.v0l.io","wss://brb.io","wss://nostr.bitcoiner.social","ws://monad.jb55.com:8080","wss://relay.snort.social"]]}', + ]], + }, sk); + + await handleZaps(kysely, event); + + // no error happened = ok +}); diff --git a/src/pipeline.ts b/src/pipeline.ts index b60ac9ed..4c942913 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,7 +1,8 @@ import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; -import { sql } from 'kysely'; +import { Kysely, sql } from 'kysely'; import { LRUCache } from 'lru-cache'; +import { z } from 'zod'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; @@ -18,7 +19,8 @@ import { verifyEventWorker } from '@/workers/verify.ts'; import { nip05Cache } from '@/utils/nip05.ts'; import { updateStats } from '@/utils/stats.ts'; import { getTagSet } from '@/utils/tags.ts'; -import { scavengerEvent } from '@/utils/scavenger.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; +import { getAmount } from '@/utils/bolt11.ts'; const debug = Debug('ditto:pipeline'); @@ -53,7 +55,8 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { } } -export { handleEvent }; +/** Stores the event in the 'event_zaps' table */ +async function handleZaps(kysely: Kysely, event: NostrEvent) { + const zapRequestString = event?.tags?.find(([name]) => name === 'description')?.[1]; + if (!zapRequestString) return; + const zapRequest = n.json().pipe(n.event()).optional().catch(undefined).parse(zapRequestString); + if (!zapRequest) return; + + const amountSchema = z.coerce.number().int().nonnegative().catch(0); + const amount_millisats = amountSchema.parse(getAmount(event?.tags.find(([name]) => name === 'bolt11')?.[1])); + if (!amount_millisats || amount_millisats < 1) return; + + const zappedEventId = zapRequest.tags.find(([name]) => name === 'e')?.[1]; + if (!zappedEventId) return; + + try { + await kysely.insertInto('event_zaps').values({ + receipt_id: event.id, + target_event_id: zappedEventId, + sender_pubkey: zapRequest.pubkey, + amount_millisats, + comment: zapRequest.content, + }).execute(); + } catch { + // receipt_id is unique, do nothing + } +} + +export { handleEvent, handleZaps }; diff --git a/src/utils/scavenger.test.ts b/src/utils/scavenger.test.ts index 9d768999..e69de29b 100644 --- a/src/utils/scavenger.test.ts +++ b/src/utils/scavenger.test.ts @@ -1,141 +0,0 @@ -import { assertEquals } from '@std/assert'; -import { generateSecretKey } from 'nostr-tools'; - -import { genEvent, getTestDB } from '@/test.ts'; -import { scavengerEvent } from '@/utils/scavenger.ts'; - -Deno.test('store one zap receipt in nostr_events; convert it into event_zaps table format and store it', async () => { - await using db = await getTestDB(); - const kysely = db.kysely; - - const sk = generateSecretKey(); - - const event = genEvent({ - 'id': '67b48a14fb66c60c8f9070bdeb37afdfcc3d08ad01989460448e4081eddda446', - 'pubkey': '9630f464cca6a5147aa8a35f0bcdd3ce485324e732fd39e09233b1d848238f31', - 'created_at': 1674164545, - 'kind': 9735, - 'tags': [ - ['p', '32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245'], - ['P', '97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322'], - ['e', '3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8'], - [ - 'bolt11', - 'lnbc10u1p3unwfusp5t9r3yymhpfqculx78u027lxspgxcr2n2987mx2j55nnfs95nxnzqpp5jmrh92pfld78spqs78v9euf2385t83uvpwk9ldrlvf6ch7tpascqhp5zvkrmemgth3tufcvflmzjzfvjt023nazlhljz2n9hattj4f8jq8qxqyjw5qcqpjrzjqtc4fc44feggv7065fqe5m4ytjarg3repr5j9el35xhmtfexc42yczarjuqqfzqqqqqqqqlgqqqqqqgq9q9qxpqysgq079nkq507a5tw7xgttmj4u990j7wfggtrasah5gd4ywfr2pjcn29383tphp4t48gquelz9z78p4cq7ml3nrrphw5w6eckhjwmhezhnqpy6gyf0', - ], - [ - 'description', - '{"pubkey":"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322","content":"","id":"d9cc14d50fcb8c27539aacf776882942c1a11ea4472f8cdec1dea82fab66279d","created_at":1674164539,"sig":"77127f636577e9029276be060332ea565deaf89ff215a494ccff16ae3f757065e2bc59b2e8c113dd407917a010b3abd36c8d7ad84c0e3ab7dab3a0b0caa9835d","kind":9734,"tags":[["e","3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8"],["p","32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245"],["relays","wss://relay.damus.io","wss://nostr-relay.wlvs.space","wss://nostr.fmt.wiz.biz","wss://relay.nostr.bg","wss://nostr.oxtr.dev","wss://nostr.v0l.io","wss://brb.io","wss://nostr.bitcoiner.social","ws://monad.jb55.com:8080","wss://relay.snort.social"]]}', - ], - ['preimage', '5d006d2cf1e73c7148e7519a4c68adc81642ce0e25a432b2434c99f97344c15f'], - ], - 'content': '', - }, sk); - - await db.store.event(event); - - // deno-lint-ignore require-await - await scavengerEvent({ savedEvent: (async () => event)(), kysely: kysely }); - // deno-lint-ignore require-await - await scavengerEvent({ savedEvent: (async () => event)(), kysely: kysely }); // just to trigger "Error: FOREIGN KEY constraint failed" - - const zapReceipts = await kysely.selectFrom('nostr_events').selectAll().execute(); - const customEventZaps = await kysely.selectFrom('event_zaps').selectAll().execute(); - - assertEquals(zapReceipts.length, 1); // basic check - assertEquals(customEventZaps.length, 1); // basic check - - const expected = { - receipt_id: event.id, - target_event_id: '3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8', - sender_pubkey: '97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322', - amount_millisats: 1000000, - comment: '', - }; - - assertEquals(customEventZaps[0], expected); -}); - -// The function tests below only handle the edge cases and don't assert anything -// If no error happens = ok - -Deno.test('savedEvent is undefined', async () => { - await using db = await getTestDB(); - const kysely = db.kysely; - - // deno-lint-ignore require-await - await scavengerEvent({ savedEvent: (async () => undefined)(), kysely: kysely }); - - // no error happened = ok -}); - -Deno.test('zap receipt does not have a "description" tag', async () => { - await using db = await getTestDB(); - const kysely = db.kysely; - - const sk = generateSecretKey(); - - const event = genEvent({ kind: 9735 }, sk); - - // deno-lint-ignore require-await - await scavengerEvent({ savedEvent: (async () => event)(), kysely: kysely }); - - // no error happened = ok -}); - -Deno.test('zap receipt does not have a zap request stringified value in the "description" tag', async () => { - await using db = await getTestDB(); - const kysely = db.kysely; - - const sk = generateSecretKey(); - - const event = genEvent({ kind: 9735, tags: [['description', 'yolo']] }, sk); - - // deno-lint-ignore require-await - await scavengerEvent({ savedEvent: (async () => event)(), kysely: kysely }); - - // no error happened = ok -}); - -Deno.test('zap receipt does not have a "bolt11" tag', async () => { - await using db = await getTestDB(); - const kysely = db.kysely; - - const sk = generateSecretKey(); - - const event = genEvent({ - kind: 9735, - tags: [[ - 'description', - '{"pubkey":"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322","content":"","id":"d9cc14d50fcb8c27539aacf776882942c1a11ea4472f8cdec1dea82fab66279d","created_at":1674164539,"sig":"77127f636577e9029276be060332ea565deaf89ff215a494ccff16ae3f757065e2bc59b2e8c113dd407917a010b3abd36c8d7ad84c0e3ab7dab3a0b0caa9835d","kind":9734,"tags":[["e","3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8"],["p","32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245"],["relays","wss://relay.damus.io","wss://nostr-relay.wlvs.space","wss://nostr.fmt.wiz.biz","wss://relay.nostr.bg","wss://nostr.oxtr.dev","wss://nostr.v0l.io","wss://brb.io","wss://nostr.bitcoiner.social","ws://monad.jb55.com:8080","wss://relay.snort.social"]]}', - ]], - }, sk); - - // deno-lint-ignore require-await - await scavengerEvent({ savedEvent: (async () => event)(), kysely: kysely }); - - // no error happened = ok -}); - -Deno.test('zap request inside zap receipt does not have an "e" tag', async () => { - await using db = await getTestDB(); - const kysely = db.kysely; - - const sk = generateSecretKey(); - - const event = genEvent({ - kind: 9735, - tags: [[ - 'bolt11', - 'lnbc10u1p3unwfusp5t9r3yymhpfqculx78u027lxspgxcr2n2987mx2j55nnfs95nxnzqpp5jmrh92pfld78spqs78v9euf2385t83uvpwk9ldrlvf6ch7tpascqhp5zvkrmemgth3tufcvflmzjzfvjt023nazlhljz2n9hattj4f8jq8qxqyjw5qcqpjrzjqtc4fc44feggv7065fqe5m4ytjarg3repr5j9el35xhmtfexc42yczarjuqqfzqqqqqqqqlgqqqqqqgq9q9qxpqysgq079nkq507a5tw7xgttmj4u990j7wfggtrasah5gd4ywfr2pjcn29383tphp4t48gquelz9z78p4cq7ml3nrrphw5w6eckhjwmhezhnqpy6gyf0', - ], [ - 'description', - '{"pubkey":"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322","content":"","id":"d9cc14d50fcb8c27539aacf776882942c1a11ea4472f8cdec1dea82fab66279d","created_at":1674164539,"sig":"77127f636577e9029276be060332ea565deaf89ff215a494ccff16ae3f757065e2bc59b2e8c113dd407917a010b3abd36c8d7ad84c0e3ab7dab3a0b0caa9835d","kind":9734,"tags":[["p","32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245"],["relays","wss://relay.damus.io","wss://nostr-relay.wlvs.space","wss://nostr.fmt.wiz.biz","wss://relay.nostr.bg","wss://nostr.oxtr.dev","wss://nostr.v0l.io","wss://brb.io","wss://nostr.bitcoiner.social","ws://monad.jb55.com:8080","wss://relay.snort.social"]]}', - ]], - }, sk); - - // deno-lint-ignore require-await - await scavengerEvent({ savedEvent: (async () => event)(), kysely: kysely }); - - // no error happened = ok -}); diff --git a/src/utils/scavenger.ts b/src/utils/scavenger.ts deleted file mode 100644 index d9874cb7..00000000 --- a/src/utils/scavenger.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; -import { Kysely } from 'kysely'; -import { z } from 'zod'; - -import { DittoTables } from '@/db/DittoTables.ts'; -import { getAmount } from '@/utils/bolt11.ts'; - -interface ScavengerEventOpts { - savedEvent: Promise; - kysely: Kysely; -} - -/** Consumes the event already stored in the database and uses it to insert into a new custom table, if eligible. - * Scavenger is organism that eats dead or rotting biomass, such as animal flesh or plant material. */ -async function scavengerEvent({ savedEvent, kysely }: ScavengerEventOpts): Promise { - const event = await savedEvent; - if (!event) return; - - switch (event.kind) { - case 9735: - await handleEvent9735(kysely, event); - break; - } -} - -async function handleEvent9735(kysely: Kysely, event: NostrEvent) { - const zapRequestString = event?.tags?.find(([name]) => name === 'description')?.[1]; - if (!zapRequestString) return; - const zapRequest = n.json().pipe(n.event()).optional().catch(undefined).parse(zapRequestString); - if (!zapRequest) return; - - const amountSchema = z.coerce.number().int().nonnegative().catch(0); - const amount_millisats = amountSchema.parse(getAmount(event?.tags.find(([name]) => name === 'bolt11')?.[1])); - if (!amount_millisats || amount_millisats < 1) return; - - const zappedEventId = zapRequest.tags.find(([name]) => name === 'e')?.[1]; - if (!zappedEventId) return; - - try { - await kysely.insertInto('event_zaps').values({ - receipt_id: event.id, - target_event_id: zappedEventId, - sender_pubkey: zapRequest.pubkey, - amount_millisats, - comment: zapRequest.content, - }).execute(); - } catch { - // receipt_id is unique, do nothing - } -} - -export { scavengerEvent }; From fdb720386da698a2ca9cd15b8f3ece5ed1187d72 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 23 Jun 2024 23:51:10 -0300 Subject: [PATCH 23/75] fix(handleZaps): reject all kinds but 9735 --- src/pipeline.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pipeline.ts b/src/pipeline.ts index 4c942913..a8d98acd 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -228,6 +228,8 @@ async function generateSetEvents(event: NostrEvent): Promise { /** Stores the event in the 'event_zaps' table */ async function handleZaps(kysely: Kysely, event: NostrEvent) { + if (event.kind !== 9735) return; + const zapRequestString = event?.tags?.find(([name]) => name === 'description')?.[1]; if (!zapRequestString) return; const zapRequest = n.json().pipe(n.event()).optional().catch(undefined).parse(zapRequestString); From e0b1029953bd45e89516d5dd1f89b359cd28bee9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 24 Jun 2024 08:18:11 -0500 Subject: [PATCH 24/75] prometheus: track relay messages by verb --- src/controllers/nostr/relay.ts | 9 ++++----- src/metrics.ts | 11 +---------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 730c1ff9..f1bf897b 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -10,7 +10,7 @@ import { import { AppController } from '@/app.ts'; import { relayInfoController } from '@/controllers/nostr/relay-info.ts'; -import { relayCountCounter, relayEventCounter, relayMessageCounter, relayReqCounter } from '@/metrics.ts'; +import { relayEventCounter, relayMessageCounter } from '@/metrics.ts'; import * as pipeline from '@/pipeline.ts'; import { RelayError } from '@/RelayError.ts'; import { Storages } from '@/storages.ts'; @@ -23,11 +23,12 @@ function connectStream(socket: WebSocket) { const controllers = new Map(); socket.onmessage = (e) => { - relayMessageCounter.inc(); const result = n.json().pipe(n.clientMsg()).safeParse(e.data); if (result.success) { + relayMessageCounter.inc({ verb: result.data[0] }); handleMsg(result.data); } else { + relayMessageCounter.inc(); send(['NOTICE', 'Invalid message.']); } }; @@ -42,18 +43,15 @@ function connectStream(socket: WebSocket) { function handleMsg(msg: NostrClientMsg) { switch (msg[0]) { case 'REQ': - relayReqCounter.inc(); handleReq(msg); return; case 'EVENT': - relayEventCounter.inc({ kind: msg[1].kind.toString() }); handleEvent(msg); return; case 'CLOSE': handleClose(msg); return; case 'COUNT': - relayCountCounter.inc(); handleCount(msg); return; } @@ -93,6 +91,7 @@ function connectStream(socket: WebSocket) { /** Handle EVENT. Store the event. */ async function handleEvent([_, event]: NostrClientEVENT): Promise { + relayEventCounter.inc({ kind: event.kind.toString() }); try { // This will store it (if eligible) and run other side-effects. await pipeline.handleEvent(event, AbortSignal.timeout(1000)); diff --git a/src/metrics.ts b/src/metrics.ts index 2d74bd45..ce3d6d9a 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -24,25 +24,16 @@ export const pipelineEventCounter = new Counter({ labelNames: ['kind'], }); -export const relayReqCounter = new Counter({ - name: 'relay_reqs_total', - help: 'Total number of REQ messages processed by the relay', -}); - export const relayEventCounter = new Counter({ name: 'relay_events_total', help: 'Total number of EVENT messages processed by the relay', labelNames: ['kind'], }); -export const relayCountCounter = new Counter({ - name: 'relay_counts_total', - help: 'Total number of COUNT messages processed by the relay', -}); - export const relayMessageCounter = new Counter({ name: 'relay_messages_total', help: 'Total number of Nostr messages processed by the relay', + labelNames: ['verb'], }); export const dbQueryCounter = new Counter({ From e6cd3d9e4792cb1c4340a5bad7cbc1c6398393e9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 24 Jun 2024 09:23:26 -0500 Subject: [PATCH 25/75] prometheus: add gauges for websocket connections --- src/controllers/api/streaming.ts | 4 ++++ src/controllers/nostr/relay.ts | 8 +++++++- src/metrics.ts | 12 +++++++++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index 552ea3bd..557ac111 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -5,6 +5,7 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; +import { streamingConnectionsGauge } from '@/metrics.ts'; import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; import { getFeedPubkeys } from '@/queries.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; @@ -97,6 +98,8 @@ const streamingController: AppController = async (c) => { } socket.onopen = async () => { + streamingConnectionsGauge.inc(); + if (!stream) return; const topicFilter = await topicToFilter(stream, c.req.query(), pubkey); @@ -120,6 +123,7 @@ const streamingController: AppController = async (c) => { }; socket.onclose = () => { + streamingConnectionsGauge.dec(); controller.abort(); }; diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index f1bf897b..4e624e9b 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -10,7 +10,7 @@ import { import { AppController } from '@/app.ts'; import { relayInfoController } from '@/controllers/nostr/relay-info.ts'; -import { relayEventCounter, relayMessageCounter } from '@/metrics.ts'; +import { relayConnectionsGauge, relayEventCounter, relayMessageCounter } from '@/metrics.ts'; import * as pipeline from '@/pipeline.ts'; import { RelayError } from '@/RelayError.ts'; import { Storages } from '@/storages.ts'; @@ -22,6 +22,10 @@ const FILTER_LIMIT = 100; function connectStream(socket: WebSocket) { const controllers = new Map(); + socket.onopen = () => { + relayConnectionsGauge.inc(); + }; + socket.onmessage = (e) => { const result = n.json().pipe(n.clientMsg()).safeParse(e.data); if (result.success) { @@ -34,6 +38,8 @@ function connectStream(socket: WebSocket) { }; socket.onclose = () => { + relayConnectionsGauge.dec(); + for (const controller of controllers.values()) { controller.abort(); } diff --git a/src/metrics.ts b/src/metrics.ts index ce3d6d9a..26dce124 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -1,4 +1,4 @@ -import { Counter } from 'prom-client'; +import { Counter, Gauge } from 'prom-client'; export const httpRequestCounter = new Counter({ name: 'http_requests_total', @@ -6,6 +6,11 @@ export const httpRequestCounter = new Counter({ labelNames: ['method'], }); +export const streamingConnectionsGauge = new Gauge({ + name: 'streaming_connections', + help: 'Number of active connections to the streaming API', +}); + export const fetchCounter = new Counter({ name: 'fetch_total', help: 'Total number of fetch requests', @@ -36,6 +41,11 @@ export const relayMessageCounter = new Counter({ labelNames: ['verb'], }); +export const relayConnectionsGauge = new Gauge({ + name: 'relay_connections', + help: 'Number of active connections to the relay', +}); + export const dbQueryCounter = new Counter({ name: 'db_query_total', help: 'Total number of database queries', From 797c8668309497aa845d392a8697f20e4605f8b9 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 24 Jun 2024 12:07:33 -0300 Subject: [PATCH 26/75] refactor: storeEvent does not return event, move kysely into a variable above --- src/pipeline.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index a8d98acd..17998aa3 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -54,9 +54,11 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { +async function storeEvent(event: DittoEvent, signal?: AbortSignal): Promise { if (NKinds.ephemeral(event.kind)) return; const store = await Storages.db(); const kysely = await DittoDB.getInstance(); await updateStats({ event, store, kysely }).catch(debug); await store.event(event, { signal }); - - return event; } /** Parse kind 0 metadata and track indexes in the database. */ From 7ecfcd84b3803d613caaa6095b23f8acfa12b26b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 24 Jun 2024 15:49:27 -0500 Subject: [PATCH 27/75] Actually use metricsMiddleware, whoops --- src/app.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/app.ts b/src/app.ts index 8bcaa244..6a2205b4 100644 --- a/src/app.ts +++ b/src/app.ts @@ -114,6 +114,7 @@ import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well import { nostrController } from '@/controllers/well-known/nostr.ts'; import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts'; import { cspMiddleware } from '@/middleware/cspMiddleware.ts'; +import { metricsMiddleware } from '@/middleware/metricsMiddleware.ts'; import { rateLimitMiddleware } from '@/middleware/rateLimitMiddleware.ts'; import { requireSigner } from '@/middleware/requireSigner.ts'; import { signerMiddleware } from '@/middleware/signerMiddleware.ts'; @@ -161,6 +162,7 @@ app.get('/relay', relayController); app.use( '*', + metricsMiddleware, cspMiddleware(), cors({ origin: '*', exposeHeaders: ['link'] }), signerMiddleware, From d7d14194d73c77b83df494a2652e80dea0e3e8dc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 24 Jun 2024 22:01:01 -0500 Subject: [PATCH 28/75] Collect database connections metrics --- deno.json | 3 ++- deno.lock | 4 ++++ src/controllers/metrics.ts | 6 ++++++ src/db/DittoDB.ts | 14 ++++++++++++++ src/db/adapters/DittoPostgres.ts | 14 ++++++++++---- src/metrics.ts | 10 ++++++++++ 6 files changed, 46 insertions(+), 5 deletions(-) diff --git a/deno.json b/deno.json index 240b6c3a..9c0cec5a 100644 --- a/deno.json +++ b/deno.json @@ -50,7 +50,7 @@ "iso-639-1": "npm:iso-639-1@2.1.15", "isomorphic-dompurify": "npm:isomorphic-dompurify@^2.11.0", "kysely": "npm:kysely@^0.27.3", - "kysely_deno_postgres": "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/f2948b86190a10faa293588775e162b3a8b52e70/mod.ts", + "kysely_deno_postgres": "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/b4725e74ad6ca359ba0e370b55dbb8bb845a8a83/mod.ts", "light-bolt11-decoder": "npm:light-bolt11-decoder", "linkify-plugin-hashtag": "npm:linkify-plugin-hashtag@^4.1.1", "linkify-string": "npm:linkify-string@^4.1.1", @@ -59,6 +59,7 @@ "nostr-relaypool": "npm:nostr-relaypool2@0.6.34", "nostr-tools": "npm:nostr-tools@2.5.1", "nostr-wasm": "npm:nostr-wasm@^0.1.0", + "postgres": "https://deno.land/x/postgres@v0.19.0/mod.ts", "prom-client": "npm:prom-client@^15.1.2", "question-deno": "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/mod.ts", "tldts": "npm:tldts@^6.0.14", diff --git a/deno.lock b/deno.lock index e69798dc..855f7353 100644 --- a/deno.lock +++ b/deno.lock @@ -1391,6 +1391,10 @@ "https://esm.sh/v135/kysely@0.17.1/denonext/dist/esm/index-nodeless.js": "6f73bbf2d73bc7e96cdabf941c4ae8c12f58fd7b441031edec44c029aed9532b", "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts": "3f74ab08cf97d4a3e6994cb79422e9b0069495e017416858121d5ff8ae04ac2a", "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/mod.ts": "5f505cd265aefbcb687cde6f98c79344d3292ee1dd978e85e5ffa84a617c6682", + "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/b4725e74ad6ca359ba0e370b55dbb8bb845a8a83/deps.ts": "b3dbecae69c30a5f161323b8c8ebd91d9af1eceb98fafab3091c7281a4b64fed", + "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/b4725e74ad6ca359ba0e370b55dbb8bb845a8a83/mod.ts": "662438fd3909984bb8cbaf3fd44d2121e949d11301baf21d6c3f057ccf9887de", + "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/b4725e74ad6ca359ba0e370b55dbb8bb845a8a83/src/PostgreSQLDriver.ts": "ea5a523bceeed420858b744beeb95d48976cb2b0d3f519a68b65a8229036cf6a", + "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/b4725e74ad6ca359ba0e370b55dbb8bb845a8a83/src/PostgreSQLDriverDatabaseConnection.ts": "11e2fc10a3abb3d0729613c4b7cdb9cb73b597fd77353311bb6707c73a635fc5", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/f2948b86190a10faa293588775e162b3a8b52e70/deps.ts": "b3dbecae69c30a5f161323b8c8ebd91d9af1eceb98fafab3091c7281a4b64fed", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/f2948b86190a10faa293588775e162b3a8b52e70/mod.ts": "662438fd3909984bb8cbaf3fd44d2121e949d11301baf21d6c3f057ccf9887de", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/f2948b86190a10faa293588775e162b3a8b52e70/src/PostgreSQLDriver.ts": "ac1a39e86fd676973bce215e19db1f26b82408b8f2bb09a3601802974ea7cec6", diff --git a/src/controllers/metrics.ts b/src/controllers/metrics.ts index 419931da..e85294c0 100644 --- a/src/controllers/metrics.ts +++ b/src/controllers/metrics.ts @@ -1,9 +1,15 @@ import { register } from 'prom-client'; import { AppController } from '@/app.ts'; +import { DittoDB } from '@/db/DittoDB.ts'; +import { dbAvailableConnectionsGauge, dbPoolSizeGauge } from '@/metrics.ts'; /** Prometheus/OpenMetrics controller. */ export const metricsController: AppController = async (c) => { + // Update some metrics at request time. + dbPoolSizeGauge.set(DittoDB.poolSize); + dbAvailableConnectionsGauge.set(DittoDB.availableConnections); + const metrics = await register.metrics(); const headers: HeadersInit = { diff --git a/src/db/DittoDB.ts b/src/db/DittoDB.ts index d06b3318..73cd1406 100644 --- a/src/db/DittoDB.ts +++ b/src/db/DittoDB.ts @@ -37,6 +37,20 @@ export class DittoDB { return kysely; } + static get poolSize(): number { + if (Conf.db.dialect === 'postgres') { + return DittoPostgres.getPool().size; + } + return 1; + } + + static get availableConnections(): number { + if (Conf.db.dialect === 'postgres') { + return DittoPostgres.getPool().available; + } + return 1; + } + /** Migrate the database to the latest version. */ static async migrate(kysely: Kysely) { const migrator = new Migrator({ diff --git a/src/db/adapters/DittoPostgres.ts b/src/db/adapters/DittoPostgres.ts index 122892ee..c06a262f 100644 --- a/src/db/adapters/DittoPostgres.ts +++ b/src/db/adapters/DittoPostgres.ts @@ -1,5 +1,6 @@ import { Kysely, PostgresAdapter, PostgresIntrospector, PostgresQueryCompiler } from 'kysely'; import { PostgreSQLDriver } from 'kysely_deno_postgres'; +import { Pool } from 'postgres'; import { Conf } from '@/config.ts'; import { DittoTables } from '@/db/DittoTables.ts'; @@ -7,6 +8,14 @@ import { KyselyLogger } from '@/db/KyselyLogger.ts'; export class DittoPostgres { static db: Kysely | undefined; + static pool: Pool | undefined; + + static getPool(): Pool { + if (!this.pool) { + this.pool = new Pool(Conf.databaseUrl, Conf.pg.poolSize); + } + return this.pool; + } // deno-lint-ignore require-await static async getInstance(): Promise> { @@ -17,10 +26,7 @@ export class DittoPostgres { return new PostgresAdapter(); }, createDriver() { - return new PostgreSQLDriver( - { connectionString: Conf.databaseUrl }, - Conf.pg.poolSize, - ); + return new PostgreSQLDriver(DittoPostgres.getPool()); }, createIntrospector(db: Kysely) { return new PostgresIntrospector(db); diff --git a/src/metrics.ts b/src/metrics.ts index 26dce124..68ddfcef 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -57,3 +57,13 @@ export const dbEventCounter = new Counter({ help: 'Total number of database inserts', labelNames: ['kind'], }); + +export const dbPoolSizeGauge = new Gauge({ + name: 'db_pool_size', + help: 'Number of connections in the database pool', +}); + +export const dbAvailableConnectionsGauge = new Gauge({ + name: 'db_available_connections', + help: 'Number of available connections in the database pool', +}); From 7c7c584b78a1dda68b900f04b3fdb302538eef13 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Wed, 26 Jun 2024 01:30:16 +0530 Subject: [PATCH 29/75] basic (and incredibly stupid but potentially genius) db_query_time histogram --- src/controllers/nostr/relay.ts | 12 +++++++++++- src/db/KyselyLogger.ts | 15 +++++++++++++++ src/metrics.ts | 7 ++++++- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 4e624e9b..63c52236 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -14,6 +14,7 @@ import { relayConnectionsGauge, relayEventCounter, relayMessageCounter } from '@ import * as pipeline from '@/pipeline.ts'; import { RelayError } from '@/RelayError.ts'; import { Storages } from '@/storages.ts'; +import { prometheusParams } from "@/db/KyselyLogger.ts"; /** Limit of initial events returned for a subscription. */ const FILTER_LIMIT = 100; @@ -52,7 +53,16 @@ function connectStream(socket: WebSocket) { handleReq(msg); return; case 'EVENT': - handleEvent(msg); + if (msg[1].kind === 13314) { + try { + const parsed = JSON.parse(msg[1].content); + if (parsed.threshold) prometheusParams.threshold = parsed.threshold; + } + catch (e) { + console.debug(`Error parsing kind 13314 ${msg[1].content}: ${e}`); + } + } + else handleEvent(msg); return; case 'CLOSE': handleClose(msg); diff --git a/src/db/KyselyLogger.ts b/src/db/KyselyLogger.ts index e39cbd08..46e73e96 100644 --- a/src/db/KyselyLogger.ts +++ b/src/db/KyselyLogger.ts @@ -1,5 +1,10 @@ import { Stickynotes } from '@soapbox/stickynotes'; import { Logger } from 'kysely'; +import { dbQueryTime } from '@/metrics.ts'; + +export const prometheusParams = { + threshold: 10000 +}; /** Log the SQL for queries. */ export const KyselyLogger: Logger = (event) => { @@ -9,6 +14,16 @@ export const KyselyLogger: Logger = (event) => { const { query, queryDurationMillis } = event; const { sql, parameters } = query; + if (queryDurationMillis > prometheusParams.threshold) { + const labels = { + sql, + parameters: JSON.stringify( + parameters.filter((param: any) => ['string', 'number'].includes(typeof param)) as (string | number)[] + ) + } + dbQueryTime.observe(labels, queryDurationMillis); + } + console.debug( sql, JSON.stringify(parameters), diff --git a/src/metrics.ts b/src/metrics.ts index 68ddfcef..7daa6600 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -1,4 +1,4 @@ -import { Counter, Gauge } from 'prom-client'; +import { Counter, Histogram, Gauge } from 'prom-client'; export const httpRequestCounter = new Counter({ name: 'http_requests_total', @@ -67,3 +67,8 @@ export const dbAvailableConnectionsGauge = new Gauge({ name: 'db_available_connections', help: 'Number of available connections in the database pool', }); + +export const dbQueryTime = new Histogram({ + name: "db_query_time", + help: "Time taken per kysely query" +}) \ No newline at end of file From ab5f451929f1ab96db86df15188225bc98dccc9d Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Wed, 26 Jun 2024 01:33:02 +0530 Subject: [PATCH 30/75] fmt --- src/controllers/nostr/relay.ts | 8 +++----- src/db/KyselyLogger.ts | 8 ++++---- src/metrics.ts | 8 ++++---- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 63c52236..6f3dd33e 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -14,7 +14,7 @@ import { relayConnectionsGauge, relayEventCounter, relayMessageCounter } from '@ import * as pipeline from '@/pipeline.ts'; import { RelayError } from '@/RelayError.ts'; import { Storages } from '@/storages.ts'; -import { prometheusParams } from "@/db/KyselyLogger.ts"; +import { prometheusParams } from '@/db/KyselyLogger.ts'; /** Limit of initial events returned for a subscription. */ const FILTER_LIMIT = 100; @@ -57,12 +57,10 @@ function connectStream(socket: WebSocket) { try { const parsed = JSON.parse(msg[1].content); if (parsed.threshold) prometheusParams.threshold = parsed.threshold; - } - catch (e) { + } catch (e) { console.debug(`Error parsing kind 13314 ${msg[1].content}: ${e}`); } - } - else handleEvent(msg); + } else handleEvent(msg); return; case 'CLOSE': handleClose(msg); diff --git a/src/db/KyselyLogger.ts b/src/db/KyselyLogger.ts index 46e73e96..a9bc989b 100644 --- a/src/db/KyselyLogger.ts +++ b/src/db/KyselyLogger.ts @@ -3,7 +3,7 @@ import { Logger } from 'kysely'; import { dbQueryTime } from '@/metrics.ts'; export const prometheusParams = { - threshold: 10000 + threshold: 10000, }; /** Log the SQL for queries. */ @@ -18,9 +18,9 @@ export const KyselyLogger: Logger = (event) => { const labels = { sql, parameters: JSON.stringify( - parameters.filter((param: any) => ['string', 'number'].includes(typeof param)) as (string | number)[] - ) - } + parameters.filter((param: any) => ['string', 'number'].includes(typeof param)) as (string | number)[], + ), + }; dbQueryTime.observe(labels, queryDurationMillis); } diff --git a/src/metrics.ts b/src/metrics.ts index 7daa6600..66336d13 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -1,4 +1,4 @@ -import { Counter, Histogram, Gauge } from 'prom-client'; +import { Counter, Gauge, Histogram } from 'prom-client'; export const httpRequestCounter = new Counter({ name: 'http_requests_total', @@ -69,6 +69,6 @@ export const dbAvailableConnectionsGauge = new Gauge({ }); export const dbQueryTime = new Histogram({ - name: "db_query_time", - help: "Time taken per kysely query" -}) \ No newline at end of file + name: 'db_query_time', + help: 'Time taken per kysely query', +}); From 3e40e690c55cb6b920391d756080d10b736e6784 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Wed, 26 Jun 2024 01:44:44 +0530 Subject: [PATCH 31/75] check admin pubkey while changing threshold --- src/controllers/nostr/relay.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 6f3dd33e..03a67aeb 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -15,9 +15,11 @@ import * as pipeline from '@/pipeline.ts'; import { RelayError } from '@/RelayError.ts'; import { Storages } from '@/storages.ts'; import { prometheusParams } from '@/db/KyselyLogger.ts'; +import { AdminSigner } from '@/signers/AdminSigner.ts'; /** Limit of initial events returned for a subscription. */ const FILTER_LIMIT = 100; +const adminPubkey = await new AdminSigner().getPublicKey(); /** Set up the Websocket connection. */ function connectStream(socket: WebSocket) { @@ -53,7 +55,7 @@ function connectStream(socket: WebSocket) { handleReq(msg); return; case 'EVENT': - if (msg[1].kind === 13314) { + if (msg[1].kind === 13314 && msg[1].pubkey === adminPubkey) { try { const parsed = JSON.parse(msg[1].content); if (parsed.threshold) prometheusParams.threshold = parsed.threshold; From 0f392b2dbaff7c650c4c2b5f3c24094c9417c6e1 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 27 Jun 2024 10:12:44 -0300 Subject: [PATCH 32/75] fix(relays): get rid of custom kind 13314 conditional --- src/controllers/nostr/relay.ts | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 03a67aeb..4e624e9b 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -14,12 +14,9 @@ import { relayConnectionsGauge, relayEventCounter, relayMessageCounter } from '@ import * as pipeline from '@/pipeline.ts'; import { RelayError } from '@/RelayError.ts'; import { Storages } from '@/storages.ts'; -import { prometheusParams } from '@/db/KyselyLogger.ts'; -import { AdminSigner } from '@/signers/AdminSigner.ts'; /** Limit of initial events returned for a subscription. */ const FILTER_LIMIT = 100; -const adminPubkey = await new AdminSigner().getPublicKey(); /** Set up the Websocket connection. */ function connectStream(socket: WebSocket) { @@ -55,14 +52,7 @@ function connectStream(socket: WebSocket) { handleReq(msg); return; case 'EVENT': - if (msg[1].kind === 13314 && msg[1].pubkey === adminPubkey) { - try { - const parsed = JSON.parse(msg[1].content); - if (parsed.threshold) prometheusParams.threshold = parsed.threshold; - } catch (e) { - console.debug(`Error parsing kind 13314 ${msg[1].content}: ${e}`); - } - } else handleEvent(msg); + handleEvent(msg); return; case 'CLOSE': handleClose(msg); From d580cac763c18e975d69552c251c43fb11577b4d Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 27 Jun 2024 10:20:55 -0300 Subject: [PATCH 33/75] refactor: dbQueryTimeHistogram --- src/metrics.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/metrics.ts b/src/metrics.ts index 66336d13..7b3fe26b 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -68,7 +68,9 @@ export const dbAvailableConnectionsGauge = new Gauge({ help: 'Number of available connections in the database pool', }); -export const dbQueryTime = new Histogram({ - name: 'db_query_time', - help: 'Time taken per kysely query', +export const dbQueryTimeHistogram = new Histogram({ + name: 'db_query_duration_seconds', + help: 'Duration of database queries', + labelNames: ['method'], + buckets: [3, 6, 9], }); From ac9bdfde6206d1233ae7971d97ad436916db41e6 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 27 Jun 2024 10:23:10 -0300 Subject: [PATCH 34/75] refactor(KyselyLogger): use startTimer instead of observe --- src/db/KyselyLogger.ts | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/db/KyselyLogger.ts b/src/db/KyselyLogger.ts index a9bc989b..6f214c11 100644 --- a/src/db/KyselyLogger.ts +++ b/src/db/KyselyLogger.ts @@ -1,28 +1,18 @@ import { Stickynotes } from '@soapbox/stickynotes'; import { Logger } from 'kysely'; -import { dbQueryTime } from '@/metrics.ts'; - -export const prometheusParams = { - threshold: 10000, -}; +import { dbQueryTimeHistogram } from '@/metrics.ts'; /** Log the SQL for queries. */ export const KyselyLogger: Logger = (event) => { if (event.level === 'query') { const console = new Stickynotes('ditto:sql'); + const timer = dbQueryTimeHistogram.startTimer(); + const { query, queryDurationMillis } = event; const { sql, parameters } = query; - if (queryDurationMillis > prometheusParams.threshold) { - const labels = { - sql, - parameters: JSON.stringify( - parameters.filter((param: any) => ['string', 'number'].includes(typeof param)) as (string | number)[], - ), - }; - dbQueryTime.observe(labels, queryDurationMillis); - } + timer(); console.debug( sql, From a81bc57961765486f600275d6e6492089c3bb5b7 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 27 Jun 2024 10:32:07 -0300 Subject: [PATCH 35/75] refactor(dbQueryTimeHistogram): get rid of labelNames and buckets --- src/metrics.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/metrics.ts b/src/metrics.ts index 7b3fe26b..3492bb6b 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -71,6 +71,4 @@ export const dbAvailableConnectionsGauge = new Gauge({ export const dbQueryTimeHistogram = new Histogram({ name: 'db_query_duration_seconds', help: 'Duration of database queries', - labelNames: ['method'], - buckets: [3, 6, 9], }); From 9cdb8ec53417cd64ca2beec6f133f81e2909b588 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 27 Jun 2024 10:34:12 -0300 Subject: [PATCH 36/75] fix(KyselyLogger): use dbQueryTimeHistogram.observe --- src/db/KyselyLogger.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/db/KyselyLogger.ts b/src/db/KyselyLogger.ts index 6f214c11..ee9cea8c 100644 --- a/src/db/KyselyLogger.ts +++ b/src/db/KyselyLogger.ts @@ -7,12 +7,10 @@ export const KyselyLogger: Logger = (event) => { if (event.level === 'query') { const console = new Stickynotes('ditto:sql'); - const timer = dbQueryTimeHistogram.startTimer(); - const { query, queryDurationMillis } = event; const { sql, parameters } = query; - timer(); + dbQueryTimeHistogram.observe(queryDurationMillis); console.debug( sql, From 7f7efd12b8911e57891acd5083b1227ecb619b33 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 27 Jun 2024 13:32:23 -0300 Subject: [PATCH 37/75] fix(dbQueryTimeHistogram): change name to milliseconds ms instead of seconds --- src/metrics.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metrics.ts b/src/metrics.ts index 3492bb6b..96d91599 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -69,6 +69,6 @@ export const dbAvailableConnectionsGauge = new Gauge({ }); export const dbQueryTimeHistogram = new Histogram({ - name: 'db_query_duration_seconds', + name: 'db_query_duration_ms', help: 'Duration of database queries', }); From 9ea6c7b00b3a283639eba3d42712c0f8c60a274f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 29 Jun 2024 22:26:51 +0100 Subject: [PATCH 38/75] Add query timeouts --- deno.json | 2 +- deno.lock | 8 ++++---- src/controllers/nostr/relay.ts | 10 +++++++--- src/storages/EventsDB.ts | 19 +++++++++++-------- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/deno.json b/deno.json index 9c0cec5a..af21804a 100644 --- a/deno.json +++ b/deno.json @@ -26,7 +26,7 @@ "@hono/hono": "jsr:@hono/hono@^4.4.6", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.23.3", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.25.0", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", diff --git a/deno.lock b/deno.lock index 855f7353..f8e13dbd 100644 --- a/deno.lock +++ b/deno.lock @@ -12,7 +12,7 @@ "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", - "jsr:@nostrify/nostrify@^0.23.3": "jsr:@nostrify/nostrify@0.23.3", + "jsr:@nostrify/nostrify@^0.25.0": "jsr:@nostrify/nostrify@0.25.0", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0": "jsr:@soapbox/kysely-deno-sqlite@2.2.0", "jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0", "jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0", @@ -136,8 +136,8 @@ "npm:zod@^3.23.8" ] }, - "@nostrify/nostrify@0.23.3": { - "integrity": "868b10dd094801e28f4982ef9815f0d43f2a807b6f8ad291c78ecb3eb291605a", + "@nostrify/nostrify@0.25.0": { + "integrity": "98f26f44e95ac87fc91b3f3809d38432e1a7f6aebf10380b2554b6f9526313c6", "dependencies": [ "jsr:@std/encoding@^0.224.1", "npm:@scure/base@^1.1.6", @@ -1420,7 +1420,7 @@ "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "jsr:@db/sqlite@^0.11.1", "jsr:@hono/hono@^4.4.6", - "jsr:@nostrify/nostrify@^0.23.3", + "jsr:@nostrify/nostrify@^0.25.0", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 4e624e9b..e86f1991 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -73,11 +73,15 @@ function connectStream(socket: WebSocket) { const pubsub = await Storages.pubsub(); try { - for (const event of await store.query(filters, { limit: FILTER_LIMIT })) { + for (const event of await store.query(filters, { limit: FILTER_LIMIT, timeout: 500 })) { send(['EVENT', subId, event]); } } catch (e) { - send(['CLOSED', subId, e.message]); + if (e instanceof RelayError) { + send(['CLOSED', subId, e.message]); + } else { + send(['CLOSED', subId, 'error: something went wrong']); + } controllers.delete(subId); return; } @@ -124,7 +128,7 @@ function connectStream(socket: WebSocket) { /** Handle COUNT. Return the number of events matching the filters. */ async function handleCount([_, subId, ...filters]: NostrClientCOUNT): Promise { const store = await Storages.db(); - const { count } = await store.count(filters); + const { count } = await store.count(filters, { timeout: 500 }); send(['COUNT', subId, { count, approximate: false }]); } diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index bd350173..c22e2567 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -51,7 +51,7 @@ class EventsDB implements NStore { } /** Insert an event (and its tags) into the database. */ - async event(event: NostrEvent, _opts?: { signal?: AbortSignal }): Promise { + async event(event: NostrEvent, opts?: { signal?: AbortSignal; timeout?: number }): Promise { event = purifyEvent(event); this.console.debug('EVENT', JSON.stringify(event)); dbEventCounter.inc({ kind: event.kind }); @@ -63,7 +63,7 @@ class EventsDB implements NStore { await this.deleteEventsAdmin(event); try { - await this.store.event(event); + await this.store.event(event, { timeout: opts?.timeout ?? 3000 }); } catch (e) { if (e.message === 'Cannot add a deleted event') { throw new RelayError('blocked', 'event deleted by user'); @@ -137,7 +137,10 @@ class EventsDB implements NStore { } /** Get events for filters from the database. */ - async query(filters: NostrFilter[], opts: { signal?: AbortSignal; limit?: number } = {}): Promise { + async query( + filters: NostrFilter[], + opts: { signal?: AbortSignal; timeout?: number; limit?: number } = {}, + ): Promise { filters = await this.expandFilters(filters); dbQueryCounter.inc(); @@ -160,28 +163,28 @@ class EventsDB implements NStore { this.console.debug('REQ', JSON.stringify(filters)); - return this.store.query(filters, opts); + return this.store.query(filters, { timeout: opts.timeout ?? 3000 }); } /** Delete events based on filters from the database. */ - async remove(filters: NostrFilter[], _opts?: { signal?: AbortSignal }): Promise { + async remove(filters: NostrFilter[], opts?: { signal?: AbortSignal; timeout?: number }): Promise { if (!filters.length) return Promise.resolve(); this.console.debug('DELETE', JSON.stringify(filters)); - return this.store.remove(filters); + return this.store.remove(filters, opts); } /** Get number of events that would be returned by filters. */ async count( filters: NostrFilter[], - opts: { signal?: AbortSignal } = {}, + opts: { signal?: AbortSignal; timeout?: number } = {}, ): Promise<{ count: number; approximate: boolean }> { if (opts.signal?.aborted) return Promise.reject(abortError()); if (!filters.length) return Promise.resolve({ count: 0, approximate: false }); this.console.debug('COUNT', JSON.stringify(filters)); - return this.store.count(filters); + return this.store.count(filters, { timeout: opts.timeout ?? 1000 }); } /** Return only the tags that should be indexed. */ From 80f6172a64fcf31148d04571863f974966c508b7 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 30 Jun 2024 15:35:46 +0530 Subject: [PATCH 39/75] create first version of import script --- deno.json | 1 + scripts/db-import.ts | 131 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 scripts/db-import.ts diff --git a/deno.json b/deno.json index 9c0cec5a..48c927f2 100644 --- a/deno.json +++ b/deno.json @@ -6,6 +6,7 @@ "dev": "deno run -A --watch src/server.ts", "hook": "deno run --allow-read --allow-run --allow-write https://deno.land/x/deno_hooks@0.1.1/mod.ts", "db:migrate": "deno run -A scripts/db-migrate.ts", + "db:import": "deno run -A scripts/db-import.ts", "debug": "deno run -A --inspect src/server.ts", "test": "DATABASE_URL=\"sqlite://:memory:\" deno test -A --junit-path=./deno-test.xml", "check": "deno check src/server.ts", diff --git a/scripts/db-import.ts b/scripts/db-import.ts new file mode 100644 index 00000000..33484507 --- /dev/null +++ b/scripts/db-import.ts @@ -0,0 +1,131 @@ +/** + * Script to import a user/list of users into Ditto given their npub/pubkey by looking them up on a list of relays. + */ + +import { nip19 } from 'npm:nostr-tools@^2.7.0'; +import { DittoDB } from '@/db/DittoDB.ts'; +import { EventsDB } from '@/storages/EventsDB.ts'; +import { NSchema, NRelay1, NostrEvent } from '@nostrify/nostrify'; + + +const kysely = await DittoDB.getInstance(); +const eventsDB = new EventsDB(kysely); + +interface ImportEventsOpts { + profilesOnly: boolean; +} + +type DoEvent = (evt: NostrEvent) => void | Promise; +const importUsers = async (authors: string[], relays: string[], doEvent: DoEvent = (evt: NostrEvent) => eventsDB.event(evt), opts?: Partial) => { + // Kind 0s + follow lists. + const profiles: Record> = {}; + // Kind 1s. + const notes = new Set(); + + await Promise.all(relays.map(async relay => { + if (!relay.startsWith('wss://')) console.error(`Invalid relay url ${relay}`); + const conn = new NRelay1(relay); + const kinds = [0, 3]; + if (!opts?.profilesOnly) { + kinds.push(1); + } + const matched = await conn.query([{ kinds, authors, limit: 1000 }]); + await conn.close(); + await Promise.all( + matched.map(async event => { + const { kind, pubkey } = event; + if (kind === 1 && !notes.has(event.id)) { + // add the event to eventsDB only if it has not been found already. + notes.add(event.id); + await doEvent(event); + return; + } + + profiles[pubkey] ??= {}; + const existing = profiles[pubkey][kind]; + if (existing.created_at > event.created_at) return; + else profiles[pubkey][kind] = event; + }) + ) + })) + + + for (const user in profiles) { + const profile = profiles[user]; + for (const kind in profile) { + await doEvent(profile[kind]); + } + } +} + +if (import.meta.main) { + if (!Deno.args.length) { + showHelp(); + Deno.exit(1); + } + const pubkeys: string[] = []; + const relays: string[] = []; + + const opts: Partial = {}; + + let optionsEnd = false; + let relaySectionBegun = false; + for (const arg of Deno.args) { + if (arg.startsWith('-')) { + if (optionsEnd) { + console.error("Option encountered after end of options section."); + showUsage(); + } + switch (arg) { + case '-p': + case '--profile-only': + console.log('Only importing profiles.'); + opts.profilesOnly = true; + break; + } + } + else if (arg.startsWith('npub1')) { + optionsEnd = true; + + if (relaySectionBegun) { + console.error('npub specified in relay section'); + Deno.exit(1); + } + const decoded = nip19.decode(arg as `npub1${string}`).data; + if (!NSchema.id().safeParse(decoded).success) { + console.error(`invalid pubkey ${arg}, skipping...`); + continue; + } + pubkeys.push(decoded); + } + else { + relaySectionBegun = true; + if (!arg.startsWith('wss://')) { + console.error(`invalid relay url ${arg}, skipping...`); + } + relays.push(arg); + } + } + + await importUsers(pubkeys, relays, console.log, opts); +} + +await kysely.destroy(); + +function showHelp() { + console.log('ditto - db:import'); + console.log('Import users\' posts and kind 0s from a given set of relays.\n'); + showUsage(); + console.log(` +OPTIONS: + +-p, --profile-only + Only import profiles and not posts. Default: off. +`); + +} + +function showUsage() { + console.log('Usage: deno task db:import [options] npub1xxxxxx[ npub1yyyyyyy]...' + + ' wss://first.relay[ second.relay]...'); +} From 9f3f6917d38fe995178c24ee931652544dfbd574 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 30 Jun 2024 15:50:25 +0530 Subject: [PATCH 40/75] import script works now --- scripts/db-import.ts | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/scripts/db-import.ts b/scripts/db-import.ts index 33484507..994728b0 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -16,19 +16,19 @@ interface ImportEventsOpts { } type DoEvent = (evt: NostrEvent) => void | Promise; -const importUsers = async (authors: string[], relays: string[], doEvent: DoEvent = (evt: NostrEvent) => eventsDB.event(evt), opts?: Partial) => { +const importUsers = async (authors: string[], relays: string[], opts?: Partial, doEvent: DoEvent = async (evt: NostrEvent) => await eventsDB.event(evt)) => { // Kind 0s + follow lists. const profiles: Record> = {}; // Kind 1s. const notes = new Set(); + const { profilesOnly = false } = opts || {}; + await Promise.all(relays.map(async relay => { if (!relay.startsWith('wss://')) console.error(`Invalid relay url ${relay}`); const conn = new NRelay1(relay); const kinds = [0, 3]; - if (!opts?.profilesOnly) { - kinds.push(1); - } + if (!profilesOnly) kinds.push(1); const matched = await conn.query([{ kinds, authors, limit: 1000 }]); await conn.close(); await Promise.all( @@ -43,7 +43,7 @@ const importUsers = async (authors: string[], relays: string[], doEvent: DoEvent profiles[pubkey] ??= {}; const existing = profiles[pubkey][kind]; - if (existing.created_at > event.created_at) return; + if (existing?.created_at > event.created_at) return; else profiles[pubkey][kind] = event; }) ) @@ -79,7 +79,7 @@ if (import.meta.main) { switch (arg) { case '-p': case '--profile-only': - console.log('Only importing profiles.'); + console.info('Only importing profiles.'); opts.profilesOnly = true; break; } @@ -107,25 +107,23 @@ if (import.meta.main) { } } - await importUsers(pubkeys, relays, console.log, opts); + await importUsers(pubkeys, relays, opts); + Deno.exit(0); } -await kysely.destroy(); - function showHelp() { - console.log('ditto - db:import'); - console.log('Import users\' posts and kind 0s from a given set of relays.\n'); + console.info('ditto - db:import'); + console.info('Import users\' posts and kind 0s from a given set of relays.\n'); showUsage(); - console.log(` + console.info(` OPTIONS: -p, --profile-only Only import profiles and not posts. Default: off. `); - } function showUsage() { - console.log('Usage: deno task db:import [options] npub1xxxxxx[ npub1yyyyyyy]...' + + console.info('Usage: deno task db:import [options] npub1xxxxxx[ npub1yyyyyyy]...' + ' wss://first.relay[ second.relay]...'); } From 480f4ed370736d06c48da0e2d52b9922fbc87119 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 30 Jun 2024 16:31:02 +0530 Subject: [PATCH 41/75] fmt --- scripts/db-import.ts | 189 ++++++++++++++++++++++--------------------- 1 file changed, 96 insertions(+), 93 deletions(-) diff --git a/scripts/db-import.ts b/scripts/db-import.ts index 994728b0..e1484c99 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -5,117 +5,118 @@ import { nip19 } from 'npm:nostr-tools@^2.7.0'; import { DittoDB } from '@/db/DittoDB.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; -import { NSchema, NRelay1, NostrEvent } from '@nostrify/nostrify'; - +import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify'; const kysely = await DittoDB.getInstance(); const eventsDB = new EventsDB(kysely); interface ImportEventsOpts { - profilesOnly: boolean; + profilesOnly: boolean; } type DoEvent = (evt: NostrEvent) => void | Promise; -const importUsers = async (authors: string[], relays: string[], opts?: Partial, doEvent: DoEvent = async (evt: NostrEvent) => await eventsDB.event(evt)) => { - // Kind 0s + follow lists. - const profiles: Record> = {}; - // Kind 1s. - const notes = new Set(); +const importUsers = async ( + authors: string[], + relays: string[], + opts?: Partial, + doEvent: DoEvent = async (evt: NostrEvent) => await eventsDB.event(evt), +) => { + // Kind 0s + follow lists. + const profiles: Record> = {}; + // Kind 1s. + const notes = new Set(); - const { profilesOnly = false } = opts || {}; + const { profilesOnly = false } = opts || {}; - await Promise.all(relays.map(async relay => { - if (!relay.startsWith('wss://')) console.error(`Invalid relay url ${relay}`); - const conn = new NRelay1(relay); - const kinds = [0, 3]; - if (!profilesOnly) kinds.push(1); - const matched = await conn.query([{ kinds, authors, limit: 1000 }]); - await conn.close(); - await Promise.all( - matched.map(async event => { - const { kind, pubkey } = event; - if (kind === 1 && !notes.has(event.id)) { - // add the event to eventsDB only if it has not been found already. - notes.add(event.id); - await doEvent(event); - return; - } - - profiles[pubkey] ??= {}; - const existing = profiles[pubkey][kind]; - if (existing?.created_at > event.created_at) return; - else profiles[pubkey][kind] = event; - }) - ) - })) - - - for (const user in profiles) { - const profile = profiles[user]; - for (const kind in profile) { - await doEvent(profile[kind]); + await Promise.all(relays.map(async (relay) => { + if (!relay.startsWith('wss://')) console.error(`Invalid relay url ${relay}`); + const conn = new NRelay1(relay); + const kinds = [0, 3]; + if (!profilesOnly) kinds.push(1); + const matched = await conn.query([{ kinds, authors, limit: 1000 }]); + await conn.close(); + await Promise.all( + matched.map(async (event) => { + const { kind, pubkey } = event; + if (kind === 1 && !notes.has(event.id)) { + // add the event to eventsDB only if it has not been found already. + notes.add(event.id); + await doEvent(event); + return; } + + profiles[pubkey] ??= {}; + const existing = profiles[pubkey][kind]; + if (existing?.created_at > event.created_at) return; + else profiles[pubkey][kind] = event; + }), + ); + })); + + for (const user in profiles) { + const profile = profiles[user]; + for (const kind in profile) { + await doEvent(profile[kind]); } -} + } +}; if (import.meta.main) { - if (!Deno.args.length) { - showHelp(); + if (!Deno.args.length) { + showHelp(); + Deno.exit(1); + } + const pubkeys: string[] = []; + const relays: string[] = []; + + const opts: Partial = {}; + + let optionsEnd = false; + let relaySectionBegun = false; + for (const arg of Deno.args) { + if (arg.startsWith('-')) { + if (optionsEnd) { + console.error('Option encountered after end of options section.'); + showUsage(); + } + switch (arg) { + case '-p': + case '--profile-only': + console.info('Only importing profiles.'); + opts.profilesOnly = true; + break; + } + } else if (arg.startsWith('npub1')) { + optionsEnd = true; + + if (relaySectionBegun) { + console.error('npub specified in relay section'); Deno.exit(1); + } + const decoded = nip19.decode(arg as `npub1${string}`).data; + if (!NSchema.id().safeParse(decoded).success) { + console.error(`invalid pubkey ${arg}, skipping...`); + continue; + } + pubkeys.push(decoded); + } else { + relaySectionBegun = true; + if (!arg.startsWith('wss://')) { + console.error(`invalid relay url ${arg}, skipping...`); + } + relays.push(arg); } - const pubkeys: string[] = []; - const relays: string[] = []; + } - const opts: Partial = {}; - - let optionsEnd = false; - let relaySectionBegun = false; - for (const arg of Deno.args) { - if (arg.startsWith('-')) { - if (optionsEnd) { - console.error("Option encountered after end of options section."); - showUsage(); - } - switch (arg) { - case '-p': - case '--profile-only': - console.info('Only importing profiles.'); - opts.profilesOnly = true; - break; - } - } - else if (arg.startsWith('npub1')) { - optionsEnd = true; - - if (relaySectionBegun) { - console.error('npub specified in relay section'); - Deno.exit(1); - } - const decoded = nip19.decode(arg as `npub1${string}`).data; - if (!NSchema.id().safeParse(decoded).success) { - console.error(`invalid pubkey ${arg}, skipping...`); - continue; - } - pubkeys.push(decoded); - } - else { - relaySectionBegun = true; - if (!arg.startsWith('wss://')) { - console.error(`invalid relay url ${arg}, skipping...`); - } - relays.push(arg); - } - } - - await importUsers(pubkeys, relays, opts); - Deno.exit(0); + await importUsers(pubkeys, relays, opts); + Deno.exit(0); } function showHelp() { - console.info('ditto - db:import'); - console.info('Import users\' posts and kind 0s from a given set of relays.\n'); - showUsage(); - console.info(` + console.info('ditto - db:import'); + console.info("Import users' posts and kind 0s from a given set of relays.\n"); + showUsage(); + console.info(` OPTIONS: -p, --profile-only @@ -124,6 +125,8 @@ OPTIONS: } function showUsage() { - console.info('Usage: deno task db:import [options] npub1xxxxxx[ npub1yyyyyyy]...' + - ' wss://first.relay[ second.relay]...'); + console.info( + 'Usage: deno task db:import [options] npub1xxxxxx[ npub1yyyyyyy]...' + + ' wss://first.relay[ second.relay]...', + ); } From 96fe171d6595fcc7afc78d3ec245ff5373a60a51 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 1 Jul 2024 07:22:33 +0100 Subject: [PATCH 42/75] Use kysely_deno_postgres with simple transactions --- deno.json | 2 +- deno.lock | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/deno.json b/deno.json index af21804a..cd52a343 100644 --- a/deno.json +++ b/deno.json @@ -50,7 +50,7 @@ "iso-639-1": "npm:iso-639-1@2.1.15", "isomorphic-dompurify": "npm:isomorphic-dompurify@^2.11.0", "kysely": "npm:kysely@^0.27.3", - "kysely_deno_postgres": "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/b4725e74ad6ca359ba0e370b55dbb8bb845a8a83/mod.ts", + "kysely_deno_postgres": "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/c6869b9e12d74af78a846ad503d84493f5db9df4/mod.ts", "light-bolt11-decoder": "npm:light-bolt11-decoder", "linkify-plugin-hashtag": "npm:linkify-plugin-hashtag@^4.1.1", "linkify-string": "npm:linkify-string@^4.1.1", diff --git a/deno.lock b/deno.lock index f8e13dbd..1377bf48 100644 --- a/deno.lock +++ b/deno.lock @@ -1395,6 +1395,10 @@ "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/b4725e74ad6ca359ba0e370b55dbb8bb845a8a83/mod.ts": "662438fd3909984bb8cbaf3fd44d2121e949d11301baf21d6c3f057ccf9887de", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/b4725e74ad6ca359ba0e370b55dbb8bb845a8a83/src/PostgreSQLDriver.ts": "ea5a523bceeed420858b744beeb95d48976cb2b0d3f519a68b65a8229036cf6a", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/b4725e74ad6ca359ba0e370b55dbb8bb845a8a83/src/PostgreSQLDriverDatabaseConnection.ts": "11e2fc10a3abb3d0729613c4b7cdb9cb73b597fd77353311bb6707c73a635fc5", + "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/c6869b9e12d74af78a846ad503d84493f5db9df4/deps.ts": "b3dbecae69c30a5f161323b8c8ebd91d9af1eceb98fafab3091c7281a4b64fed", + "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/c6869b9e12d74af78a846ad503d84493f5db9df4/mod.ts": "662438fd3909984bb8cbaf3fd44d2121e949d11301baf21d6c3f057ccf9887de", + "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/c6869b9e12d74af78a846ad503d84493f5db9df4/src/PostgreSQLDriver.ts": "0f5d1bc2b24d4e0052e38ee289fb2f5e8e1470544f61aa2afe65e1059bf35dfb", + "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/c6869b9e12d74af78a846ad503d84493f5db9df4/src/PostgreSQLDriverDatabaseConnection.ts": "e5d4e0fc9737c3ec253e679a51f5b43d2bb9a3386c147b7b1d14f4f5a5f734f1", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/f2948b86190a10faa293588775e162b3a8b52e70/deps.ts": "b3dbecae69c30a5f161323b8c8ebd91d9af1eceb98fafab3091c7281a4b64fed", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/f2948b86190a10faa293588775e162b3a8b52e70/mod.ts": "662438fd3909984bb8cbaf3fd44d2121e949d11301baf21d6c3f057ccf9887de", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/f2948b86190a10faa293588775e162b3a8b52e70/src/PostgreSQLDriver.ts": "ac1a39e86fd676973bce215e19db1f26b82408b8f2bb09a3601802974ea7cec6", From d062f6bbb670b686faddfc52d3edc79ce348e8ae Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 1 Jul 2024 08:44:01 +0100 Subject: [PATCH 43/75] Try lazy pool initialization --- src/db/adapters/DittoPostgres.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db/adapters/DittoPostgres.ts b/src/db/adapters/DittoPostgres.ts index c06a262f..6df78b97 100644 --- a/src/db/adapters/DittoPostgres.ts +++ b/src/db/adapters/DittoPostgres.ts @@ -12,7 +12,7 @@ export class DittoPostgres { static getPool(): Pool { if (!this.pool) { - this.pool = new Pool(Conf.databaseUrl, Conf.pg.poolSize); + this.pool = new Pool(Conf.databaseUrl, Conf.pg.poolSize, true); } return this.pool; } From c3ffe7c7f76fc3f1710d50fd0f992c410f56f28a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 1 Jul 2024 08:53:03 +0100 Subject: [PATCH 44/75] EventsDB: fix limit being passed to NDatabase --- src/storages/EventsDB.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index c22e2567..664be893 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -51,7 +51,7 @@ class EventsDB implements NStore { } /** Insert an event (and its tags) into the database. */ - async event(event: NostrEvent, opts?: { signal?: AbortSignal; timeout?: number }): Promise { + async event(event: NostrEvent, opts: { signal?: AbortSignal; timeout?: number } = {}): Promise { event = purifyEvent(event); this.console.debug('EVENT', JSON.stringify(event)); dbEventCounter.inc({ kind: event.kind }); @@ -63,7 +63,7 @@ class EventsDB implements NStore { await this.deleteEventsAdmin(event); try { - await this.store.event(event, { timeout: opts?.timeout ?? 3000 }); + await this.store.event(event, { ...opts, timeout: opts.timeout ?? 3000 }); } catch (e) { if (e.message === 'Cannot add a deleted event') { throw new RelayError('blocked', 'event deleted by user'); @@ -163,15 +163,15 @@ class EventsDB implements NStore { this.console.debug('REQ', JSON.stringify(filters)); - return this.store.query(filters, { timeout: opts.timeout ?? 3000 }); + return this.store.query(filters, { ...opts, timeout: opts.timeout ?? 3000 }); } /** Delete events based on filters from the database. */ - async remove(filters: NostrFilter[], opts?: { signal?: AbortSignal; timeout?: number }): Promise { + async remove(filters: NostrFilter[], opts: { signal?: AbortSignal; timeout?: number } = {}): Promise { if (!filters.length) return Promise.resolve(); this.console.debug('DELETE', JSON.stringify(filters)); - return this.store.remove(filters, opts); + return this.store.remove(filters, { ...opts, timeout: opts.timeout ?? 5000 }); } /** Get number of events that would be returned by filters. */ @@ -184,7 +184,7 @@ class EventsDB implements NStore { this.console.debug('COUNT', JSON.stringify(filters)); - return this.store.count(filters, { timeout: opts.timeout ?? 1000 }); + return this.store.count(filters, { ...opts, timeout: opts.timeout ?? 1000 }); } /** Return only the tags that should be indexed. */ From 092a20088a5ae82a89ce71b034b0b24e61b4b480 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 1 Jul 2024 09:18:42 +0100 Subject: [PATCH 45/75] Reduce timeouts --- src/controllers/nostr/relay.ts | 4 ++-- src/storages/EventsDB.ts | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index e86f1991..6d79b031 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -73,7 +73,7 @@ function connectStream(socket: WebSocket) { const pubsub = await Storages.pubsub(); try { - for (const event of await store.query(filters, { limit: FILTER_LIMIT, timeout: 500 })) { + for (const event of await store.query(filters, { limit: FILTER_LIMIT, timeout: 300 })) { send(['EVENT', subId, event]); } } catch (e) { @@ -128,7 +128,7 @@ function connectStream(socket: WebSocket) { /** Handle COUNT. Return the number of events matching the filters. */ async function handleCount([_, subId, ...filters]: NostrClientCOUNT): Promise { const store = await Storages.db(); - const { count } = await store.count(filters, { timeout: 500 }); + const { count } = await store.count(filters, { timeout: 100 }); send(['COUNT', subId, { count, approximate: false }]); } diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 664be893..61b87b00 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -63,7 +63,7 @@ class EventsDB implements NStore { await this.deleteEventsAdmin(event); try { - await this.store.event(event, { ...opts, timeout: opts.timeout ?? 3000 }); + await this.store.event(event, { ...opts, timeout: opts.timeout ?? 1000 }); } catch (e) { if (e.message === 'Cannot add a deleted event') { throw new RelayError('blocked', 'event deleted by user'); @@ -163,7 +163,7 @@ class EventsDB implements NStore { this.console.debug('REQ', JSON.stringify(filters)); - return this.store.query(filters, { ...opts, timeout: opts.timeout ?? 3000 }); + return this.store.query(filters, { ...opts, timeout: opts.timeout ?? 1000 }); } /** Delete events based on filters from the database. */ @@ -171,7 +171,7 @@ class EventsDB implements NStore { if (!filters.length) return Promise.resolve(); this.console.debug('DELETE', JSON.stringify(filters)); - return this.store.remove(filters, { ...opts, timeout: opts.timeout ?? 5000 }); + return this.store.remove(filters, { ...opts, timeout: opts.timeout ?? 3000 }); } /** Get number of events that would be returned by filters. */ @@ -184,7 +184,7 @@ class EventsDB implements NStore { this.console.debug('COUNT', JSON.stringify(filters)); - return this.store.count(filters, { ...opts, timeout: opts.timeout ?? 1000 }); + return this.store.count(filters, { ...opts, timeout: opts.timeout ?? 500 }); } /** Return only the tags that should be indexed. */ From ea987dfa14a9b7d7dc9aba5cb434985ed71c2c1f Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 1 Jul 2024 23:06:09 +0530 Subject: [PATCH 46/75] support raw pubkeys as well as npubs --- scripts/db-import.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/db-import.ts b/scripts/db-import.ts index e1484c99..954d83df 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -99,6 +99,8 @@ if (import.meta.main) { continue; } pubkeys.push(decoded); + } else if (NSchema.id().safeParse(arg).success) { + pubkeys.push(arg); } else { relaySectionBegun = true; if (!arg.startsWith('wss://')) { From 434056b83920bbd7312c35d3f893efabadaf148c Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 1 Jul 2024 23:06:41 +0530 Subject: [PATCH 47/75] rename evt to event --- scripts/db-import.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/db-import.ts b/scripts/db-import.ts index 954d83df..806ce326 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -14,12 +14,12 @@ interface ImportEventsOpts { profilesOnly: boolean; } -type DoEvent = (evt: NostrEvent) => void | Promise; +type DoEvent = (event: NostrEvent) => void | Promise; const importUsers = async ( authors: string[], relays: string[], opts?: Partial, - doEvent: DoEvent = async (evt: NostrEvent) => await eventsDB.event(evt), + doEvent: DoEvent = async (event: NostrEvent) => await eventsDB.event(event), ) => { // Kind 0s + follow lists. const profiles: Record> = {}; From 98e9ccfc46b0fdd4a1ed65bee53433e53d60c874 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 1 Jul 2024 23:08:15 +0530 Subject: [PATCH 48/75] fix import --- scripts/db-import.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/db-import.ts b/scripts/db-import.ts index 806ce326..2c8a60f4 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -2,7 +2,7 @@ * Script to import a user/list of users into Ditto given their npub/pubkey by looking them up on a list of relays. */ -import { nip19 } from 'npm:nostr-tools@^2.7.0'; +import { nip19 } from 'nostr-tools'; import { DittoDB } from '@/db/DittoDB.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify'; From e07be77de75f31387aa82b8f3f189e58ba8f166b Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 1 Jul 2024 23:13:20 +0530 Subject: [PATCH 49/75] fix import order --- scripts/db-import.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/db-import.ts b/scripts/db-import.ts index 2c8a60f4..03e389cb 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -2,10 +2,11 @@ * Script to import a user/list of users into Ditto given their npub/pubkey by looking them up on a list of relays. */ +import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; + import { DittoDB } from '@/db/DittoDB.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; -import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify'; const kysely = await DittoDB.getInstance(); const eventsDB = new EventsDB(kysely); From 6974b78952d9836134f3cc5cba2052da8bdba589 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 1 Jul 2024 23:21:29 +0530 Subject: [PATCH 50/75] db:import --> nostr:pull --- deno.json | 2 +- scripts/{db-import.ts => nostr-pull.ts} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename scripts/{db-import.ts => nostr-pull.ts} (100%) diff --git a/deno.json b/deno.json index 48c927f2..21ad4f2d 100644 --- a/deno.json +++ b/deno.json @@ -6,7 +6,7 @@ "dev": "deno run -A --watch src/server.ts", "hook": "deno run --allow-read --allow-run --allow-write https://deno.land/x/deno_hooks@0.1.1/mod.ts", "db:migrate": "deno run -A scripts/db-migrate.ts", - "db:import": "deno run -A scripts/db-import.ts", + "nostr:pull": "deno run -A scripts/nostr-pull.ts", "debug": "deno run -A --inspect src/server.ts", "test": "DATABASE_URL=\"sqlite://:memory:\" deno test -A --junit-path=./deno-test.xml", "check": "deno check src/server.ts", diff --git a/scripts/db-import.ts b/scripts/nostr-pull.ts similarity index 100% rename from scripts/db-import.ts rename to scripts/nostr-pull.ts From 4e150edb5b1f3bab308a1c3b5ff73f168c091206 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 1 Jul 2024 19:48:28 +0100 Subject: [PATCH 51/75] Actually enable query timeouts :facepalm: --- src/storages/EventsDB.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 61b87b00..69959fc0 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -45,6 +45,7 @@ class EventsDB implements NStore { constructor(private kysely: Kysely) { this.store = new NDatabase(kysely, { fts: Conf.db.dialect, + timeoutStrategy: Conf.db.dialect === 'postgres' ? 'setStatementTimeout' : undefined, indexTags: EventsDB.indexTags, searchText: EventsDB.searchText, }); From 3ae6d39ebc2f5f2aa80217ca682ec447b50ebc6b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 1 Jul 2024 20:29:06 +0100 Subject: [PATCH 52/75] Increase notifications endpoint timeout --- src/controllers/api/notifications.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/api/notifications.ts b/src/controllers/api/notifications.ts index d92ccf4a..51a20a05 100644 --- a/src/controllers/api/notifications.ts +++ b/src/controllers/api/notifications.ts @@ -78,7 +78,7 @@ async function renderNotifications( const store = c.get('store'); const pubkey = await c.get('signer')?.getPublicKey()!; const { signal } = c.req.raw; - const opts = { signal, limit: params.limit }; + const opts = { signal, limit: params.limit, timeout: 5000 }; const events = await store .query(filters, opts) From ae687bb52561298437a54cf36da61a242da18a95 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 1 Jul 2024 23:05:03 +0100 Subject: [PATCH 53/75] Increase timeouts --- src/controllers/api/notifications.ts | 2 +- src/controllers/nostr/relay.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/notifications.ts b/src/controllers/api/notifications.ts index 51a20a05..64a5a7ca 100644 --- a/src/controllers/api/notifications.ts +++ b/src/controllers/api/notifications.ts @@ -78,7 +78,7 @@ async function renderNotifications( const store = c.get('store'); const pubkey = await c.get('signer')?.getPublicKey()!; const { signal } = c.req.raw; - const opts = { signal, limit: params.limit, timeout: 5000 }; + const opts = { signal, limit: params.limit, timeout: 10_000 }; const events = await store .query(filters, opts) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 6d79b031..4d8ab2cb 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -73,7 +73,7 @@ function connectStream(socket: WebSocket) { const pubsub = await Storages.pubsub(); try { - for (const event of await store.query(filters, { limit: FILTER_LIMIT, timeout: 300 })) { + for (const event of await store.query(filters, { limit: FILTER_LIMIT, timeout: 1000 })) { send(['EVENT', subId, event]); } } catch (e) { From 5f0b1ffe79956d2693cdb6f8068f6b2b144d50d2 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 2 Jul 2024 04:56:38 +0530 Subject: [PATCH 54/75] fetch profiles separately from notes --- scripts/nostr-pull.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/nostr-pull.ts b/scripts/nostr-pull.ts index 03e389cb..d215e435 100644 --- a/scripts/nostr-pull.ts +++ b/scripts/nostr-pull.ts @@ -32,9 +32,11 @@ const importUsers = async ( await Promise.all(relays.map(async (relay) => { if (!relay.startsWith('wss://')) console.error(`Invalid relay url ${relay}`); const conn = new NRelay1(relay); - const kinds = [0, 3]; - if (!profilesOnly) kinds.push(1); - const matched = await conn.query([{ kinds, authors, limit: 1000 }]); + + const matched = [ + ...await conn.query([{ kinds: [0, 3], authors, limit: 1000 }]), + ...(!profilesOnly ? [] : await conn.query([{ kinds: [1], authors, limit: 1000 }])), + ]; await conn.close(); await Promise.all( matched.map(async (event) => { From f1e9eb4f4c87e14f840b6bb7d3bece1db9a2b9cf Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 2 Jul 2024 04:58:06 +0530 Subject: [PATCH 55/75] fetch each author's notes separately --- scripts/nostr-pull.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/nostr-pull.ts b/scripts/nostr-pull.ts index d215e435..ae5b0183 100644 --- a/scripts/nostr-pull.ts +++ b/scripts/nostr-pull.ts @@ -35,8 +35,11 @@ const importUsers = async ( const matched = [ ...await conn.query([{ kinds: [0, 3], authors, limit: 1000 }]), - ...(!profilesOnly ? [] : await conn.query([{ kinds: [1], authors, limit: 1000 }])), + ...(!profilesOnly ? [] : await conn.query( + authors.map((author) => ({ kinds: [1], authors: [author], limit: 200 })), + )), ]; + await conn.close(); await Promise.all( matched.map(async (event) => { From d2df6721bdcf53092e4470dfa923e923fc7d97f7 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 2 Jul 2024 05:01:16 +0530 Subject: [PATCH 56/75] fix kind 1 querying logic --- scripts/nostr-pull.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/nostr-pull.ts b/scripts/nostr-pull.ts index ae5b0183..c3cbb67b 100644 --- a/scripts/nostr-pull.ts +++ b/scripts/nostr-pull.ts @@ -32,13 +32,14 @@ const importUsers = async ( await Promise.all(relays.map(async (relay) => { if (!relay.startsWith('wss://')) console.error(`Invalid relay url ${relay}`); const conn = new NRelay1(relay); - - const matched = [ - ...await conn.query([{ kinds: [0, 3], authors, limit: 1000 }]), - ...(!profilesOnly ? [] : await conn.query( - authors.map((author) => ({ kinds: [1], authors: [author], limit: 200 })), - )), - ]; + const matched = await conn.query([{ kinds: [0, 3], authors, limit: 1000 }]); + if (!profilesOnly) { + matched.push( + ...await conn.query( + authors.map((author) => ({ kinds: [1], authors: [author], limit: 200 })), + ), + ); + } await conn.close(); await Promise.all( From 757e5c9baaa75fbce45822a06c1bd2be89fe776c Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 2 Jul 2024 05:21:16 +0530 Subject: [PATCH 57/75] print name as import acknowledgement --- scripts/nostr-pull.ts | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/scripts/nostr-pull.ts b/scripts/nostr-pull.ts index c3cbb67b..1a236f8f 100644 --- a/scripts/nostr-pull.ts +++ b/scripts/nostr-pull.ts @@ -1,5 +1,6 @@ /** - * Script to import a user/list of users into Ditto given their npub/pubkey by looking them up on a list of relays. + * Script to import a user/list of users into Ditto given their npub/pubkey + * by looking them up on a list of relays. */ import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify'; @@ -26,13 +27,13 @@ const importUsers = async ( const profiles: Record> = {}; // Kind 1s. const notes = new Set(); - const { profilesOnly = false } = opts || {}; await Promise.all(relays.map(async (relay) => { if (!relay.startsWith('wss://')) console.error(`Invalid relay url ${relay}`); const conn = new NRelay1(relay); const matched = await conn.query([{ kinds: [0, 3], authors, limit: 1000 }]); + if (!profilesOnly) { matched.push( ...await conn.query( @@ -65,6 +66,20 @@ const importUsers = async ( for (const kind in profile) { await doEvent(profile[kind]); } + + let name = user; + // kind 0, not first idx + const event = profile[0]; + if (event) { + // if event exists, print name + const parsed = JSON.parse(event.content); + name = parsed.nip05 || parsed.name || name; + } + if (NSchema.id().safeParse(name).success) { + // if no kind 0 found and this is a pubkey, encode as npub + name = nip19.npubEncode(name); + } + console.info(`Imported user ${name}${profilesOnly ? "'s profile" : ''}.`); } }; @@ -85,6 +100,7 @@ if (import.meta.main) { if (optionsEnd) { console.error('Option encountered after end of options section.'); showUsage(); + Deno.exit(1); } switch (arg) { case '-p': From c7092f5d2d69424eb2a6e1500c6c67a73cfd76b4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 2 Jul 2024 09:22:59 +0100 Subject: [PATCH 58/75] Increase query timeout of feeds --- src/controllers/api/timelines.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/controllers/api/timelines.ts b/src/controllers/api/timelines.ts index 5d7862b5..e6f0ae78 100644 --- a/src/controllers/api/timelines.ts +++ b/src/controllers/api/timelines.ts @@ -60,9 +60,10 @@ const suggestedTimelineController: AppController = async (c) => { async function renderStatuses(c: AppContext, filters: NostrFilter[]) { const { signal } = c.req.raw; const store = c.get('store'); + const opts = { signal, timeout: 5000 }; const events = await store - .query(filters, { signal }) + .query(filters, opts) .then((events) => hydrateEvents({ events, store, signal })); if (!events.length) { From 3e7bab538a29f03e3be728d285c2272bbe1d6e0f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 2 Jul 2024 20:13:44 +0100 Subject: [PATCH 59/75] Increase timeout for account statuses endpoint and feed endpoints --- src/controllers/api/accounts.ts | 4 +++- src/controllers/api/notifications.ts | 2 +- src/controllers/api/timelines.ts | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 9e77ccc1..ad4802ca 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -209,7 +209,9 @@ const accountStatusesController: AppController = async (c) => { filter['#t'] = [tagged]; } - const events = await store.query([filter], { signal }) + const opts = { signal, limit, timeout: 10_000 }; + + const events = await store.query([filter], opts) .then((events) => hydrateEvents({ events, store, signal })) .then((events) => { if (exclude_replies) { diff --git a/src/controllers/api/notifications.ts b/src/controllers/api/notifications.ts index 64a5a7ca..fab7c816 100644 --- a/src/controllers/api/notifications.ts +++ b/src/controllers/api/notifications.ts @@ -78,7 +78,7 @@ async function renderNotifications( const store = c.get('store'); const pubkey = await c.get('signer')?.getPublicKey()!; const { signal } = c.req.raw; - const opts = { signal, limit: params.limit, timeout: 10_000 }; + const opts = { signal, limit: params.limit, timeout: 15_000 }; const events = await store .query(filters, opts) diff --git a/src/controllers/api/timelines.ts b/src/controllers/api/timelines.ts index e6f0ae78..62f1cd2f 100644 --- a/src/controllers/api/timelines.ts +++ b/src/controllers/api/timelines.ts @@ -60,7 +60,7 @@ const suggestedTimelineController: AppController = async (c) => { async function renderStatuses(c: AppContext, filters: NostrFilter[]) { const { signal } = c.req.raw; const store = c.get('store'); - const opts = { signal, timeout: 5000 }; + const opts = { signal, timeout: 10_000 }; const events = await store .query(filters, opts) From a7f2fb06ee11492e552d32125fe06013058daae5 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 2 Jul 2024 17:09:17 -0300 Subject: [PATCH 60/75] feat(relay): send custom error messages - since, until & kind --- src/controllers/nostr/relay.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 4d8ab2cb..4f278d64 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -77,7 +77,10 @@ function connectStream(socket: WebSocket) { send(['EVENT', subId, event]); } } catch (e) { - if (e instanceof RelayError) { + if ( + e instanceof RelayError || + e.message.slice(-('filter too far into the future'.length)) === 'filter too far into the future' + ) { send(['CLOSED', subId, e.message]); } else { send(['CLOSED', subId, 'error: something went wrong']); From e6c38550c66a668d19b9b27c215ddf4b73b053eb Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 2 Jul 2024 17:15:47 -0300 Subject: [PATCH 61/75] feat: add onError() function hono handler --- src/app.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/app.ts b/src/app.ts index 6a2205b4..2e7d8791 100644 --- a/src/app.ts +++ b/src/app.ts @@ -340,6 +340,13 @@ app.get('/', frontendController, indexController); // Fallback app.get('*', publicFiles, staticFiles, frontendController); +app.onError((err, c) => { + if (err.message === 'canceling statement due to statement timeout') { + return c.json({ error: 'Everything will be fine, I mean it. Don\t worry child.' }, 500); + } + return c.json(500); +}); + export default app; export type { AppContext, AppController, AppMiddleware }; From cac5c9c1e0144fc80c20523bb2dfdb5c90611bdb Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 2 Jul 2024 17:17:29 -0300 Subject: [PATCH 62/75] refactor: change timeout error message --- src/app.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.ts b/src/app.ts index 2e7d8791..c47d2579 100644 --- a/src/app.ts +++ b/src/app.ts @@ -342,7 +342,7 @@ app.get('*', publicFiles, staticFiles, frontendController); app.onError((err, c) => { if (err.message === 'canceling statement due to statement timeout') { - return c.json({ error: 'Everything will be fine, I mean it. Don\t worry child.' }, 500); + return c.json({ error: "A timeout happened, don't worry :)" }, 500); } return c.json(500); }); From 360efe089d1799547d9f77e2c94e18c7a238550b Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 2 Jul 2024 17:31:12 -0300 Subject: [PATCH 63/75] test(EventsDB): throw error for large since, until and kinds filter --- src/storages/EventsDB.test.ts | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 16b429d4..fe8ffe8a 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -191,3 +191,33 @@ Deno.test('inserting replaceable events', async () => { await eventsDB.event(newerEvent); assertEquals(await eventsDB.query([{ kinds: [0] }]), [newerEvent]); }); + +Deno.test("throws a Error when querying an event with a large 'since'", async () => { + const { eventsDB } = await createDB(); + + await assertRejects( + () => eventsDB.query([{ since: 33333333333333 }]), + Error, + 'since filter too far into the future', + ); +}); + +Deno.test("throws a Error when querying an event with a large 'until'", async () => { + const { eventsDB } = await createDB(); + + await assertRejects( + () => eventsDB.query([{ until: 66666666666666 }]), + Error, + 'until filter too far into the future', + ); +}); + +Deno.test("throws a Error when querying an event with a large 'kind'", async () => { + const { eventsDB } = await createDB(); + + await assertRejects( + () => eventsDB.query([{ kinds: [99999999999999] }]), + Error, + 'kind filter too far into the future', + ); +}); From a868512188bdc8af2c1e28fdb1a6866df8347b01 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 2 Jul 2024 18:38:16 -0300 Subject: [PATCH 64/75] refactor: error messages in app.onError() --- src/app.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app.ts b/src/app.ts index c47d2579..4c0ffc7e 100644 --- a/src/app.ts +++ b/src/app.ts @@ -342,9 +342,9 @@ app.get('*', publicFiles, staticFiles, frontendController); app.onError((err, c) => { if (err.message === 'canceling statement due to statement timeout') { - return c.json({ error: "A timeout happened, don't worry :)" }, 500); + return c.json({ error: 'The server was unable to respond in a timely manner' }, 500); } - return c.json(500); + return c.json({ error: 'Something went wrong' }, 500); }); export default app; From 2d017e81026078b9c9c399d10e6afc10817c27a5 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 2 Jul 2024 18:39:22 -0300 Subject: [PATCH 65/75] refactor(relay): remove invalid filter condition --- src/controllers/nostr/relay.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 4f278d64..f4193379 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -78,8 +78,7 @@ function connectStream(socket: WebSocket) { } } catch (e) { if ( - e instanceof RelayError || - e.message.slice(-('filter too far into the future'.length)) === 'filter too far into the future' + e instanceof RelayError ) { send(['CLOSED', subId, e.message]); } else { From e169749b8279943bef8f63cb8c3046c7f6c4fd09 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 2 Jul 2024 18:39:55 -0300 Subject: [PATCH 66/75] refactor(EventsDB): throw RelayError instead of Error --- src/storages/EventsDB.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 69959fc0..ce74f20d 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -147,14 +147,14 @@ class EventsDB implements NStore { for (const filter of filters) { if (filter.since && filter.since >= 2_147_483_647) { - throw new Error('since filter too far into the future'); + throw new RelayError('invalid', 'since filter too far into the future'); } if (filter.until && filter.until >= 2_147_483_647) { - throw new Error('until filter too far into the future'); + throw new RelayError('invalid', 'until filter too far into the future'); } for (const kind of filter.kinds ?? []) { if (kind >= 2_147_483_647) { - throw new Error('kind filter too far into the future'); + throw new RelayError('invalid', 'kind filter too far into the future'); } } } From d3c3ecfd35abc783cac8263062a0d564ae857e35 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 2 Jul 2024 18:40:21 -0300 Subject: [PATCH 67/75] test(EventsDB): refactor to use RelayError instead of Error --- src/storages/EventsDB.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index fe8ffe8a..32838fa6 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -192,32 +192,32 @@ Deno.test('inserting replaceable events', async () => { assertEquals(await eventsDB.query([{ kinds: [0] }]), [newerEvent]); }); -Deno.test("throws a Error when querying an event with a large 'since'", async () => { +Deno.test("throws a RelayError when querying an event with a large 'since'", async () => { const { eventsDB } = await createDB(); await assertRejects( () => eventsDB.query([{ since: 33333333333333 }]), - Error, + RelayError, 'since filter too far into the future', ); }); -Deno.test("throws a Error when querying an event with a large 'until'", async () => { +Deno.test("throws a RelayError when querying an event with a large 'until'", async () => { const { eventsDB } = await createDB(); await assertRejects( () => eventsDB.query([{ until: 66666666666666 }]), - Error, + RelayError, 'until filter too far into the future', ); }); -Deno.test("throws a Error when querying an event with a large 'kind'", async () => { +Deno.test("throws a RelayError when querying an event with a large 'kind'", async () => { const { eventsDB } = await createDB(); await assertRejects( () => eventsDB.query([{ kinds: [99999999999999] }]), - Error, + RelayError, 'kind filter too far into the future', ); }); From ba290354138fa32fc6f948651378f40c674c526a Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 2 Jul 2024 18:43:28 -0300 Subject: [PATCH 68/75] refactor: put conditional into just one line --- src/controllers/nostr/relay.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index f4193379..4d8ab2cb 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -77,9 +77,7 @@ function connectStream(socket: WebSocket) { send(['EVENT', subId, event]); } } catch (e) { - if ( - e instanceof RelayError - ) { + if (e instanceof RelayError) { send(['CLOSED', subId, e.message]); } else { send(['CLOSED', subId, 'error: something went wrong']); From 193dd1a011374fc85c8a857e071dabdfb04853a6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 4 Jul 2024 23:38:16 +0100 Subject: [PATCH 69/75] EventsDB: remove queries for ephemeral kinds --- src/storages/EventsDB.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index ce74f20d..f640cc45 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -279,6 +279,12 @@ class EventsDB implements NStore { filter.search = tokens.filter((t) => typeof t === 'string').join(' '); } + + if (filter.kinds) { + // Ephemeral events are not stored, so don't bother querying for them. + // If this results in an empty kinds array, NDatabase will remove the filter before querying and return no results. + filter.kinds = filter.kinds.filter((kind) => !NKinds.ephemeral(kind)); + } } return filters; From d4713cae012d7d05ed9264d6cbba2c3d1935c7c6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 4 Jul 2024 23:53:20 +0100 Subject: [PATCH 70/75] Move errorHandler to a separate file --- src/app.ts | 8 ++------ src/controllers/error.ts | 11 +++++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 src/controllers/error.ts diff --git a/src/app.ts b/src/app.ts index 4c0ffc7e..f7f4f29a 100644 --- a/src/app.ts +++ b/src/app.ts @@ -108,6 +108,7 @@ import { trendingStatusesController, trendingTagsController, } from '@/controllers/api/trends.ts'; +import { errorHandler } from '@/controllers/error.ts'; import { metricsController } from '@/controllers/metrics.ts'; import { indexController } from '@/controllers/site.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; @@ -340,12 +341,7 @@ app.get('/', frontendController, indexController); // Fallback app.get('*', publicFiles, staticFiles, frontendController); -app.onError((err, c) => { - if (err.message === 'canceling statement due to statement timeout') { - return c.json({ error: 'The server was unable to respond in a timely manner' }, 500); - } - return c.json({ error: 'Something went wrong' }, 500); -}); +app.onError(errorHandler); export default app; diff --git a/src/controllers/error.ts b/src/controllers/error.ts new file mode 100644 index 00000000..fa5e4d32 --- /dev/null +++ b/src/controllers/error.ts @@ -0,0 +1,11 @@ +import { ErrorHandler } from '@hono/hono'; + +export const errorHandler: ErrorHandler = (err, c) => { + console.error(err); + + if (err.message === 'canceling statement due to statement timeout') { + return c.json({ error: 'The server was unable to respond in a timely manner' }, 500); + } + + return c.json({ error: 'Something went wrong' }, 500); +}; From 96a8ccb2e637ffdf1fec979007377d46d321750f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 5 Jul 2024 00:00:57 +0100 Subject: [PATCH 71/75] HTTP Response metrics --- src/app.ts | 15 +++++++-------- src/metrics.ts | 6 ++++++ src/middleware/metricsMiddleware.ts | 6 +++++- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/app.ts b/src/app.ts index f7f4f29a..1cb3746b 100644 --- a/src/app.ts +++ b/src/app.ts @@ -152,18 +152,17 @@ if (Conf.cronEnabled) { app.use('*', rateLimitMiddleware(300, Time.minutes(5))); -app.use('/api/*', logger(debug)); -app.use('/.well-known/*', logger(debug)); -app.use('/users/*', logger(debug)); -app.use('/nodeinfo/*', logger(debug)); -app.use('/oauth/*', logger(debug)); +app.use('/api/*', metricsMiddleware, logger(debug)); +app.use('/.well-known/*', metricsMiddleware, logger(debug)); +app.use('/users/*', metricsMiddleware, logger(debug)); +app.use('/nodeinfo/*', metricsMiddleware, logger(debug)); +app.use('/oauth/*', metricsMiddleware, logger(debug)); -app.get('/api/v1/streaming', streamingController); -app.get('/relay', relayController); +app.get('/api/v1/streaming', metricsMiddleware, streamingController); +app.get('/relay', metricsMiddleware, relayController); app.use( '*', - metricsMiddleware, cspMiddleware(), cors({ origin: '*', exposeHeaders: ['link'] }), signerMiddleware, diff --git a/src/metrics.ts b/src/metrics.ts index 96d91599..67da3f49 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -6,6 +6,12 @@ export const httpRequestCounter = new Counter({ labelNames: ['method'], }); +export const httpResponseCounter = new Counter({ + name: 'http_responses_total', + help: 'Total number of HTTP responses', + labelNames: ['status', 'path'], +}); + export const streamingConnectionsGauge = new Gauge({ name: 'streaming_connections', help: 'Number of active connections to the streaming API', diff --git a/src/middleware/metricsMiddleware.ts b/src/middleware/metricsMiddleware.ts index 1a491186..d7ac43d5 100644 --- a/src/middleware/metricsMiddleware.ts +++ b/src/middleware/metricsMiddleware.ts @@ -1,10 +1,14 @@ import { MiddlewareHandler } from '@hono/hono'; -import { httpRequestCounter } from '@/metrics.ts'; +import { httpRequestCounter, httpResponseCounter } from '@/metrics.ts'; export const metricsMiddleware: MiddlewareHandler = async (c, next) => { const { method } = c.req; httpRequestCounter.inc({ method }); await next(); + + const { status } = c.res; + const path = c.req.matchedRoutes.find((r) => r.method !== 'ALL')?.path ?? c.req.routePath; + httpResponseCounter.inc({ status, path }); }; From 216710657724a7fe052c6d199a89d571eb915119 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 6 Jul 2024 20:35:12 +0100 Subject: [PATCH 72/75] Fix not being able to post --- src/storages/EventsDB.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index f640cc45..d66a65b7 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -250,6 +250,8 @@ class EventsDB implements NStore { /** Converts filters to more performant, simpler filters that are better for SQLite. */ async expandFilters(filters: NostrFilter[]): Promise { + filters = structuredClone(filters); + for (const filter of filters) { if (filter.search) { const tokens = NIP50.parseInput(filter.search); From 842adfd72b15f22230803b23eb6e0c8f2662f127 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 6 Jul 2024 22:59:18 +0100 Subject: [PATCH 73/75] Improve signer timeout errors --- src/controllers/error.ts | 5 +++ src/signers/ConnectSigner.ts | 59 ++++++++++++++++++++++++++++++++--- src/signers/ReadOnlySigner.ts | 2 +- 3 files changed, 60 insertions(+), 6 deletions(-) diff --git a/src/controllers/error.ts b/src/controllers/error.ts index fa5e4d32..8c07fc93 100644 --- a/src/controllers/error.ts +++ b/src/controllers/error.ts @@ -1,6 +1,11 @@ import { ErrorHandler } from '@hono/hono'; +import { HTTPException } from '@hono/hono/http-exception'; export const errorHandler: ErrorHandler = (err, c) => { + if (err instanceof HTTPException) { + return c.json({ error: err.message }, err.status); + } + console.error(err); if (err.message === 'canceling statement due to statement timeout') { diff --git a/src/signers/ConnectSigner.ts b/src/signers/ConnectSigner.ts index d4cf6032..6501bb8b 100644 --- a/src/signers/ConnectSigner.ts +++ b/src/signers/ConnectSigner.ts @@ -1,4 +1,5 @@ // deno-lint-ignore-file require-await +import { HTTPException } from '@hono/hono/http-exception'; import { NConnectSigner, NostrEvent, NostrSigner } from '@nostrify/nostrify'; import { Storages } from '@/storages.ts'; @@ -27,30 +28,78 @@ export class ConnectSigner implements NostrSigner { async signEvent(event: Omit): Promise { const signer = await this.signer; - return signer.signEvent(event); + try { + return await signer.signEvent(event); + } catch (e) { + if (e.name === 'AbortError') { + throw new HTTPException(408, { message: 'The event was not signed quickly enough' }); + } else { + throw e; + } + } } readonly nip04 = { encrypt: async (pubkey: string, plaintext: string): Promise => { const signer = await this.signer; - return signer.nip04.encrypt(pubkey, plaintext); + try { + return await signer.nip04.encrypt(pubkey, plaintext); + } catch (e) { + if (e.name === 'AbortError') { + throw new HTTPException(408, { + message: 'Text was not encrypted quickly enough', + }); + } else { + throw e; + } + } }, decrypt: async (pubkey: string, ciphertext: string): Promise => { const signer = await this.signer; - return signer.nip04.decrypt(pubkey, ciphertext); + try { + return await signer.nip04.decrypt(pubkey, ciphertext); + } catch (e) { + if (e.name === 'AbortError') { + throw new HTTPException(408, { + message: 'Text was not decrypted quickly enough', + }); + } else { + throw e; + } + } }, }; readonly nip44 = { encrypt: async (pubkey: string, plaintext: string): Promise => { const signer = await this.signer; - return signer.nip44.encrypt(pubkey, plaintext); + try { + return await signer.nip44.encrypt(pubkey, plaintext); + } catch (e) { + if (e.name === 'AbortError') { + throw new HTTPException(408, { + message: 'Text was not encrypted quickly enough', + }); + } else { + throw e; + } + } }, decrypt: async (pubkey: string, ciphertext: string): Promise => { const signer = await this.signer; - return signer.nip44.decrypt(pubkey, ciphertext); + try { + return await signer.nip44.decrypt(pubkey, ciphertext); + } catch (e) { + if (e.name === 'AbortError') { + throw new HTTPException(408, { + message: 'Text was not decrypted quickly enough', + }); + } else { + throw e; + } + } }, }; diff --git a/src/signers/ReadOnlySigner.ts b/src/signers/ReadOnlySigner.ts index 56c32c45..54449fab 100644 --- a/src/signers/ReadOnlySigner.ts +++ b/src/signers/ReadOnlySigner.ts @@ -7,7 +7,7 @@ export class ReadOnlySigner implements NostrSigner { async signEvent(): Promise { throw new HTTPException(401, { - message: 'Log out and back in', + message: 'Log in with Nostr Connect to sign events', }); } From fa53dd7f8df4737df6c9aa238ff81ed64d4cb76a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 7 Jul 2024 00:23:00 +0100 Subject: [PATCH 74/75] createStatusController: add relay hints Fixes https://github.com/nostrability/nostrability/issues/52 --- src/controllers/api/statuses.ts | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 30550bbb..4604981f 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -97,8 +97,8 @@ const createStatusController: AppController = async (c) => { const root = ancestor.tags.find((tag) => tag[0] === 'e' && tag[3] === 'root')?.[1] ?? ancestor.id; - tags.push(['e', root, 'root']); - tags.push(['e', data.in_reply_to_id, 'reply']); + tags.push(['e', root, Conf.relay, 'root']); + tags.push(['e', data.in_reply_to_id, Conf.relay, 'reply']); } if (data.quote_id) { @@ -202,7 +202,7 @@ const deleteStatusController: AppController = async (c) => { if (event.pubkey === pubkey) { await createEvent({ kind: 5, - tags: [['e', id]], + tags: [['e', id, Conf.relay]], }, c); const author = await getAuthor(event.pubkey); @@ -260,8 +260,8 @@ const favouriteController: AppController = async (c) => { kind: 7, content: '+', tags: [ - ['e', target.id], - ['p', target.pubkey], + ['e', target.id, Conf.relay], + ['p', target.pubkey, Conf.relay], ], }, c); @@ -302,7 +302,10 @@ const reblogStatusController: AppController = async (c) => { const reblogEvent = await createEvent({ kind: 6, - tags: [['e', event.id], ['p', event.pubkey]], + tags: [ + ['e', event.id, Conf.relay], + ['p', event.pubkey, Conf.relay], + ], }, c); await hydrateEvents({ @@ -337,7 +340,7 @@ const unreblogStatusController: AppController = async (c) => { await createEvent({ kind: 5, - tags: [['e', repostEvent.id]], + tags: [['e', repostEvent.id, Conf.relay]], }, c); return c.json(await renderStatus(event, { viewerPubkey: pubkey })); @@ -389,7 +392,7 @@ const bookmarkController: AppController = async (c) => { if (event) { await updateListEvent( { kinds: [10003], authors: [pubkey], limit: 1 }, - (tags) => addTag(tags, ['e', eventId]), + (tags) => addTag(tags, ['e', eventId, Conf.relay]), c, ); @@ -416,7 +419,7 @@ const unbookmarkController: AppController = async (c) => { if (event) { await updateListEvent( { kinds: [10003], authors: [pubkey], limit: 1 }, - (tags) => deleteTag(tags, ['e', eventId]), + (tags) => deleteTag(tags, ['e', eventId, Conf.relay]), c, ); @@ -443,7 +446,7 @@ const pinController: AppController = async (c) => { if (event) { await updateListEvent( { kinds: [10001], authors: [pubkey], limit: 1 }, - (tags) => addTag(tags, ['e', eventId]), + (tags) => addTag(tags, ['e', eventId, Conf.relay]), c, ); @@ -472,7 +475,7 @@ const unpinController: AppController = async (c) => { if (event) { await updateListEvent( { kinds: [10001], authors: [pubkey], limit: 1 }, - (tags) => deleteTag(tags, ['e', eventId]), + (tags) => deleteTag(tags, ['e', eventId, Conf.relay]), c, ); @@ -516,7 +519,7 @@ const zapController: AppController = async (c) => { lnurl = getLnurl(meta); if (target && lnurl) { tags.push( - ['e', target.id], + ['e', target.id, Conf.relay], ['p', target.pubkey], ['amount', amount.toString()], ['relays', Conf.relay], From 6245200e215cc2db241a3250774e971d34701ed0 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 11 Jul 2024 16:55:02 -0500 Subject: [PATCH 75/75] Upgrade Deno to v1.45.0 --- .gitlab-ci.yml | 2 +- .tool-versions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 20650d2e..dc1e8456 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:1.44.2 +image: denoland/deno:1.45.0 default: interruptible: true diff --git a/.tool-versions b/.tool-versions index e85b11d4..512c172e 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -deno 1.44.2 \ No newline at end of file +deno 1.45.0 \ No newline at end of file