From 148e71486362cfa3ade52fed35813a04ef5be0a9 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 19 Sep 2024 10:07:04 -0300 Subject: [PATCH 01/44] fix(pg_trgm): use <% instead of % --- src/utils/search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/search.ts b/src/utils/search.ts index b0be761b..5ca43417 100644 --- a/src/utils/search.ts +++ b/src/utils/search.ts @@ -16,7 +16,7 @@ export async function getPubkeysBySearch( 'search', eb.fn('word_similarity', [sql`${q}`, 'search']).as('sml'), ]) - .where(() => sql`${q} % search`) + .where(() => sql`${q} <% search`) .orderBy(['sml desc', 'search']) .limit(limit); From 2f27e22b4ceb8166025ff69722dca6345ff6a1f2 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 19 Sep 2024 10:15:41 -0300 Subject: [PATCH 02/44] test(getPubkeysBySearch): change search query --- src/utils/search.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/search.test.ts b/src/utils/search.test.ts index 1f01a157..7dc37750 100644 --- a/src/utils/search.test.ts +++ b/src/utils/search.test.ts @@ -13,7 +13,7 @@ Deno.test('fuzzy search works', async () => { assertEquals(await getPubkeysBySearch(db.kysely, { q: 'pat rick', limit: 1, followedPubkeys: new Set() }), new Set()); assertEquals( - await getPubkeysBySearch(db.kysely, { q: 'patrick dos reis', limit: 1, followedPubkeys: new Set() }), + await getPubkeysBySearch(db.kysely, { q: 'patrick dosreis', limit: 1, followedPubkeys: new Set() }), new Set([ '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', ]), From 017c17c8a273842a2a92b908f5ece2bd702cee34 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 19 Sep 2024 19:35:05 -0300 Subject: [PATCH 03/44] refactor: remove author_search table, put search in author_stats --- .../034_move_author_search_to_author_stats.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 src/db/migrations/034_move_author_search_to_author_stats.ts diff --git a/src/db/migrations/034_move_author_search_to_author_stats.ts b/src/db/migrations/034_move_author_search_to_author_stats.ts new file mode 100644 index 00000000..6d21ca39 --- /dev/null +++ b/src/db/migrations/034_move_author_search_to_author_stats.ts @@ -0,0 +1,32 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .alterTable('author_stats') + .addColumn('search', 'text', (col) => col.notNull().defaultTo('')) + .execute(); + + await sql`CREATE INDEX author_stats_search_idx ON author_stats USING GIN (search gin_trgm_ops)`.execute(db); + + await db.insertInto('author_stats') + .columns(['pubkey', 'search']) + .expression( + db.selectFrom('author_search') + .select(['pubkey', 'search']), + ) + .onConflict((oc) => + oc.column('pubkey') + .doUpdateSet((eb) => ({ + search: eb.ref('excluded.search'), + })) + ) + .execute(); + + await db.schema.dropIndex('author_search_search_idx').ifExists().execute(); + await db.schema.dropTable('author_search').execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropIndex('author_stats_search_idx').ifExists().execute(); + await db.schema.alterTable('author_stats').dropColumn('search').execute(); +} From f063da1b86204702eb33721a8849932f60203ce1 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 19 Sep 2024 19:35:38 -0300 Subject: [PATCH 04/44] refactor: add search field to AuthorStatsRow, remove AuthorSearch interface --- src/db/DittoTables.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index 642db484..c05ffe66 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -9,7 +9,6 @@ export interface DittoTables extends NPostgresSchema { event_stats: EventStatsRow; pubkey_domains: PubkeyDomainRow; event_zaps: EventZapRow; - author_search: AuthorSearch; } type NostrEventsRow = NPostgresSchema['nostr_events'] & { @@ -21,6 +20,7 @@ interface AuthorStatsRow { followers_count: number; following_count: number; notes_count: number; + search: string; } interface EventStatsRow { @@ -55,8 +55,3 @@ interface EventZapRow { amount_millisats: number; comment: string; } - -interface AuthorSearch { - pubkey: string; - search: string; -} From 2727523540c3df6a6abc0a1911b43899de2fe1a5 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 19 Sep 2024 19:36:13 -0300 Subject: [PATCH 05/44] feat: order search by followers count also --- src/utils/search.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/search.ts b/src/utils/search.ts index 5ca43417..e17be135 100644 --- a/src/utils/search.ts +++ b/src/utils/search.ts @@ -10,13 +10,14 @@ export async function getPubkeysBySearch( const { q, limit, followedPubkeys } = opts; let query = kysely - .selectFrom('author_search') + .selectFrom('author_stats') .select((eb) => [ 'pubkey', 'search', eb.fn('word_similarity', [sql`${q}`, 'search']).as('sml'), ]) .where(() => sql`${q} <% search`) + .orderBy(['followers_count desc']) .orderBy(['sml desc', 'search']) .limit(limit); From 1b6e9160ec7db3df9a01e30a0bc4e7d2ca8ec0da Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 19 Sep 2024 19:36:44 -0300 Subject: [PATCH 06/44] test: update to use author_stats table --- src/utils/search.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/utils/search.test.ts b/src/utils/search.test.ts index 7dc37750..1acd2f60 100644 --- a/src/utils/search.test.ts +++ b/src/utils/search.test.ts @@ -6,9 +6,12 @@ import { getPubkeysBySearch } from '@/utils/search.ts'; Deno.test('fuzzy search works', async () => { await using db = await createTestDB(); - await db.kysely.insertInto('author_search').values({ + await db.kysely.insertInto('author_stats').values({ pubkey: '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', search: 'patrickReiis patrickdosreis.com', + notes_count: 0, + followers_count: 0, + following_count: 0, }).execute(); assertEquals(await getPubkeysBySearch(db.kysely, { q: 'pat rick', limit: 1, followedPubkeys: new Set() }), new Set()); From ebeb150463d5ff211fd7c9e68c1f440ff327843a Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 19 Sep 2024 19:37:13 -0300 Subject: [PATCH 07/44] refactor: use search in author_stats --- src/pipeline.ts | 4 ++-- src/storages/hydrate.ts | 1 + src/utils/stats.test.ts | 11 ++++++++++- src/utils/stats.ts | 13 ++++++++++--- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index aaa6ca07..d56653c4 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -157,8 +157,8 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise oc.column('pubkey').doUpdateSet({ search })) .execute(); } diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 7b11cfb8..5948018b 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -303,6 +303,7 @@ async function gatherAuthorStats( followers_count: Math.max(0, row.followers_count), following_count: Math.max(0, row.following_count), notes_count: Math.max(0, row.notes_count), + search: row.search, })); } diff --git a/src/utils/stats.test.ts b/src/utils/stats.test.ts index 69633ae3..797f78da 100644 --- a/src/utils/stats.test.ts +++ b/src/utils/stats.test.ts @@ -171,7 +171,16 @@ Deno.test('countAuthorStats counts author stats from the database', async () => await db.store.event(genEvent({ kind: 1, content: 'yolo' }, sk)); await db.store.event(genEvent({ kind: 3, tags: [['p', pubkey]] })); - const stats = await countAuthorStats(db.store, pubkey); + await db.kysely.insertInto('author_stats').values({ + pubkey, + search: 'Yolo Lolo', + notes_count: 0, + followers_count: 0, + following_count: 0, + }).onConflict((oc) => oc.column('pubkey').doUpdateSet({ 'search': 'baka' })) + .execute(); + + const stats = await countAuthorStats({ store: db.store, pubkey, kysely: db.kysely }); assertEquals(stats!.notes_count, 2); assertEquals(stats!.followers_count, 1); diff --git a/src/utils/stats.ts b/src/utils/stats.ts index e4d4d3f2..4946be3a 100644 --- a/src/utils/stats.ts +++ b/src/utils/stats.ts @@ -194,6 +194,7 @@ export async function updateAuthorStats( followers_count: 0, following_count: 0, notes_count: 0, + search: '', }; const prev = await kysely @@ -268,8 +269,7 @@ export async function updateEventStats( /** Calculate author stats from the database. */ export async function countAuthorStats( - store: SetRequired, - pubkey: string, + { pubkey, kysely, store }: RefreshAuthorStatsOpts, ): Promise { const [{ count: followers_count }, { count: notes_count }, [followList]] = await Promise.all([ store.count([{ kinds: [3], '#p': [pubkey] }]), @@ -277,11 +277,18 @@ export async function countAuthorStats( store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]), ]); + const [{ search }] = await kysely + .selectFrom('author_stats') + .select('search') + .where('pubkey', '=', [pubkey]) + .execute(); + return { pubkey, followers_count, following_count: getTagSet(followList?.tags ?? [], 'p').size, notes_count, + search, }; } @@ -295,7 +302,7 @@ export interface RefreshAuthorStatsOpts { export async function refreshAuthorStats( { pubkey, kysely, store }: RefreshAuthorStatsOpts, ): Promise { - const stats = await countAuthorStats(store, pubkey); + const stats = await countAuthorStats({ store, pubkey, kysely }); await kysely.insertInto('author_stats') .values(stats) From b53c6dab626a5723e193b168d040df28035f037c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 19 Sep 2024 21:51:48 -0500 Subject: [PATCH 08/44] Add FIREHOSE_KINDS variable --- src/config.ts | 6 ++++++ src/firehose.ts | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/config.ts b/src/config.ts index f007341f..955371d4 100644 --- a/src/config.ts +++ b/src/config.ts @@ -209,6 +209,12 @@ class Conf { static get firehoseConcurrency(): number { return Math.ceil(Number(Deno.env.get('FIREHOSE_CONCURRENCY') ?? (Conf.pg.poolSize * 0.25))); } + /** Nostr event kinds of events to listen for on the firehose. */ + static get firehoseKinds(): number[] { + return (Deno.env.get('FIREHOSE_KINDS') ?? '0, 1, 3, 5, 6, 7, 9735, 10002') + .split(/[, ]+/g) + .map(Number); + } /** Whether to enable Ditto cron jobs. */ static get cronEnabled(): boolean { return optionalBooleanSchema.parse(Deno.env.get('CRON_ENABLED')) ?? true; diff --git a/src/firehose.ts b/src/firehose.ts index 85e3dc89..da8ab9c1 100644 --- a/src/firehose.ts +++ b/src/firehose.ts @@ -19,7 +19,7 @@ const sem = new Semaphore(Conf.firehoseConcurrency); export async function startFirehose(): Promise { const store = await Storages.client(); - for await (const msg of store.req([{ kinds: [0, 1, 3, 5, 6, 7, 9735, 10002], limit: 0, since: nostrNow() }])) { + for await (const msg of store.req([{ kinds: Conf.firehoseKinds, limit: 0, since: nostrNow() }])) { if (msg[0] === 'EVENT') { const event = msg[2]; console.debug(`NostrEvent<${event.kind}> ${event.id}`); From 5dca8d49507d58ca1652f0fd85dac58f404f7886 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 19 Sep 2024 21:57:59 -0500 Subject: [PATCH 09/44] pipeline: let ditto admin skip the policy --- src/pipeline.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index aaa6ca07..41ff0d64 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -48,7 +48,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise Date: Fri, 20 Sep 2024 09:53:30 -0300 Subject: [PATCH 10/44] refactor: countAuthorStats() function does not return search --- src/utils/stats.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/utils/stats.ts b/src/utils/stats.ts index 4946be3a..a44f8882 100644 --- a/src/utils/stats.ts +++ b/src/utils/stats.ts @@ -270,25 +270,18 @@ export async function updateEventStats( /** Calculate author stats from the database. */ export async function countAuthorStats( { pubkey, kysely, store }: RefreshAuthorStatsOpts, -): Promise { +): Promise> { const [{ count: followers_count }, { count: notes_count }, [followList]] = await Promise.all([ store.count([{ kinds: [3], '#p': [pubkey] }]), store.count([{ kinds: [1], authors: [pubkey] }]), store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]), ]); - const [{ search }] = await kysely - .selectFrom('author_stats') - .select('search') - .where('pubkey', '=', [pubkey]) - .execute(); - return { pubkey, followers_count, following_count: getTagSet(followList?.tags ?? [], 'p').size, notes_count, - search, }; } From b13e92400105881705ed66147166721f8731fd5c Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 20 Sep 2024 10:24:41 -0300 Subject: [PATCH 11/44] fix: build search and also return it in countAuthorStats() function --- src/utils/stats.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/utils/stats.ts b/src/utils/stats.ts index a44f8882..4573bb60 100644 --- a/src/utils/stats.ts +++ b/src/utils/stats.ts @@ -269,19 +269,27 @@ export async function updateEventStats( /** Calculate author stats from the database. */ export async function countAuthorStats( - { pubkey, kysely, store }: RefreshAuthorStatsOpts, -): Promise> { - const [{ count: followers_count }, { count: notes_count }, [followList]] = await Promise.all([ + { pubkey, store }: RefreshAuthorStatsOpts, +): Promise { + const [{ count: followers_count }, { count: notes_count }, [followList], [kind0]] = await Promise.all([ store.count([{ kinds: [3], '#p': [pubkey] }]), store.count([{ kinds: [1], authors: [pubkey] }]), store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]), + store.query([{ kinds: [0], authors: [pubkey], limit: 1 }]), ]); + let search: string = ''; + const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(kind0?.content); + if (metadata.success) { + const { name, nip05 } = metadata.data; + search = [name, nip05].filter(Boolean).join(' ').trim(); + } return { pubkey, followers_count, following_count: getTagSet(followList?.tags ?? [], 'p').size, notes_count, + search, }; } From c582b1c520f643b3072635670dc6cc19e964e93e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 20 Sep 2024 09:16:09 -0500 Subject: [PATCH 12/44] author_stats: add index on followers_count --- .../035_author_stats_followers_index.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 src/db/migrations/035_author_stats_followers_index.ts diff --git a/src/db/migrations/035_author_stats_followers_index.ts b/src/db/migrations/035_author_stats_followers_index.ts new file mode 100644 index 00000000..0509d403 --- /dev/null +++ b/src/db/migrations/035_author_stats_followers_index.ts @@ -0,0 +1,17 @@ +import { Kysely } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .createIndex('author_stats_followers_count_idx') + .ifNotExists() + .on('author_stats') + .column('followers_count desc') + .execute(); + + // This index should have never been added, because pubkey is the primary key. + await db.schema.dropIndex('idx_author_stats_pubkey').ifExists().execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropIndex('author_stats_followers_count_idx').ifExists().execute(); +} From 2b2cdca28270dfbbdf91e70c8195687440dcbeca Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 20 Sep 2024 09:58:46 -0500 Subject: [PATCH 13/44] Use char(64) for stats primary keys --- src/db/migrations/036_stats64.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 src/db/migrations/036_stats64.ts diff --git a/src/db/migrations/036_stats64.ts b/src/db/migrations/036_stats64.ts new file mode 100644 index 00000000..bad63ba4 --- /dev/null +++ b/src/db/migrations/036_stats64.ts @@ -0,0 +1,11 @@ +import { Kysely } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema.alterTable('author_stats').alterColumn('pubkey', (col) => col.setDataType('char(64)')).execute(); + await db.schema.alterTable('event_stats').alterColumn('event_id', (col) => col.setDataType('char(64)')).execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.alterTable('author_stats').alterColumn('pubkey', (col) => col.setDataType('text')).execute(); + await db.schema.alterTable('event_stats').alterColumn('event_id', (col) => col.setDataType('text')).execute(); +} From f0c8096498e7646f596896502877523a07248a96 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 20 Sep 2024 10:06:17 -0500 Subject: [PATCH 14/44] stats64: delete invalid rows first --- src/db/migrations/036_stats64.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/db/migrations/036_stats64.ts b/src/db/migrations/036_stats64.ts index bad63ba4..fa9d357e 100644 --- a/src/db/migrations/036_stats64.ts +++ b/src/db/migrations/036_stats64.ts @@ -1,11 +1,14 @@ -import { Kysely } from 'kysely'; +import { Kysely, sql } from 'kysely'; export async function up(db: Kysely): Promise { - await db.schema.alterTable('author_stats').alterColumn('pubkey', (col) => col.setDataType('char(64)')).execute(); + await db.deleteFrom('event_stats').where(sql`length(event_id)`, '>', 64).execute(); + await db.deleteFrom('author_stats').where(sql`length(pubkey)`, '>', 64).execute(); + await db.schema.alterTable('event_stats').alterColumn('event_id', (col) => col.setDataType('char(64)')).execute(); + await db.schema.alterTable('author_stats').alterColumn('pubkey', (col) => col.setDataType('char(64)')).execute(); } export async function down(db: Kysely): Promise { - await db.schema.alterTable('author_stats').alterColumn('pubkey', (col) => col.setDataType('text')).execute(); await db.schema.alterTable('event_stats').alterColumn('event_id', (col) => col.setDataType('text')).execute(); + await db.schema.alterTable('author_stats').alterColumn('pubkey', (col) => col.setDataType('text')).execute(); } From 084e3c8341b49455352494f5e556b7147ed54cbd Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 20 Sep 2024 11:46:14 -0500 Subject: [PATCH 15/44] note: test that mentions with commas get parsed correctly --- src/utils/note.test.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/utils/note.test.ts b/src/utils/note.test.ts index e8d17e89..699c4c5e 100644 --- a/src/utils/note.test.ts +++ b/src/utils/note.test.ts @@ -36,6 +36,27 @@ Deno.test('parseNoteContent parses mentions with apostrophes', () => { ); }); +Deno.test('parseNoteContent parses mentions with commas', () => { + const { html } = parseNoteContent( + `Sim. Hi nostr:npub1q3sle0kvfsehgsuexttt3ugjd8xdklxfwwkh559wxckmzddywnws6cd26p and nostr:npub1gujeqakgt7fyp6zjggxhyy7ft623qtcaay5lkc8n8gkry4cvnrzqd3f67z, any chance to have Cobrafuma as PWA?`, + [{ + id: '0461fcbecc4c3374439932d6b8f11269ccdb7cc973ad7a50ae362db135a474dd', + username: 'alex', + acct: 'alex@gleasonator.dev', + url: 'https://gleasonator.dev/@alex', + }, { + id: '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', + username: 'patrick', + acct: 'patrick@patrickdosreis.com', + url: 'https://gleasonator.dev/@patrick@patrickdosreis.com', + }], + ); + assertEquals( + html, + 'Sim. Hi @alex@gleasonator.dev and @patrick@patrickdosreis.com, any chance to have Cobrafuma as PWA?', + ); +}); + Deno.test("parseNoteContent doesn't parse invalid nostr URIs", () => { const { html } = parseNoteContent('nip19 has URIs like nostr:npub and nostr:nevent, etc.', []); assertEquals(html, 'nip19 has URIs like nostr:npub and nostr:nevent, etc.'); From 58bf286ce0bf3a9d833d6206c691a1306e2c5db5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 20 Sep 2024 12:34:54 -0500 Subject: [PATCH 16/44] Add cache metrics --- src/metrics.ts | 20 ++++++++++++++++++++ src/utils/SimpleLRU.ts | 11 ++++++++++- src/utils/favicon.ts | 3 ++- src/utils/lnurl.ts | 3 ++- src/utils/nip05.ts | 3 ++- src/utils/unfurl.ts | 2 ++ 6 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/metrics.ts b/src/metrics.ts index ac1db2ee..ccb4d382 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -84,3 +84,23 @@ export const dbQueryDurationHistogram = new Histogram({ name: 'ditto_db_query_duration_ms', help: 'Duration of database queries', }); + +export const cachedFaviconsSizeGauge = new Gauge({ + name: 'ditto_cached_favicons_size', + help: 'Number of domain favicons in cache', +}); + +export const cachedLnurlsSizeGauge = new Gauge({ + name: 'ditto_cached_lnurls_size', + help: 'Number of LNURL details in cache', +}); + +export const cachedNip05sSizeGauge = new Gauge({ + name: 'ditto_cached_nip05s_size', + help: 'Number of NIP-05 results in cache', +}); + +export const cachedLinkPreviewSizeGauge = new Gauge({ + name: 'ditto_cached_link_previews_size', + help: 'Number of link previews in cache', +}); diff --git a/src/utils/SimpleLRU.ts b/src/utils/SimpleLRU.ts index 0f2b5b37..c48b8d0c 100644 --- a/src/utils/SimpleLRU.ts +++ b/src/utils/SimpleLRU.ts @@ -1,6 +1,7 @@ // deno-lint-ignore-file ban-types import { LRUCache } from 'lru-cache'; +import { type Gauge } from 'prom-client'; type FetchFn = (key: K, opts: O) => Promise; @@ -8,6 +9,10 @@ interface FetchFnOpts { signal?: AbortSignal | null; } +type SimpleLRUOpts = LRUCache.Options & { + gauge?: Gauge; +}; + export class SimpleLRU< K extends {}, V extends {}, @@ -15,7 +20,7 @@ export class SimpleLRU< > { protected cache: LRUCache; - constructor(fetchFn: FetchFn, opts: LRUCache.Options) { + constructor(fetchFn: FetchFn, private opts: SimpleLRUOpts) { this.cache = new LRUCache({ fetchMethod: (key, _staleValue, { signal }) => fetchFn(key, { signal: signal as unknown as AbortSignal }), ...opts, @@ -24,9 +29,13 @@ export class SimpleLRU< async fetch(key: K, opts?: O): Promise { const result = await this.cache.fetch(key, opts); + + this.opts.gauge?.set(this.cache.size); + if (result === undefined) { throw new Error('SimpleLRU: fetch failed'); } + return result; } diff --git a/src/utils/favicon.ts b/src/utils/favicon.ts index 1fd0640a..49ca525a 100644 --- a/src/utils/favicon.ts +++ b/src/utils/favicon.ts @@ -2,6 +2,7 @@ import { DOMParser } from '@b-fuze/deno-dom'; import Debug from '@soapbox/stickynotes/debug'; import tldts from 'tldts'; +import { cachedFaviconsSizeGauge } from '@/metrics.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; import { Time } from '@/utils/time.ts'; import { fetchWorker } from '@/workers/fetch.ts'; @@ -37,7 +38,7 @@ const faviconCache = new SimpleLRU( throw new Error(`Favicon not found: ${key}`); }, - { max: 500, ttl: Time.hours(1) }, + { max: 500, ttl: Time.hours(1), gauge: cachedFaviconsSizeGauge }, ); export { faviconCache }; diff --git a/src/utils/lnurl.ts b/src/utils/lnurl.ts index ca7e1256..64e10fe3 100644 --- a/src/utils/lnurl.ts +++ b/src/utils/lnurl.ts @@ -1,6 +1,7 @@ import { LNURL, LNURLDetails } from '@nostrify/nostrify/ln'; import Debug from '@soapbox/stickynotes/debug'; +import { cachedLnurlsSizeGauge } from '@/metrics.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; import { Time } from '@/utils/time.ts'; import { fetchWorker } from '@/workers/fetch.ts'; @@ -20,7 +21,7 @@ const lnurlCache = new SimpleLRU( throw e; } }, - { max: 1000, ttl: Time.minutes(30) }, + { max: 1000, ttl: Time.minutes(30), gauge: cachedLnurlsSizeGauge }, ); /** Get an LNURL from a lud06 or lud16. */ diff --git a/src/utils/nip05.ts b/src/utils/nip05.ts index e9dd78cc..696b5078 100644 --- a/src/utils/nip05.ts +++ b/src/utils/nip05.ts @@ -4,6 +4,7 @@ import Debug from '@soapbox/stickynotes/debug'; import tldts from 'tldts'; import { Conf } from '@/config.ts'; +import { cachedNip05sSizeGauge } from '@/metrics.ts'; import { Storages } from '@/storages.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; import { Time } from '@/utils/time.ts'; @@ -43,7 +44,7 @@ const nip05Cache = new SimpleLRU( throw e; } }, - { max: 500, ttl: Time.hours(1) }, + { max: 500, ttl: Time.hours(1), gauge: cachedNip05sSizeGauge }, ); async function localNip05Lookup(store: NStore, localpart: string): Promise { diff --git a/src/utils/unfurl.ts b/src/utils/unfurl.ts index 8123c423..e749dbcb 100644 --- a/src/utils/unfurl.ts +++ b/src/utils/unfurl.ts @@ -5,6 +5,7 @@ import { unfurl } from 'unfurl.js'; import { Conf } from '@/config.ts'; import { PreviewCard } from '@/entities/PreviewCard.ts'; +import { cachedLinkPreviewSizeGauge } from '@/metrics.ts'; import { Time } from '@/utils/time.ts'; import { fetchWorker } from '@/workers/fetch.ts'; @@ -67,6 +68,7 @@ function unfurlCardCached(url: string, signal = AbortSignal.timeout(1000)): Prom } else { const card = unfurlCard(url, signal); previewCardCache.set(url, card); + cachedLinkPreviewSizeGauge.set(previewCardCache.size); return card; } } From efb37701f0250aa54324a40300f02b84c19ad31c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 20 Sep 2024 13:03:30 -0500 Subject: [PATCH 17/44] grafana: add cache section --- grafana/Ditto-Dashboard.json | 469 ++++++++++++++++++++++++++++++++--- 1 file changed, 428 insertions(+), 41 deletions(-) diff --git a/grafana/Ditto-Dashboard.json b/grafana/Ditto-Dashboard.json index 4b632806..52fd539f 100644 --- a/grafana/Ditto-Dashboard.json +++ b/grafana/Ditto-Dashboard.json @@ -967,8 +967,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1168,8 +1167,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1369,8 +1367,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1570,8 +1567,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1771,8 +1767,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1972,8 +1967,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2173,8 +2167,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2374,8 +2367,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2575,8 +2567,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2776,8 +2767,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2977,8 +2967,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3178,8 +3167,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3335,13 +3323,415 @@ "type": "row" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, + "id": 38, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of link previews cached for URLs shared in statuses.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 29 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "ditto_cached_link_previews_size", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Cached Link Previews", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of NIP-05 results cached for usernames that have been looked up.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 29 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "ditto_cached_nip05s_size", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Cached NIP-05 Results", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of LNURL details cached for Lightning addresses.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 29 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "ditto_cached_lnurls_size", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Cached LNURL Details", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of favicons cached for domain names.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 29 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "ditto_cached_favicons_size", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Cached Domain Favicons", + "type": "timeseries" + } + ], + "title": "Cache", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, "id": 21, "panels": [], "title": "Database", @@ -3523,7 +3913,7 @@ "h": 11, "w": 15, "x": 0, - "y": 29 + "y": 30 }, "id": 13, "options": { @@ -3679,7 +4069,7 @@ "h": 11, "w": 9, "x": 15, - "y": 29 + "y": 30 }, "id": 14, "options": { @@ -3756,7 +4146,7 @@ "h": 9, "w": 7, "x": 0, - "y": 40 + "y": 41 }, "id": 16, "options": { @@ -3853,7 +4243,7 @@ "h": 9, "w": 8, "x": 7, - "y": 40 + "y": 41 }, "id": 17, "options": { @@ -4018,7 +4408,7 @@ "h": 9, "w": 9, "x": 15, - "y": 40 + "y": 41 }, "id": 18, "options": { @@ -4073,7 +4463,7 @@ "h": 1, "w": 24, "x": 0, - "y": 49 + "y": 50 }, "id": 23, "panels": [], @@ -4130,8 +4520,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4210,7 +4599,7 @@ "h": 12, "w": 24, "x": 0, - "y": 50 + "y": 51 }, "id": 9, "options": { @@ -4295,8 +4684,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4348,7 +4736,7 @@ "h": 12, "w": 24, "x": 0, - "y": 62 + "y": 63 }, "id": 5, "options": { @@ -4431,8 +4819,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4447,7 +4834,7 @@ "h": 11, "w": 24, "x": 0, - "y": 74 + "y": 75 }, "id": 19, "options": { @@ -4565,6 +4952,6 @@ "timezone": "browser", "title": "Ditto", "uid": "ddps3ap51fv28d", - "version": 7, + "version": 8, "weekStart": "" } \ No newline at end of file From 099fec6e3109f0b504b4da8b755a3fb6fef73264 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 20 Sep 2024 13:13:26 -0500 Subject: [PATCH 18/44] Increase nip05 cache to 3000, link previews to 1000 --- src/utils/nip05.ts | 2 +- src/utils/unfurl.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/nip05.ts b/src/utils/nip05.ts index 696b5078..8f453c93 100644 --- a/src/utils/nip05.ts +++ b/src/utils/nip05.ts @@ -44,7 +44,7 @@ const nip05Cache = new SimpleLRU( throw e; } }, - { max: 500, ttl: Time.hours(1), gauge: cachedNip05sSizeGauge }, + { max: 3000, ttl: Time.hours(1), gauge: cachedNip05sSizeGauge }, ); async function localNip05Lookup(store: NStore, localpart: string): Promise { diff --git a/src/utils/unfurl.ts b/src/utils/unfurl.ts index e749dbcb..5763c151 100644 --- a/src/utils/unfurl.ts +++ b/src/utils/unfurl.ts @@ -57,7 +57,7 @@ async function unfurlCard(url: string, signal: AbortSignal): Promise>({ ttl: Time.hours(12), - max: 500, + max: 1000, }); /** Unfurl card from cache if available, otherwise fetch it. */ From 807bc784721e38fddd999d5e1ee35983b5567aca Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 20 Sep 2024 13:22:45 -0500 Subject: [PATCH 19/44] Let caches be configurable --- src/config.ts | 24 ++++++++++++++++++++++++ src/utils/favicon.ts | 3 ++- src/utils/nip05.ts | 2 +- src/utils/unfurl.ts | 6 +----- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/config.ts b/src/config.ts index 955371d4..21fbbe01 100644 --- a/src/config.ts +++ b/src/config.ts @@ -247,6 +247,30 @@ class Conf { static get zapSplitsEnabled(): boolean { return optionalBooleanSchema.parse(Deno.env.get('ZAP_SPLITS_ENABLED')) ?? false; } + /** Cache settings. */ + static caches = { + /** NIP-05 cache settings. */ + get nip05(): { max: number; ttl: number } { + return { + max: Number(Deno.env.get('DITTO_CACHE_NIP05_MAX') || 3000), + ttl: Number(Deno.env.get('DITTO_CACHE_NIP05_TTL') || 1 * 60 * 60 * 1000), + }; + }, + /** Favicon cache settings. */ + get favicon(): { max: number; ttl: number } { + return { + max: Number(Deno.env.get('DITTO_CACHE_FAVICON_MAX') || 500), + ttl: Number(Deno.env.get('DITTO_CACHE_FAVICON_TTL') || 1 * 60 * 60 * 1000), + }; + }, + /** Link preview cache settings. */ + get linkPreview(): { max: number; ttl: number } { + return { + max: Number(Deno.env.get('DITTO_CACHE_LINK_PREVIEW_MAX') || 1000), + ttl: Number(Deno.env.get('DITTO_CACHE_LINK_PREVIEW_TTL') || 12 * 60 * 60 * 1000), + }; + }, + }; } const optionalBooleanSchema = z diff --git a/src/utils/favicon.ts b/src/utils/favicon.ts index 49ca525a..2be327a6 100644 --- a/src/utils/favicon.ts +++ b/src/utils/favicon.ts @@ -6,6 +6,7 @@ import { cachedFaviconsSizeGauge } from '@/metrics.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; import { Time } from '@/utils/time.ts'; import { fetchWorker } from '@/workers/fetch.ts'; +import { Conf } from '@/config.ts'; const debug = Debug('ditto:favicon'); @@ -38,7 +39,7 @@ const faviconCache = new SimpleLRU( throw new Error(`Favicon not found: ${key}`); }, - { max: 500, ttl: Time.hours(1), gauge: cachedFaviconsSizeGauge }, + { ...Conf.caches.favicon, gauge: cachedFaviconsSizeGauge }, ); export { faviconCache }; diff --git a/src/utils/nip05.ts b/src/utils/nip05.ts index 8f453c93..388ec37f 100644 --- a/src/utils/nip05.ts +++ b/src/utils/nip05.ts @@ -44,7 +44,7 @@ const nip05Cache = new SimpleLRU( throw e; } }, - { max: 3000, ttl: Time.hours(1), gauge: cachedNip05sSizeGauge }, + { ...Conf.caches.nip05, gauge: cachedNip05sSizeGauge }, ); async function localNip05Lookup(store: NStore, localpart: string): Promise { diff --git a/src/utils/unfurl.ts b/src/utils/unfurl.ts index 5763c151..b5f5c4eb 100644 --- a/src/utils/unfurl.ts +++ b/src/utils/unfurl.ts @@ -6,7 +6,6 @@ import { unfurl } from 'unfurl.js'; import { Conf } from '@/config.ts'; import { PreviewCard } from '@/entities/PreviewCard.ts'; import { cachedLinkPreviewSizeGauge } from '@/metrics.ts'; -import { Time } from '@/utils/time.ts'; import { fetchWorker } from '@/workers/fetch.ts'; const debug = Debug('ditto:unfurl'); @@ -55,10 +54,7 @@ async function unfurlCard(url: string, signal: AbortSignal): Promise>({ - ttl: Time.hours(12), - max: 1000, -}); +const previewCardCache = new TTLCache>(Conf.caches.linkPreview); /** Unfurl card from cache if available, otherwise fetch it. */ function unfurlCardCached(url: string, signal = AbortSignal.timeout(1000)): Promise { From 454b362825d0207ffaf31adaaa0a7b737707b4ea Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 20 Sep 2024 13:24:33 -0500 Subject: [PATCH 20/44] deno lint --- src/utils/favicon.ts | 3 +-- src/utils/nip05.ts | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/utils/favicon.ts b/src/utils/favicon.ts index 2be327a6..dfe82d1b 100644 --- a/src/utils/favicon.ts +++ b/src/utils/favicon.ts @@ -2,11 +2,10 @@ import { DOMParser } from '@b-fuze/deno-dom'; import Debug from '@soapbox/stickynotes/debug'; import tldts from 'tldts'; +import { Conf } from '@/config.ts'; import { cachedFaviconsSizeGauge } from '@/metrics.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; -import { Time } from '@/utils/time.ts'; import { fetchWorker } from '@/workers/fetch.ts'; -import { Conf } from '@/config.ts'; const debug = Debug('ditto:favicon'); diff --git a/src/utils/nip05.ts b/src/utils/nip05.ts index 388ec37f..cd763d92 100644 --- a/src/utils/nip05.ts +++ b/src/utils/nip05.ts @@ -7,7 +7,6 @@ import { Conf } from '@/config.ts'; import { cachedNip05sSizeGauge } from '@/metrics.ts'; import { Storages } from '@/storages.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; -import { Time } from '@/utils/time.ts'; import { Nip05, parseNip05 } from '@/utils.ts'; import { fetchWorker } from '@/workers/fetch.ts'; From 9c9b87bc94f345de7fb522da0ff9d5900cf15b64 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 21 Sep 2024 11:36:08 -0500 Subject: [PATCH 21/44] grafana: make caches into gauges --- grafana/Ditto-Dashboard.json | 327 +++++++++++------------------------ 1 file changed, 97 insertions(+), 230 deletions(-) diff --git a/grafana/Ditto-Dashboard.json b/grafana/Ditto-Dashboard.json index 52fd539f..16162b09 100644 --- a/grafana/Ditto-Dashboard.json +++ b/grafana/Ditto-Dashboard.json @@ -1043,7 +1043,7 @@ "h": 7, "w": 6, "x": 0, - "y": 28 + "y": 33 }, "id": 34, "options": { @@ -1243,7 +1243,7 @@ "h": 7, "w": 6, "x": 6, - "y": 28 + "y": 33 }, "id": 25, "options": { @@ -1443,7 +1443,7 @@ "h": 7, "w": 6, "x": 12, - "y": 28 + "y": 33 }, "id": 31, "options": { @@ -1643,7 +1643,7 @@ "h": 7, "w": 6, "x": 18, - "y": 28 + "y": 33 }, "id": 32, "options": { @@ -1843,7 +1843,7 @@ "h": 7, "w": 6, "x": 0, - "y": 35 + "y": 40 }, "id": 30, "options": { @@ -2043,7 +2043,7 @@ "h": 7, "w": 6, "x": 6, - "y": 35 + "y": 40 }, "id": 35, "options": { @@ -2243,7 +2243,7 @@ "h": 7, "w": 6, "x": 12, - "y": 35 + "y": 40 }, "id": 36, "options": { @@ -2443,7 +2443,7 @@ "h": 7, "w": 6, "x": 18, - "y": 35 + "y": 40 }, "id": 28, "options": { @@ -2643,7 +2643,7 @@ "h": 7, "w": 6, "x": 0, - "y": 42 + "y": 47 }, "id": 26, "options": { @@ -2843,7 +2843,7 @@ "h": 7, "w": 6, "x": 6, - "y": 42 + "y": 47 }, "id": 29, "options": { @@ -3043,7 +3043,7 @@ "h": 7, "w": 6, "x": 12, - "y": 42 + "y": 47 }, "id": 37, "options": { @@ -3243,7 +3243,7 @@ "h": 7, "w": 6, "x": 18, - "y": 42 + "y": 47 }, "id": 33, "options": { @@ -3341,40 +3341,7 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], "thresholds": { @@ -3382,10 +3349,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] } @@ -3393,23 +3356,26 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 6, + "h": 4, + "w": 3, "x": 0, - "y": 29 + "y": 28 }, "id": 42, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" }, "pluginVersion": "11.2.0", "targets": [ @@ -3421,13 +3387,14 @@ "editorMode": "code", "expr": "ditto_cached_link_previews_size", "instant": false, - "legendFormat": "__auto", + "legendFormat": "Values", "range": true, "refId": "A" } ], - "title": "Cached Link Previews", - "type": "timeseries" + "title": "Link Previews", + "transparent": true, + "type": "gauge" }, { "datasource": { @@ -3438,40 +3405,7 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], "thresholds": { @@ -3479,10 +3413,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] } @@ -3490,23 +3420,26 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 29 + "h": 4, + "w": 3, + "x": 3, + "y": 28 }, "id": 41, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" }, "pluginVersion": "11.2.0", "targets": [ @@ -3518,13 +3451,14 @@ "editorMode": "code", "expr": "ditto_cached_nip05s_size", "instant": false, - "legendFormat": "__auto", + "legendFormat": "Values", "range": true, "refId": "A" } ], - "title": "Cached NIP-05 Results", - "type": "timeseries" + "title": "NIP-05 Results", + "transparent": true, + "type": "gauge" }, { "datasource": { @@ -3535,40 +3469,7 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], "thresholds": { @@ -3576,10 +3477,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] } @@ -3587,23 +3484,26 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 29 + "h": 4, + "w": 3, + "x": 6, + "y": 28 }, "id": 40, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" }, "pluginVersion": "11.2.0", "targets": [ @@ -3615,13 +3515,14 @@ "editorMode": "code", "expr": "ditto_cached_lnurls_size", "instant": false, - "legendFormat": "__auto", + "legendFormat": "Values", "range": true, "refId": "A" } ], - "title": "Cached LNURL Details", - "type": "timeseries" + "title": "LNURL Details", + "transparent": true, + "type": "gauge" }, { "datasource": { @@ -3632,40 +3533,7 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], "thresholds": { @@ -3673,10 +3541,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] } @@ -3684,23 +3548,26 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 29 + "h": 4, + "w": 3, + "x": 9, + "y": 28 }, "id": 39, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" }, "pluginVersion": "11.2.0", "targets": [ @@ -3712,13 +3579,14 @@ "editorMode": "code", "expr": "ditto_cached_favicons_size", "instant": false, - "legendFormat": "__auto", + "legendFormat": "Values", "range": true, "refId": "A" } ], - "title": "Cached Domain Favicons", - "type": "timeseries" + "title": "Domain Favicons", + "transparent": true, + "type": "gauge" } ], "title": "Cache", @@ -4335,8 +4203,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4952,6 +4819,6 @@ "timezone": "browser", "title": "Ditto", "uid": "ddps3ap51fv28d", - "version": 8, + "version": 9, "weekStart": "" } \ No newline at end of file From 5a0a2087e55698c86c476c68627e2c8032086a9f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 21 Sep 2024 14:48:50 -0500 Subject: [PATCH 22/44] relay: fix connection metrics --- src/controllers/nostr/relay.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 9f47b382..b2bb8dba 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -26,14 +26,16 @@ const LIMITER_LIMIT = 300; const limiter = new TTLCache(); +/** Connections for metrics purposes. */ +const connections = new Set(); + /** Set up the Websocket connection. */ function connectStream(socket: WebSocket, ip: string | undefined) { - let opened = false; const controllers = new Map(); socket.onopen = () => { - opened = true; - relayConnectionsGauge.inc(); + connections.add(socket); + relayConnectionsGauge.set(connections.size); }; socket.onmessage = (e) => { @@ -63,9 +65,8 @@ function connectStream(socket: WebSocket, ip: string | undefined) { }; socket.onclose = () => { - if (opened) { - relayConnectionsGauge.dec(); - } + connections.delete(socket); + relayConnectionsGauge.set(connections.size); for (const controller of controllers.values()) { controller.abort(); From 6f487c0891ade242c4c3a87037e54bf8d05feaaf Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 21 Sep 2024 15:10:59 -0500 Subject: [PATCH 23/44] grafana: fix datasource variables --- grafana/Ditto-Dashboard.json | 152 ++++++++++++++--------------------- 1 file changed, 62 insertions(+), 90 deletions(-) diff --git a/grafana/Ditto-Dashboard.json b/grafana/Ditto-Dashboard.json index 16162b09..1f595700 100644 --- a/grafana/Ditto-Dashboard.json +++ b/grafana/Ditto-Dashboard.json @@ -1,22 +1,5 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - }, - { - "name": "DS_DITTO-PG", - "label": "ditto-pg", - "description": "", - "type": "datasource", - "pluginId": "grafana-postgresql-datasource", - "pluginName": "PostgreSQL" - } - ], + "__inputs": [], "__elements": {}, "__requires": [ { @@ -253,7 +236,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"5..\",path!=\"/relay\"}[$__rate_interval])", @@ -279,7 +262,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"3..\",path!=\"/relay\"}[$__rate_interval])", @@ -309,7 +292,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "Number of idle database connections available to the server. Higher is better. At 0, the site stops working.", "fieldConfig": { @@ -385,7 +368,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "Usage of system resources.", "fieldConfig": { @@ -443,7 +426,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])))", @@ -472,7 +455,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "Number of individual database calls.", "fieldConfig": { @@ -570,7 +553,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "Total number of Nostr clients currently connected to the Nostr relay.", "fieldConfig": { @@ -652,7 +635,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "ditto_relay_connections", @@ -668,7 +651,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "Number of Nostr events that are accepted or rejected by the custom policy script.", "fieldConfig": { @@ -793,7 +776,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_policy_events_total{ok=\"false\"}[$__rate_interval])", @@ -894,7 +877,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_db_query_duration_ms_sum[$__rate_interval])", @@ -1065,7 +1048,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v1/timelines/home\"}[$__rate_interval])", @@ -1091,7 +1074,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v1/timelines/home\"}[$__rate_interval])", @@ -1121,7 +1104,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "", "fieldConfig": { @@ -1265,7 +1248,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v1/notifications\"}[$__rate_interval])", @@ -1291,7 +1274,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v1/notifications\"}[$__rate_interval])", @@ -1321,7 +1304,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "", "fieldConfig": { @@ -1465,7 +1448,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v1/accounts/verify_credentials\"}[$__rate_interval])", @@ -1491,7 +1474,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v1/accounts/verify_credentials\"}[$__rate_interval])", @@ -1521,7 +1504,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "", "fieldConfig": { @@ -1665,7 +1648,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"POST\",path=\"/oauth/token\"}[$__rate_interval])", @@ -1691,7 +1674,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"POST\",path=\"/oauth/token\"}[$__rate_interval])", @@ -1721,7 +1704,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "", "fieldConfig": { @@ -1865,7 +1848,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v1/instance\"}[$__rate_interval])", @@ -1891,7 +1874,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v1/instance\"}[$__rate_interval])", @@ -1921,7 +1904,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "", "fieldConfig": { @@ -2065,7 +2048,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v1/timelines/public\"}[$__rate_interval])", @@ -2091,7 +2074,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v1/timelines/public\"}[$__rate_interval])", @@ -2121,7 +2104,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "", "fieldConfig": { @@ -2265,7 +2248,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v1/timelines/tag/:hashtag\"}[$__rate_interval])", @@ -2291,7 +2274,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v1/timelines/tag/:hashtag\"}[$__rate_interval])", @@ -2321,7 +2304,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "", "fieldConfig": { @@ -2465,7 +2448,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v2/search\"}[$__rate_interval])", @@ -2491,7 +2474,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v2/search\"}[$__rate_interval])", @@ -2521,7 +2504,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "", "fieldConfig": { @@ -2665,7 +2648,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"POST\",path=\"/api/v1/statuses\"}[$__rate_interval])", @@ -2691,7 +2674,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"POST\",path=\"/api/v1/statuses\"}[$__rate_interval])", @@ -2721,7 +2704,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "", "fieldConfig": { @@ -2865,7 +2848,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/favourite\"}[$__rate_interval])", @@ -2891,7 +2874,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/favourite\"}[$__rate_interval])", @@ -2921,7 +2904,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "", "fieldConfig": { @@ -3065,7 +3048,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/reblog\"}[$__rate_interval])", @@ -3091,7 +3074,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/reblog\"}[$__rate_interval])", @@ -3121,7 +3104,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "", "fieldConfig": { @@ -3265,7 +3248,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"POST\",path=\"/api/v1/ditto/zap\"}[$__rate_interval])", @@ -3291,7 +3274,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"POST\",path=\"/api/v1/ditto/zap\"}[$__rate_interval])", @@ -3335,7 +3318,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "Number of link previews cached for URLs shared in statuses.", "fieldConfig": { @@ -3399,7 +3382,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "Number of NIP-05 results cached for usernames that have been looked up.", "fieldConfig": { @@ -3463,7 +3446,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "Number of LNURL details cached for Lightning addresses.", "fieldConfig": { @@ -3527,7 +3510,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "description": "Number of favicons cached for domain names.", "fieldConfig": { @@ -3608,7 +3591,7 @@ { "datasource": { "type": "grafana-postgresql-datasource", - "uid": "${DS_DITTO-PG}" + "uid": "${postgres}" }, "description": "SQL queries ranked by total time.", "fieldConfig": { @@ -3808,7 +3791,7 @@ { "datasource": { "type": "grafana-postgresql-datasource", - "uid": "${DS_DITTO-PG}" + "uid": "${postgres}" }, "editorMode": "code", "format": "table", @@ -3957,7 +3940,7 @@ { "datasource": { "type": "grafana-postgresql-datasource", - "uid": "${DS_DITTO-PG}" + "uid": "${postgres}" }, "editorMode": "code", "format": "table", @@ -4048,7 +4031,7 @@ { "datasource": { "type": "grafana-postgresql-datasource", - "uid": "${DS_DITTO-PG}" + "uid": "${postgres}" }, "editorMode": "code", "format": "table", @@ -4144,7 +4127,7 @@ { "datasource": { "type": "grafana-postgresql-datasource", - "uid": "${DS_DITTO-PG}" + "uid": "${postgres}" }, "editorMode": "code", "format": "table", @@ -4295,7 +4278,7 @@ { "datasource": { "type": "grafana-postgresql-datasource", - "uid": "${DS_DITTO-PG}" + "uid": "${postgres}" }, "editorMode": "code", "format": "table", @@ -4486,7 +4469,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "exemplar": false, @@ -4623,7 +4606,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "increase(ditto_pipeline_events_total[$__rate_interval])", @@ -4721,7 +4704,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "sum(increase(ditto_pipeline_events_total[$__rate_interval]))", @@ -4746,7 +4729,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${prometheus}" }, "editorMode": "code", "expr": "sum(increase(ditto_firehose_events_total[$__rate_interval]))", @@ -4766,17 +4749,6 @@ "tags": [], "templating": { "list": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "cdtcs576ar7cwb" - }, - "filters": [], - "hide": 0, - "name": "Filters", - "skipUrlSync": false, - "type": "adhoc" - }, { "current": {}, "description": "Prometheus datasource", @@ -4819,6 +4791,6 @@ "timezone": "browser", "title": "Ditto", "uid": "ddps3ap51fv28d", - "version": 9, + "version": 10, "weekStart": "" } \ No newline at end of file From 7ec4be50116901d1970a7d36b6944f5a65a2c20e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 21 Sep 2024 16:56:01 -0500 Subject: [PATCH 24/44] Add gleasonator-policy 0.6.0 to deno.lock --- deno.lock | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/deno.lock b/deno.lock index 657162fd..e0f85fc8 100644 --- a/deno.lock +++ b/deno.lock @@ -13,6 +13,7 @@ "jsr:@gleasonator/policy@0.5.0": "jsr:@gleasonator/policy@0.5.0", "jsr:@gleasonator/policy@0.5.1": "jsr:@gleasonator/policy@0.5.1", "jsr:@gleasonator/policy@0.5.2": "jsr:@gleasonator/policy@0.5.2", + "jsr:@gleasonator/policy@0.6.0": "jsr:@gleasonator/policy@0.6.0", "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.11", "jsr:@lambdalisue/async@^2.1.1": "jsr:@lambdalisue/async@2.1.1", "jsr:@nostrify/db@^0.32.2": "jsr:@nostrify/db@0.32.2", @@ -22,8 +23,10 @@ "jsr:@nostrify/nostrify@^0.30.0": "jsr:@nostrify/nostrify@0.30.1", "jsr:@nostrify/nostrify@^0.30.1": "jsr:@nostrify/nostrify@0.30.1", "jsr:@nostrify/nostrify@^0.31.0": "jsr:@nostrify/nostrify@0.31.0", + "jsr:@nostrify/nostrify@^0.32.0": "jsr:@nostrify/nostrify@0.32.0", "jsr:@nostrify/policies@^0.33.0": "jsr:@nostrify/policies@0.33.0", "jsr:@nostrify/policies@^0.33.1": "jsr:@nostrify/policies@0.33.1", + "jsr:@nostrify/policies@^0.34.0": "jsr:@nostrify/policies@0.34.0", "jsr:@nostrify/types@^0.30.0": "jsr:@nostrify/types@0.30.1", "jsr:@nostrify/types@^0.30.1": "jsr:@nostrify/types@0.30.1", "jsr:@soapbox/kysely-pglite@^0.0.1": "jsr:@soapbox/kysely-pglite@0.0.1", @@ -171,6 +174,13 @@ "jsr:@nostrify/policies@^0.33.1" ] }, + "@gleasonator/policy@0.6.0": { + "integrity": "77f52bb245255a61070a4970c50e2ea8e82345c1de2fef12b9d8887a20b46e6d", + "dependencies": [ + "jsr:@nostrify/nostrify@^0.32.0", + "jsr:@nostrify/policies@^0.34.0" + ] + }, "@hono/hono@4.4.6": { "integrity": "aa557ca9930787ee86b9ca1730691f1ce1c379174c2cb244d5934db2b6314453" }, @@ -279,6 +289,19 @@ "npm:zod@^3.23.8" ] }, + "@nostrify/nostrify@0.32.0": { + "integrity": "2d3b7a9cce275c150355f8e566c11f14044afd0b889afcb48e883da9467bdaa9", + "dependencies": [ + "jsr:@nostrify/types@^0.30.1", + "jsr:@std/encoding@^0.224.1", + "npm:@scure/bip32@^1.4.0", + "npm:@scure/bip39@^1.3.0", + "npm:lru-cache@^10.2.0", + "npm:nostr-tools@^2.7.0", + "npm:websocket-ts@^2.1.5", + "npm:zod@^3.23.8" + ] + }, "@nostrify/policies@0.33.0": { "integrity": "c946b06d0527298b4d7c9819d142a10f522ba09eee76c37525aa4acfc5d87aee", "dependencies": [ @@ -293,6 +316,14 @@ "npm:nostr-tools@^2.7.0" ] }, + "@nostrify/policies@0.34.0": { + "integrity": "27eb8fb36106a29e982ec7fc6bbb91bd6989f8ce11113a3ef6c528b4c2deceee", + "dependencies": [ + "jsr:@nostrify/nostrify@^0.32.0", + "jsr:@nostrify/types@^0.30.1", + "npm:nostr-tools@^2.7.0" + ] + }, "@nostrify/types@0.30.0": { "integrity": "1f38fa849cff930bd709edbf94ef9ac02f46afb8b851f86c8736517b354616da" }, From 4598782a38a282f40499774d36559f2d9e015ad4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 21 Sep 2024 18:12:57 -0500 Subject: [PATCH 25/44] Upgrade @gleasonator/policy in deno.lock --- deno.lock | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deno.lock b/deno.lock index e0f85fc8..f36c99a2 100644 --- a/deno.lock +++ b/deno.lock @@ -14,6 +14,7 @@ "jsr:@gleasonator/policy@0.5.1": "jsr:@gleasonator/policy@0.5.1", "jsr:@gleasonator/policy@0.5.2": "jsr:@gleasonator/policy@0.5.2", "jsr:@gleasonator/policy@0.6.0": "jsr:@gleasonator/policy@0.6.0", + "jsr:@gleasonator/policy@0.6.1": "jsr:@gleasonator/policy@0.6.1", "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.11", "jsr:@lambdalisue/async@^2.1.1": "jsr:@lambdalisue/async@2.1.1", "jsr:@nostrify/db@^0.32.2": "jsr:@nostrify/db@0.32.2", @@ -181,6 +182,13 @@ "jsr:@nostrify/policies@^0.34.0" ] }, + "@gleasonator/policy@0.6.1": { + "integrity": "ba763d69332a736678b068b4063709874bc64010dfc3f974818218a41deb2291", + "dependencies": [ + "jsr:@nostrify/nostrify@^0.32.0", + "jsr:@nostrify/policies@^0.34.0" + ] + }, "@hono/hono@4.4.6": { "integrity": "aa557ca9930787ee86b9ca1730691f1ce1c379174c2cb244d5934db2b6314453" }, From 323d97e5e0fca2a02e2a050510132c0ea88eaba4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 21 Sep 2024 21:01:11 -0500 Subject: [PATCH 26/44] Upgrade @gleasonator/policy in deno.lock --- deno.lock | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deno.lock b/deno.lock index f36c99a2..39156925 100644 --- a/deno.lock +++ b/deno.lock @@ -15,6 +15,7 @@ "jsr:@gleasonator/policy@0.5.2": "jsr:@gleasonator/policy@0.5.2", "jsr:@gleasonator/policy@0.6.0": "jsr:@gleasonator/policy@0.6.0", "jsr:@gleasonator/policy@0.6.1": "jsr:@gleasonator/policy@0.6.1", + "jsr:@gleasonator/policy@0.6.3": "jsr:@gleasonator/policy@0.6.3", "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.11", "jsr:@lambdalisue/async@^2.1.1": "jsr:@lambdalisue/async@2.1.1", "jsr:@nostrify/db@^0.32.2": "jsr:@nostrify/db@0.32.2", @@ -189,6 +190,13 @@ "jsr:@nostrify/policies@^0.34.0" ] }, + "@gleasonator/policy@0.6.3": { + "integrity": "7126c52edd3de21488714e66ec71f31ba9b14f8afc761ab73ac7c3ecc936625c", + "dependencies": [ + "jsr:@nostrify/nostrify@^0.32.0", + "jsr:@nostrify/policies@^0.34.0" + ] + }, "@hono/hono@4.4.6": { "integrity": "aa557ca9930787ee86b9ca1730691f1ce1c379174c2cb244d5934db2b6314453" }, From fc7228e183174ab83e9813417c1fa1eb6c022238 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 21 Sep 2024 21:40:52 -0500 Subject: [PATCH 27/44] Streaming metrics --- src/controllers/api/streaming.ts | 8 ++++++-- src/metrics.ts | 5 +++++ src/storages.ts | 3 ++- src/storages/InternalRelay.ts | 9 +++++++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index aafa9915..31e8a398 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -61,6 +61,8 @@ const LIMITER_LIMIT = 100; const limiter = new TTLCache(); +const connections = new Set(); + const streamingController: AppController = async (c) => { const upgrade = c.req.header('upgrade'); const token = c.req.header('sec-websocket-protocol'); @@ -126,7 +128,8 @@ const streamingController: AppController = async (c) => { } socket.onopen = async () => { - streamingConnectionsGauge.inc(); + connections.add(socket); + streamingConnectionsGauge.set(connections.size); if (!stream) return; const topicFilter = await topicToFilter(stream, c.req.query(), pubkey); @@ -186,7 +189,8 @@ const streamingController: AppController = async (c) => { }; socket.onclose = () => { - streamingConnectionsGauge.dec(); + connections.delete(socket); + streamingConnectionsGauge.set(connections.size); controller.abort(); }; diff --git a/src/metrics.ts b/src/metrics.ts index ccb4d382..8d1a6531 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -104,3 +104,8 @@ export const cachedLinkPreviewSizeGauge = new Gauge({ name: 'ditto_cached_link_previews_size', help: 'Number of link previews in cache', }); + +export const internalSubscriptionsSizeGauge = new Gauge({ + name: 'ditto_internal_subscriptions_size', + help: "Number of active subscriptions to Ditto's internal relay", +}); diff --git a/src/storages.ts b/src/storages.ts index 073b6135..8c7f2d28 100644 --- a/src/storages.ts +++ b/src/storages.ts @@ -2,6 +2,7 @@ import { Conf } from '@/config.ts'; import { DittoDatabase } from '@/db/DittoDatabase.ts'; import { DittoDB } from '@/db/DittoDB.ts'; +import { internalSubscriptionsSizeGauge } from '@/metrics.ts'; import { AdminStore } from '@/storages/AdminStore.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; import { SearchStore } from '@/storages/search-store.ts'; @@ -61,7 +62,7 @@ export class Storages { /** Internal pubsub relay between controllers and the pipeline. */ public static async pubsub(): Promise { if (!this._pubsub) { - this._pubsub = Promise.resolve(new InternalRelay()); + this._pubsub = Promise.resolve(new InternalRelay({ gauge: internalSubscriptionsSizeGauge })); } return this._pubsub; } diff --git a/src/storages/InternalRelay.ts b/src/storages/InternalRelay.ts index 93a480e1..71a13b31 100644 --- a/src/storages/InternalRelay.ts +++ b/src/storages/InternalRelay.ts @@ -10,10 +10,15 @@ import { } from '@nostrify/nostrify'; import { Machina } from '@nostrify/nostrify/utils'; import { matchFilter } from 'nostr-tools'; +import { Gauge } from 'prom-client'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { purifyEvent } from '@/utils/purify.ts'; +interface InternalRelayOpts { + gauge?: Gauge; +} + /** * PubSub event store for streaming events within the application. * The pipeline should push events to it, then anything in the application can subscribe to it. @@ -21,6 +26,8 @@ import { purifyEvent } from '@/utils/purify.ts'; export class InternalRelay implements NRelay { private subs = new Map }>(); + constructor(private opts: InternalRelayOpts = {}) {} + async *req( filters: NostrFilter[], opts?: { signal?: AbortSignal }, @@ -31,6 +38,7 @@ export class InternalRelay implements NRelay { yield ['EOSE', id]; this.subs.set(id, { filters, machina }); + this.opts.gauge?.set(this.subs.size); try { for await (const event of machina) { @@ -38,6 +46,7 @@ export class InternalRelay implements NRelay { } } finally { this.subs.delete(id); + this.opts.gauge?.set(this.subs.size); } } From 195cf9f44e9000c41fbd4d0e81360879eff93639 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 21 Sep 2024 21:44:24 -0500 Subject: [PATCH 28/44] metrics: add messages sent and received by streaming API --- src/controllers/api/streaming.ts | 9 ++++++++- src/metrics.ts | 10 ++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index 31e8a398..4bbe6177 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -4,7 +4,11 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { streamingConnectionsGauge } from '@/metrics.ts'; +import { + streamingClientMessagesCounter, + streamingConnectionsGauge, + streamingServerMessagesCounter, +} from '@/metrics.ts'; import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; import { getFeedPubkeys } from '@/queries.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; @@ -96,6 +100,7 @@ const streamingController: AppController = async (c) => { function send(e: StreamingEvent) { if (socket.readyState === WebSocket.OPEN) { debug('send', e.event, e.payload); + streamingServerMessagesCounter.inc(); socket.send(JSON.stringify(e)); } } @@ -172,6 +177,8 @@ const streamingController: AppController = async (c) => { }; socket.onmessage = (e) => { + streamingClientMessagesCounter.inc(); + if (ip) { const count = limiter.get(ip) ?? 0; limiter.set(ip, count + 1, { ttl: LIMITER_WINDOW }); diff --git a/src/metrics.ts b/src/metrics.ts index 8d1a6531..1e20747b 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -17,6 +17,16 @@ export const streamingConnectionsGauge = new Gauge({ help: 'Number of active connections to the streaming API', }); +export const streamingServerMessagesCounter = new Counter({ + name: 'ditto_streaming_server_messages_total', + help: 'Total number of messages sent from the streaming API', +}); + +export const streamingClientMessagesCounter = new Counter({ + name: 'ditto_streaming_client_messages_total', + help: 'Total number of messages received by the streaming API', +}); + export const fetchCounter = new Counter({ name: 'ditto_fetch_total', help: 'Total number of fetch requests', From 2587b57794c67d7d0975b1a2988e2229fd0d0466 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 21 Sep 2024 22:27:00 -0500 Subject: [PATCH 29/44] grafana: add streaming API section --- grafana/Ditto-Dashboard.json | 315 ++++++++++++++++++++++++++++------- 1 file changed, 259 insertions(+), 56 deletions(-) diff --git a/grafana/Ditto-Dashboard.json b/grafana/Ditto-Dashboard.json index 1f595700..d722d5d9 100644 --- a/grafana/Ditto-Dashboard.json +++ b/grafana/Ditto-Dashboard.json @@ -792,7 +792,6 @@ }, { "datasource": { - "default": false, "type": "prometheus", "uid": "${prometheus}" }, @@ -902,7 +901,6 @@ "panels": [ { "datasource": { - "default": false, "type": "prometheus", "uid": "${prometheus}" }, @@ -950,7 +948,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1026,7 +1025,7 @@ "h": 7, "w": 6, "x": 0, - "y": 33 + "y": 28 }, "id": 34, "options": { @@ -1150,7 +1149,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1226,7 +1226,7 @@ "h": 7, "w": 6, "x": 6, - "y": 33 + "y": 28 }, "id": 25, "options": { @@ -1350,7 +1350,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1426,7 +1427,7 @@ "h": 7, "w": 6, "x": 12, - "y": 33 + "y": 28 }, "id": 31, "options": { @@ -1550,7 +1551,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1626,7 +1628,7 @@ "h": 7, "w": 6, "x": 18, - "y": 33 + "y": 28 }, "id": 32, "options": { @@ -1750,7 +1752,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1826,7 +1829,7 @@ "h": 7, "w": 6, "x": 0, - "y": 40 + "y": 35 }, "id": 30, "options": { @@ -1950,7 +1953,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2026,7 +2030,7 @@ "h": 7, "w": 6, "x": 6, - "y": 40 + "y": 35 }, "id": 35, "options": { @@ -2150,7 +2154,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2226,7 +2231,7 @@ "h": 7, "w": 6, "x": 12, - "y": 40 + "y": 35 }, "id": 36, "options": { @@ -2350,7 +2355,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2426,7 +2432,7 @@ "h": 7, "w": 6, "x": 18, - "y": 40 + "y": 35 }, "id": 28, "options": { @@ -2550,7 +2556,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2626,7 +2633,7 @@ "h": 7, "w": 6, "x": 0, - "y": 47 + "y": 42 }, "id": 26, "options": { @@ -2750,7 +2757,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2826,7 +2834,7 @@ "h": 7, "w": 6, "x": 6, - "y": 47 + "y": 42 }, "id": 29, "options": { @@ -2950,7 +2958,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3026,7 +3035,7 @@ "h": 7, "w": 6, "x": 12, - "y": 47 + "y": 42 }, "id": 37, "options": { @@ -3150,7 +3159,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3226,7 +3236,7 @@ "h": 7, "w": 6, "x": 18, - "y": 47 + "y": 42 }, "id": 33, "options": { @@ -3313,6 +3323,198 @@ "x": 0, "y": 28 }, + "id": 43, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "description": "Number of active connections opened by clients to the Streaming API.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 0, + "y": 50 + }, + "id": 44, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "ditto_streaming_connections", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Streaming Clients", + "type": "gauge" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${prometheus}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 21, + "x": 3, + "y": 50 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_streaming_server_messages_total[$__rate_interval])", + "instant": false, + "legendFormat": "Server", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_streaming_client_messages_total[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "Client", + "range": true, + "refId": "B" + } + ], + "title": "Streaming Messages", + "type": "timeseries" + } + ], + "title": "Streaming", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, "id": 38, "panels": [ { @@ -3331,7 +3533,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] } @@ -3342,7 +3545,7 @@ "h": 4, "w": 3, "x": 0, - "y": 28 + "y": 51 }, "id": 42, "options": { @@ -3395,7 +3598,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] } @@ -3406,7 +3610,7 @@ "h": 4, "w": 3, "x": 3, - "y": 28 + "y": 51 }, "id": 41, "options": { @@ -3459,7 +3663,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] } @@ -3470,7 +3675,7 @@ "h": 4, "w": 3, "x": 6, - "y": 28 + "y": 51 }, "id": 40, "options": { @@ -3523,7 +3728,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] } @@ -3534,7 +3740,7 @@ "h": 4, "w": 3, "x": 9, - "y": 28 + "y": 51 }, "id": 39, "options": { @@ -3581,7 +3787,7 @@ "h": 1, "w": 24, "x": 0, - "y": 29 + "y": 30 }, "id": 21, "panels": [], @@ -3764,7 +3970,7 @@ "h": 11, "w": 15, "x": 0, - "y": 30 + "y": 31 }, "id": 13, "options": { @@ -3823,7 +4029,6 @@ }, { "datasource": { - "name": "${postgres}", "type": "grafana-postgresql-datasource", "uid": "${postgres}" }, @@ -3920,7 +4125,7 @@ "h": 11, "w": 9, "x": 15, - "y": 30 + "y": 31 }, "id": 14, "options": { @@ -3971,7 +4176,6 @@ }, { "datasource": { - "name": "${postgres}", "type": "grafana-postgresql-datasource", "uid": "${postgres}" }, @@ -3997,7 +4201,7 @@ "h": 9, "w": 7, "x": 0, - "y": 41 + "y": 42 }, "id": 16, "options": { @@ -4068,7 +4272,6 @@ }, { "datasource": { - "name": "${postgres}", "type": "grafana-postgresql-datasource", "uid": "${postgres}" }, @@ -4094,7 +4297,7 @@ "h": 9, "w": 8, "x": 7, - "y": 41 + "y": 42 }, "id": 17, "options": { @@ -4164,7 +4367,6 @@ }, { "datasource": { - "name": "${postgres}", "type": "grafana-postgresql-datasource", "uid": "${postgres}" }, @@ -4186,7 +4388,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4258,7 +4461,7 @@ "h": 9, "w": 9, "x": 15, - "y": 41 + "y": 42 }, "id": 18, "options": { @@ -4313,7 +4516,7 @@ "h": 1, "w": 24, "x": 0, - "y": 50 + "y": 51 }, "id": 23, "panels": [], @@ -4322,7 +4525,6 @@ }, { "datasource": { - "default": false, "type": "prometheus", "uid": "${prometheus}" }, @@ -4370,7 +4572,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4449,7 +4652,7 @@ "h": 12, "w": 24, "x": 0, - "y": 51 + "y": 52 }, "id": 9, "options": { @@ -4486,7 +4689,6 @@ }, { "datasource": { - "default": false, "type": "prometheus", "uid": "${prometheus}" }, @@ -4534,7 +4736,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4586,7 +4789,7 @@ "h": 12, "w": 24, "x": 0, - "y": 63 + "y": 64 }, "id": 5, "options": { @@ -4621,7 +4824,6 @@ }, { "datasource": { - "default": false, "type": "prometheus", "uid": "${prometheus}" }, @@ -4669,7 +4871,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4684,7 +4887,7 @@ "h": 11, "w": 24, "x": 0, - "y": 75 + "y": 76 }, "id": 19, "options": { @@ -4791,6 +4994,6 @@ "timezone": "browser", "title": "Ditto", "uid": "ddps3ap51fv28d", - "version": 10, + "version": 11, "weekStart": "" } \ No newline at end of file From c4391ccd99cfd79bea6040a93e21cf118857543f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 22 Sep 2024 11:49:37 -0500 Subject: [PATCH 30/44] pipeline: fix events not being pushed through streaming API Fixes https://gitlab.com/soapbox-pub/ditto/-/issues/206 --- src/pipeline.ts | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index e94ed21e..d89c6cef 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -62,14 +62,17 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { From ce562b3b6a3b3ba7cbfd329cafaeb665ae2f9ded Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 22 Sep 2024 14:41:34 -0500 Subject: [PATCH 31/44] Upgrade @gleasonator/policy in deno.lock --- deno.lock | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deno.lock b/deno.lock index 39156925..ff81d4bb 100644 --- a/deno.lock +++ b/deno.lock @@ -16,6 +16,7 @@ "jsr:@gleasonator/policy@0.6.0": "jsr:@gleasonator/policy@0.6.0", "jsr:@gleasonator/policy@0.6.1": "jsr:@gleasonator/policy@0.6.1", "jsr:@gleasonator/policy@0.6.3": "jsr:@gleasonator/policy@0.6.3", + "jsr:@gleasonator/policy@0.6.4": "jsr:@gleasonator/policy@0.6.4", "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.11", "jsr:@lambdalisue/async@^2.1.1": "jsr:@lambdalisue/async@2.1.1", "jsr:@nostrify/db@^0.32.2": "jsr:@nostrify/db@0.32.2", @@ -197,6 +198,13 @@ "jsr:@nostrify/policies@^0.34.0" ] }, + "@gleasonator/policy@0.6.4": { + "integrity": "fd91c94546edd1de1faa80cb3248699b2f010ef1bdd89818dbc4a03e7606e0bb", + "dependencies": [ + "jsr:@nostrify/nostrify@^0.32.0", + "jsr:@nostrify/policies@^0.34.0" + ] + }, "@hono/hono@4.4.6": { "integrity": "aa557ca9930787ee86b9ca1730691f1ce1c379174c2cb244d5934db2b6314453" }, From 6745e96c64cf037266042b4557fab26300aa2b20 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 22 Sep 2024 16:06:10 -0500 Subject: [PATCH 32/44] SimpleLRU: fix repeated calls fetching --- src/utils/SimpleLRU.test.ts | 21 +++++++++++++++++++++ src/utils/SimpleLRU.ts | 10 ++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 src/utils/SimpleLRU.test.ts diff --git a/src/utils/SimpleLRU.test.ts b/src/utils/SimpleLRU.test.ts new file mode 100644 index 00000000..a73e4f36 --- /dev/null +++ b/src/utils/SimpleLRU.test.ts @@ -0,0 +1,21 @@ +import { SimpleLRU } from '@/utils/SimpleLRU.ts'; +import { assertEquals, assertRejects } from '@std/assert'; + +Deno.test("SimpleLRU doesn't repeat failed calls", async () => { + let calls = 0; + + const cache = new SimpleLRU( + // deno-lint-ignore require-await + async () => { + calls++; + throw new Error('gg'); + }, + { max: 100 }, + ); + + await assertRejects(() => cache.fetch('foo')); + assertEquals(calls, 1); + + await assertRejects(() => cache.fetch('foo')); + assertEquals(calls, 1); +}); diff --git a/src/utils/SimpleLRU.ts b/src/utils/SimpleLRU.ts index c48b8d0c..f18a6211 100644 --- a/src/utils/SimpleLRU.ts +++ b/src/utils/SimpleLRU.ts @@ -22,7 +22,13 @@ export class SimpleLRU< constructor(fetchFn: FetchFn, private opts: SimpleLRUOpts) { this.cache = new LRUCache({ - fetchMethod: (key, _staleValue, { signal }) => fetchFn(key, { signal: signal as unknown as AbortSignal }), + async fetchMethod(key, _staleValue, { signal }) { + try { + return await fetchFn(key, { signal: signal as unknown as AbortSignal }); + } catch { + return null as unknown as V; + } + }, ...opts, }); } @@ -32,7 +38,7 @@ export class SimpleLRU< this.opts.gauge?.set(this.cache.size); - if (result === undefined) { + if (result === undefined || result === null) { throw new Error('SimpleLRU: fetch failed'); } From cd66234af7769b27d071bfa5903111f3d91974c8 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 22 Sep 2024 17:35:04 -0500 Subject: [PATCH 33/44] Ugrade to Deno 2.0 --- .gitlab-ci.yml | 2 +- scripts/db-export.ts | 2 +- src/pipeline.test.ts | 125 ----------------------------------- src/storages/EventsDB.ts | 16 ++--- src/workers/policy.ts | 2 +- src/workers/policy.worker.ts | 2 +- 6 files changed, 12 insertions(+), 137 deletions(-) delete mode 100644 src/pipeline.test.ts diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 48c5b253..d25f61ba 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:1.46.3 +image: denoland/deno:2.0.0-rc.4 default: interruptible: true diff --git a/scripts/db-export.ts b/scripts/db-export.ts index 71939105..780a5ebc 100644 --- a/scripts/db-export.ts +++ b/scripts/db-export.ts @@ -102,7 +102,7 @@ async function exportEvents(args: ExportFilter) { let filter: NostrFilter = {}; try { filter = buildFilter(args); - } catch (e) { + } catch (e: any) { die(1, e.message || e.toString()); } diff --git a/src/pipeline.test.ts b/src/pipeline.test.ts deleted file mode 100644 index 76b1fe51..00000000 --- a/src/pipeline.test.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { assertEquals } from '@std/assert'; -import { generateSecretKey } from 'nostr-tools'; - -import { createTestDB, genEvent } from '@/test.ts'; -import { handleZaps } from '@/pipeline.ts'; - -Deno.test('store one zap receipt in nostr_events; convert it into event_zaps table format and store it', async () => { - await using db = await createTestDB(); - const kysely = db.kysely; - - const sk = generateSecretKey(); - - const event = genEvent({ - 'id': '67b48a14fb66c60c8f9070bdeb37afdfcc3d08ad01989460448e4081eddda446', - 'pubkey': '9630f464cca6a5147aa8a35f0bcdd3ce485324e732fd39e09233b1d848238f31', - 'created_at': 1674164545, - 'kind': 9735, - 'tags': [ - ['p', '32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245'], - ['P', '97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322'], - ['e', '3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8'], - [ - 'bolt11', - 'lnbc10u1p3unwfusp5t9r3yymhpfqculx78u027lxspgxcr2n2987mx2j55nnfs95nxnzqpp5jmrh92pfld78spqs78v9euf2385t83uvpwk9ldrlvf6ch7tpascqhp5zvkrmemgth3tufcvflmzjzfvjt023nazlhljz2n9hattj4f8jq8qxqyjw5qcqpjrzjqtc4fc44feggv7065fqe5m4ytjarg3repr5j9el35xhmtfexc42yczarjuqqfzqqqqqqqqlgqqqqqqgq9q9qxpqysgq079nkq507a5tw7xgttmj4u990j7wfggtrasah5gd4ywfr2pjcn29383tphp4t48gquelz9z78p4cq7ml3nrrphw5w6eckhjwmhezhnqpy6gyf0', - ], - [ - 'description', - '{"pubkey":"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322","content":"","id":"d9cc14d50fcb8c27539aacf776882942c1a11ea4472f8cdec1dea82fab66279d","created_at":1674164539,"sig":"77127f636577e9029276be060332ea565deaf89ff215a494ccff16ae3f757065e2bc59b2e8c113dd407917a010b3abd36c8d7ad84c0e3ab7dab3a0b0caa9835d","kind":9734,"tags":[["e","3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8"],["p","32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245"],["relays","wss://relay.damus.io","wss://nostr-relay.wlvs.space","wss://nostr.fmt.wiz.biz","wss://relay.nostr.bg","wss://nostr.oxtr.dev","wss://nostr.v0l.io","wss://brb.io","wss://nostr.bitcoiner.social","ws://monad.jb55.com:8080","wss://relay.snort.social"]]}', - ], - ['preimage', '5d006d2cf1e73c7148e7519a4c68adc81642ce0e25a432b2434c99f97344c15f'], - ], - 'content': '', - }, sk); - - await db.store.event(event); - - await handleZaps(kysely, event); - await handleZaps(kysely, event); - - const zapReceipts = await db.store.query([{}]); - const customEventZaps = await kysely.selectFrom('event_zaps').selectAll().execute(); - - assertEquals(zapReceipts.length, 1); // basic check - assertEquals(customEventZaps.length, 1); // basic check - - const expected = { - receipt_id: event.id, - target_event_id: '3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8', - sender_pubkey: '97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322', - amount_millisats: 1000000, - comment: '', - }; - - assertEquals(customEventZaps[0], expected); -}); - -// The function tests below only handle the edge cases and don't assert anything -// If no error happens = ok - -Deno.test('zap receipt does not have a "description" tag', async () => { - await using db = await createTestDB(); - const kysely = db.kysely; - - const sk = generateSecretKey(); - - const event = genEvent({ kind: 9735 }, sk); - - await handleZaps(kysely, event); - - // no error happened = ok -}); - -Deno.test('zap receipt does not have a zap request stringified value in the "description" tag', async () => { - await using db = await createTestDB(); - const kysely = db.kysely; - - const sk = generateSecretKey(); - - const event = genEvent({ kind: 9735, tags: [['description', 'yolo']] }, sk); - - await handleZaps(kysely, event); - - // no error happened = ok -}); - -Deno.test('zap receipt does not have a "bolt11" tag', async () => { - await using db = await createTestDB(); - const kysely = db.kysely; - - const sk = generateSecretKey(); - - const event = genEvent({ - kind: 9735, - tags: [[ - 'description', - '{"pubkey":"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322","content":"","id":"d9cc14d50fcb8c27539aacf776882942c1a11ea4472f8cdec1dea82fab66279d","created_at":1674164539,"sig":"77127f636577e9029276be060332ea565deaf89ff215a494ccff16ae3f757065e2bc59b2e8c113dd407917a010b3abd36c8d7ad84c0e3ab7dab3a0b0caa9835d","kind":9734,"tags":[["e","3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8"],["p","32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245"],["relays","wss://relay.damus.io","wss://nostr-relay.wlvs.space","wss://nostr.fmt.wiz.biz","wss://relay.nostr.bg","wss://nostr.oxtr.dev","wss://nostr.v0l.io","wss://brb.io","wss://nostr.bitcoiner.social","ws://monad.jb55.com:8080","wss://relay.snort.social"]]}', - ]], - }, sk); - - await handleZaps(kysely, event); - - // no error happened = ok -}); - -Deno.test('zap request inside zap receipt does not have an "e" tag', async () => { - await using db = await createTestDB(); - const kysely = db.kysely; - - const sk = generateSecretKey(); - - const event = genEvent({ - kind: 9735, - tags: [[ - 'bolt11', - 'lnbc10u1p3unwfusp5t9r3yymhpfqculx78u027lxspgxcr2n2987mx2j55nnfs95nxnzqpp5jmrh92pfld78spqs78v9euf2385t83uvpwk9ldrlvf6ch7tpascqhp5zvkrmemgth3tufcvflmzjzfvjt023nazlhljz2n9hattj4f8jq8qxqyjw5qcqpjrzjqtc4fc44feggv7065fqe5m4ytjarg3repr5j9el35xhmtfexc42yczarjuqqfzqqqqqqqqlgqqqqqqgq9q9qxpqysgq079nkq507a5tw7xgttmj4u990j7wfggtrasah5gd4ywfr2pjcn29383tphp4t48gquelz9z78p4cq7ml3nrrphw5w6eckhjwmhezhnqpy6gyf0', - ], [ - 'description', - '{"pubkey":"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322","content":"","id":"d9cc14d50fcb8c27539aacf776882942c1a11ea4472f8cdec1dea82fab66279d","created_at":1674164539,"sig":"77127f636577e9029276be060332ea565deaf89ff215a494ccff16ae3f757065e2bc59b2e8c113dd407917a010b3abd36c8d7ad84c0e3ab7dab3a0b0caa9835d","kind":9734,"tags":[["p","32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245"],["relays","wss://relay.damus.io","wss://nostr-relay.wlvs.space","wss://nostr.fmt.wiz.biz","wss://relay.nostr.bg","wss://nostr.oxtr.dev","wss://nostr.v0l.io","wss://brb.io","wss://nostr.bitcoiner.social","ws://monad.jb55.com:8080","wss://relay.snort.social"]]}', - ]], - }, sk); - - await handleZaps(kysely, event); - - // no error happened = ok -}); diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 7d61c2de..1bf3cd86 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -59,7 +59,7 @@ class EventsDB extends NPostgres { } /** Insert an event (and its tags) into the database. */ - async event(event: NostrEvent, opts: { signal?: AbortSignal; timeout?: number } = {}): Promise { + override async event(event: NostrEvent, opts: { signal?: AbortSignal; timeout?: number } = {}): Promise { event = purifyEvent(event); this.console.debug('EVENT', JSON.stringify(event)); dbEventsCounter.inc({ kind: event.kind }); @@ -72,7 +72,7 @@ class EventsDB extends NPostgres { try { await super.event(event, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); - } catch (e) { + } catch (e: any) { if (e.message === 'Cannot add a deleted event') { throw new RelayError('blocked', 'event deleted by user'); } else if (e.message === 'Cannot replace an event with an older event') { @@ -144,7 +144,7 @@ class EventsDB extends NPostgres { } } - protected getFilterQuery(trx: Kysely, filter: NostrFilter) { + protected override getFilterQuery(trx: Kysely, filter: NostrFilter) { if (filter.search) { const tokens = NIP50.parseInput(filter.search); @@ -172,7 +172,7 @@ class EventsDB extends NPostgres { } /** Get events for filters from the database. */ - async query( + override async query( filters: NostrFilter[], opts: { signal?: AbortSignal; timeout?: number; limit?: number } = {}, ): Promise { @@ -200,13 +200,13 @@ class EventsDB extends NPostgres { } /** Delete events based on filters from the database. */ - async remove(filters: NostrFilter[], opts: { signal?: AbortSignal; timeout?: number } = {}): Promise { + override async remove(filters: NostrFilter[], opts: { signal?: AbortSignal; timeout?: number } = {}): Promise { this.console.debug('DELETE', JSON.stringify(filters)); return super.remove(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } /** Get number of events that would be returned by filters. */ - async count( + override async count( filters: NostrFilter[], opts: { signal?: AbortSignal; timeout?: number } = {}, ): Promise<{ count: number; approximate: any }> { @@ -218,7 +218,7 @@ class EventsDB extends NPostgres { } /** Return only the tags that should be indexed. */ - static indexTags(event: NostrEvent): string[][] { + static override indexTags(event: NostrEvent): string[][] { const tagCounts: Record = {}; function getCount(name: string) { @@ -325,7 +325,7 @@ class EventsDB extends NPostgres { return filters; } - async transaction(callback: (store: NPostgres, kysely: Kysely) => Promise): Promise { + override async transaction(callback: (store: NPostgres, kysely: Kysely) => Promise): Promise { return super.transaction((store, kysely) => callback(store, kysely as unknown as Kysely)); } } diff --git a/src/workers/policy.ts b/src/workers/policy.ts index f86f9d9b..7bf3f7bf 100644 --- a/src/workers/policy.ts +++ b/src/workers/policy.ts @@ -31,7 +31,7 @@ try { adminPubkey: Conf.pubkey, }); console.debug(`Using custom policy: ${Conf.policy}`); -} catch (e) { +} catch (e: any) { if (e.message.includes('Module not found')) { console.debug('Custom policy not found '); } else { diff --git a/src/workers/policy.worker.ts b/src/workers/policy.worker.ts index 1d65f405..c7a16e30 100644 --- a/src/workers/policy.worker.ts +++ b/src/workers/policy.worker.ts @@ -45,7 +45,7 @@ export class CustomPolicy implements NPolicy { try { const Policy = (await import(path)).default; this.policy = new Policy({ store }); - } catch (e) { + } catch (e: any) { if (e.message.includes('Module not found')) { this.policy = new NoOpPolicy(); } From b6e7a5529f862f8fe28e89e12c901da8686f13c2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 22 Sep 2024 17:38:31 -0500 Subject: [PATCH 34/44] Downgrade GitLab CI to rc3 for now --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d25f61ba..2ec0892f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:2.0.0-rc.4 +image: denoland/deno:2.0.0-rc.3 default: interruptible: true From ebce4a8b1d75de9a256e1a299eb9a92882171049 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 22 Sep 2024 17:46:21 -0500 Subject: [PATCH 35/44] Fix a few more type errors, whoops --- src/controllers/nostr/relay.ts | 2 +- src/signers/ConnectSigner.ts | 10 +++++----- src/trends.ts | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index b2bb8dba..3fd6b29a 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -104,7 +104,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) { for (const event of await store.query(filters, { limit: FILTER_LIMIT, timeout: Conf.db.timeouts.relay })) { send(['EVENT', subId, event]); } - } catch (e) { + } catch (e: any) { if (e instanceof RelayError) { send(['CLOSED', subId, e.message]); } else if (e.message.includes('timeout')) { diff --git a/src/signers/ConnectSigner.ts b/src/signers/ConnectSigner.ts index 6501bb8b..26a9cbc9 100644 --- a/src/signers/ConnectSigner.ts +++ b/src/signers/ConnectSigner.ts @@ -30,7 +30,7 @@ export class ConnectSigner implements NostrSigner { const signer = await this.signer; try { return await signer.signEvent(event); - } catch (e) { + } catch (e: any) { if (e.name === 'AbortError') { throw new HTTPException(408, { message: 'The event was not signed quickly enough' }); } else { @@ -44,7 +44,7 @@ export class ConnectSigner implements NostrSigner { const signer = await this.signer; try { return await signer.nip04.encrypt(pubkey, plaintext); - } catch (e) { + } catch (e: any) { if (e.name === 'AbortError') { throw new HTTPException(408, { message: 'Text was not encrypted quickly enough', @@ -59,7 +59,7 @@ export class ConnectSigner implements NostrSigner { const signer = await this.signer; try { return await signer.nip04.decrypt(pubkey, ciphertext); - } catch (e) { + } catch (e: any) { if (e.name === 'AbortError') { throw new HTTPException(408, { message: 'Text was not decrypted quickly enough', @@ -76,7 +76,7 @@ export class ConnectSigner implements NostrSigner { const signer = await this.signer; try { return await signer.nip44.encrypt(pubkey, plaintext); - } catch (e) { + } catch (e: any) { if (e.name === 'AbortError') { throw new HTTPException(408, { message: 'Text was not encrypted quickly enough', @@ -91,7 +91,7 @@ export class ConnectSigner implements NostrSigner { const signer = await this.signer; try { return await signer.nip44.decrypt(pubkey, ciphertext); - } catch (e) { + } catch (e: any) { if (e.name === 'AbortError') { throw new HTTPException(408, { message: 'Text was not decrypted quickly enough', diff --git a/src/trends.ts b/src/trends.ts index de91a33d..23f7ea4d 100644 --- a/src/trends.ts +++ b/src/trends.ts @@ -106,7 +106,7 @@ export async function updateTrendingTags( await handleEvent(label, signal); console.info(`Trending ${l} updated.`); - } catch (e) { + } catch (e: any) { console.error(`Error updating trending ${l}: ${e.message}`); } } From 8267916466fed6a5889383d530130de24649b0bf Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 22 Sep 2024 18:01:07 -0500 Subject: [PATCH 36/44] Add back pipeline test --- src/pipeline.test.ts | 125 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 src/pipeline.test.ts diff --git a/src/pipeline.test.ts b/src/pipeline.test.ts new file mode 100644 index 00000000..76b1fe51 --- /dev/null +++ b/src/pipeline.test.ts @@ -0,0 +1,125 @@ +import { assertEquals } from '@std/assert'; +import { generateSecretKey } from 'nostr-tools'; + +import { createTestDB, genEvent } from '@/test.ts'; +import { handleZaps } from '@/pipeline.ts'; + +Deno.test('store one zap receipt in nostr_events; convert it into event_zaps table format and store it', async () => { + await using db = await createTestDB(); + const kysely = db.kysely; + + const sk = generateSecretKey(); + + const event = genEvent({ + 'id': '67b48a14fb66c60c8f9070bdeb37afdfcc3d08ad01989460448e4081eddda446', + 'pubkey': '9630f464cca6a5147aa8a35f0bcdd3ce485324e732fd39e09233b1d848238f31', + 'created_at': 1674164545, + 'kind': 9735, + 'tags': [ + ['p', '32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245'], + ['P', '97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322'], + ['e', '3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8'], + [ + 'bolt11', + 'lnbc10u1p3unwfusp5t9r3yymhpfqculx78u027lxspgxcr2n2987mx2j55nnfs95nxnzqpp5jmrh92pfld78spqs78v9euf2385t83uvpwk9ldrlvf6ch7tpascqhp5zvkrmemgth3tufcvflmzjzfvjt023nazlhljz2n9hattj4f8jq8qxqyjw5qcqpjrzjqtc4fc44feggv7065fqe5m4ytjarg3repr5j9el35xhmtfexc42yczarjuqqfzqqqqqqqqlgqqqqqqgq9q9qxpqysgq079nkq507a5tw7xgttmj4u990j7wfggtrasah5gd4ywfr2pjcn29383tphp4t48gquelz9z78p4cq7ml3nrrphw5w6eckhjwmhezhnqpy6gyf0', + ], + [ + 'description', + '{"pubkey":"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322","content":"","id":"d9cc14d50fcb8c27539aacf776882942c1a11ea4472f8cdec1dea82fab66279d","created_at":1674164539,"sig":"77127f636577e9029276be060332ea565deaf89ff215a494ccff16ae3f757065e2bc59b2e8c113dd407917a010b3abd36c8d7ad84c0e3ab7dab3a0b0caa9835d","kind":9734,"tags":[["e","3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8"],["p","32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245"],["relays","wss://relay.damus.io","wss://nostr-relay.wlvs.space","wss://nostr.fmt.wiz.biz","wss://relay.nostr.bg","wss://nostr.oxtr.dev","wss://nostr.v0l.io","wss://brb.io","wss://nostr.bitcoiner.social","ws://monad.jb55.com:8080","wss://relay.snort.social"]]}', + ], + ['preimage', '5d006d2cf1e73c7148e7519a4c68adc81642ce0e25a432b2434c99f97344c15f'], + ], + 'content': '', + }, sk); + + await db.store.event(event); + + await handleZaps(kysely, event); + await handleZaps(kysely, event); + + const zapReceipts = await db.store.query([{}]); + const customEventZaps = await kysely.selectFrom('event_zaps').selectAll().execute(); + + assertEquals(zapReceipts.length, 1); // basic check + assertEquals(customEventZaps.length, 1); // basic check + + const expected = { + receipt_id: event.id, + target_event_id: '3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8', + sender_pubkey: '97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322', + amount_millisats: 1000000, + comment: '', + }; + + assertEquals(customEventZaps[0], expected); +}); + +// The function tests below only handle the edge cases and don't assert anything +// If no error happens = ok + +Deno.test('zap receipt does not have a "description" tag', async () => { + await using db = await createTestDB(); + const kysely = db.kysely; + + const sk = generateSecretKey(); + + const event = genEvent({ kind: 9735 }, sk); + + await handleZaps(kysely, event); + + // no error happened = ok +}); + +Deno.test('zap receipt does not have a zap request stringified value in the "description" tag', async () => { + await using db = await createTestDB(); + const kysely = db.kysely; + + const sk = generateSecretKey(); + + const event = genEvent({ kind: 9735, tags: [['description', 'yolo']] }, sk); + + await handleZaps(kysely, event); + + // no error happened = ok +}); + +Deno.test('zap receipt does not have a "bolt11" tag', async () => { + await using db = await createTestDB(); + const kysely = db.kysely; + + const sk = generateSecretKey(); + + const event = genEvent({ + kind: 9735, + tags: [[ + 'description', + '{"pubkey":"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322","content":"","id":"d9cc14d50fcb8c27539aacf776882942c1a11ea4472f8cdec1dea82fab66279d","created_at":1674164539,"sig":"77127f636577e9029276be060332ea565deaf89ff215a494ccff16ae3f757065e2bc59b2e8c113dd407917a010b3abd36c8d7ad84c0e3ab7dab3a0b0caa9835d","kind":9734,"tags":[["e","3624762a1274dd9636e0c552b53086d70bc88c165bc4dc0f9e836a1eaf86c3b8"],["p","32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245"],["relays","wss://relay.damus.io","wss://nostr-relay.wlvs.space","wss://nostr.fmt.wiz.biz","wss://relay.nostr.bg","wss://nostr.oxtr.dev","wss://nostr.v0l.io","wss://brb.io","wss://nostr.bitcoiner.social","ws://monad.jb55.com:8080","wss://relay.snort.social"]]}', + ]], + }, sk); + + await handleZaps(kysely, event); + + // no error happened = ok +}); + +Deno.test('zap request inside zap receipt does not have an "e" tag', async () => { + await using db = await createTestDB(); + const kysely = db.kysely; + + const sk = generateSecretKey(); + + const event = genEvent({ + kind: 9735, + tags: [[ + 'bolt11', + 'lnbc10u1p3unwfusp5t9r3yymhpfqculx78u027lxspgxcr2n2987mx2j55nnfs95nxnzqpp5jmrh92pfld78spqs78v9euf2385t83uvpwk9ldrlvf6ch7tpascqhp5zvkrmemgth3tufcvflmzjzfvjt023nazlhljz2n9hattj4f8jq8qxqyjw5qcqpjrzjqtc4fc44feggv7065fqe5m4ytjarg3repr5j9el35xhmtfexc42yczarjuqqfzqqqqqqqqlgqqqqqqgq9q9qxpqysgq079nkq507a5tw7xgttmj4u990j7wfggtrasah5gd4ywfr2pjcn29383tphp4t48gquelz9z78p4cq7ml3nrrphw5w6eckhjwmhezhnqpy6gyf0', + ], [ + 'description', + '{"pubkey":"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322","content":"","id":"d9cc14d50fcb8c27539aacf776882942c1a11ea4472f8cdec1dea82fab66279d","created_at":1674164539,"sig":"77127f636577e9029276be060332ea565deaf89ff215a494ccff16ae3f757065e2bc59b2e8c113dd407917a010b3abd36c8d7ad84c0e3ab7dab3a0b0caa9835d","kind":9734,"tags":[["p","32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245"],["relays","wss://relay.damus.io","wss://nostr-relay.wlvs.space","wss://nostr.fmt.wiz.biz","wss://relay.nostr.bg","wss://nostr.oxtr.dev","wss://nostr.v0l.io","wss://brb.io","wss://nostr.bitcoiner.social","ws://monad.jb55.com:8080","wss://relay.snort.social"]]}', + ]], + }, sk); + + await handleZaps(kysely, event); + + // no error happened = ok +}); From 1a1adc1654f31da5bc9347c26bfb21e9de599600 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 23 Sep 2024 14:47:17 -0300 Subject: [PATCH 37/44] feat: add ditto:relay in /relay - sticky notes --- src/controllers/nostr/relay.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 3fd6b29a..0cdd8b3a 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -1,3 +1,4 @@ +import Debug from '@soapbox/stickynotes/debug'; import TTLCache from '@isaacs/ttlcache'; import { NostrClientCLOSE, @@ -29,6 +30,8 @@ const limiter = new TTLCache(); /** Connections for metrics purposes. */ const connections = new Set(); +const debug = Debug('ditto:relay'); + /** Set up the Websocket connection. */ function connectStream(socket: WebSocket, ip: string | undefined) { const controllers = new Map(); @@ -141,7 +144,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) { send(['OK', event.id, false, e.message]); } else { send(['OK', event.id, false, 'error: something went wrong']); - console.error(e); + debug(e); } } } From 60ebff45d52c74653b361ab31328748006d03457 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 23 Sep 2024 15:33:14 -0300 Subject: [PATCH 38/44] refactor: use correct Stickynotes, and not legacy Debug --- src/controllers/nostr/relay.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 0cdd8b3a..2a38e751 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -1,4 +1,4 @@ -import Debug from '@soapbox/stickynotes/debug'; +import { Stickynotes } from '@soapbox/stickynotes'; import TTLCache from '@isaacs/ttlcache'; import { NostrClientCLOSE, @@ -30,7 +30,7 @@ const limiter = new TTLCache(); /** Connections for metrics purposes. */ const connections = new Set(); -const debug = Debug('ditto:relay'); +const console = new Stickynotes('ditto:relay'); /** Set up the Websocket connection. */ function connectStream(socket: WebSocket, ip: string | undefined) { @@ -144,7 +144,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) { send(['OK', event.id, false, e.message]); } else { send(['OK', event.id, false, 'error: something went wrong']); - debug(e); + console.error(e); } } } From 92d8f9b8c23f4ac48f4ebbb8e592894440534de1 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 23 Sep 2024 15:36:05 -0300 Subject: [PATCH 39/44] refactor(pipeline.ts): use correct Stickynotes, and not legacy Debug --- src/pipeline.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index d89c6cef..f8fb6eed 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,5 +1,5 @@ import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify'; -import Debug from '@soapbox/stickynotes/debug'; +import { Stickynotes } from '@soapbox/stickynotes'; import ISO6391 from 'iso-639-1'; import { Kysely, sql } from 'kysely'; import lande from 'lande'; @@ -23,7 +23,7 @@ import { purifyEvent } from '@/utils/purify.ts'; import { updateStats } from '@/utils/stats.ts'; import { getTagSet } from '@/utils/tags.ts'; -const debug = Debug('ditto:pipeline'); +const console = new Stickynotes('ditto:pipeline'); /** * Common pipeline function to process (and maybe store) events. @@ -41,7 +41,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise ${event.id}`); + console.info(`NostrEvent<${event.kind}> ${event.id}`); pipelineEventsCounter.inc({ kind: event.kind }); if (isProtectedEvent(event)) { @@ -76,18 +76,18 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { - const debug = Debug('ditto:policy'); + const console = new Stickynotes('ditto:policy'); try { const result = await policyWorker.call(event); policyEventsCounter.inc({ ok: String(result[2]) }); - debug(JSON.stringify(result)); + console.log(JSON.stringify(result)); RelayError.assert(result); } catch (e) { if (e instanceof RelayError) { throw e; } else { - console.error('POLICY ERROR:', e); + console.error(e); throw new RelayError('blocked', 'policy error'); } } From 42371bcc31fa6350f7d2e2d4c2dbfe0e7cbe8781 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 23 Sep 2024 17:47:55 -0300 Subject: [PATCH 40/44] fix: catch updateStats() function --- src/pipeline.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index f8fb6eed..db43915a 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -136,7 +136,7 @@ async function storeEvent(event: DittoEvent, signal?: AbortSignal): Promise { - await updateStats({ event, store, kysely }); + await updateStats({ event, store, kysely }).catch((e) => console.error(e)); await store.event(event, { signal }); }); } From 6d0611dafe3d19f97c75533ae6b66d8db14baa6e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 23 Sep 2024 18:04:12 -0500 Subject: [PATCH 41/44] Pass signal into policy --- src/pipeline.ts | 6 +++--- src/workers/policy.ts | 2 ++ src/workers/policy.worker.ts | 6 ++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index d89c6cef..eb06c12b 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -49,7 +49,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { +async function policyFilter(event: NostrEvent, signal: AbortSignal): Promise { const debug = Debug('ditto:policy'); try { - const result = await policyWorker.call(event); + const result = await policyWorker.call(event, signal); policyEventsCounter.inc({ ok: String(result[2]) }); debug(JSON.stringify(result)); RelayError.assert(result); diff --git a/src/workers/policy.ts b/src/workers/policy.ts index 7bf3f7bf..65a6e79a 100644 --- a/src/workers/policy.ts +++ b/src/workers/policy.ts @@ -4,6 +4,8 @@ import * as Comlink from 'comlink'; import { Conf } from '@/config.ts'; import type { CustomPolicy } from '@/workers/policy.worker.ts'; +import '@/workers/handlers/abortsignal.ts'; + const console = new Stickynotes('ditto:policy'); export const policyWorker = Comlink.wrap( diff --git a/src/workers/policy.worker.ts b/src/workers/policy.worker.ts index c7a16e30..0c115821 100644 --- a/src/workers/policy.worker.ts +++ b/src/workers/policy.worker.ts @@ -6,6 +6,8 @@ import * as Comlink from 'comlink'; import { DittoDB } from '@/db/DittoDB.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; +import '@/workers/handlers/abortsignal.ts'; + // @ts-ignore Don't try to access the env from this worker. Deno.env = new Map(); @@ -25,8 +27,8 @@ export class CustomPolicy implements NPolicy { private policy: NPolicy = new ReadOnlyPolicy(); // deno-lint-ignore require-await - async call(event: NostrEvent): Promise { - return this.policy.call(event); + async call(event: NostrEvent, signal?: AbortSignal): Promise { + return this.policy.call(event, signal); } async init({ path, cwd, databaseUrl, adminPubkey }: PolicyInit): Promise { From 6a8b22d018a6d2a4d492fb4dfd2d921e417a50de Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 23 Sep 2024 21:29:50 -0500 Subject: [PATCH 42/44] metrics: improve fetch response metrics --- src/metrics.ts | 6 +++--- src/workers/fetch.ts | 13 ++++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/metrics.ts b/src/metrics.ts index 1e20747b..7bed083d 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -27,10 +27,10 @@ export const streamingClientMessagesCounter = new Counter({ help: 'Total number of messages received by the streaming API', }); -export const fetchCounter = new Counter({ - name: 'ditto_fetch_total', +export const fetchResponsesCounter = new Counter({ + name: 'ditto_fetch_responses_total', help: 'Total number of fetch requests', - labelNames: ['method'], + labelNames: ['method', 'status'], }); export const firehoseEventsCounter = new Counter({ diff --git a/src/workers/fetch.ts b/src/workers/fetch.ts index 3ed98fbb..4fbc57bb 100644 --- a/src/workers/fetch.ts +++ b/src/workers/fetch.ts @@ -3,7 +3,7 @@ import * as Comlink from 'comlink'; import { FetchWorker } from './fetch.worker.ts'; import './handlers/abortsignal.ts'; -import { fetchCounter } from '@/metrics.ts'; +import { fetchResponsesCounter } from '@/metrics.ts'; const worker = new Worker(new URL('./fetch.worker.ts', import.meta.url), { type: 'module' }); const client = Comlink.wrap(worker); @@ -23,11 +23,18 @@ const ready = new Promise((resolve) => { */ const fetchWorker: typeof fetch = async (...args) => { await ready; + const [url, init] = serializeFetchArgs(args); const { body, signal, ...rest } = init; - fetchCounter.inc({ method: init.method }); + const result = await client.fetch(url, { ...rest, body: await prepareBodyForWorker(body) }, signal); - return new Response(...result); + const response = new Response(...result); + + const { method } = init; + const { status } = response; + fetchResponsesCounter.inc({ method, status }); + + return response; }; /** Take arguments to `fetch`, and turn them into something we can send over Comlink. */ From fc73cb29611f18a7eadbca59a93ff7c9645b633f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 23 Sep 2024 22:16:03 -0500 Subject: [PATCH 43/44] Upgrade Nostrify --- deno.json | 5 +- deno.lock | 88 ++++++++++++++++++++++++----------- src/storages.ts | 4 +- src/storages/InternalRelay.ts | 4 ++ src/workers/policy.worker.ts | 2 +- 5 files changed, 72 insertions(+), 31 deletions(-) diff --git a/deno.json b/deno.json index 80c72b20..b702e9e5 100644 --- a/deno.json +++ b/deno.json @@ -32,8 +32,9 @@ "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/db": "jsr:@nostrify/db@^0.32.2", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.30.1", + "@nostrify/db": "jsr:@nostrify/db@^0.35.0", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.36.0", + "@nostrify/policies": "jsr:@nostrify/policies@^0.35.0", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-pglite": "jsr:@soapbox/kysely-pglite@^0.0.1", diff --git a/deno.lock b/deno.lock index ff81d4bb..088afce8 100644 --- a/deno.lock +++ b/deno.lock @@ -17,21 +17,23 @@ "jsr:@gleasonator/policy@0.6.1": "jsr:@gleasonator/policy@0.6.1", "jsr:@gleasonator/policy@0.6.3": "jsr:@gleasonator/policy@0.6.3", "jsr:@gleasonator/policy@0.6.4": "jsr:@gleasonator/policy@0.6.4", - "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.11", + "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.6.2", "jsr:@lambdalisue/async@^2.1.1": "jsr:@lambdalisue/async@2.1.1", - "jsr:@nostrify/db@^0.32.2": "jsr:@nostrify/db@0.32.2", + "jsr:@nostrify/db@^0.35.0": "jsr:@nostrify/db@0.35.0", "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", - "jsr:@nostrify/nostrify@^0.30.0": "jsr:@nostrify/nostrify@0.30.1", - "jsr:@nostrify/nostrify@^0.30.1": "jsr:@nostrify/nostrify@0.30.1", "jsr:@nostrify/nostrify@^0.31.0": "jsr:@nostrify/nostrify@0.31.0", "jsr:@nostrify/nostrify@^0.32.0": "jsr:@nostrify/nostrify@0.32.0", + "jsr:@nostrify/nostrify@^0.35.0": "jsr:@nostrify/nostrify@0.35.0", + "jsr:@nostrify/nostrify@^0.36.0": "jsr:@nostrify/nostrify@0.36.0", "jsr:@nostrify/policies@^0.33.0": "jsr:@nostrify/policies@0.33.0", "jsr:@nostrify/policies@^0.33.1": "jsr:@nostrify/policies@0.33.1", "jsr:@nostrify/policies@^0.34.0": "jsr:@nostrify/policies@0.34.0", + "jsr:@nostrify/policies@^0.35.0": "jsr:@nostrify/policies@0.35.0", "jsr:@nostrify/types@^0.30.0": "jsr:@nostrify/types@0.30.1", "jsr:@nostrify/types@^0.30.1": "jsr:@nostrify/types@0.30.1", + "jsr:@nostrify/types@^0.35.0": "jsr:@nostrify/types@0.35.0", "jsr:@soapbox/kysely-pglite@^0.0.1": "jsr:@soapbox/kysely-pglite@0.0.1", "jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0", "jsr:@std/assert@^0.213.1": "jsr:@std/assert@0.213.1", @@ -55,7 +57,7 @@ "jsr:@std/fs@^0.229.3": "jsr:@std/fs@0.229.3", "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.3", "jsr:@std/io@^0.223.0": "jsr:@std/io@0.223.0", - "jsr:@std/io@^0.224": "jsr:@std/io@0.224.7", + "jsr:@std/io@^0.224": "jsr:@std/io@0.224.8", "jsr:@std/json@^0.223.0": "jsr:@std/json@0.223.0", "jsr:@std/media-types@^0.224.1": "jsr:@std/media-types@0.224.1", "jsr:@std/path@0.213.1": "jsr:@std/path@0.213.1", @@ -229,14 +231,17 @@ "@hono/hono@4.5.9": { "integrity": "47f561e67aedbd6d1e21e3a1ae26c1b80ffdb62a51c161d502e75bee17ca40af" }, + "@hono/hono@4.6.2": { + "integrity": "35fcf3be4687825080b01bed7bbe2ac66f8d8b8939f0bad459661bf3b46d916f" + }, "@lambdalisue/async@2.1.1": { "integrity": "1fc9bc6f4ed50215cd2f7217842b18cea80f81c25744f88f8c5eb4be5a1c9ab4" }, - "@nostrify/db@0.32.2": { - "integrity": "265fb41e9d5810b99f1003ce56c89e4b468e6d0c04e7b9d9e3126c4efd49c1c2", + "@nostrify/db@0.35.0": { + "integrity": "637191c41812544e361b7997dc44ea098f8bd7efebb28f37a8a7142a0ecada8d", "dependencies": [ - "jsr:@nostrify/nostrify@^0.31.0", - "jsr:@nostrify/types@^0.30.1", + "jsr:@nostrify/nostrify@^0.35.0", + "jsr:@nostrify/types@^0.35.0", "npm:kysely@^0.27.3", "npm:nostr-tools@^2.7.0" ] @@ -285,21 +290,6 @@ "npm:zod@^3.23.8" ] }, - "@nostrify/nostrify@0.30.1": { - "integrity": "fcc923707e87a9fbecc82dbb18756d1d3d134cd0763f4b1254c4bce709e811eb", - "dependencies": [ - "jsr:@nostrify/types@^0.30.0", - "jsr:@std/crypto@^0.224.0", - "jsr:@std/encoding@^0.224.1", - "npm:@scure/base@^1.1.6", - "npm:@scure/bip32@^1.4.0", - "npm:@scure/bip39@^1.3.0", - "npm:lru-cache@^10.2.0", - "npm:nostr-tools@^2.7.0", - "npm:websocket-ts@^2.1.5", - "npm:zod@^3.23.8" - ] - }, "@nostrify/nostrify@0.31.0": { "integrity": "1c1b686bb9ca3ad8d19807e3b96ef3793a65d70fd0f433fe6ef8b3fdb9f45557", "dependencies": [ @@ -326,6 +316,34 @@ "npm:zod@^3.23.8" ] }, + "@nostrify/nostrify@0.35.0": { + "integrity": "9bfef4883838b8b4cb2e2b28a60b72de95391ca5b789bc7206a2baea054dea55", + "dependencies": [ + "jsr:@nostrify/types@^0.35.0", + "jsr:@std/encoding@^0.224.1", + "npm:@scure/bip32@^1.4.0", + "npm:@scure/bip39@^1.3.0", + "npm:lru-cache@^10.2.0", + "npm:nostr-tools@^2.7.0", + "npm:websocket-ts@^2.1.5", + "npm:zod@^3.23.8" + ] + }, + "@nostrify/nostrify@0.36.0": { + "integrity": "f00dbff1f02a2c496c5e85eeeb7a84101b7dd874d87456449dc71b6d037e40fc", + "dependencies": [ + "jsr:@nostrify/types@^0.35.0", + "jsr:@std/crypto@^0.224.0", + "jsr:@std/encoding@^0.224.1", + "npm:@scure/base@^1.1.6", + "npm:@scure/bip32@^1.4.0", + "npm:@scure/bip39@^1.3.0", + "npm:lru-cache@^10.2.0", + "npm:nostr-tools@^2.7.0", + "npm:websocket-ts@^2.1.5", + "npm:zod@^3.23.8" + ] + }, "@nostrify/policies@0.33.0": { "integrity": "c946b06d0527298b4d7c9819d142a10f522ba09eee76c37525aa4acfc5d87aee", "dependencies": [ @@ -348,12 +366,23 @@ "npm:nostr-tools@^2.7.0" ] }, + "@nostrify/policies@0.35.0": { + "integrity": "b828fac9f253e460a9587c05588b7dae6a0a32c5a9c9083e449219887b9e8e20", + "dependencies": [ + "jsr:@nostrify/nostrify@^0.35.0", + "jsr:@nostrify/types@^0.35.0", + "npm:nostr-tools@^2.7.0" + ] + }, "@nostrify/types@0.30.0": { "integrity": "1f38fa849cff930bd709edbf94ef9ac02f46afb8b851f86c8736517b354616da" }, "@nostrify/types@0.30.1": { "integrity": "245da176f6893a43250697db51ad32bfa29bf9b1cdc1ca218043d9abf6de5ae5" }, + "@nostrify/types@0.35.0": { + "integrity": "b8d515563d467072694557d5626fa1600f74e83197eef45dd86a9a99c64f7fe6" + }, "@soapbox/kysely-pglite@0.0.1": { "integrity": "7a4221aa780aad6fba9747c45c59dfb1c62017ba8cad9db5607f6e5822c058d5", "dependencies": [ @@ -483,6 +512,12 @@ "jsr:@std/bytes@^1.0.2" ] }, + "@std/io@0.224.8": { + "integrity": "f525d05d51fd873de6352b9afcf35cab9ab5dc448bf3c20e0c8b521ded9be392", + "dependencies": [ + "jsr:@std/bytes@^1.0.2" + ] + }, "@std/json@0.223.0": { "integrity": "9a4a255931dd0397924c6b10bb6a72fe3e28ddd876b981ada2e3b8dd0764163f", "dependencies": [ @@ -2064,8 +2099,9 @@ "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "jsr:@hono/hono@^4.4.6", "jsr:@lambdalisue/async@^2.1.1", - "jsr:@nostrify/db@^0.32.2", - "jsr:@nostrify/nostrify@^0.30.1", + "jsr:@nostrify/db@^0.35.0", + "jsr:@nostrify/nostrify@^0.36.0", + "jsr:@nostrify/policies@^0.35.0", "jsr:@soapbox/kysely-pglite@^0.0.1", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", diff --git a/src/storages.ts b/src/storages.ts index 8c7f2d28..643de7a5 100644 --- a/src/storages.ts +++ b/src/storages.ts @@ -15,7 +15,7 @@ export class Storages { private static _db: Promise | undefined; private static _database: Promise | undefined; private static _admin: Promise | undefined; - private static _client: Promise | undefined; + private static _client: Promise> | undefined; private static _pubsub: Promise | undefined; private static _search: Promise | undefined; @@ -68,7 +68,7 @@ export class Storages { } /** Relay pool storage. */ - public static async client(): Promise { + public static async client(): Promise> { if (!this._client) { this._client = (async () => { const db = await this.db(); diff --git a/src/storages/InternalRelay.ts b/src/storages/InternalRelay.ts index 71a13b31..4400b562 100644 --- a/src/storages/InternalRelay.ts +++ b/src/storages/InternalRelay.ts @@ -79,4 +79,8 @@ export class InternalRelay implements NRelay { async query(): Promise { return []; } + + async close(): Promise { + return Promise.resolve(); + } } diff --git a/src/workers/policy.worker.ts b/src/workers/policy.worker.ts index 0c115821..ae6ef8b1 100644 --- a/src/workers/policy.worker.ts +++ b/src/workers/policy.worker.ts @@ -1,6 +1,6 @@ import 'deno-safe-fetch/load'; import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify'; -import { NoOpPolicy, ReadOnlyPolicy } from '@nostrify/nostrify/policies'; +import { NoOpPolicy, ReadOnlyPolicy } from '@nostrify/policies'; import * as Comlink from 'comlink'; import { DittoDB } from '@/db/DittoDB.ts'; From d72ec843cff73001adae383f620fe8c41ad9bc65 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 23 Sep 2024 22:28:01 -0500 Subject: [PATCH 44/44] Add relay connections metrics --- src/controllers/metrics.ts | 17 ++++++++++++++++- src/metrics.ts | 11 +++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/controllers/metrics.ts b/src/controllers/metrics.ts index 4ef378a0..d168243b 100644 --- a/src/controllers/metrics.ts +++ b/src/controllers/metrics.ts @@ -1,17 +1,32 @@ import { register } from 'prom-client'; import { AppController } from '@/app.ts'; -import { dbAvailableConnectionsGauge, dbPoolSizeGauge } from '@/metrics.ts'; +import { + dbAvailableConnectionsGauge, + dbPoolSizeGauge, + relayPoolRelaysSizeGauge, + relayPoolSubscriptionsSizeGauge, +} from '@/metrics.ts'; import { Storages } from '@/storages.ts'; /** Prometheus/OpenMetrics controller. */ export const metricsController: AppController = async (c) => { const db = await Storages.database(); + const pool = await Storages.client(); // Update some metrics at request time. dbPoolSizeGauge.set(db.poolSize); dbAvailableConnectionsGauge.set(db.availableConnections); + relayPoolRelaysSizeGauge.reset(); + relayPoolSubscriptionsSizeGauge.reset(); + + for (const relay of pool.relays.values()) { + relayPoolRelaysSizeGauge.inc({ ready_state: relay.socket.readyState }); + relayPoolSubscriptionsSizeGauge.inc(relay.subscriptions.length); + } + + // Serve the metrics. const metrics = await register.metrics(); const headers: HeadersInit = { diff --git a/src/metrics.ts b/src/metrics.ts index 7bed083d..633d72f0 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -119,3 +119,14 @@ export const internalSubscriptionsSizeGauge = new Gauge({ name: 'ditto_internal_subscriptions_size', help: "Number of active subscriptions to Ditto's internal relay", }); + +export const relayPoolRelaysSizeGauge = new Gauge({ + name: 'ditto_relay_pool_relays_size', + help: 'Number of relays in the relay pool', + labelNames: ['ready_state'], +}); + +export const relayPoolSubscriptionsSizeGauge = new Gauge({ + name: 'ditto_relay_pool_subscriptions_size', + help: 'Number of active subscriptions to the relay pool', +});