From 0f5c28deeb9558337510659e873a6f00026d690b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 17 Jul 2024 19:58:12 -0500 Subject: [PATCH 1/6] EventsDB: query tags by converting to ids filter --- src/storages/EventsDB.ts | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index abf076c7..656c17fc 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -287,10 +287,42 @@ class EventsDB implements NStore { // If this results in an empty kinds array, NDatabase will remove the filter before querying and return no results. filter.kinds = filter.kinds.filter((kind) => !NKinds.ephemeral(kind)); } + + // Convert tag filters into `ids` filter for performance reasons. + const tagEntries = Object.entries(filter).filter(EventsDB.isTagEntry); + if (tagEntries.length) { + const tagIds: string[] = await tagEntries + .map(([key, value]) => + this.kysely + .selectFrom('nostr_tags') + .select('event_id') + .distinct() + .where('name', '=', key.replace(/^#/, '')) + .where('value', 'in', value) + ) + .reduce((result, query) => result.intersect(query)) + .execute() + .then((rows) => rows.map(({ event_id }) => event_id)); + + if (tagIds.length) { + filter.ids = filter.ids ?? []; + filter.ids.push(...tagIds); + } + + for (const [key] of tagEntries) { + delete filter[key]; + } + } } return filters; } + + /** Check if the object entry is a valid tag filter. */ + private static isTagEntry(entry: [string, unknown]): entry is [`#${string}`, string[]] { + const [key, value] = entry; + return key.startsWith('#') && Array.isArray(value); + } } export { EventsDB }; From ff8d7ef9d4a65e5008b07c0e8e2123a574ce672c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 17 Jul 2024 20:04:19 -0500 Subject: [PATCH 2/6] EventsDB: normalizeFilters before expanding tag queries --- src/storages/EventsDB.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 656c17fc..1265a98b 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -7,6 +7,7 @@ import { nip27 } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { DittoTables } from '@/db/DittoTables.ts'; +import { normalizeFilters } from '@/filter.ts'; import { dbEventCounter, dbQueryCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; @@ -250,7 +251,7 @@ class EventsDB implements NStore { /** Converts filters to more performant, simpler filters that are better for SQLite. */ async expandFilters(filters: NostrFilter[]): Promise { - filters = structuredClone(filters); + filters = normalizeFilters(structuredClone(filters)); for (const filter of filters) { if (filter.search) { From 2d239fcaec3a7de1d485eaa81dd19f10229f68d1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 18 Jul 2024 04:11:04 +0000 Subject: [PATCH 3/6] Revert "Merge branch 'tag-perf' into 'main'" This reverts merge request !425 --- src/storages/EventsDB.ts | 35 +---------------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 1265a98b..abf076c7 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -7,7 +7,6 @@ import { nip27 } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { DittoTables } from '@/db/DittoTables.ts'; -import { normalizeFilters } from '@/filter.ts'; import { dbEventCounter, dbQueryCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; @@ -251,7 +250,7 @@ class EventsDB implements NStore { /** Converts filters to more performant, simpler filters that are better for SQLite. */ async expandFilters(filters: NostrFilter[]): Promise { - filters = normalizeFilters(structuredClone(filters)); + filters = structuredClone(filters); for (const filter of filters) { if (filter.search) { @@ -288,42 +287,10 @@ class EventsDB implements NStore { // If this results in an empty kinds array, NDatabase will remove the filter before querying and return no results. filter.kinds = filter.kinds.filter((kind) => !NKinds.ephemeral(kind)); } - - // Convert tag filters into `ids` filter for performance reasons. - const tagEntries = Object.entries(filter).filter(EventsDB.isTagEntry); - if (tagEntries.length) { - const tagIds: string[] = await tagEntries - .map(([key, value]) => - this.kysely - .selectFrom('nostr_tags') - .select('event_id') - .distinct() - .where('name', '=', key.replace(/^#/, '')) - .where('value', 'in', value) - ) - .reduce((result, query) => result.intersect(query)) - .execute() - .then((rows) => rows.map(({ event_id }) => event_id)); - - if (tagIds.length) { - filter.ids = filter.ids ?? []; - filter.ids.push(...tagIds); - } - - for (const [key] of tagEntries) { - delete filter[key]; - } - } } return filters; } - - /** Check if the object entry is a valid tag filter. */ - private static isTagEntry(entry: [string, unknown]): entry is [`#${string}`, string[]] { - const [key, value] = entry; - return key.startsWith('#') && Array.isArray(value); - } } export { EventsDB }; From 7793db3e2c38397ca082a73b85dc010131e23b9a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 17 Jul 2024 23:13:04 -0500 Subject: [PATCH 4/6] Upgrade Nostrify to v0.26.1 --- deno.json | 2 +- deno.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deno.json b/deno.json index 3affdef9..9e48c1d7 100644 --- a/deno.json +++ b/deno.json @@ -27,7 +27,7 @@ "@hono/hono": "jsr:@hono/hono@^4.4.6", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.26.0", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.26.1", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", diff --git a/deno.lock b/deno.lock index 21898cf5..0f1ee264 100644 --- a/deno.lock +++ b/deno.lock @@ -12,7 +12,7 @@ "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", - "jsr:@nostrify/nostrify@^0.26.0": "jsr:@nostrify/nostrify@0.26.0", + "jsr:@nostrify/nostrify@^0.26.1": "jsr:@nostrify/nostrify@0.26.1", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0": "jsr:@soapbox/kysely-deno-sqlite@2.2.0", "jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0", "jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0", @@ -142,8 +142,8 @@ "npm:zod@^3.23.8" ] }, - "@nostrify/nostrify@0.26.0": { - "integrity": "8a4a49326f2d2bdf27e9ac6dc8c052bb37b31ae892350daa1df470070fddacd1", + "@nostrify/nostrify@0.26.1": { + "integrity": "7079fc00fa46900808931e26ef9af509863014d4028d526652e2a7fbae89a6e6", "dependencies": [ "jsr:@std/crypto@^0.224.0", "jsr:@std/encoding@^0.224.1", @@ -1748,7 +1748,7 @@ "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "jsr:@db/sqlite@^0.11.1", "jsr:@hono/hono@^4.4.6", - "jsr:@nostrify/nostrify@^0.26.0", + "jsr:@nostrify/nostrify@^0.26.1", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", From 13f0d3f49e2be648e2efb4e72e7689a5f8836f7b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 18 Jul 2024 22:01:16 -0500 Subject: [PATCH 5/6] Upgrade Nostrify to v0.26.2 --- deno.json | 2 +- deno.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deno.json b/deno.json index 9e48c1d7..bca8aa1d 100644 --- a/deno.json +++ b/deno.json @@ -27,7 +27,7 @@ "@hono/hono": "jsr:@hono/hono@^4.4.6", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.26.1", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.26.2", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", diff --git a/deno.lock b/deno.lock index 0f1ee264..6c0bc580 100644 --- a/deno.lock +++ b/deno.lock @@ -12,7 +12,7 @@ "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", - "jsr:@nostrify/nostrify@^0.26.1": "jsr:@nostrify/nostrify@0.26.1", + "jsr:@nostrify/nostrify@^0.26.2": "jsr:@nostrify/nostrify@0.26.2", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0": "jsr:@soapbox/kysely-deno-sqlite@2.2.0", "jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0", "jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0", @@ -142,8 +142,8 @@ "npm:zod@^3.23.8" ] }, - "@nostrify/nostrify@0.26.1": { - "integrity": "7079fc00fa46900808931e26ef9af509863014d4028d526652e2a7fbae89a6e6", + "@nostrify/nostrify@0.26.2": { + "integrity": "41a2aa076a92ea60dcf188c5be23c0809f73e753996c6456768c02d4fd6cfede", "dependencies": [ "jsr:@std/crypto@^0.224.0", "jsr:@std/encoding@^0.224.1", @@ -1748,7 +1748,7 @@ "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "jsr:@db/sqlite@^0.11.1", "jsr:@hono/hono@^4.4.6", - "jsr:@nostrify/nostrify@^0.26.1", + "jsr:@nostrify/nostrify@^0.26.2", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", From 8ec5feae13191878e5abff5001d8341bd8f53c7a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 18 Jul 2024 22:48:11 -0500 Subject: [PATCH 6/6] Sort events by id after created_at --- deno.json | 2 +- deno.lock | 8 +++--- src/db/migrations/028_stable_sort.ts | 39 ++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 src/db/migrations/028_stable_sort.ts diff --git a/deno.json b/deno.json index bca8aa1d..51f24cf0 100644 --- a/deno.json +++ b/deno.json @@ -27,7 +27,7 @@ "@hono/hono": "jsr:@hono/hono@^4.4.6", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.26.2", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.26.3", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", diff --git a/deno.lock b/deno.lock index 6c0bc580..104ea5c9 100644 --- a/deno.lock +++ b/deno.lock @@ -12,7 +12,7 @@ "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", - "jsr:@nostrify/nostrify@^0.26.2": "jsr:@nostrify/nostrify@0.26.2", + "jsr:@nostrify/nostrify@^0.26.3": "jsr:@nostrify/nostrify@0.26.3", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0": "jsr:@soapbox/kysely-deno-sqlite@2.2.0", "jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0", "jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0", @@ -142,8 +142,8 @@ "npm:zod@^3.23.8" ] }, - "@nostrify/nostrify@0.26.2": { - "integrity": "41a2aa076a92ea60dcf188c5be23c0809f73e753996c6456768c02d4fd6cfede", + "@nostrify/nostrify@0.26.3": { + "integrity": "3e13e30f4fa3f76dcbcf9178630a9b2871186eb1d226d66234c0cdfd4841f548", "dependencies": [ "jsr:@std/crypto@^0.224.0", "jsr:@std/encoding@^0.224.1", @@ -1748,7 +1748,7 @@ "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "jsr:@db/sqlite@^0.11.1", "jsr:@hono/hono@^4.4.6", - "jsr:@nostrify/nostrify@^0.26.2", + "jsr:@nostrify/nostrify@^0.26.3", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", diff --git a/src/db/migrations/028_stable_sort.ts b/src/db/migrations/028_stable_sort.ts new file mode 100644 index 00000000..191f32ca --- /dev/null +++ b/src/db/migrations/028_stable_sort.ts @@ -0,0 +1,39 @@ +import { Kysely } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .createIndex('nostr_events_created_at_kind') + .on('nostr_events') + .ifNotExists() + .columns(['created_at desc', 'id asc', 'kind']) + .execute(); + + await db.schema + .createIndex('nostr_events_kind_pubkey_created_at') + .on('nostr_events') + .ifNotExists() + .columns(['kind', 'pubkey', 'created_at desc', 'id asc']) + .execute(); + + await db.schema.dropIndex('idx_events_created_at_kind').execute(); + await db.schema.dropIndex('idx_events_kind_pubkey_created_at').execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropIndex('nostr_events_created_at_kind').execute(); + await db.schema.dropIndex('nostr_events_kind_pubkey_created_at').execute(); + + await db.schema + .createIndex('idx_events_created_at_kind') + .on('nostr_events') + .ifNotExists() + .columns(['created_at desc', 'kind']) + .execute(); + + await db.schema + .createIndex('idx_events_kind_pubkey_created_at') + .on('nostr_events') + .ifNotExists() + .columns(['kind', 'pubkey', 'created_at desc']) + .execute(); +}