From 78d41bbd6d51236d6ac3b70129e9a14f137db495 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Wed, 3 Jul 2024 20:41:48 -0300 Subject: [PATCH 01/24] feat(EventsDB): add nip05 split in buildUserSearchContent() --- src/storages/EventsDB.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index ce74f20d..c74fedb6 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -240,7 +240,9 @@ class EventsDB implements NStore { /** Build search content for a user. */ static buildUserSearchContent(event: NostrEvent): string { const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(event.content); - return [name, nip05].filter(Boolean).join('\n'); + const nip05splitted = nip05 ? nip05.split(/[_@.]/) : []; + + return [name, nip05, ...nip05splitted].filter(Boolean).join('\n'); } /** Build search content from tag values. */ From d8247dc76aac6aa627cb6bdfbac4a1730626d85f Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 4 Jul 2024 18:08:03 -0300 Subject: [PATCH 02/24] build: remove DATABASE_URL in test script --- deno.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deno.json b/deno.json index 084b527b..14212d9f 100644 --- a/deno.json +++ b/deno.json @@ -8,7 +8,7 @@ "db:migrate": "deno run -A scripts/db-migrate.ts", "nostr:pull": "deno run -A scripts/nostr-pull.ts", "debug": "deno run -A --inspect src/server.ts", - "test": "DATABASE_URL=\"sqlite://:memory:\" deno test -A --junit-path=./deno-test.xml", + "test": "deno test -A --junit-path=./deno-test.xml", "check": "deno check src/server.ts", "nsec": "deno run scripts/nsec.ts", "admin:event": "deno run -A scripts/admin-event.ts", From cbb48867c12aa876b40c226c4bdccdb117cb916b Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 4 Jul 2024 18:13:31 -0300 Subject: [PATCH 03/24] test: attempt to use Postgres --- src/storages/EventsDB.test.ts | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 32838fa6..818a9b14 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -1,25 +1,16 @@ -import { Database as Sqlite } from '@db/sqlite'; -import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite'; import { assertEquals, assertRejects } from '@std/assert'; -import { Kysely } from 'kysely'; import { generateSecretKey } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; import { RelayError } from '@/RelayError.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; import { eventFixture, genEvent } from '@/test.ts'; -/** Create in-memory database for testing. */ +/** Create an database for testing. */ const createDB = async () => { - const kysely = new Kysely({ - dialect: new DenoSqlite3Dialect({ - database: new Sqlite(':memory:'), - }), - }); + const kysely = await DittoDB.getInstance(); const eventsDB = new EventsDB(kysely); - await DittoDB.migrate(kysely); return { eventsDB, kysely }; }; @@ -221,3 +212,7 @@ Deno.test("throws a RelayError when querying an event with a large 'kind'", asyn 'kind filter too far into the future', ); }); + +Deno.test('query user by NIP-05 search filter', async () => { + // implement +}); From a9f79176a23411fa7a7d2d80d8411dabd791bc9e Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 4 Jul 2024 22:43:56 -0300 Subject: [PATCH 04/24] test: support sqlite and postgres --- src/storages/EventsDB.test.ts | 101 ++++++++++++++++++++++++++++------ 1 file changed, 85 insertions(+), 16 deletions(-) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 818a9b14..a9fe18c5 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -1,21 +1,80 @@ import { assertEquals, assertRejects } from '@std/assert'; import { generateSecretKey } from 'nostr-tools'; +import { Database as Sqlite } from '@db/sqlite'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; import { RelayError } from '@/RelayError.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; import { eventFixture, genEvent } from '@/test.ts'; +import { EventsDB } from '@/storages/EventsDB.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; +import { Kysely } from 'kysely'; +import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite'; + +const databaseUrl = Deno.env.get('DATABASE_URL') ?? 'sqlite://:memory:'; + +const dialect: 'sqlite' | 'postgres' = (() => { + const protocol = databaseUrl.split(':')[0]; + switch (protocol) { + case 'sqlite': + return 'sqlite'; + case 'postgres': + return protocol; + case 'postgresql': + return 'postgres'; + default: + throw new Error(`Unsupported protocol: ${protocol}`); + } +})(); /** Create an database for testing. */ const createDB = async () => { - const kysely = await DittoDB.getInstance(); - const eventsDB = new EventsDB(kysely); - return { eventsDB, kysely }; -}; + let kysely: Kysely; + if (dialect === 'sqlite') { + kysely = new Kysely({ + dialect: new DenoSqlite3Dialect({ + database: new Sqlite(':memory:'), + }), + }); + await DittoDB.migrate(kysely); + } else { + kysely = await DittoDB.getInstance(); + } + + const eventsDB = new EventsDB(kysely); + + return { + eventsDB, + kysely, + [Symbol.asyncDispose]: async () => { + if (dialect === 'postgres') { + for ( + const table of [ + 'author_stats', + 'event_stats', + 'event_zaps', + 'kysely_migration', + 'kysely_migration_lock', + 'nip46_tokens', + 'pubkey_domains', + 'unattached_media', + 'nostr_events', + 'nostr_tags', + 'nostr_pgfts', + ] + ) { + await kysely.schema.dropTable(table).ifExists().cascade().execute(); + } + await kysely.destroy(); + } + }, + }; +}; Deno.test('count filters', async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; + const event1 = await eventFixture('event-1'); assertEquals((await eventsDB.count([{ kinds: [1] }])).count, 0); @@ -24,7 +83,8 @@ Deno.test('count filters', async () => { }); Deno.test('insert and filter events', async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; const event1 = await eventFixture('event-1'); await eventsDB.event(event1); @@ -59,7 +119,8 @@ Deno.test('query events with domain search filter', async () => { }); Deno.test('delete events', async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; const [one, two] = [ { id: '1', kind: 1, pubkey: 'abc', content: 'hello world', created_at: 1, sig: '', tags: [] }, @@ -86,7 +147,8 @@ Deno.test('delete events', async () => { }); Deno.test("user cannot delete another user's event", async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; const event = { id: '1', kind: 1, pubkey: 'abc', content: 'hello world', created_at: 1, sig: '', tags: [] }; await eventsDB.event(event); @@ -108,7 +170,8 @@ Deno.test("user cannot delete another user's event", async () => { }); Deno.test('admin can delete any event', async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; const [one, two] = [ { id: '1', kind: 1, pubkey: 'abc', content: 'hello world', created_at: 1, sig: '', tags: [] }, @@ -135,7 +198,8 @@ Deno.test('admin can delete any event', async () => { }); Deno.test('throws a RelayError when inserting an event deleted by the admin', async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; const event = genEvent(); await eventsDB.event(event); @@ -151,7 +215,8 @@ Deno.test('throws a RelayError when inserting an event deleted by the admin', as }); Deno.test('throws a RelayError when inserting an event deleted by a user', async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; const sk = generateSecretKey(); @@ -169,7 +234,8 @@ Deno.test('throws a RelayError when inserting an event deleted by a user', async }); Deno.test('inserting replaceable events', async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; const event = await eventFixture('event-0'); await eventsDB.event(event); @@ -184,7 +250,8 @@ Deno.test('inserting replaceable events', async () => { }); Deno.test("throws a RelayError when querying an event with a large 'since'", async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; await assertRejects( () => eventsDB.query([{ since: 33333333333333 }]), @@ -194,7 +261,8 @@ Deno.test("throws a RelayError when querying an event with a large 'since'", asy }); Deno.test("throws a RelayError when querying an event with a large 'until'", async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; await assertRejects( () => eventsDB.query([{ until: 66666666666666 }]), @@ -204,7 +272,8 @@ Deno.test("throws a RelayError when querying an event with a large 'until'", asy }); Deno.test("throws a RelayError when querying an event with a large 'kind'", async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; await assertRejects( () => eventsDB.query([{ kinds: [99999999999999] }]), From 12b2032d8732c2760f6a02cc0aa985d1d191ddff Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 4 Jul 2024 22:50:12 -0300 Subject: [PATCH 05/24] refactor(EventsDB.test): add one blank line between functions --- src/storages/EventsDB.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index a9fe18c5..749ca777 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -71,6 +71,7 @@ const createDB = async () => { }, }; }; + Deno.test('count filters', async () => { await using db = await createDB(); const { eventsDB } = db; From 2154cf011bd41254a4a0eb92b44f2d488ed4bc5d Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 17:26:27 -0300 Subject: [PATCH 06/24] test: create createTestDB function, supports both Sqlite and Postgres --- src/test.ts | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) diff --git a/src/test.ts b/src/test.ts index b0172063..7aecab7a 100644 --- a/src/test.ts +++ b/src/test.ts @@ -4,11 +4,23 @@ import path from 'node:path'; import { Database as Sqlite } from '@db/sqlite'; import { NDatabase, NostrEvent } from '@nostrify/nostrify'; import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite'; -import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; +import { + FileMigrationProvider, + Kysely, + Migrator, + PostgresAdapter, + PostgresIntrospector, + PostgresQueryCompiler, +} from 'kysely'; import { finalizeEvent, generateSecretKey } from 'nostr-tools'; +import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; +import { PostgreSQLDriver } from 'kysely_deno_postgres'; +import { Pool } from 'postgres'; +import { KyselyLogger } from '@/db/KyselyLogger.ts'; +import { EventsDB } from '@/storages/EventsDB.ts'; /** Import an event fixture by name in tests. */ export async function eventFixture(name: string): Promise { @@ -62,3 +74,84 @@ export async function getTestDB() { [Symbol.asyncDispose]: () => kysely.destroy(), }; } + +/** Create an database for testing. */ +export const createTestDB = async (databaseUrl?: string) => { + databaseUrl ??= Deno.env.get('DATABASE_URL') ?? 'sqlite://:memory:'; + + const dialect: 'sqlite' | 'postgres' = (() => { + const protocol = databaseUrl.split(':')[0]; + switch (protocol) { + case 'sqlite': + return 'sqlite'; + case 'postgres': + return protocol; + case 'postgresql': + return 'postgres'; + default: + throw new Error(`Unsupported protocol: ${protocol}`); + } + })(); + + let kysely: Kysely; + + if (dialect === 'sqlite') { + Deno.env.set('DATABASE_URL', 'sqlite://:memory:'); // hack, refactor all, 021 migration + + kysely = new Kysely({ + dialect: new DenoSqlite3Dialect({ + database: new Sqlite(':memory:'), + }), + }); + } else { + //kysely = await DittoDB.getInstance(); + kysely = new Kysely({ + dialect: { + createAdapter() { + return new PostgresAdapter(); + }, + createDriver() { + return new PostgreSQLDriver(new Pool(databaseUrl, 10, true)); + }, + createIntrospector(db: Kysely) { + return new PostgresIntrospector(db); + }, + createQueryCompiler() { + return new PostgresQueryCompiler(); + }, + }, + log: KyselyLogger, + }); + } + await DittoDB.migrate(kysely); + + const store = new EventsDB(kysely); + + return { + store, + kysely, + [Symbol.asyncDispose]: async () => { + if (dialect === 'postgres') { + for ( + const table of [ + 'author_stats', + 'event_stats', + 'event_zaps', + 'kysely_migration', + 'kysely_migration_lock', + 'nip46_tokens', + 'pubkey_domains', + 'unattached_media', + 'nostr_events', + 'nostr_tags', + 'nostr_pgfts', + 'event_zaps', + ] + ) { + await kysely.schema.dropTable(table).ifExists().cascade().execute(); + } + await kysely.destroy(); + } + }, + }; +}; From 8e1826020c138c04c6ce6488ed68ae38b1159ba8 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 17:35:25 -0300 Subject: [PATCH 07/24] test(EventsDB): remove createDB function --- src/storages/EventsDB.test.ts | 203 ++++++++++++---------------------- 1 file changed, 69 insertions(+), 134 deletions(-) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 749ca777..1cc9bbf5 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -1,140 +1,75 @@ import { assertEquals, assertRejects } from '@std/assert'; import { generateSecretKey } from 'nostr-tools'; -import { Database as Sqlite } from '@db/sqlite'; -import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { RelayError } from '@/RelayError.ts'; import { eventFixture, genEvent } from '@/test.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; -import { Kysely } from 'kysely'; -import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite'; - -const databaseUrl = Deno.env.get('DATABASE_URL') ?? 'sqlite://:memory:'; - -const dialect: 'sqlite' | 'postgres' = (() => { - const protocol = databaseUrl.split(':')[0]; - switch (protocol) { - case 'sqlite': - return 'sqlite'; - case 'postgres': - return protocol; - case 'postgresql': - return 'postgres'; - default: - throw new Error(`Unsupported protocol: ${protocol}`); - } -})(); - -/** Create an database for testing. */ -const createDB = async () => { - let kysely: Kysely; - - if (dialect === 'sqlite') { - kysely = new Kysely({ - dialect: new DenoSqlite3Dialect({ - database: new Sqlite(':memory:'), - }), - }); - await DittoDB.migrate(kysely); - } else { - kysely = await DittoDB.getInstance(); - } - - const eventsDB = new EventsDB(kysely); - - return { - eventsDB, - kysely, - [Symbol.asyncDispose]: async () => { - if (dialect === 'postgres') { - for ( - const table of [ - 'author_stats', - 'event_stats', - 'event_zaps', - 'kysely_migration', - 'kysely_migration_lock', - 'nip46_tokens', - 'pubkey_domains', - 'unattached_media', - 'nostr_events', - 'nostr_tags', - 'nostr_pgfts', - ] - ) { - await kysely.schema.dropTable(table).ifExists().cascade().execute(); - } - await kysely.destroy(); - } - }, - }; -}; +import { Conf } from '@/config.ts'; +import { createTestDB } from '@/test.ts'; Deno.test('count filters', async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; const event1 = await eventFixture('event-1'); - assertEquals((await eventsDB.count([{ kinds: [1] }])).count, 0); - await eventsDB.event(event1); - assertEquals((await eventsDB.count([{ kinds: [1] }])).count, 1); + assertEquals((await store.count([{ kinds: [1] }])).count, 0); + await store.event(event1); + assertEquals((await store.count([{ kinds: [1] }])).count, 1); }); Deno.test('insert and filter events', async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; const event1 = await eventFixture('event-1'); - await eventsDB.event(event1); + await store.event(event1); - assertEquals(await eventsDB.query([{ kinds: [1] }]), [event1]); - assertEquals(await eventsDB.query([{ kinds: [3] }]), []); - assertEquals(await eventsDB.query([{ since: 1691091000 }]), [event1]); - assertEquals(await eventsDB.query([{ until: 1691091000 }]), []); + assertEquals(await store.query([{ kinds: [1] }]), [event1]); + assertEquals(await store.query([{ kinds: [3] }]), []); + assertEquals(await store.query([{ since: 1691091000 }]), [event1]); + assertEquals(await store.query([{ until: 1691091000 }]), []); assertEquals( - await eventsDB.query([{ '#proxy': ['https://gleasonator.com/objects/8f6fac53-4f66-4c6e-ac7d-92e5e78c3e79'] }]), + await store.query([{ '#proxy': ['https://gleasonator.com/objects/8f6fac53-4f66-4c6e-ac7d-92e5e78c3e79'] }]), [event1], ); }); Deno.test('query events with domain search filter', async () => { - const { eventsDB, kysely } = await createDB(); + await using db = await createTestDB(); + const { store, kysely } = db; const event1 = await eventFixture('event-1'); - await eventsDB.event(event1); + await store.event(event1); - assertEquals(await eventsDB.query([{}]), [event1]); - assertEquals(await eventsDB.query([{ search: 'domain:localhost:4036' }]), []); - assertEquals(await eventsDB.query([{ search: '' }]), [event1]); + assertEquals(await store.query([{}]), [event1]); + assertEquals(await store.query([{ search: 'domain:localhost:4036' }]), []); + assertEquals(await store.query([{ search: '' }]), [event1]); await kysely .insertInto('pubkey_domains') .values({ pubkey: event1.pubkey, domain: 'localhost:4036', last_updated_at: event1.created_at }) .execute(); - assertEquals(await eventsDB.query([{ kinds: [1], search: 'domain:localhost:4036' }]), [event1]); - assertEquals(await eventsDB.query([{ kinds: [1], search: 'domain:example.com' }]), []); + assertEquals(await store.query([{ kinds: [1], search: 'domain:localhost:4036' }]), [event1]); + assertEquals(await store.query([{ kinds: [1], search: 'domain:example.com' }]), []); }); Deno.test('delete events', async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; const [one, two] = [ { id: '1', kind: 1, pubkey: 'abc', content: 'hello world', created_at: 1, sig: '', tags: [] }, { id: '2', kind: 1, pubkey: 'abc', content: 'yolo fam', created_at: 2, sig: '', tags: [] }, ]; - await eventsDB.event(one); - await eventsDB.event(two); + await store.event(one); + await store.event(two); // Sanity check - assertEquals(await eventsDB.query([{ kinds: [1] }]), [two, one]); + assertEquals(await store.query([{ kinds: [1] }]), [two, one]); - await eventsDB.event({ + await store.event({ kind: 5, pubkey: one.pubkey, tags: [['e', one.id]], @@ -144,20 +79,20 @@ Deno.test('delete events', async () => { sig: '', }); - assertEquals(await eventsDB.query([{ kinds: [1] }]), [two]); + assertEquals(await store.query([{ kinds: [1] }]), [two]); }); Deno.test("user cannot delete another user's event", async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; const event = { id: '1', kind: 1, pubkey: 'abc', content: 'hello world', created_at: 1, sig: '', tags: [] }; - await eventsDB.event(event); + await store.event(event); // Sanity check - assertEquals(await eventsDB.query([{ kinds: [1] }]), [event]); + assertEquals(await store.query([{ kinds: [1] }]), [event]); - await eventsDB.event({ + await store.event({ kind: 5, pubkey: 'def', // different pubkey tags: [['e', event.id]], @@ -167,25 +102,25 @@ Deno.test("user cannot delete another user's event", async () => { sig: '', }); - assertEquals(await eventsDB.query([{ kinds: [1] }]), [event]); + assertEquals(await store.query([{ kinds: [1] }]), [event]); }); Deno.test('admin can delete any event', async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; const [one, two] = [ { id: '1', kind: 1, pubkey: 'abc', content: 'hello world', created_at: 1, sig: '', tags: [] }, { id: '2', kind: 1, pubkey: 'abc', content: 'yolo fam', created_at: 2, sig: '', tags: [] }, ]; - await eventsDB.event(one); - await eventsDB.event(two); + await store.event(one); + await store.event(two); // Sanity check - assertEquals(await eventsDB.query([{ kinds: [1] }]), [two, one]); + assertEquals(await store.query([{ kinds: [1] }]), [two, one]); - await eventsDB.event({ + await store.event({ kind: 5, pubkey: Conf.pubkey, // Admin pubkey tags: [['e', one.id]], @@ -195,89 +130,89 @@ Deno.test('admin can delete any event', async () => { sig: '', }); - assertEquals(await eventsDB.query([{ kinds: [1] }]), [two]); + assertEquals(await store.query([{ kinds: [1] }]), [two]); }); Deno.test('throws a RelayError when inserting an event deleted by the admin', async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; const event = genEvent(); - await eventsDB.event(event); + await store.event(event); const deletion = genEvent({ kind: 5, tags: [['e', event.id]] }, Conf.seckey); - await eventsDB.event(deletion); + await store.event(deletion); await assertRejects( - () => eventsDB.event(event), + () => store.event(event), RelayError, 'event deleted by admin', ); }); Deno.test('throws a RelayError when inserting an event deleted by a user', async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; const sk = generateSecretKey(); const event = genEvent({}, sk); - await eventsDB.event(event); + await store.event(event); const deletion = genEvent({ kind: 5, tags: [['e', event.id]] }, sk); - await eventsDB.event(deletion); + await store.event(deletion); await assertRejects( - () => eventsDB.event(event), + () => store.event(event), RelayError, 'event deleted by user', ); }); Deno.test('inserting replaceable events', async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; const event = await eventFixture('event-0'); - await eventsDB.event(event); + await store.event(event); const olderEvent = { ...event, id: '123', created_at: event.created_at - 1 }; - await eventsDB.event(olderEvent); - assertEquals(await eventsDB.query([{ kinds: [0], authors: [event.pubkey] }]), [event]); + await store.event(olderEvent); + assertEquals(await store.query([{ kinds: [0], authors: [event.pubkey] }]), [event]); const newerEvent = { ...event, id: '123', created_at: event.created_at + 1 }; - await eventsDB.event(newerEvent); - assertEquals(await eventsDB.query([{ kinds: [0] }]), [newerEvent]); + await store.event(newerEvent); + assertEquals(await store.query([{ kinds: [0] }]), [newerEvent]); }); Deno.test("throws a RelayError when querying an event with a large 'since'", async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; await assertRejects( - () => eventsDB.query([{ since: 33333333333333 }]), + () => store.query([{ since: 33333333333333 }]), RelayError, 'since filter too far into the future', ); }); Deno.test("throws a RelayError when querying an event with a large 'until'", async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; await assertRejects( - () => eventsDB.query([{ until: 66666666666666 }]), + () => store.query([{ until: 66666666666666 }]), RelayError, 'until filter too far into the future', ); }); Deno.test("throws a RelayError when querying an event with a large 'kind'", async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; await assertRejects( - () => eventsDB.query([{ kinds: [99999999999999] }]), + () => store.query([{ kinds: [99999999999999] }]), RelayError, 'kind filter too far into the future', ); From 85fd7909e6159ada7190208d113a79442c260000 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 17:42:19 -0300 Subject: [PATCH 08/24] feat: add optional kysely in HydrateOpts interface --- src/storages/hydrate.ts | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 3c264320..00ae59a4 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -7,16 +7,19 @@ import { Conf } from '@/config.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { findQuoteTag } from '@/utils/tags.ts'; import { findQuoteInContent } from '@/utils/note.ts'; +import { Kysely } from 'kysely'; interface HydrateOpts { events: DittoEvent[]; store: NStore; signal?: AbortSignal; + kysely?: Kysely; } /** Hydrate events using the provided storage. */ async function hydrateEvents(opts: HydrateOpts): Promise { - const { events, store, signal } = opts; + const { events, store, signal, kysely = await DittoDB.getInstance() } = opts; + console.log(kysely); if (!events.length) { return events; @@ -57,8 +60,8 @@ async function hydrateEvents(opts: HydrateOpts): Promise { } const stats = { - authors: await gatherAuthorStats(cache), - events: await gatherEventStats(cache), + authors: await gatherAuthorStats(cache, kysely), + events: await gatherEventStats(cache, kysely), }; // Dedupe events. @@ -276,7 +279,10 @@ function gatherReportedProfiles({ events, store, signal }: HydrateOpts): Promise } /** Collect author stats from the events. */ -async function gatherAuthorStats(events: DittoEvent[]): Promise { +async function gatherAuthorStats( + events: DittoEvent[], + kysely: Kysely, +): Promise { const pubkeys = new Set( events .filter((event) => event.kind === 0) @@ -287,8 +293,6 @@ async function gatherAuthorStats(events: DittoEvent[]): Promise { +async function gatherEventStats( + events: DittoEvent[], + kysely: Kysely, +): Promise { const ids = new Set( events .filter((event) => event.kind === 1) @@ -315,8 +322,6 @@ async function gatherEventStats(events: DittoEvent[]): Promise Date: Fri, 5 Jul 2024 17:43:47 -0300 Subject: [PATCH 09/24] test(hydrate): refactor to only use sqlite --- src/storages/hydrate.test.ts | 68 +++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/src/storages/hydrate.test.ts b/src/storages/hydrate.test.ts index e3b1cf2e..6730315b 100644 --- a/src/storages/hydrate.test.ts +++ b/src/storages/hydrate.test.ts @@ -3,21 +3,23 @@ import { assertEquals } from '@std/assert'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { eventFixture } from '@/test.ts'; +import { createTestDB, eventFixture } from '@/test.ts'; Deno.test('hydrateEvents(): author --- WITHOUT stats', async () => { - const db = new MockRelay(); + const relay = new MockRelay(); + await using db = await createTestDB('sqlite://:memory:'); const event0 = await eventFixture('event-0'); const event1 = await eventFixture('event-1'); // Save events to database - await db.event(event0); - await db.event(event1); + await relay.event(event0); + await relay.event(event1); await hydrateEvents({ events: [event1], - store: db, + store: relay, + kysely: db.kysely, }); const expectedEvent = { ...event1, author: event0 }; @@ -25,7 +27,8 @@ Deno.test('hydrateEvents(): author --- WITHOUT stats', async () => { }); Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { - const db = new MockRelay(); + const relay = new MockRelay(); + await using db = await createTestDB('sqlite://:memory:'); const event0madePost = await eventFixture('event-0-the-one-who-post-and-users-repost'); const event0madeRepost = await eventFixture('event-0-the-one-who-repost'); @@ -33,14 +36,15 @@ Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { const event6 = await eventFixture('event-6'); // Save events to database - await db.event(event0madePost); - await db.event(event0madeRepost); - await db.event(event1reposted); - await db.event(event6); + await relay.event(event0madePost); + await relay.event(event0madeRepost); + await relay.event(event1reposted); + await relay.event(event6); await hydrateEvents({ events: [event6], - store: db, + store: relay, + kysely: db.kysely, }); const expectedEvent6 = { @@ -52,7 +56,8 @@ Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { }); Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { - const db = new MockRelay(); + const relay = new MockRelay(); + await using db = await createTestDB('sqlite://:memory:'); const event0madeQuoteRepost = await eventFixture('event-0-the-one-who-quote-repost'); const event0 = await eventFixture('event-0'); @@ -60,14 +65,15 @@ Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { const event1willBeQuoteReposted = await eventFixture('event-1-that-will-be-quote-reposted'); // Save events to database - await db.event(event0madeQuoteRepost); - await db.event(event0); - await db.event(event1quoteRepost); - await db.event(event1willBeQuoteReposted); + await relay.event(event0madeQuoteRepost); + await relay.event(event0); + await relay.event(event1quoteRepost); + await relay.event(event1willBeQuoteReposted); await hydrateEvents({ events: [event1quoteRepost], - store: db, + store: relay, + kysely: db.kysely, }); const expectedEvent1quoteRepost = { @@ -80,7 +86,8 @@ Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { }); Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () => { - const db = new MockRelay(); + const relay = new MockRelay(); + await using db = await createTestDB('sqlite://:memory:'); const author = await eventFixture('event-0-makes-repost-with-quote-repost'); const event1 = await eventFixture('event-1-will-be-reposted-with-quote-repost'); @@ -88,14 +95,15 @@ Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () const event1quote = await eventFixture('event-1-quote-repost-will-be-reposted'); // Save events to database - await db.event(author); - await db.event(event1); - await db.event(event1quote); - await db.event(event6); + await relay.event(author); + await relay.event(event1); + await relay.event(event1quote); + await relay.event(event6); await hydrateEvents({ events: [event6], - store: db, + store: relay, + kysely: db.kysely, }); const expectedEvent6 = { @@ -107,7 +115,8 @@ Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () }); Deno.test('hydrateEvents(): report pubkey and post // kind 1984 --- WITHOUT stats', async () => { - const db = new MockRelay(); + const relay = new MockRelay(); + await using db = await createTestDB('sqlite://:memory:'); const authorDictator = await eventFixture('kind-0-dictator'); const authorVictim = await eventFixture('kind-0-george-orwell'); @@ -115,14 +124,15 @@ Deno.test('hydrateEvents(): report pubkey and post // kind 1984 --- WITHOUT stat const event1 = await eventFixture('kind-1-author-george-orwell'); // Save events to database - await db.event(authorDictator); - await db.event(authorVictim); - await db.event(reportEvent); - await db.event(event1); + await relay.event(authorDictator); + await relay.event(authorVictim); + await relay.event(reportEvent); + await relay.event(event1); await hydrateEvents({ events: [reportEvent], - store: db, + store: relay, + kysely: db.kysely, }); const expectedEvent: DittoEvent = { From f380120cd39c9a87dc7f557eb97b2985a3997ff7 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 17:48:06 -0300 Subject: [PATCH 10/24] test: refactor to use createTestDB instead of getTestDB --- src/utils/stats.test.ts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/utils/stats.test.ts b/src/utils/stats.test.ts index c3e26595..69633ae3 100644 --- a/src/utils/stats.test.ts +++ b/src/utils/stats.test.ts @@ -1,11 +1,11 @@ import { assertEquals } from '@std/assert'; import { generateSecretKey, getPublicKey } from 'nostr-tools'; -import { genEvent, getTestDB } from '@/test.ts'; +import { createTestDB, genEvent } from '@/test.ts'; import { countAuthorStats, getAuthorStats, getEventStats, getFollowDiff, updateStats } from '@/utils/stats.ts'; Deno.test('updateStats with kind 1 increments notes count', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const sk = generateSecretKey(); const pubkey = getPublicKey(sk); @@ -18,7 +18,7 @@ Deno.test('updateStats with kind 1 increments notes count', async () => { }); Deno.test('updateStats with kind 1 increments replies count', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const sk = generateSecretKey(); @@ -36,7 +36,7 @@ Deno.test('updateStats with kind 1 increments replies count', async () => { }); Deno.test('updateStats with kind 5 decrements notes count', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const sk = generateSecretKey(); const pubkey = getPublicKey(sk); @@ -54,7 +54,7 @@ Deno.test('updateStats with kind 5 decrements notes count', async () => { }); Deno.test('updateStats with kind 3 increments followers count', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); await updateStats({ ...db, event: genEvent({ kind: 3, tags: [['p', 'alex']] }) }); await updateStats({ ...db, event: genEvent({ kind: 3, tags: [['p', 'alex']] }) }); @@ -66,7 +66,7 @@ Deno.test('updateStats with kind 3 increments followers count', async () => { }); Deno.test('updateStats with kind 3 decrements followers count', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const sk = generateSecretKey(); const follow = genEvent({ kind: 3, tags: [['p', 'alex']], created_at: 0 }, sk); @@ -92,7 +92,7 @@ Deno.test('getFollowDiff returns added and removed followers', () => { }); Deno.test('updateStats with kind 6 increments reposts count', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const note = genEvent({ kind: 1 }); await updateStats({ ...db, event: note }); @@ -108,7 +108,7 @@ Deno.test('updateStats with kind 6 increments reposts count', async () => { }); Deno.test('updateStats with kind 5 decrements reposts count', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const note = genEvent({ kind: 1 }); await updateStats({ ...db, event: note }); @@ -127,7 +127,7 @@ Deno.test('updateStats with kind 5 decrements reposts count', async () => { }); Deno.test('updateStats with kind 7 increments reactions count', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const note = genEvent({ kind: 1 }); await updateStats({ ...db, event: note }); @@ -143,7 +143,7 @@ Deno.test('updateStats with kind 7 increments reactions count', async () => { }); Deno.test('updateStats with kind 5 decrements reactions count', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const note = genEvent({ kind: 1 }); await updateStats({ ...db, event: note }); @@ -162,7 +162,7 @@ Deno.test('updateStats with kind 5 decrements reactions count', async () => { }); Deno.test('countAuthorStats counts author stats from the database', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const sk = generateSecretKey(); const pubkey = getPublicKey(sk); From 34bbf6e39a10077dd6b42692b20fde6a18df40a7 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 17:48:34 -0300 Subject: [PATCH 11/24] test(pipeline): refactor to use createTestDB instead of getTestDB --- src/pipeline.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipeline.test.ts b/src/pipeline.test.ts index 64cb523b..3a196c16 100644 --- a/src/pipeline.test.ts +++ b/src/pipeline.test.ts @@ -1,11 +1,11 @@ import { assertEquals } from '@std/assert'; import { generateSecretKey } from 'nostr-tools'; -import { genEvent, getTestDB } from '@/test.ts'; +import { createTestDB, genEvent, getTestDB } from '@/test.ts'; import { handleZaps } from '@/pipeline.ts'; Deno.test('store one zap receipt in nostr_events; convert it into event_zaps table format and store it', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const kysely = db.kysely; const sk = generateSecretKey(); From 1af229da45dbf04b1c09ebd9f73139d2f60efd70 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 17:52:13 -0300 Subject: [PATCH 12/24] build(gitlab-ci): use deno task test in postgres ci --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 20650d2e..88829e92 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,10 +32,10 @@ test: postgres: stage: test - script: deno task db:migrate + script: deno task db:migrate && deno task test services: - postgres:16 variables: DITTO_NSEC: nsec1zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs4rm7hz DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres - POSTGRES_HOST_AUTH_METHOD: trust \ No newline at end of file + POSTGRES_HOST_AUTH_METHOD: trust From 7e965f4aded194cda350555745ac51cba6883dbe Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 17:59:12 -0300 Subject: [PATCH 13/24] refactor(test.ts): explain why set DATABASE_URL to sqlite://:memory: --- src/test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test.ts b/src/test.ts index 7aecab7a..ae2af52a 100644 --- a/src/test.ts +++ b/src/test.ts @@ -96,7 +96,10 @@ export const createTestDB = async (databaseUrl?: string) => { let kysely: Kysely; if (dialect === 'sqlite') { - Deno.env.set('DATABASE_URL', 'sqlite://:memory:'); // hack, refactor all, 021 migration + // migration 021_pgfts_index.ts calls 'Conf.db.dialect', + // and this calls the DATABASE_URL environment variable. + // The following line ensures to NOT use the DATABASE_URL that may exist in an .env file. + Deno.env.set('DATABASE_URL', 'sqlite://:memory:'); kysely = new Kysely({ dialect: new DenoSqlite3Dialect({ @@ -104,7 +107,6 @@ export const createTestDB = async (databaseUrl?: string) => { }), }); } else { - //kysely = await DittoDB.getInstance(); kysely = new Kysely({ dialect: { createAdapter() { From fed0c70f69c7d76366a993cae845b4e012e88577 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 18:33:29 -0300 Subject: [PATCH 14/24] feat: only run tests with Postgres if ALLOW_TO_USE_DATABASE_URL is set to true --- src/test.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/test.ts b/src/test.ts index ae2af52a..e6967596 100644 --- a/src/test.ts +++ b/src/test.ts @@ -79,7 +79,7 @@ export async function getTestDB() { export const createTestDB = async (databaseUrl?: string) => { databaseUrl ??= Deno.env.get('DATABASE_URL') ?? 'sqlite://:memory:'; - const dialect: 'sqlite' | 'postgres' = (() => { + let dialect: 'sqlite' | 'postgres' = (() => { const protocol = databaseUrl.split(':')[0]; switch (protocol) { case 'sqlite': @@ -93,6 +93,15 @@ export const createTestDB = async (databaseUrl?: string) => { } })(); + const allowToUseDATABASE_URL = Deno.env.get('ALLOW_TO_USE_DATABASE_URL')?.toLowerCase() ?? ''; + if (allowToUseDATABASE_URL !== 'true' && dialect === 'postgres') { + console.warn( + '%cRunning tests with sqlite, if you meant to use Postgres, run again with ALLOW_TO_USE_DATABASE_URL environment variable set to true', + 'color: yellow;', + ); + dialect = 'sqlite'; + } + let kysely: Kysely; if (dialect === 'sqlite') { From 1b3082cd228ddd8e7bdcc1372a0339a44c9d297d Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 18:34:28 -0300 Subject: [PATCH 15/24] build: set ALLOW_TO_USE_DATABASE_URL in postgres ci --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 88829e92..e7de7aef 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,3 +39,4 @@ postgres: DITTO_NSEC: nsec1zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs4rm7hz DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres POSTGRES_HOST_AUTH_METHOD: trust + ALLOW_TO_USE_DATABASE_URL: true From 13d346afbd8ecab4d0a3164288c12cc67ac6f40f Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 18:43:14 -0300 Subject: [PATCH 16/24] refactor: remove console.log --- src/storages/hydrate.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 00ae59a4..6fe8df6f 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -19,7 +19,6 @@ interface HydrateOpts { /** Hydrate events using the provided storage. */ async function hydrateEvents(opts: HydrateOpts): Promise { const { events, store, signal, kysely = await DittoDB.getInstance() } = opts; - console.log(kysely); if (!events.length) { return events; From 01966496c155036c0dc6b769e6c2fb73eae8e877 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 18:43:52 -0300 Subject: [PATCH 17/24] feat: print which dialect is being used when calling createTestDB --- src/test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test.ts b/src/test.ts index e6967596..0a579220 100644 --- a/src/test.ts +++ b/src/test.ts @@ -102,6 +102,8 @@ export const createTestDB = async (databaseUrl?: string) => { dialect = 'sqlite'; } + console.warn(`Using: ${dialect}`); + let kysely: Kysely; if (dialect === 'sqlite') { From 92231dc4520359e20da0d8fffbb47942b2950342 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 20:42:11 -0300 Subject: [PATCH 18/24] test: query by NIP-05 --- src/storages/EventsDB.test.ts | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 1cc9bbf5..942cf849 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -218,6 +218,23 @@ Deno.test("throws a RelayError when querying an event with a large 'kind'", asyn ); }); -Deno.test('query user by NIP-05 search filter', async () => { - // implement -}); +Deno.test( + 'query user by NIP-05 search filter', + { ignore: Deno.env.get('DATABASE_URL')?.slice(0, 8) !== 'postgres' }, + async () => { + await using db = await createTestDB(); + const { store } = db; + + const event0 = await eventFixture('event-0'); + await store.event(event0); + + assertEquals(await store.query([{}]), [event0]); + assertEquals(await store.query([{ search: 'sonator.dev' }]), []); + assertEquals(await store.query([{ search: 'alex' }]), [event0]); + assertEquals(await store.query([{ search: 'gleasonator' }]), [event0]); + assertEquals(await store.query([{ search: 'com' }]), [event0]); + assertEquals(await store.query([{ search: 'mostr' }]), [event0]); + assertEquals(await store.query([{ search: 'pub' }]), [event0]); + assertEquals(await store.query([{ search: 'mostr.pub' }]), [event0]); + }, +); From 5412d2a07b211a5c402cf101813894b297506556 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 20:43:13 -0300 Subject: [PATCH 19/24] fix(EventsDB): also add a split only by '@' at symbol --- src/storages/EventsDB.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index c74fedb6..3ade7e2e 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -241,8 +241,9 @@ class EventsDB implements NStore { static buildUserSearchContent(event: NostrEvent): string { const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(event.content); const nip05splitted = nip05 ? nip05.split(/[_@.]/) : []; + const nip05splitted2 = nip05 ? nip05.split(/[@]/) : []; - return [name, nip05, ...nip05splitted].filter(Boolean).join('\n'); + return [name, nip05, ...nip05splitted, ...nip05splitted2].filter(Boolean).join('\n'); } /** Build search content from tag values. */ From 71bdfea317b35552eb85e2a2e5ea6ce96efa3bc6 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sat, 6 Jul 2024 11:11:41 -0300 Subject: [PATCH 20/24] refactor: get database dialect from Conf.db.dialect --- src/storages/EventsDB.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 942cf849..1b4b66e0 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -220,7 +220,7 @@ Deno.test("throws a RelayError when querying an event with a large 'kind'", asyn Deno.test( 'query user by NIP-05 search filter', - { ignore: Deno.env.get('DATABASE_URL')?.slice(0, 8) !== 'postgres' }, + { ignore: Conf.db.dialect !== 'postgres' }, async () => { await using db = await createTestDB(); const { store } = db; From 8afc3bca3d131a68028b3e42b75b644b3f5dbe1c Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 16 Jul 2024 14:06:32 -0300 Subject: [PATCH 21/24] refactor: order of imports --- src/test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test.ts b/src/test.ts index 808fc9ad..5de5430b 100644 --- a/src/test.ts +++ b/src/test.ts @@ -2,8 +2,11 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { Database as Sqlite } from '@db/sqlite'; -import { NDatabase, NostrEvent } from '@nostrify/nostrify'; import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite'; +import { finalizeEvent, generateSecretKey } from 'nostr-tools'; +import { PostgreSQLDriver } from 'kysely_deno_postgres'; +import { Pool } from 'postgres'; +import { NDatabase, NostrEvent } from '@nostrify/nostrify'; import { FileMigrationProvider, Kysely, @@ -12,13 +15,10 @@ import { PostgresIntrospector, PostgresQueryCompiler, } from 'kysely'; -import { finalizeEvent, generateSecretKey } from 'nostr-tools'; import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; -import { PostgreSQLDriver } from 'kysely_deno_postgres'; -import { Pool } from 'postgres'; import { KyselyLogger } from '@/db/KyselyLogger.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; From 842b527273a7c33b5d475fb56d123fd9223da085 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 16 Jul 2024 14:47:14 -0300 Subject: [PATCH 22/24] test: update to use kysely-postgres-js --- src/test.ts | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/src/test.ts b/src/test.ts index 5de5430b..a82d9e16 100644 --- a/src/test.ts +++ b/src/test.ts @@ -4,23 +4,17 @@ import path from 'node:path'; import { Database as Sqlite } from '@db/sqlite'; import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite'; import { finalizeEvent, generateSecretKey } from 'nostr-tools'; -import { PostgreSQLDriver } from 'kysely_deno_postgres'; -import { Pool } from 'postgres'; import { NDatabase, NostrEvent } from '@nostrify/nostrify'; -import { - FileMigrationProvider, - Kysely, - Migrator, - PostgresAdapter, - PostgresIntrospector, - PostgresQueryCompiler, -} from 'kysely'; +import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; +import postgres from 'postgres'; +import { PostgresJSDialect, PostgresJSDialectConfig } from 'kysely-postgres-js'; import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; import { KyselyLogger } from '@/db/KyselyLogger.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; +import { Conf } from '@/config.ts'; /** Import an event fixture by name in tests. */ export async function eventFixture(name: string): Promise { @@ -119,20 +113,11 @@ export const createTestDB = async (databaseUrl?: string) => { }); } else { kysely = new Kysely({ - dialect: { - createAdapter() { - return new PostgresAdapter(); - }, - createDriver() { - return new PostgreSQLDriver(new Pool(databaseUrl, 10, true)); - }, - createIntrospector(db: Kysely) { - return new PostgresIntrospector(db); - }, - createQueryCompiler() { - return new PostgresQueryCompiler(); - }, - }, + dialect: new PostgresJSDialect({ + postgres: postgres(Conf.databaseUrl, { + max: Conf.pg.poolSize, + }) as unknown as PostgresJSDialectConfig['postgres'], + }), log: KyselyLogger, }); } From 9c02a2d555d6d02ea0e4ee01a5ba5c3ea4769f35 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 26 Jul 2024 18:49:31 -0300 Subject: [PATCH 23/24] refactor: remove nip05 enhancement and tests --- src/storages/EventsDB.test.ts | 21 --------------------- src/storages/EventsDB.ts | 5 +---- 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 1b4b66e0..82c4e34a 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -217,24 +217,3 @@ Deno.test("throws a RelayError when querying an event with a large 'kind'", asyn 'kind filter too far into the future', ); }); - -Deno.test( - 'query user by NIP-05 search filter', - { ignore: Conf.db.dialect !== 'postgres' }, - async () => { - await using db = await createTestDB(); - const { store } = db; - - const event0 = await eventFixture('event-0'); - await store.event(event0); - - assertEquals(await store.query([{}]), [event0]); - assertEquals(await store.query([{ search: 'sonator.dev' }]), []); - assertEquals(await store.query([{ search: 'alex' }]), [event0]); - assertEquals(await store.query([{ search: 'gleasonator' }]), [event0]); - assertEquals(await store.query([{ search: 'com' }]), [event0]); - assertEquals(await store.query([{ search: 'mostr' }]), [event0]); - assertEquals(await store.query([{ search: 'pub' }]), [event0]); - assertEquals(await store.query([{ search: 'mostr.pub' }]), [event0]); - }, -); diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index d5171bc2..abf076c7 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -240,10 +240,7 @@ class EventsDB implements NStore { /** Build search content for a user. */ static buildUserSearchContent(event: NostrEvent): string { const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(event.content); - const nip05splitted = nip05 ? nip05.split(/[_@.]/) : []; - const nip05splitted2 = nip05 ? nip05.split(/[@]/) : []; - - return [name, nip05, ...nip05splitted, ...nip05splitted2].filter(Boolean).join('\n'); + return [name, nip05].filter(Boolean).join('\n'); } /** Build search content from tag values. */ From a271340ffc6199fb2f88837f12d5d5af914cd942 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 26 Jul 2024 19:01:18 -0300 Subject: [PATCH 24/24] test: use postgres in hydrate.test.ts --- src/storages/hydrate.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/storages/hydrate.test.ts b/src/storages/hydrate.test.ts index 6730315b..3eb70bf8 100644 --- a/src/storages/hydrate.test.ts +++ b/src/storages/hydrate.test.ts @@ -7,7 +7,7 @@ import { createTestDB, eventFixture } from '@/test.ts'; Deno.test('hydrateEvents(): author --- WITHOUT stats', async () => { const relay = new MockRelay(); - await using db = await createTestDB('sqlite://:memory:'); + await using db = await createTestDB(); const event0 = await eventFixture('event-0'); const event1 = await eventFixture('event-1'); @@ -28,7 +28,7 @@ Deno.test('hydrateEvents(): author --- WITHOUT stats', async () => { Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { const relay = new MockRelay(); - await using db = await createTestDB('sqlite://:memory:'); + await using db = await createTestDB(); const event0madePost = await eventFixture('event-0-the-one-who-post-and-users-repost'); const event0madeRepost = await eventFixture('event-0-the-one-who-repost'); @@ -57,7 +57,7 @@ Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { const relay = new MockRelay(); - await using db = await createTestDB('sqlite://:memory:'); + await using db = await createTestDB(); const event0madeQuoteRepost = await eventFixture('event-0-the-one-who-quote-repost'); const event0 = await eventFixture('event-0'); @@ -87,7 +87,7 @@ Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () => { const relay = new MockRelay(); - await using db = await createTestDB('sqlite://:memory:'); + await using db = await createTestDB(); const author = await eventFixture('event-0-makes-repost-with-quote-repost'); const event1 = await eventFixture('event-1-will-be-reposted-with-quote-repost'); @@ -116,7 +116,7 @@ Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () Deno.test('hydrateEvents(): report pubkey and post // kind 1984 --- WITHOUT stats', async () => { const relay = new MockRelay(); - await using db = await createTestDB('sqlite://:memory:'); + await using db = await createTestDB(); const authorDictator = await eventFixture('kind-0-dictator'); const authorVictim = await eventFixture('kind-0-george-orwell');