From 78d41bbd6d51236d6ac3b70129e9a14f137db495 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Wed, 3 Jul 2024 20:41:48 -0300 Subject: [PATCH 01/59] feat(EventsDB): add nip05 split in buildUserSearchContent() --- src/storages/EventsDB.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index ce74f20d..c74fedb6 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -240,7 +240,9 @@ class EventsDB implements NStore { /** Build search content for a user. */ static buildUserSearchContent(event: NostrEvent): string { const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(event.content); - return [name, nip05].filter(Boolean).join('\n'); + const nip05splitted = nip05 ? nip05.split(/[_@.]/) : []; + + return [name, nip05, ...nip05splitted].filter(Boolean).join('\n'); } /** Build search content from tag values. */ From d8247dc76aac6aa627cb6bdfbac4a1730626d85f Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 4 Jul 2024 18:08:03 -0300 Subject: [PATCH 02/59] build: remove DATABASE_URL in test script --- deno.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deno.json b/deno.json index 084b527b..14212d9f 100644 --- a/deno.json +++ b/deno.json @@ -8,7 +8,7 @@ "db:migrate": "deno run -A scripts/db-migrate.ts", "nostr:pull": "deno run -A scripts/nostr-pull.ts", "debug": "deno run -A --inspect src/server.ts", - "test": "DATABASE_URL=\"sqlite://:memory:\" deno test -A --junit-path=./deno-test.xml", + "test": "deno test -A --junit-path=./deno-test.xml", "check": "deno check src/server.ts", "nsec": "deno run scripts/nsec.ts", "admin:event": "deno run -A scripts/admin-event.ts", From cbb48867c12aa876b40c226c4bdccdb117cb916b Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 4 Jul 2024 18:13:31 -0300 Subject: [PATCH 03/59] test: attempt to use Postgres --- src/storages/EventsDB.test.ts | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 32838fa6..818a9b14 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -1,25 +1,16 @@ -import { Database as Sqlite } from '@db/sqlite'; -import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite'; import { assertEquals, assertRejects } from '@std/assert'; -import { Kysely } from 'kysely'; import { generateSecretKey } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; import { RelayError } from '@/RelayError.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; import { eventFixture, genEvent } from '@/test.ts'; -/** Create in-memory database for testing. */ +/** Create an database for testing. */ const createDB = async () => { - const kysely = new Kysely({ - dialect: new DenoSqlite3Dialect({ - database: new Sqlite(':memory:'), - }), - }); + const kysely = await DittoDB.getInstance(); const eventsDB = new EventsDB(kysely); - await DittoDB.migrate(kysely); return { eventsDB, kysely }; }; @@ -221,3 +212,7 @@ Deno.test("throws a RelayError when querying an event with a large 'kind'", asyn 'kind filter too far into the future', ); }); + +Deno.test('query user by NIP-05 search filter', async () => { + // implement +}); From a9f79176a23411fa7a7d2d80d8411dabd791bc9e Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 4 Jul 2024 22:43:56 -0300 Subject: [PATCH 04/59] test: support sqlite and postgres --- src/storages/EventsDB.test.ts | 101 ++++++++++++++++++++++++++++------ 1 file changed, 85 insertions(+), 16 deletions(-) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 818a9b14..a9fe18c5 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -1,21 +1,80 @@ import { assertEquals, assertRejects } from '@std/assert'; import { generateSecretKey } from 'nostr-tools'; +import { Database as Sqlite } from '@db/sqlite'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; import { RelayError } from '@/RelayError.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; import { eventFixture, genEvent } from '@/test.ts'; +import { EventsDB } from '@/storages/EventsDB.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; +import { Kysely } from 'kysely'; +import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite'; + +const databaseUrl = Deno.env.get('DATABASE_URL') ?? 'sqlite://:memory:'; + +const dialect: 'sqlite' | 'postgres' = (() => { + const protocol = databaseUrl.split(':')[0]; + switch (protocol) { + case 'sqlite': + return 'sqlite'; + case 'postgres': + return protocol; + case 'postgresql': + return 'postgres'; + default: + throw new Error(`Unsupported protocol: ${protocol}`); + } +})(); /** Create an database for testing. */ const createDB = async () => { - const kysely = await DittoDB.getInstance(); - const eventsDB = new EventsDB(kysely); - return { eventsDB, kysely }; -}; + let kysely: Kysely; + if (dialect === 'sqlite') { + kysely = new Kysely({ + dialect: new DenoSqlite3Dialect({ + database: new Sqlite(':memory:'), + }), + }); + await DittoDB.migrate(kysely); + } else { + kysely = await DittoDB.getInstance(); + } + + const eventsDB = new EventsDB(kysely); + + return { + eventsDB, + kysely, + [Symbol.asyncDispose]: async () => { + if (dialect === 'postgres') { + for ( + const table of [ + 'author_stats', + 'event_stats', + 'event_zaps', + 'kysely_migration', + 'kysely_migration_lock', + 'nip46_tokens', + 'pubkey_domains', + 'unattached_media', + 'nostr_events', + 'nostr_tags', + 'nostr_pgfts', + ] + ) { + await kysely.schema.dropTable(table).ifExists().cascade().execute(); + } + await kysely.destroy(); + } + }, + }; +}; Deno.test('count filters', async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; + const event1 = await eventFixture('event-1'); assertEquals((await eventsDB.count([{ kinds: [1] }])).count, 0); @@ -24,7 +83,8 @@ Deno.test('count filters', async () => { }); Deno.test('insert and filter events', async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; const event1 = await eventFixture('event-1'); await eventsDB.event(event1); @@ -59,7 +119,8 @@ Deno.test('query events with domain search filter', async () => { }); Deno.test('delete events', async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; const [one, two] = [ { id: '1', kind: 1, pubkey: 'abc', content: 'hello world', created_at: 1, sig: '', tags: [] }, @@ -86,7 +147,8 @@ Deno.test('delete events', async () => { }); Deno.test("user cannot delete another user's event", async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; const event = { id: '1', kind: 1, pubkey: 'abc', content: 'hello world', created_at: 1, sig: '', tags: [] }; await eventsDB.event(event); @@ -108,7 +170,8 @@ Deno.test("user cannot delete another user's event", async () => { }); Deno.test('admin can delete any event', async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; const [one, two] = [ { id: '1', kind: 1, pubkey: 'abc', content: 'hello world', created_at: 1, sig: '', tags: [] }, @@ -135,7 +198,8 @@ Deno.test('admin can delete any event', async () => { }); Deno.test('throws a RelayError when inserting an event deleted by the admin', async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; const event = genEvent(); await eventsDB.event(event); @@ -151,7 +215,8 @@ Deno.test('throws a RelayError when inserting an event deleted by the admin', as }); Deno.test('throws a RelayError when inserting an event deleted by a user', async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; const sk = generateSecretKey(); @@ -169,7 +234,8 @@ Deno.test('throws a RelayError when inserting an event deleted by a user', async }); Deno.test('inserting replaceable events', async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; const event = await eventFixture('event-0'); await eventsDB.event(event); @@ -184,7 +250,8 @@ Deno.test('inserting replaceable events', async () => { }); Deno.test("throws a RelayError when querying an event with a large 'since'", async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; await assertRejects( () => eventsDB.query([{ since: 33333333333333 }]), @@ -194,7 +261,8 @@ Deno.test("throws a RelayError when querying an event with a large 'since'", asy }); Deno.test("throws a RelayError when querying an event with a large 'until'", async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; await assertRejects( () => eventsDB.query([{ until: 66666666666666 }]), @@ -204,7 +272,8 @@ Deno.test("throws a RelayError when querying an event with a large 'until'", asy }); Deno.test("throws a RelayError when querying an event with a large 'kind'", async () => { - const { eventsDB } = await createDB(); + await using db = await createDB(); + const { eventsDB } = db; await assertRejects( () => eventsDB.query([{ kinds: [99999999999999] }]), From 12b2032d8732c2760f6a02cc0aa985d1d191ddff Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 4 Jul 2024 22:50:12 -0300 Subject: [PATCH 05/59] refactor(EventsDB.test): add one blank line between functions --- src/storages/EventsDB.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index a9fe18c5..749ca777 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -71,6 +71,7 @@ const createDB = async () => { }, }; }; + Deno.test('count filters', async () => { await using db = await createDB(); const { eventsDB } = db; From 2154cf011bd41254a4a0eb92b44f2d488ed4bc5d Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 17:26:27 -0300 Subject: [PATCH 06/59] test: create createTestDB function, supports both Sqlite and Postgres --- src/test.ts | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) diff --git a/src/test.ts b/src/test.ts index b0172063..7aecab7a 100644 --- a/src/test.ts +++ b/src/test.ts @@ -4,11 +4,23 @@ import path from 'node:path'; import { Database as Sqlite } from '@db/sqlite'; import { NDatabase, NostrEvent } from '@nostrify/nostrify'; import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite'; -import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; +import { + FileMigrationProvider, + Kysely, + Migrator, + PostgresAdapter, + PostgresIntrospector, + PostgresQueryCompiler, +} from 'kysely'; import { finalizeEvent, generateSecretKey } from 'nostr-tools'; +import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; +import { PostgreSQLDriver } from 'kysely_deno_postgres'; +import { Pool } from 'postgres'; +import { KyselyLogger } from '@/db/KyselyLogger.ts'; +import { EventsDB } from '@/storages/EventsDB.ts'; /** Import an event fixture by name in tests. */ export async function eventFixture(name: string): Promise { @@ -62,3 +74,84 @@ export async function getTestDB() { [Symbol.asyncDispose]: () => kysely.destroy(), }; } + +/** Create an database for testing. */ +export const createTestDB = async (databaseUrl?: string) => { + databaseUrl ??= Deno.env.get('DATABASE_URL') ?? 'sqlite://:memory:'; + + const dialect: 'sqlite' | 'postgres' = (() => { + const protocol = databaseUrl.split(':')[0]; + switch (protocol) { + case 'sqlite': + return 'sqlite'; + case 'postgres': + return protocol; + case 'postgresql': + return 'postgres'; + default: + throw new Error(`Unsupported protocol: ${protocol}`); + } + })(); + + let kysely: Kysely; + + if (dialect === 'sqlite') { + Deno.env.set('DATABASE_URL', 'sqlite://:memory:'); // hack, refactor all, 021 migration + + kysely = new Kysely({ + dialect: new DenoSqlite3Dialect({ + database: new Sqlite(':memory:'), + }), + }); + } else { + //kysely = await DittoDB.getInstance(); + kysely = new Kysely({ + dialect: { + createAdapter() { + return new PostgresAdapter(); + }, + createDriver() { + return new PostgreSQLDriver(new Pool(databaseUrl, 10, true)); + }, + createIntrospector(db: Kysely) { + return new PostgresIntrospector(db); + }, + createQueryCompiler() { + return new PostgresQueryCompiler(); + }, + }, + log: KyselyLogger, + }); + } + await DittoDB.migrate(kysely); + + const store = new EventsDB(kysely); + + return { + store, + kysely, + [Symbol.asyncDispose]: async () => { + if (dialect === 'postgres') { + for ( + const table of [ + 'author_stats', + 'event_stats', + 'event_zaps', + 'kysely_migration', + 'kysely_migration_lock', + 'nip46_tokens', + 'pubkey_domains', + 'unattached_media', + 'nostr_events', + 'nostr_tags', + 'nostr_pgfts', + 'event_zaps', + ] + ) { + await kysely.schema.dropTable(table).ifExists().cascade().execute(); + } + await kysely.destroy(); + } + }, + }; +}; From 8e1826020c138c04c6ce6488ed68ae38b1159ba8 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 17:35:25 -0300 Subject: [PATCH 07/59] test(EventsDB): remove createDB function --- src/storages/EventsDB.test.ts | 203 ++++++++++++---------------------- 1 file changed, 69 insertions(+), 134 deletions(-) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 749ca777..1cc9bbf5 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -1,140 +1,75 @@ import { assertEquals, assertRejects } from '@std/assert'; import { generateSecretKey } from 'nostr-tools'; -import { Database as Sqlite } from '@db/sqlite'; -import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { RelayError } from '@/RelayError.ts'; import { eventFixture, genEvent } from '@/test.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; -import { Kysely } from 'kysely'; -import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite'; - -const databaseUrl = Deno.env.get('DATABASE_URL') ?? 'sqlite://:memory:'; - -const dialect: 'sqlite' | 'postgres' = (() => { - const protocol = databaseUrl.split(':')[0]; - switch (protocol) { - case 'sqlite': - return 'sqlite'; - case 'postgres': - return protocol; - case 'postgresql': - return 'postgres'; - default: - throw new Error(`Unsupported protocol: ${protocol}`); - } -})(); - -/** Create an database for testing. */ -const createDB = async () => { - let kysely: Kysely; - - if (dialect === 'sqlite') { - kysely = new Kysely({ - dialect: new DenoSqlite3Dialect({ - database: new Sqlite(':memory:'), - }), - }); - await DittoDB.migrate(kysely); - } else { - kysely = await DittoDB.getInstance(); - } - - const eventsDB = new EventsDB(kysely); - - return { - eventsDB, - kysely, - [Symbol.asyncDispose]: async () => { - if (dialect === 'postgres') { - for ( - const table of [ - 'author_stats', - 'event_stats', - 'event_zaps', - 'kysely_migration', - 'kysely_migration_lock', - 'nip46_tokens', - 'pubkey_domains', - 'unattached_media', - 'nostr_events', - 'nostr_tags', - 'nostr_pgfts', - ] - ) { - await kysely.schema.dropTable(table).ifExists().cascade().execute(); - } - await kysely.destroy(); - } - }, - }; -}; +import { Conf } from '@/config.ts'; +import { createTestDB } from '@/test.ts'; Deno.test('count filters', async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; const event1 = await eventFixture('event-1'); - assertEquals((await eventsDB.count([{ kinds: [1] }])).count, 0); - await eventsDB.event(event1); - assertEquals((await eventsDB.count([{ kinds: [1] }])).count, 1); + assertEquals((await store.count([{ kinds: [1] }])).count, 0); + await store.event(event1); + assertEquals((await store.count([{ kinds: [1] }])).count, 1); }); Deno.test('insert and filter events', async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; const event1 = await eventFixture('event-1'); - await eventsDB.event(event1); + await store.event(event1); - assertEquals(await eventsDB.query([{ kinds: [1] }]), [event1]); - assertEquals(await eventsDB.query([{ kinds: [3] }]), []); - assertEquals(await eventsDB.query([{ since: 1691091000 }]), [event1]); - assertEquals(await eventsDB.query([{ until: 1691091000 }]), []); + assertEquals(await store.query([{ kinds: [1] }]), [event1]); + assertEquals(await store.query([{ kinds: [3] }]), []); + assertEquals(await store.query([{ since: 1691091000 }]), [event1]); + assertEquals(await store.query([{ until: 1691091000 }]), []); assertEquals( - await eventsDB.query([{ '#proxy': ['https://gleasonator.com/objects/8f6fac53-4f66-4c6e-ac7d-92e5e78c3e79'] }]), + await store.query([{ '#proxy': ['https://gleasonator.com/objects/8f6fac53-4f66-4c6e-ac7d-92e5e78c3e79'] }]), [event1], ); }); Deno.test('query events with domain search filter', async () => { - const { eventsDB, kysely } = await createDB(); + await using db = await createTestDB(); + const { store, kysely } = db; const event1 = await eventFixture('event-1'); - await eventsDB.event(event1); + await store.event(event1); - assertEquals(await eventsDB.query([{}]), [event1]); - assertEquals(await eventsDB.query([{ search: 'domain:localhost:4036' }]), []); - assertEquals(await eventsDB.query([{ search: '' }]), [event1]); + assertEquals(await store.query([{}]), [event1]); + assertEquals(await store.query([{ search: 'domain:localhost:4036' }]), []); + assertEquals(await store.query([{ search: '' }]), [event1]); await kysely .insertInto('pubkey_domains') .values({ pubkey: event1.pubkey, domain: 'localhost:4036', last_updated_at: event1.created_at }) .execute(); - assertEquals(await eventsDB.query([{ kinds: [1], search: 'domain:localhost:4036' }]), [event1]); - assertEquals(await eventsDB.query([{ kinds: [1], search: 'domain:example.com' }]), []); + assertEquals(await store.query([{ kinds: [1], search: 'domain:localhost:4036' }]), [event1]); + assertEquals(await store.query([{ kinds: [1], search: 'domain:example.com' }]), []); }); Deno.test('delete events', async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; const [one, two] = [ { id: '1', kind: 1, pubkey: 'abc', content: 'hello world', created_at: 1, sig: '', tags: [] }, { id: '2', kind: 1, pubkey: 'abc', content: 'yolo fam', created_at: 2, sig: '', tags: [] }, ]; - await eventsDB.event(one); - await eventsDB.event(two); + await store.event(one); + await store.event(two); // Sanity check - assertEquals(await eventsDB.query([{ kinds: [1] }]), [two, one]); + assertEquals(await store.query([{ kinds: [1] }]), [two, one]); - await eventsDB.event({ + await store.event({ kind: 5, pubkey: one.pubkey, tags: [['e', one.id]], @@ -144,20 +79,20 @@ Deno.test('delete events', async () => { sig: '', }); - assertEquals(await eventsDB.query([{ kinds: [1] }]), [two]); + assertEquals(await store.query([{ kinds: [1] }]), [two]); }); Deno.test("user cannot delete another user's event", async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; const event = { id: '1', kind: 1, pubkey: 'abc', content: 'hello world', created_at: 1, sig: '', tags: [] }; - await eventsDB.event(event); + await store.event(event); // Sanity check - assertEquals(await eventsDB.query([{ kinds: [1] }]), [event]); + assertEquals(await store.query([{ kinds: [1] }]), [event]); - await eventsDB.event({ + await store.event({ kind: 5, pubkey: 'def', // different pubkey tags: [['e', event.id]], @@ -167,25 +102,25 @@ Deno.test("user cannot delete another user's event", async () => { sig: '', }); - assertEquals(await eventsDB.query([{ kinds: [1] }]), [event]); + assertEquals(await store.query([{ kinds: [1] }]), [event]); }); Deno.test('admin can delete any event', async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; const [one, two] = [ { id: '1', kind: 1, pubkey: 'abc', content: 'hello world', created_at: 1, sig: '', tags: [] }, { id: '2', kind: 1, pubkey: 'abc', content: 'yolo fam', created_at: 2, sig: '', tags: [] }, ]; - await eventsDB.event(one); - await eventsDB.event(two); + await store.event(one); + await store.event(two); // Sanity check - assertEquals(await eventsDB.query([{ kinds: [1] }]), [two, one]); + assertEquals(await store.query([{ kinds: [1] }]), [two, one]); - await eventsDB.event({ + await store.event({ kind: 5, pubkey: Conf.pubkey, // Admin pubkey tags: [['e', one.id]], @@ -195,89 +130,89 @@ Deno.test('admin can delete any event', async () => { sig: '', }); - assertEquals(await eventsDB.query([{ kinds: [1] }]), [two]); + assertEquals(await store.query([{ kinds: [1] }]), [two]); }); Deno.test('throws a RelayError when inserting an event deleted by the admin', async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; const event = genEvent(); - await eventsDB.event(event); + await store.event(event); const deletion = genEvent({ kind: 5, tags: [['e', event.id]] }, Conf.seckey); - await eventsDB.event(deletion); + await store.event(deletion); await assertRejects( - () => eventsDB.event(event), + () => store.event(event), RelayError, 'event deleted by admin', ); }); Deno.test('throws a RelayError when inserting an event deleted by a user', async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; const sk = generateSecretKey(); const event = genEvent({}, sk); - await eventsDB.event(event); + await store.event(event); const deletion = genEvent({ kind: 5, tags: [['e', event.id]] }, sk); - await eventsDB.event(deletion); + await store.event(deletion); await assertRejects( - () => eventsDB.event(event), + () => store.event(event), RelayError, 'event deleted by user', ); }); Deno.test('inserting replaceable events', async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; const event = await eventFixture('event-0'); - await eventsDB.event(event); + await store.event(event); const olderEvent = { ...event, id: '123', created_at: event.created_at - 1 }; - await eventsDB.event(olderEvent); - assertEquals(await eventsDB.query([{ kinds: [0], authors: [event.pubkey] }]), [event]); + await store.event(olderEvent); + assertEquals(await store.query([{ kinds: [0], authors: [event.pubkey] }]), [event]); const newerEvent = { ...event, id: '123', created_at: event.created_at + 1 }; - await eventsDB.event(newerEvent); - assertEquals(await eventsDB.query([{ kinds: [0] }]), [newerEvent]); + await store.event(newerEvent); + assertEquals(await store.query([{ kinds: [0] }]), [newerEvent]); }); Deno.test("throws a RelayError when querying an event with a large 'since'", async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; await assertRejects( - () => eventsDB.query([{ since: 33333333333333 }]), + () => store.query([{ since: 33333333333333 }]), RelayError, 'since filter too far into the future', ); }); Deno.test("throws a RelayError when querying an event with a large 'until'", async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; await assertRejects( - () => eventsDB.query([{ until: 66666666666666 }]), + () => store.query([{ until: 66666666666666 }]), RelayError, 'until filter too far into the future', ); }); Deno.test("throws a RelayError when querying an event with a large 'kind'", async () => { - await using db = await createDB(); - const { eventsDB } = db; + await using db = await createTestDB(); + const { store } = db; await assertRejects( - () => eventsDB.query([{ kinds: [99999999999999] }]), + () => store.query([{ kinds: [99999999999999] }]), RelayError, 'kind filter too far into the future', ); From 85fd7909e6159ada7190208d113a79442c260000 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 17:42:19 -0300 Subject: [PATCH 08/59] feat: add optional kysely in HydrateOpts interface --- src/storages/hydrate.ts | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 3c264320..00ae59a4 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -7,16 +7,19 @@ import { Conf } from '@/config.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { findQuoteTag } from '@/utils/tags.ts'; import { findQuoteInContent } from '@/utils/note.ts'; +import { Kysely } from 'kysely'; interface HydrateOpts { events: DittoEvent[]; store: NStore; signal?: AbortSignal; + kysely?: Kysely; } /** Hydrate events using the provided storage. */ async function hydrateEvents(opts: HydrateOpts): Promise { - const { events, store, signal } = opts; + const { events, store, signal, kysely = await DittoDB.getInstance() } = opts; + console.log(kysely); if (!events.length) { return events; @@ -57,8 +60,8 @@ async function hydrateEvents(opts: HydrateOpts): Promise { } const stats = { - authors: await gatherAuthorStats(cache), - events: await gatherEventStats(cache), + authors: await gatherAuthorStats(cache, kysely), + events: await gatherEventStats(cache, kysely), }; // Dedupe events. @@ -276,7 +279,10 @@ function gatherReportedProfiles({ events, store, signal }: HydrateOpts): Promise } /** Collect author stats from the events. */ -async function gatherAuthorStats(events: DittoEvent[]): Promise { +async function gatherAuthorStats( + events: DittoEvent[], + kysely: Kysely, +): Promise { const pubkeys = new Set( events .filter((event) => event.kind === 0) @@ -287,8 +293,6 @@ async function gatherAuthorStats(events: DittoEvent[]): Promise { +async function gatherEventStats( + events: DittoEvent[], + kysely: Kysely, +): Promise { const ids = new Set( events .filter((event) => event.kind === 1) @@ -315,8 +322,6 @@ async function gatherEventStats(events: DittoEvent[]): Promise Date: Fri, 5 Jul 2024 17:43:47 -0300 Subject: [PATCH 09/59] test(hydrate): refactor to only use sqlite --- src/storages/hydrate.test.ts | 68 +++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/src/storages/hydrate.test.ts b/src/storages/hydrate.test.ts index e3b1cf2e..6730315b 100644 --- a/src/storages/hydrate.test.ts +++ b/src/storages/hydrate.test.ts @@ -3,21 +3,23 @@ import { assertEquals } from '@std/assert'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { eventFixture } from '@/test.ts'; +import { createTestDB, eventFixture } from '@/test.ts'; Deno.test('hydrateEvents(): author --- WITHOUT stats', async () => { - const db = new MockRelay(); + const relay = new MockRelay(); + await using db = await createTestDB('sqlite://:memory:'); const event0 = await eventFixture('event-0'); const event1 = await eventFixture('event-1'); // Save events to database - await db.event(event0); - await db.event(event1); + await relay.event(event0); + await relay.event(event1); await hydrateEvents({ events: [event1], - store: db, + store: relay, + kysely: db.kysely, }); const expectedEvent = { ...event1, author: event0 }; @@ -25,7 +27,8 @@ Deno.test('hydrateEvents(): author --- WITHOUT stats', async () => { }); Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { - const db = new MockRelay(); + const relay = new MockRelay(); + await using db = await createTestDB('sqlite://:memory:'); const event0madePost = await eventFixture('event-0-the-one-who-post-and-users-repost'); const event0madeRepost = await eventFixture('event-0-the-one-who-repost'); @@ -33,14 +36,15 @@ Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { const event6 = await eventFixture('event-6'); // Save events to database - await db.event(event0madePost); - await db.event(event0madeRepost); - await db.event(event1reposted); - await db.event(event6); + await relay.event(event0madePost); + await relay.event(event0madeRepost); + await relay.event(event1reposted); + await relay.event(event6); await hydrateEvents({ events: [event6], - store: db, + store: relay, + kysely: db.kysely, }); const expectedEvent6 = { @@ -52,7 +56,8 @@ Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { }); Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { - const db = new MockRelay(); + const relay = new MockRelay(); + await using db = await createTestDB('sqlite://:memory:'); const event0madeQuoteRepost = await eventFixture('event-0-the-one-who-quote-repost'); const event0 = await eventFixture('event-0'); @@ -60,14 +65,15 @@ Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { const event1willBeQuoteReposted = await eventFixture('event-1-that-will-be-quote-reposted'); // Save events to database - await db.event(event0madeQuoteRepost); - await db.event(event0); - await db.event(event1quoteRepost); - await db.event(event1willBeQuoteReposted); + await relay.event(event0madeQuoteRepost); + await relay.event(event0); + await relay.event(event1quoteRepost); + await relay.event(event1willBeQuoteReposted); await hydrateEvents({ events: [event1quoteRepost], - store: db, + store: relay, + kysely: db.kysely, }); const expectedEvent1quoteRepost = { @@ -80,7 +86,8 @@ Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { }); Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () => { - const db = new MockRelay(); + const relay = new MockRelay(); + await using db = await createTestDB('sqlite://:memory:'); const author = await eventFixture('event-0-makes-repost-with-quote-repost'); const event1 = await eventFixture('event-1-will-be-reposted-with-quote-repost'); @@ -88,14 +95,15 @@ Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () const event1quote = await eventFixture('event-1-quote-repost-will-be-reposted'); // Save events to database - await db.event(author); - await db.event(event1); - await db.event(event1quote); - await db.event(event6); + await relay.event(author); + await relay.event(event1); + await relay.event(event1quote); + await relay.event(event6); await hydrateEvents({ events: [event6], - store: db, + store: relay, + kysely: db.kysely, }); const expectedEvent6 = { @@ -107,7 +115,8 @@ Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () }); Deno.test('hydrateEvents(): report pubkey and post // kind 1984 --- WITHOUT stats', async () => { - const db = new MockRelay(); + const relay = new MockRelay(); + await using db = await createTestDB('sqlite://:memory:'); const authorDictator = await eventFixture('kind-0-dictator'); const authorVictim = await eventFixture('kind-0-george-orwell'); @@ -115,14 +124,15 @@ Deno.test('hydrateEvents(): report pubkey and post // kind 1984 --- WITHOUT stat const event1 = await eventFixture('kind-1-author-george-orwell'); // Save events to database - await db.event(authorDictator); - await db.event(authorVictim); - await db.event(reportEvent); - await db.event(event1); + await relay.event(authorDictator); + await relay.event(authorVictim); + await relay.event(reportEvent); + await relay.event(event1); await hydrateEvents({ events: [reportEvent], - store: db, + store: relay, + kysely: db.kysely, }); const expectedEvent: DittoEvent = { From f380120cd39c9a87dc7f557eb97b2985a3997ff7 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 17:48:06 -0300 Subject: [PATCH 10/59] test: refactor to use createTestDB instead of getTestDB --- src/utils/stats.test.ts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/utils/stats.test.ts b/src/utils/stats.test.ts index c3e26595..69633ae3 100644 --- a/src/utils/stats.test.ts +++ b/src/utils/stats.test.ts @@ -1,11 +1,11 @@ import { assertEquals } from '@std/assert'; import { generateSecretKey, getPublicKey } from 'nostr-tools'; -import { genEvent, getTestDB } from '@/test.ts'; +import { createTestDB, genEvent } from '@/test.ts'; import { countAuthorStats, getAuthorStats, getEventStats, getFollowDiff, updateStats } from '@/utils/stats.ts'; Deno.test('updateStats with kind 1 increments notes count', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const sk = generateSecretKey(); const pubkey = getPublicKey(sk); @@ -18,7 +18,7 @@ Deno.test('updateStats with kind 1 increments notes count', async () => { }); Deno.test('updateStats with kind 1 increments replies count', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const sk = generateSecretKey(); @@ -36,7 +36,7 @@ Deno.test('updateStats with kind 1 increments replies count', async () => { }); Deno.test('updateStats with kind 5 decrements notes count', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const sk = generateSecretKey(); const pubkey = getPublicKey(sk); @@ -54,7 +54,7 @@ Deno.test('updateStats with kind 5 decrements notes count', async () => { }); Deno.test('updateStats with kind 3 increments followers count', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); await updateStats({ ...db, event: genEvent({ kind: 3, tags: [['p', 'alex']] }) }); await updateStats({ ...db, event: genEvent({ kind: 3, tags: [['p', 'alex']] }) }); @@ -66,7 +66,7 @@ Deno.test('updateStats with kind 3 increments followers count', async () => { }); Deno.test('updateStats with kind 3 decrements followers count', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const sk = generateSecretKey(); const follow = genEvent({ kind: 3, tags: [['p', 'alex']], created_at: 0 }, sk); @@ -92,7 +92,7 @@ Deno.test('getFollowDiff returns added and removed followers', () => { }); Deno.test('updateStats with kind 6 increments reposts count', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const note = genEvent({ kind: 1 }); await updateStats({ ...db, event: note }); @@ -108,7 +108,7 @@ Deno.test('updateStats with kind 6 increments reposts count', async () => { }); Deno.test('updateStats with kind 5 decrements reposts count', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const note = genEvent({ kind: 1 }); await updateStats({ ...db, event: note }); @@ -127,7 +127,7 @@ Deno.test('updateStats with kind 5 decrements reposts count', async () => { }); Deno.test('updateStats with kind 7 increments reactions count', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const note = genEvent({ kind: 1 }); await updateStats({ ...db, event: note }); @@ -143,7 +143,7 @@ Deno.test('updateStats with kind 7 increments reactions count', async () => { }); Deno.test('updateStats with kind 5 decrements reactions count', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const note = genEvent({ kind: 1 }); await updateStats({ ...db, event: note }); @@ -162,7 +162,7 @@ Deno.test('updateStats with kind 5 decrements reactions count', async () => { }); Deno.test('countAuthorStats counts author stats from the database', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const sk = generateSecretKey(); const pubkey = getPublicKey(sk); From 34bbf6e39a10077dd6b42692b20fde6a18df40a7 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 17:48:34 -0300 Subject: [PATCH 11/59] test(pipeline): refactor to use createTestDB instead of getTestDB --- src/pipeline.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipeline.test.ts b/src/pipeline.test.ts index 64cb523b..3a196c16 100644 --- a/src/pipeline.test.ts +++ b/src/pipeline.test.ts @@ -1,11 +1,11 @@ import { assertEquals } from '@std/assert'; import { generateSecretKey } from 'nostr-tools'; -import { genEvent, getTestDB } from '@/test.ts'; +import { createTestDB, genEvent, getTestDB } from '@/test.ts'; import { handleZaps } from '@/pipeline.ts'; Deno.test('store one zap receipt in nostr_events; convert it into event_zaps table format and store it', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const kysely = db.kysely; const sk = generateSecretKey(); From 1af229da45dbf04b1c09ebd9f73139d2f60efd70 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 17:52:13 -0300 Subject: [PATCH 12/59] build(gitlab-ci): use deno task test in postgres ci --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 20650d2e..88829e92 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,10 +32,10 @@ test: postgres: stage: test - script: deno task db:migrate + script: deno task db:migrate && deno task test services: - postgres:16 variables: DITTO_NSEC: nsec1zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs4rm7hz DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres - POSTGRES_HOST_AUTH_METHOD: trust \ No newline at end of file + POSTGRES_HOST_AUTH_METHOD: trust From 7e965f4aded194cda350555745ac51cba6883dbe Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 17:59:12 -0300 Subject: [PATCH 13/59] refactor(test.ts): explain why set DATABASE_URL to sqlite://:memory: --- src/test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test.ts b/src/test.ts index 7aecab7a..ae2af52a 100644 --- a/src/test.ts +++ b/src/test.ts @@ -96,7 +96,10 @@ export const createTestDB = async (databaseUrl?: string) => { let kysely: Kysely; if (dialect === 'sqlite') { - Deno.env.set('DATABASE_URL', 'sqlite://:memory:'); // hack, refactor all, 021 migration + // migration 021_pgfts_index.ts calls 'Conf.db.dialect', + // and this calls the DATABASE_URL environment variable. + // The following line ensures to NOT use the DATABASE_URL that may exist in an .env file. + Deno.env.set('DATABASE_URL', 'sqlite://:memory:'); kysely = new Kysely({ dialect: new DenoSqlite3Dialect({ @@ -104,7 +107,6 @@ export const createTestDB = async (databaseUrl?: string) => { }), }); } else { - //kysely = await DittoDB.getInstance(); kysely = new Kysely({ dialect: { createAdapter() { From fed0c70f69c7d76366a993cae845b4e012e88577 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 18:33:29 -0300 Subject: [PATCH 14/59] feat: only run tests with Postgres if ALLOW_TO_USE_DATABASE_URL is set to true --- src/test.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/test.ts b/src/test.ts index ae2af52a..e6967596 100644 --- a/src/test.ts +++ b/src/test.ts @@ -79,7 +79,7 @@ export async function getTestDB() { export const createTestDB = async (databaseUrl?: string) => { databaseUrl ??= Deno.env.get('DATABASE_URL') ?? 'sqlite://:memory:'; - const dialect: 'sqlite' | 'postgres' = (() => { + let dialect: 'sqlite' | 'postgres' = (() => { const protocol = databaseUrl.split(':')[0]; switch (protocol) { case 'sqlite': @@ -93,6 +93,15 @@ export const createTestDB = async (databaseUrl?: string) => { } })(); + const allowToUseDATABASE_URL = Deno.env.get('ALLOW_TO_USE_DATABASE_URL')?.toLowerCase() ?? ''; + if (allowToUseDATABASE_URL !== 'true' && dialect === 'postgres') { + console.warn( + '%cRunning tests with sqlite, if you meant to use Postgres, run again with ALLOW_TO_USE_DATABASE_URL environment variable set to true', + 'color: yellow;', + ); + dialect = 'sqlite'; + } + let kysely: Kysely; if (dialect === 'sqlite') { From 1b3082cd228ddd8e7bdcc1372a0339a44c9d297d Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 18:34:28 -0300 Subject: [PATCH 15/59] build: set ALLOW_TO_USE_DATABASE_URL in postgres ci --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 88829e92..e7de7aef 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,3 +39,4 @@ postgres: DITTO_NSEC: nsec1zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs4rm7hz DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres POSTGRES_HOST_AUTH_METHOD: trust + ALLOW_TO_USE_DATABASE_URL: true From 13d346afbd8ecab4d0a3164288c12cc67ac6f40f Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 18:43:14 -0300 Subject: [PATCH 16/59] refactor: remove console.log --- src/storages/hydrate.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 00ae59a4..6fe8df6f 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -19,7 +19,6 @@ interface HydrateOpts { /** Hydrate events using the provided storage. */ async function hydrateEvents(opts: HydrateOpts): Promise { const { events, store, signal, kysely = await DittoDB.getInstance() } = opts; - console.log(kysely); if (!events.length) { return events; From 01966496c155036c0dc6b769e6c2fb73eae8e877 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 18:43:52 -0300 Subject: [PATCH 17/59] feat: print which dialect is being used when calling createTestDB --- src/test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test.ts b/src/test.ts index e6967596..0a579220 100644 --- a/src/test.ts +++ b/src/test.ts @@ -102,6 +102,8 @@ export const createTestDB = async (databaseUrl?: string) => { dialect = 'sqlite'; } + console.warn(`Using: ${dialect}`); + let kysely: Kysely; if (dialect === 'sqlite') { From 92231dc4520359e20da0d8fffbb47942b2950342 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 20:42:11 -0300 Subject: [PATCH 18/59] test: query by NIP-05 --- src/storages/EventsDB.test.ts | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 1cc9bbf5..942cf849 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -218,6 +218,23 @@ Deno.test("throws a RelayError when querying an event with a large 'kind'", asyn ); }); -Deno.test('query user by NIP-05 search filter', async () => { - // implement -}); +Deno.test( + 'query user by NIP-05 search filter', + { ignore: Deno.env.get('DATABASE_URL')?.slice(0, 8) !== 'postgres' }, + async () => { + await using db = await createTestDB(); + const { store } = db; + + const event0 = await eventFixture('event-0'); + await store.event(event0); + + assertEquals(await store.query([{}]), [event0]); + assertEquals(await store.query([{ search: 'sonator.dev' }]), []); + assertEquals(await store.query([{ search: 'alex' }]), [event0]); + assertEquals(await store.query([{ search: 'gleasonator' }]), [event0]); + assertEquals(await store.query([{ search: 'com' }]), [event0]); + assertEquals(await store.query([{ search: 'mostr' }]), [event0]); + assertEquals(await store.query([{ search: 'pub' }]), [event0]); + assertEquals(await store.query([{ search: 'mostr.pub' }]), [event0]); + }, +); From 5412d2a07b211a5c402cf101813894b297506556 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 5 Jul 2024 20:43:13 -0300 Subject: [PATCH 19/59] fix(EventsDB): also add a split only by '@' at symbol --- src/storages/EventsDB.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index c74fedb6..3ade7e2e 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -241,8 +241,9 @@ class EventsDB implements NStore { static buildUserSearchContent(event: NostrEvent): string { const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(event.content); const nip05splitted = nip05 ? nip05.split(/[_@.]/) : []; + const nip05splitted2 = nip05 ? nip05.split(/[@]/) : []; - return [name, nip05, ...nip05splitted].filter(Boolean).join('\n'); + return [name, nip05, ...nip05splitted, ...nip05splitted2].filter(Boolean).join('\n'); } /** Build search content from tag values. */ From 71bdfea317b35552eb85e2a2e5ea6ce96efa3bc6 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sat, 6 Jul 2024 11:11:41 -0300 Subject: [PATCH 20/59] refactor: get database dialect from Conf.db.dialect --- src/storages/EventsDB.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 942cf849..1b4b66e0 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -220,7 +220,7 @@ Deno.test("throws a RelayError when querying an event with a large 'kind'", asyn Deno.test( 'query user by NIP-05 search filter', - { ignore: Deno.env.get('DATABASE_URL')?.slice(0, 8) !== 'postgres' }, + { ignore: Conf.db.dialect !== 'postgres' }, async () => { await using db = await createTestDB(); const { store } = db; From 8afc3bca3d131a68028b3e42b75b644b3f5dbe1c Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 16 Jul 2024 14:06:32 -0300 Subject: [PATCH 21/59] refactor: order of imports --- src/test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test.ts b/src/test.ts index 808fc9ad..5de5430b 100644 --- a/src/test.ts +++ b/src/test.ts @@ -2,8 +2,11 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { Database as Sqlite } from '@db/sqlite'; -import { NDatabase, NostrEvent } from '@nostrify/nostrify'; import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite'; +import { finalizeEvent, generateSecretKey } from 'nostr-tools'; +import { PostgreSQLDriver } from 'kysely_deno_postgres'; +import { Pool } from 'postgres'; +import { NDatabase, NostrEvent } from '@nostrify/nostrify'; import { FileMigrationProvider, Kysely, @@ -12,13 +15,10 @@ import { PostgresIntrospector, PostgresQueryCompiler, } from 'kysely'; -import { finalizeEvent, generateSecretKey } from 'nostr-tools'; import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; -import { PostgreSQLDriver } from 'kysely_deno_postgres'; -import { Pool } from 'postgres'; import { KyselyLogger } from '@/db/KyselyLogger.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; From 842b527273a7c33b5d475fb56d123fd9223da085 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 16 Jul 2024 14:47:14 -0300 Subject: [PATCH 22/59] test: update to use kysely-postgres-js --- src/test.ts | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/src/test.ts b/src/test.ts index 5de5430b..a82d9e16 100644 --- a/src/test.ts +++ b/src/test.ts @@ -4,23 +4,17 @@ import path from 'node:path'; import { Database as Sqlite } from '@db/sqlite'; import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite'; import { finalizeEvent, generateSecretKey } from 'nostr-tools'; -import { PostgreSQLDriver } from 'kysely_deno_postgres'; -import { Pool } from 'postgres'; import { NDatabase, NostrEvent } from '@nostrify/nostrify'; -import { - FileMigrationProvider, - Kysely, - Migrator, - PostgresAdapter, - PostgresIntrospector, - PostgresQueryCompiler, -} from 'kysely'; +import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; +import postgres from 'postgres'; +import { PostgresJSDialect, PostgresJSDialectConfig } from 'kysely-postgres-js'; import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; import { KyselyLogger } from '@/db/KyselyLogger.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; +import { Conf } from '@/config.ts'; /** Import an event fixture by name in tests. */ export async function eventFixture(name: string): Promise { @@ -119,20 +113,11 @@ export const createTestDB = async (databaseUrl?: string) => { }); } else { kysely = new Kysely({ - dialect: { - createAdapter() { - return new PostgresAdapter(); - }, - createDriver() { - return new PostgreSQLDriver(new Pool(databaseUrl, 10, true)); - }, - createIntrospector(db: Kysely) { - return new PostgresIntrospector(db); - }, - createQueryCompiler() { - return new PostgresQueryCompiler(); - }, - }, + dialect: new PostgresJSDialect({ + postgres: postgres(Conf.databaseUrl, { + max: Conf.pg.poolSize, + }) as unknown as PostgresJSDialectConfig['postgres'], + }), log: KyselyLogger, }); } From dcec2ecdd0f59b75fa87aef5fe2dabc1d6713ddb Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 19 Jul 2024 13:19:04 -0300 Subject: [PATCH 23/59] feat: create isNumberFrom1To100 function --- src/utils.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/utils.ts b/src/utils.ts index e9213ed1..96ae74c4 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,6 +1,6 @@ import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { z } from 'zod'; +import { boolean, z } from 'zod'; /** Get the current time in Nostr format. */ const nostrNow = (): number => Math.floor(Date.now() / 1000); @@ -93,12 +93,17 @@ function isURL(value: unknown): boolean { return z.string().url().safeParse(value).success; } +function isNumberFrom1To100(value: unknown): boolean { + return z.coerce.number().int().gte(1).lte(100).safeParse(value).success; +} + export { bech32ToPubkey, dedupeEvents, eventAge, findTag, isNostrId, + isNumberFrom1To100, isURL, type Nip05, nostrDate, From 11809637ee835fdb8b2c641fbd34cd77ce278eb4 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 19 Jul 2024 13:21:17 -0300 Subject: [PATCH 24/59] test(utils.ts): isNumberFrom1To100 function --- src/utils.test.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/utils.test.ts diff --git a/src/utils.test.ts b/src/utils.test.ts new file mode 100644 index 00000000..a48d0832 --- /dev/null +++ b/src/utils.test.ts @@ -0,0 +1,19 @@ +import { isNumberFrom1To100 } from '@/utils.ts'; +import { assertEquals } from 'https://deno.land/std@0.132.0/testing/asserts.ts'; + +Deno.test('Value is any number from 1 to 100', () => { + assertEquals(isNumberFrom1To100('latvia'), false); + assertEquals(isNumberFrom1To100(1.5), false); + assertEquals(isNumberFrom1To100(Infinity), false); + assertEquals(isNumberFrom1To100('Infinity'), false); + assertEquals(isNumberFrom1To100('0'), false); + assertEquals(isNumberFrom1To100(0), false); + assertEquals(isNumberFrom1To100(-1), false); + assertEquals(isNumberFrom1To100('-10'), false); + + for (let i = 1; i < 100; i++) { + assertEquals(isNumberFrom1To100(String(i)), true); + } + + assertEquals(isNumberFrom1To100('1e1'), true); +}); From c2225da8dd33d5e6d871705274c8510c66685cdd Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 19 Jul 2024 13:34:45 -0300 Subject: [PATCH 25/59] test(utls.ts): add more cases, isNumberFrom1To100 function --- src/utils.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/utils.test.ts b/src/utils.test.ts index a48d0832..8adf2935 100644 --- a/src/utils.test.ts +++ b/src/utils.test.ts @@ -10,6 +10,8 @@ Deno.test('Value is any number from 1 to 100', () => { assertEquals(isNumberFrom1To100(0), false); assertEquals(isNumberFrom1To100(-1), false); assertEquals(isNumberFrom1To100('-10'), false); + assertEquals(isNumberFrom1To100([]), false); + assertEquals(isNumberFrom1To100(undefined), false); for (let i = 1; i < 100; i++) { assertEquals(isNumberFrom1To100(String(i)), true); From cc6441d2397d45c963518e9ca3d9603f50cc1622 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 19 Jul 2024 15:22:35 -0300 Subject: [PATCH 26/59] refactor(utils.ts): remove unused imports --- src/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.ts b/src/utils.ts index 96ae74c4..a1c3b1d1 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,6 +1,6 @@ import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { boolean, z } from 'zod'; +import { z } from 'zod'; /** Get the current time in Nostr format. */ const nostrNow = (): number => Math.floor(Date.now() / 1000); From c17db5844889801970eb6ade964f516f8c33e9d6 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 19 Jul 2024 16:26:50 -0300 Subject: [PATCH 27/59] feat(utils.ts): create isObjectEmpty function --- src/utils.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/utils.ts b/src/utils.ts index a1c3b1d1..de826ad6 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -97,6 +97,13 @@ function isNumberFrom1To100(value: unknown): boolean { return z.coerce.number().int().gte(1).lte(100).safeParse(value).success; } +function isObjectEmpty(obj: object): boolean { + for (const prop in obj) { + if (Object.hasOwn(obj, prop)) return false; + } + return true; +} + export { bech32ToPubkey, dedupeEvents, @@ -104,6 +111,7 @@ export { findTag, isNostrId, isNumberFrom1To100, + isObjectEmpty, isURL, type Nip05, nostrDate, From 0c9e3e2b47913580bee76cbe5eed4b38b193d193 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 19 Jul 2024 16:27:10 -0300 Subject: [PATCH 28/59] test(utils.ts): isObjectEmpty function --- src/utils.test.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/utils.test.ts b/src/utils.test.ts index 8adf2935..1023c633 100644 --- a/src/utils.test.ts +++ b/src/utils.test.ts @@ -1,5 +1,5 @@ -import { isNumberFrom1To100 } from '@/utils.ts'; -import { assertEquals } from 'https://deno.land/std@0.132.0/testing/asserts.ts'; +import { isNumberFrom1To100, isObjectEmpty } from '@/utils.ts'; +import { assertEquals } from '@std/assert'; Deno.test('Value is any number from 1 to 100', () => { assertEquals(isNumberFrom1To100('latvia'), false); @@ -19,3 +19,11 @@ Deno.test('Value is any number from 1 to 100', () => { assertEquals(isNumberFrom1To100('1e1'), true); }); + +Deno.test('Object is empty', () => { + assertEquals(isObjectEmpty([1]), false); + assertEquals(isObjectEmpty({ 'yolo': 'no yolo' }), false); + + assertEquals(isObjectEmpty([]), true); + assertEquals(isObjectEmpty({}), true); +}); From 76a591ab6db38500d4ffded99938e2fdc8a0897b Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 19 Jul 2024 17:20:31 -0300 Subject: [PATCH 29/59] feat: create getZapSplits function --- src/utils/zap_split.ts | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 src/utils/zap_split.ts diff --git a/src/utils/zap_split.ts b/src/utils/zap_split.ts new file mode 100644 index 00000000..68aabc8a --- /dev/null +++ b/src/utils/zap_split.ts @@ -0,0 +1,35 @@ +import { NSchema as n, NStore } from '@nostrify/nostrify'; +import { isNumberFrom1To100 } from '@/utils.ts'; + +type Pubkey = string; +type ExtraMessage = string; +/** Number from 1 to 100, stringified. */ +type splitPercentages = string; + +type DittoZapSplits = { + [key: Pubkey]: [splitPercentages, ExtraMessage]; +}; + +/** Gets zap splits from NIP-78 in DittoZapSplits format. */ +export async function getZapSplits(store: NStore, pubkey: string): Promise { + const zapSplits: DittoZapSplits = {}; + + const [event] = await store.query([{ + authors: [pubkey], + kinds: [30078], + '#d': ['pub.ditto.zapSplits'], + limit: 1, + }]); + if (!event) return {}; + + for (const tag of event.tags) { + if ( + tag[0] === 'p' && n.id().safeParse(tag[1]).success && + isNumberFrom1To100(tag[2]) + ) { + zapSplits[tag[1]] = [tag[2], tag[3] ?? '']; + } + } + + return zapSplits; +} From 3bc3e7675d5d0d9f5748fe62ef409e7bb6b02b3b Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 19 Jul 2024 17:35:34 -0300 Subject: [PATCH 30/59] test: getZapSplits function --- src/utils/zap_split.test.ts | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/utils/zap_split.test.ts diff --git a/src/utils/zap_split.test.ts b/src/utils/zap_split.test.ts new file mode 100644 index 00000000..a439e3aa --- /dev/null +++ b/src/utils/zap_split.test.ts @@ -0,0 +1,36 @@ +import { assertEquals } from '@std/assert'; +import { generateSecretKey, getPublicKey } from 'nostr-tools'; + +import { genEvent } from '@/test.ts'; +import { getZapSplits } from '@/utils/zap_split.ts'; +import { getTestDB } from '@/test.ts'; + +Deno.test('Get zap splits in DittoZapSplits format', async () => { + const { store } = await getTestDB(); + + const sk = generateSecretKey(); + const pubkey = getPublicKey(sk); + + const event = genEvent({ + kind: 30078, + tags: [ + ['d', 'pub.ditto.zapSplits'], + ['p', '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', '2', 'Patrick developer'], + ['p', '0461fcbecc4c3374439932d6b8f11269ccdb7cc973ad7a50ae362db135a474dd', '3', 'Alex creator of Ditto'], + ], + }, sk); + await store.event(event); + + const eventFromDb = await store.query([{ kinds: [30078], authors: [pubkey] }]); + + assertEquals(eventFromDb.length, 1); + + const zapSplits = await getZapSplits(store, pubkey); + + assertEquals(zapSplits, { + '0461fcbecc4c3374439932d6b8f11269ccdb7cc973ad7a50ae362db135a474dd': ['3', 'Alex creator of Ditto'], + '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4': ['2', 'Patrick developer'], + }); + + assertEquals(await getZapSplits(store, 'garbage'), {}); +}); From beee0e76e78f3c0548c8b42bc7227fc516356a48 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 19 Jul 2024 19:19:04 -0300 Subject: [PATCH 31/59] refactor: allow to return undefined in getZapSplits function --- src/utils/zap_split.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils/zap_split.ts b/src/utils/zap_split.ts index 68aabc8a..38eecba4 100644 --- a/src/utils/zap_split.ts +++ b/src/utils/zap_split.ts @@ -6,12 +6,12 @@ type ExtraMessage = string; /** Number from 1 to 100, stringified. */ type splitPercentages = string; -type DittoZapSplits = { +export type DittoZapSplits = { [key: Pubkey]: [splitPercentages, ExtraMessage]; }; /** Gets zap splits from NIP-78 in DittoZapSplits format. */ -export async function getZapSplits(store: NStore, pubkey: string): Promise { +export async function getZapSplits(store: NStore, pubkey: string): Promise { const zapSplits: DittoZapSplits = {}; const [event] = await store.query([{ @@ -20,7 +20,7 @@ export async function getZapSplits(store: NStore, pubkey: string): Promise Date: Fri, 19 Jul 2024 19:19:35 -0300 Subject: [PATCH 32/59] test: update so getZapSplits function returns undefined --- src/utils/zap_split.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/zap_split.test.ts b/src/utils/zap_split.test.ts index a439e3aa..9678aba9 100644 --- a/src/utils/zap_split.test.ts +++ b/src/utils/zap_split.test.ts @@ -32,5 +32,5 @@ Deno.test('Get zap splits in DittoZapSplits format', async () => { '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4': ['2', 'Patrick developer'], }); - assertEquals(await getZapSplits(store, 'garbage'), {}); + assertEquals(await getZapSplits(store, 'garbage'), undefined); }); From 1f9896bdbf951e552b6eff09efc40e4979af6cae Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 19 Jul 2024 19:30:31 -0300 Subject: [PATCH 33/59] test: getZapSplits function return empty object --- src/utils/zap_split.test.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/utils/zap_split.test.ts b/src/utils/zap_split.test.ts index 9678aba9..6ccf1ada 100644 --- a/src/utils/zap_split.test.ts +++ b/src/utils/zap_split.test.ts @@ -34,3 +34,27 @@ Deno.test('Get zap splits in DittoZapSplits format', async () => { assertEquals(await getZapSplits(store, 'garbage'), undefined); }); + +Deno.test('Zap split is empty', async () => { + const { store } = await getTestDB(); + + const sk = generateSecretKey(); + const pubkey = getPublicKey(sk); + + const event = genEvent({ + kind: 30078, + tags: [ + ['d', 'pub.ditto.zapSplits'], + ['p', 'baka'], + ], + }, sk); + await store.event(event); + + const eventFromDb = await store.query([{ kinds: [30078], authors: [pubkey] }]); + + assertEquals(eventFromDb.length, 1); + + const zapSplits = await getZapSplits(store, pubkey); + + assertEquals(zapSplits, {}); +}); From 2fe9b9f98d7e1a103179365905a5828ca936959f Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 21 Jul 2024 19:22:15 -0300 Subject: [PATCH 34/59] feat: get zap splits and return it in api/v1/instance endpoint --- src/controllers/api/instance.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/controllers/api/instance.ts b/src/controllers/api/instance.ts index faca9c9f..1d97b1e9 100644 --- a/src/controllers/api/instance.ts +++ b/src/controllers/api/instance.ts @@ -4,12 +4,29 @@ import { AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { Storages } from '@/storages.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; +import { DittoZapSplits, getZapSplits } from '@/utils/zap_split.ts'; +import { createAdminEvent } from '@/utils/api.ts'; const version = `3.0.0 (compatible; Ditto ${denoJson.version})`; const instanceV1Controller: AppController = async (c) => { const { host, protocol } = Conf.url; const meta = await getInstanceMetadata(await Storages.db(), c.req.raw.signal); + const store = c.get('store'); + + let zap_split: DittoZapSplits | undefined = await getZapSplits(store, Conf.pubkey); + if (!zap_split) { + const officialDittoAccountPubkey = '781a1527055f74c1f70230f10384609b34548f8ab6a0a6caa74025827f9fdae5'; + const officialDittoAccountMsg = 'Official Ditto Account'; + await createAdminEvent({ + kind: 30078, + tags: [ + ['d', 'pub.ditto.zapSplits'], + ['p', officialDittoAccountPubkey, '5', officialDittoAccountMsg], + ], + }, c); + zap_split = { [officialDittoAccountPubkey]: ['5', officialDittoAccountMsg] }; + } /** Protocol to use for WebSocket URLs, depending on the protocol of the `LOCAL_DOMAIN`. */ const wsProtocol = protocol === 'http:' ? 'ws:' : 'wss:'; @@ -68,6 +85,9 @@ const instanceV1Controller: AppController = async (c) => { }, }, rules: [], + ditto: { + zap_split, + }, }); }; From 2e66af26db2d5bed2cce39593afebc2b7f0b6111 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 21 Jul 2024 19:21:33 -0300 Subject: [PATCH 35/59] feat: create updateZapSplitsController --- src/app.ts | 4 ++++ src/controllers/api/ditto.ts | 41 ++++++++++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/src/app.ts b/src/app.ts index 1cb3746b..7538fea2 100644 --- a/src/app.ts +++ b/src/app.ts @@ -44,6 +44,7 @@ import { adminSetRelaysController, nameRequestController, nameRequestsController, + updateZapSplitsController, } from '@/controllers/api/ditto.ts'; import { emptyArrayController, emptyObjectController, notImplementedController } from '@/controllers/api/fallback.ts'; import { @@ -270,6 +271,9 @@ app.put('/api/v1/admin/ditto/relays', requireRole('admin'), adminSetRelaysContro app.post('/api/v1/ditto/names', requireSigner, nameRequestController); app.get('/api/v1/ditto/names', requireSigner, nameRequestsController); +app.put('/api/v1/admin/ditto/zap_splits', requireRole('admin'), updateZapSplitsController); +//app.delete('/api/v1/admin/ditto/zap_splits', requireRole('admin'), deleteZapSplitsController); + app.post('/api/v1/ditto/zap', requireSigner, zapController); app.get('/api/v1/ditto/statuses/:id{[0-9a-f]{64}}/zapped_by', zappedByController); diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index 841eb861..c3381a66 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -1,14 +1,17 @@ -import { NostrEvent, NostrFilter } from '@nostrify/nostrify'; +import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { z } from 'zod'; import { AppController } from '@/app.ts'; -import { Conf } from '@/config.ts'; -import { booleanParamSchema } from '@/schema.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; +import { booleanParamSchema } from '@/schema.ts'; +import { Conf } from '@/config.ts'; import { Storages } from '@/storages.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { createEvent, paginated, paginationSchema } from '@/utils/api.ts'; +import { createEvent, paginated, paginationSchema, parseBody } from '@/utils/api.ts'; import { renderNameRequest } from '@/views/ditto.ts'; +import { getZapSplits } from '@/utils/zap_split.ts'; +import { updateListAdminEvent } from '@/utils/api.ts'; +import { addTag } from '@/utils/tags.ts'; const markerSchema = z.enum(['read', 'write']); @@ -148,3 +151,33 @@ export const nameRequestsController: AppController = async (c) => { return paginated(c, orig, nameRequests); }; + +const zapSplitSchema = z.array(z.tuple([n.id(), z.number().int().min(1).max(100), z.string().max(500)])).min(1); + +export const updateZapSplitsController: AppController = async (c) => { + const body = await parseBody(c.req.raw); + const result = zapSplitSchema.safeParse(body); + const store = c.get('store'); + + if (!result.success) { + return c.json({ error: result.error }, 400); + } + + const zap_split = await getZapSplits(store, Conf.pubkey); + if (!zap_split) { + return c.json({ error: 'Zap split not activated, visit `/api/v1/instance` to activate it.' }, 404); + } + + const { data } = result; + + await updateListAdminEvent( + { kinds: [30078], authors: [Conf.pubkey], '#d': ['pub.ditto.zapSplits'], limit: 1 }, + (tags) => + data.reduce((accumulator, currentValue) => { + return addTag(accumulator, ['p', currentValue[0], String(currentValue[1]), currentValue[2]]); + }, tags), + c, + ); + + return c.json(200); +}; From 449a3497ba5aac442b2c335ddd46c86f12260e06 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 21 Jul 2024 19:40:55 -0300 Subject: [PATCH 36/59] feat: create deleteZapSplitsController --- src/app.ts | 3 ++- src/controllers/api/ditto.ts | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/app.ts b/src/app.ts index 7538fea2..f9dad138 100644 --- a/src/app.ts +++ b/src/app.ts @@ -42,6 +42,7 @@ import { bookmarksController } from '@/controllers/api/bookmarks.ts'; import { adminRelaysController, adminSetRelaysController, + deleteZapSplitsController, nameRequestController, nameRequestsController, updateZapSplitsController, @@ -272,7 +273,7 @@ app.post('/api/v1/ditto/names', requireSigner, nameRequestController); app.get('/api/v1/ditto/names', requireSigner, nameRequestsController); app.put('/api/v1/admin/ditto/zap_splits', requireRole('admin'), updateZapSplitsController); -//app.delete('/api/v1/admin/ditto/zap_splits', requireRole('admin'), deleteZapSplitsController); +app.delete('/api/v1/admin/ditto/zap_splits', requireRole('admin'), deleteZapSplitsController); app.post('/api/v1/ditto/zap', requireSigner, zapController); app.get('/api/v1/ditto/statuses/:id{[0-9a-f]{64}}/zapped_by', zappedByController); diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index c3381a66..aebf3427 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -12,6 +12,7 @@ import { renderNameRequest } from '@/views/ditto.ts'; import { getZapSplits } from '@/utils/zap_split.ts'; import { updateListAdminEvent } from '@/utils/api.ts'; import { addTag } from '@/utils/tags.ts'; +import { deleteTag } from '@/utils/tags.ts'; const markerSchema = z.enum(['read', 'write']); @@ -181,3 +182,33 @@ export const updateZapSplitsController: AppController = async (c) => { return c.json(200); }; + +const deleteZapSplitSchema = z.array(n.id()).min(1); + +export const deleteZapSplitsController: AppController = async (c) => { + const body = await parseBody(c.req.raw); + const result = deleteZapSplitSchema.safeParse(body); + const store = c.get('store'); + + if (!result.success) { + return c.json({ error: result.error }, 400); + } + + const zap_split = await getZapSplits(store, Conf.pubkey); + if (!zap_split) { + return c.json({ error: 'Zap split not activated, visit `/api/v1/instance` to activate it.' }, 404); + } + + const { data } = result; + + await updateListAdminEvent( + { kinds: [30078], authors: [Conf.pubkey], '#d': ['pub.ditto.zapSplits'], limit: 1 }, + (tags) => + data.reduce((accumulator, currentValue) => { + return deleteTag(accumulator, ['p', currentValue]); + }, tags), + c, + ); + + return c.json(200); +}; From 5a5e8b7c5d522f00f02fad69c92a3429cb636d0b Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 21 Jul 2024 20:11:26 -0300 Subject: [PATCH 37/59] feat(createStatusController): add 'zap' tag to event --- src/controllers/api/statuses.ts | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 4604981f..c56de586 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -29,6 +29,8 @@ import { addTag, deleteTag } from '@/utils/tags.ts'; import { asyncReplaceAll } from '@/utils/text.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; +import { isObjectEmpty } from '@/utils.ts'; +import { getZapSplits } from '@/utils/zap_split.ts'; const createStatusSchema = z.object({ in_reply_to_id: n.id().nullish(), @@ -71,6 +73,7 @@ const createStatusController: AppController = async (c) => { const body = await parseBody(c.req.raw); const result = createStatusSchema.safeParse(body); const kysely = await DittoDB.getInstance(); + const store = c.get('store'); if (!result.success) { return c.json({ error: 'Bad request', schema: result.error }, 400); @@ -173,14 +176,26 @@ const createStatusController: AppController = async (c) => { const quoteCompat = data.quote_id ? `\n\nnostr:${nip19.noteEncode(data.quote_id)}` : ''; const mediaCompat = mediaUrls.length ? `\n\n${mediaUrls.join('\n')}` : ''; + const author = await getAuthor(await c.get('signer')?.getPublicKey() as string); + + const meta = n.json().pipe(n.metadata()).catch({}).parse(author?.content); + const lnurl = getLnurl(meta); + const zap_split = await getZapSplits(store, Conf.pubkey); + if (lnurl && zap_split && isObjectEmpty(zap_split) === false) { + let totalSplit = 0; + for (const pubkey in zap_split) { + totalSplit += Number(zap_split[pubkey][0]); + tags.push(['zap', pubkey, Conf.relay, zap_split[pubkey][0]]); + } + tags.push(['zap', author?.pubkey as string, Conf.relay, Math.max(0, 100 - totalSplit).toString()]); + } + const event = await createEvent({ kind: 1, content: content + quoteCompat + mediaCompat, tags, }, c); - const author = await getAuthor(event.pubkey); - if (data.quote_id) { await hydrateEvents({ events: [event], @@ -189,7 +204,7 @@ const createStatusController: AppController = async (c) => { }); } - return c.json(await renderStatus({ ...event, author }, { viewerPubkey: await c.get('signer')?.getPublicKey() })); + return c.json(await renderStatus({ ...event, author }, { viewerPubkey: author?.pubkey })); }; const deleteStatusController: AppController = async (c) => { From 80e14c65c0ba9382c8b33af58d0ca945dd543875 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 23 Jul 2024 18:14:22 -0300 Subject: [PATCH 38/59] refactor: rename zap_split.ts to zap-split.ts, rename tests as well --- src/controllers/api/ditto.ts | 2 +- src/controllers/api/instance.ts | 2 +- src/controllers/api/statuses.ts | 2 +- src/utils/{zap_split.test.ts => zap-split.test.ts} | 2 +- src/utils/{zap_split.ts => zap-split.ts} | 0 5 files changed, 4 insertions(+), 4 deletions(-) rename src/utils/{zap_split.test.ts => zap-split.test.ts} (97%) rename src/utils/{zap_split.ts => zap-split.ts} (100%) diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index aebf3427..a22dd1e5 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -9,7 +9,7 @@ import { Storages } from '@/storages.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { createEvent, paginated, paginationSchema, parseBody } from '@/utils/api.ts'; import { renderNameRequest } from '@/views/ditto.ts'; -import { getZapSplits } from '@/utils/zap_split.ts'; +import { getZapSplits } from '@/utils/zap-split.ts'; import { updateListAdminEvent } from '@/utils/api.ts'; import { addTag } from '@/utils/tags.ts'; import { deleteTag } from '@/utils/tags.ts'; diff --git a/src/controllers/api/instance.ts b/src/controllers/api/instance.ts index 1d97b1e9..6b8a86a3 100644 --- a/src/controllers/api/instance.ts +++ b/src/controllers/api/instance.ts @@ -4,7 +4,7 @@ import { AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { Storages } from '@/storages.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; -import { DittoZapSplits, getZapSplits } from '@/utils/zap_split.ts'; +import { DittoZapSplits, getZapSplits } from '@/utils/zap-split.ts'; import { createAdminEvent } from '@/utils/api.ts'; const version = `3.0.0 (compatible; Ditto ${denoJson.version})`; diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index c56de586..7b0b7820 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -30,7 +30,7 @@ import { asyncReplaceAll } from '@/utils/text.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { isObjectEmpty } from '@/utils.ts'; -import { getZapSplits } from '@/utils/zap_split.ts'; +import { getZapSplits } from '@/utils/zap-split.ts'; const createStatusSchema = z.object({ in_reply_to_id: n.id().nullish(), diff --git a/src/utils/zap_split.test.ts b/src/utils/zap-split.test.ts similarity index 97% rename from src/utils/zap_split.test.ts rename to src/utils/zap-split.test.ts index 6ccf1ada..bc9527df 100644 --- a/src/utils/zap_split.test.ts +++ b/src/utils/zap-split.test.ts @@ -2,7 +2,7 @@ import { assertEquals } from '@std/assert'; import { generateSecretKey, getPublicKey } from 'nostr-tools'; import { genEvent } from '@/test.ts'; -import { getZapSplits } from '@/utils/zap_split.ts'; +import { getZapSplits } from '@/utils/zap-split.ts'; import { getTestDB } from '@/test.ts'; Deno.test('Get zap splits in DittoZapSplits format', async () => { diff --git a/src/utils/zap_split.ts b/src/utils/zap-split.ts similarity index 100% rename from src/utils/zap_split.ts rename to src/utils/zap-split.ts From de32930c441f15498d34269180c2e39bef563a0c Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 23 Jul 2024 18:26:06 -0300 Subject: [PATCH 39/59] refactor: use exclamation mark (bang) instead of 'as string' cast --- src/controllers/api/statuses.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 7b0b7820..7823949d 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -176,7 +176,7 @@ const createStatusController: AppController = async (c) => { const quoteCompat = data.quote_id ? `\n\nnostr:${nip19.noteEncode(data.quote_id)}` : ''; const mediaCompat = mediaUrls.length ? `\n\n${mediaUrls.join('\n')}` : ''; - const author = await getAuthor(await c.get('signer')?.getPublicKey() as string); + const author = await getAuthor(await c.get('signer')?.getPublicKey()!); const meta = n.json().pipe(n.metadata()).catch({}).parse(author?.content); const lnurl = getLnurl(meta); From a0c952b9b9a7bd764c28b0f7003ae2baa2a733cc Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 23 Jul 2024 20:15:44 -0300 Subject: [PATCH 40/59] refactor: do not use isObjectEmpty function in if condition, zap tag --- src/controllers/api/statuses.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 7823949d..4b29aea2 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -29,7 +29,6 @@ import { addTag, deleteTag } from '@/utils/tags.ts'; import { asyncReplaceAll } from '@/utils/text.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; -import { isObjectEmpty } from '@/utils.ts'; import { getZapSplits } from '@/utils/zap-split.ts'; const createStatusSchema = z.object({ @@ -181,13 +180,15 @@ const createStatusController: AppController = async (c) => { const meta = n.json().pipe(n.metadata()).catch({}).parse(author?.content); const lnurl = getLnurl(meta); const zap_split = await getZapSplits(store, Conf.pubkey); - if (lnurl && zap_split && isObjectEmpty(zap_split) === false) { + if (lnurl && zap_split) { let totalSplit = 0; for (const pubkey in zap_split) { totalSplit += Number(zap_split[pubkey][0]); tags.push(['zap', pubkey, Conf.relay, zap_split[pubkey][0]]); } - tags.push(['zap', author?.pubkey as string, Conf.relay, Math.max(0, 100 - totalSplit).toString()]); + if (totalSplit) { + tags.push(['zap', author?.pubkey as string, Conf.relay, Math.max(0, 100 - totalSplit).toString()]); + } } const event = await createEvent({ From 7cdfb67b993ce66cb2071be33154f82c31c8f18a Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 23 Jul 2024 20:25:02 -0300 Subject: [PATCH 41/59] refactor: rename officialDittoAccountPubkey to dittoPubkey & officialDittoAccountMsg to dittoMsg --- src/controllers/api/instance.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/controllers/api/instance.ts b/src/controllers/api/instance.ts index 6b8a86a3..305f9e96 100644 --- a/src/controllers/api/instance.ts +++ b/src/controllers/api/instance.ts @@ -16,16 +16,16 @@ const instanceV1Controller: AppController = async (c) => { let zap_split: DittoZapSplits | undefined = await getZapSplits(store, Conf.pubkey); if (!zap_split) { - const officialDittoAccountPubkey = '781a1527055f74c1f70230f10384609b34548f8ab6a0a6caa74025827f9fdae5'; - const officialDittoAccountMsg = 'Official Ditto Account'; + const dittoPubkey = '781a1527055f74c1f70230f10384609b34548f8ab6a0a6caa74025827f9fdae5'; + const dittoMsg = 'Official Ditto Account'; await createAdminEvent({ kind: 30078, tags: [ ['d', 'pub.ditto.zapSplits'], - ['p', officialDittoAccountPubkey, '5', officialDittoAccountMsg], + ['p', dittoPubkey, '5', dittoMsg], ], }, c); - zap_split = { [officialDittoAccountPubkey]: ['5', officialDittoAccountMsg] }; + zap_split = { [dittoPubkey]: ['5', dittoMsg] }; } /** Protocol to use for WebSocket URLs, depending on the protocol of the `LOCAL_DOMAIN`. */ From 2ebaee880749c16898fcaeafaa81c1162f371966 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 23 Jul 2024 20:39:23 -0300 Subject: [PATCH 42/59] refactor: change DittoZapSplits data structure to use object fields instead of array --- src/utils/zap-split.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils/zap-split.ts b/src/utils/zap-split.ts index 38eecba4..33cd9c4e 100644 --- a/src/utils/zap-split.ts +++ b/src/utils/zap-split.ts @@ -4,10 +4,10 @@ import { isNumberFrom1To100 } from '@/utils.ts'; type Pubkey = string; type ExtraMessage = string; /** Number from 1 to 100, stringified. */ -type splitPercentages = string; +type splitPercentages = number; export type DittoZapSplits = { - [key: Pubkey]: [splitPercentages, ExtraMessage]; + [key: Pubkey]: { amount: splitPercentages; message: ExtraMessage }; }; /** Gets zap splits from NIP-78 in DittoZapSplits format. */ @@ -27,7 +27,7 @@ export async function getZapSplits(store: NStore, pubkey: string): Promise Date: Tue, 23 Jul 2024 20:39:51 -0300 Subject: [PATCH 43/59] test(zap-split): update to be in accord with new data structure --- src/utils/zap-split.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/zap-split.test.ts b/src/utils/zap-split.test.ts index bc9527df..08454160 100644 --- a/src/utils/zap-split.test.ts +++ b/src/utils/zap-split.test.ts @@ -28,8 +28,8 @@ Deno.test('Get zap splits in DittoZapSplits format', async () => { const zapSplits = await getZapSplits(store, pubkey); assertEquals(zapSplits, { - '0461fcbecc4c3374439932d6b8f11269ccdb7cc973ad7a50ae362db135a474dd': ['3', 'Alex creator of Ditto'], - '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4': ['2', 'Patrick developer'], + '0461fcbecc4c3374439932d6b8f11269ccdb7cc973ad7a50ae362db135a474dd': { amount: 3, message: 'Alex creator of Ditto' }, + '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4': { amount: 2, message: 'Patrick developer' }, }); assertEquals(await getZapSplits(store, 'garbage'), undefined); From c49460e37d854ac662b6f79b10bcc3d1267922b8 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 23 Jul 2024 20:41:02 -0300 Subject: [PATCH 44/59] refactor(instance.ts): use new zap_split data structure in ditto hard coded pubkey --- src/controllers/api/instance.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/api/instance.ts b/src/controllers/api/instance.ts index 305f9e96..120f6801 100644 --- a/src/controllers/api/instance.ts +++ b/src/controllers/api/instance.ts @@ -25,7 +25,7 @@ const instanceV1Controller: AppController = async (c) => { ['p', dittoPubkey, '5', dittoMsg], ], }, c); - zap_split = { [dittoPubkey]: ['5', dittoMsg] }; + zap_split = { [dittoPubkey]: { amount: 5, message: dittoMsg } }; } /** Protocol to use for WebSocket URLs, depending on the protocol of the `LOCAL_DOMAIN`. */ From 0e43d1e8a765f22477949d69191b84810bc1ae51 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 23 Jul 2024 22:40:38 -0300 Subject: [PATCH 45/59] refactor: add zap tag with new data structure --- src/controllers/api/statuses.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 4b29aea2..e222608c 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -183,8 +183,8 @@ const createStatusController: AppController = async (c) => { if (lnurl && zap_split) { let totalSplit = 0; for (const pubkey in zap_split) { - totalSplit += Number(zap_split[pubkey][0]); - tags.push(['zap', pubkey, Conf.relay, zap_split[pubkey][0]]); + totalSplit += zap_split[pubkey].amount; + tags.push(['zap', pubkey, Conf.relay, zap_split[pubkey].amount.toString()]); } if (totalSplit) { tags.push(['zap', author?.pubkey as string, Conf.relay, Math.max(0, 100 - totalSplit).toString()]); From 86874e3a0869f21a40f690b64fa55a42fb140a29 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Wed, 24 Jul 2024 10:53:32 -0300 Subject: [PATCH 46/59] feat: create createZapSplitsIfNotExists() function --- src/utils/zap-split.ts | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/utils/zap-split.ts b/src/utils/zap-split.ts index 33cd9c4e..54fe04a5 100644 --- a/src/utils/zap-split.ts +++ b/src/utils/zap-split.ts @@ -1,5 +1,9 @@ +import { Conf } from '@/config.ts'; import { NSchema as n, NStore } from '@nostrify/nostrify'; -import { isNumberFrom1To100 } from '@/utils.ts'; +import { isNumberFrom1To100, nostrNow } from '@/utils.ts'; +import { Storages } from '@/storages.ts'; +import { AdminSigner } from '@/signers/AdminSigner.ts'; +import { handleEvent } from '@/pipeline.ts'; type Pubkey = string; type ExtraMessage = string; @@ -33,3 +37,25 @@ export async function getZapSplits(store: NStore, pubkey: string): Promise Date: Wed, 24 Jul 2024 10:54:20 -0300 Subject: [PATCH 47/59] feat: create startup file --- src/startup.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 src/startup.ts diff --git a/src/startup.ts b/src/startup.ts new file mode 100644 index 00000000..4ec75e03 --- /dev/null +++ b/src/startup.ts @@ -0,0 +1,16 @@ +// Starts up applications required to run before the HTTP server is on. + +import { Conf } from '@/config.ts'; +import { createZapSplitsIfNotExists } from '@/utils/zap-split.ts'; +import { cron } from '@/cron.ts'; +import { startFirehose } from '@/firehose.ts'; + +if (Conf.firehoseEnabled) { + startFirehose(); +} + +if (Conf.cronEnabled) { + cron(); +} + +await createZapSplitsIfNotExists(); From 23bb24929c0d693d3186904d21e1b5606e9412fd Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Wed, 24 Jul 2024 10:54:56 -0300 Subject: [PATCH 48/59] refactor: remove zap split creation from instanceV1Controller endpoint --- src/controllers/api/instance.ts | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/controllers/api/instance.ts b/src/controllers/api/instance.ts index 120f6801..5f7054ee 100644 --- a/src/controllers/api/instance.ts +++ b/src/controllers/api/instance.ts @@ -5,7 +5,6 @@ import { Conf } from '@/config.ts'; import { Storages } from '@/storages.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; import { DittoZapSplits, getZapSplits } from '@/utils/zap-split.ts'; -import { createAdminEvent } from '@/utils/api.ts'; const version = `3.0.0 (compatible; Ditto ${denoJson.version})`; @@ -14,19 +13,7 @@ const instanceV1Controller: AppController = async (c) => { const meta = await getInstanceMetadata(await Storages.db(), c.req.raw.signal); const store = c.get('store'); - let zap_split: DittoZapSplits | undefined = await getZapSplits(store, Conf.pubkey); - if (!zap_split) { - const dittoPubkey = '781a1527055f74c1f70230f10384609b34548f8ab6a0a6caa74025827f9fdae5'; - const dittoMsg = 'Official Ditto Account'; - await createAdminEvent({ - kind: 30078, - tags: [ - ['d', 'pub.ditto.zapSplits'], - ['p', dittoPubkey, '5', dittoMsg], - ], - }, c); - zap_split = { [dittoPubkey]: { amount: 5, message: dittoMsg } }; - } + const zap_split: DittoZapSplits | undefined = await getZapSplits(store, Conf.pubkey) ?? {}; /** Protocol to use for WebSocket URLs, depending on the protocol of the `LOCAL_DOMAIN`. */ const wsProtocol = protocol === 'http:' ? 'ws:' : 'wss:'; From ddd1972c831b025a0a891188bcee6df69c6560f3 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Wed, 24 Jul 2024 10:55:30 -0300 Subject: [PATCH 49/59] refactor(app.ts): move cron function and startFirehose function to startup.ts --- src/app.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/app.ts b/src/app.ts index f9dad138..690862eb 100644 --- a/src/app.ts +++ b/src/app.ts @@ -5,9 +5,6 @@ import { logger } from '@hono/hono/logger'; import { NostrEvent, NostrSigner, NStore, NUploader } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; -import { Conf } from '@/config.ts'; -import { cron } from '@/cron.ts'; -import { startFirehose } from '@/firehose.ts'; import { Time } from '@/utils/time.ts'; import { @@ -145,13 +142,6 @@ const app = new Hono({ strict: false }); const debug = Debug('ditto:http'); -if (Conf.firehoseEnabled) { - startFirehose(); -} -if (Conf.cronEnabled) { - cron(); -} - app.use('*', rateLimitMiddleware(300, Time.minutes(5))); app.use('/api/*', metricsMiddleware, logger(debug)); From 961da0f52cb09654955550d9dfc2a3cee12e7dd6 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Wed, 24 Jul 2024 10:56:12 -0300 Subject: [PATCH 50/59] refactor: change error message in updateZapSplitsController & deleteZapSplitsController --- src/controllers/api/ditto.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index a22dd1e5..f90579dd 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -166,7 +166,7 @@ export const updateZapSplitsController: AppController = async (c) => { const zap_split = await getZapSplits(store, Conf.pubkey); if (!zap_split) { - return c.json({ error: 'Zap split not activated, visit `/api/v1/instance` to activate it.' }, 404); + return c.json({ error: 'Zap split not activated, restart the server.' }, 404); } const { data } = result; @@ -196,7 +196,7 @@ export const deleteZapSplitsController: AppController = async (c) => { const zap_split = await getZapSplits(store, Conf.pubkey); if (!zap_split) { - return c.json({ error: 'Zap split not activated, visit `/api/v1/instance` to activate it.' }, 404); + return c.json({ error: 'Zap split not activated, restart the server.' }, 404); } const { data } = result; From 85806f7ea8aef84ed642d799acb29a1ed5548343 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Wed, 24 Jul 2024 11:14:42 -0300 Subject: [PATCH 51/59] refactor: use object fields instead of tuple in zapSplitSchema --- src/controllers/api/ditto.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index f90579dd..0f815350 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -153,7 +153,11 @@ export const nameRequestsController: AppController = async (c) => { return paginated(c, orig, nameRequests); }; -const zapSplitSchema = z.array(z.tuple([n.id(), z.number().int().min(1).max(100), z.string().max(500)])).min(1); +const zapSplitSchema = z.array(z.object({ + pubkey: n.id(), + amount: z.number().int().min(1).max(100), + message: z.string().max(500), +})).min(1); export const updateZapSplitsController: AppController = async (c) => { const body = await parseBody(c.req.raw); @@ -175,7 +179,7 @@ export const updateZapSplitsController: AppController = async (c) => { { kinds: [30078], authors: [Conf.pubkey], '#d': ['pub.ditto.zapSplits'], limit: 1 }, (tags) => data.reduce((accumulator, currentValue) => { - return addTag(accumulator, ['p', currentValue[0], String(currentValue[1]), currentValue[2]]); + return addTag(accumulator, ['p', currentValue.pubkey, currentValue.amount.toString(), currentValue.message]); }, tags), c, ); From f94ae7606c9c2c029e8f54a03e0d34bf7c652bc0 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 25 Jul 2024 11:11:44 -0300 Subject: [PATCH 52/59] refactor: change zapSplitSchema to z.record()) --- src/controllers/api/ditto.ts | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index 0f815350..d1ba002b 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -153,11 +153,13 @@ export const nameRequestsController: AppController = async (c) => { return paginated(c, orig, nameRequests); }; -const zapSplitSchema = z.array(z.object({ - pubkey: n.id(), - amount: z.number().int().min(1).max(100), - message: z.string().max(500), -})).min(1); +const zapSplitSchema = z.record( + n.id(), + z.object({ + amount: z.number().int().min(1).max(100), + message: z.string().max(500), + }), +); export const updateZapSplitsController: AppController = async (c) => { const body = await parseBody(c.req.raw); @@ -174,12 +176,17 @@ export const updateZapSplitsController: AppController = async (c) => { } const { data } = result; + const pubkeys = Object.keys(data); + + if (pubkeys.length < 1) { + return c.json(200); + } await updateListAdminEvent( { kinds: [30078], authors: [Conf.pubkey], '#d': ['pub.ditto.zapSplits'], limit: 1 }, (tags) => - data.reduce((accumulator, currentValue) => { - return addTag(accumulator, ['p', currentValue.pubkey, currentValue.amount.toString(), currentValue.message]); + pubkeys.reduce((accumulator, pubkey) => { + return addTag(accumulator, ['p', pubkey, data[pubkey].amount.toString(), data[pubkey].message]); }, tags), c, ); From 541b5b1c39aaf3bbdafeba7756b5c43231309a9e Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 25 Jul 2024 14:52:03 -0300 Subject: [PATCH 53/59] refactor: imports in alphabetical order --- src/controllers/api/statuses.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index e222608c..6a9ed1f5 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -6,10 +6,15 @@ import { nip19 } from 'nostr-tools'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; +import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; +import { addTag, deleteTag } from '@/utils/tags.ts'; +import { asyncReplaceAll } from '@/utils/text.ts'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; +import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { getAncestors, getAuthor, getDescendants, getEvent } from '@/queries.ts'; import { getUnattachedMediaByIds } from '@/db/unattached-media.ts'; +import { lookupPubkey } from '@/utils/lookup.ts'; import { renderEventAccounts } from '@/views.ts'; import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts'; import { Storages } from '@/storages.ts'; @@ -24,11 +29,6 @@ import { updateListEvent, } from '@/utils/api.ts'; import { getInvoice, getLnurl } from '@/utils/lnurl.ts'; -import { lookupPubkey } from '@/utils/lookup.ts'; -import { addTag, deleteTag } from '@/utils/tags.ts'; -import { asyncReplaceAll } from '@/utils/text.ts'; -import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { getZapSplits } from '@/utils/zap-split.ts'; const createStatusSchema = z.object({ From f3d521356d1ca64a3472ce1652755703763f9ad7 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 25 Jul 2024 14:52:52 -0300 Subject: [PATCH 54/59] refactor: get rid of isObjectEmpty function --- src/utils.test.ts | 11 ++--------- src/utils.ts | 8 -------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/src/utils.test.ts b/src/utils.test.ts index 1023c633..d89472c6 100644 --- a/src/utils.test.ts +++ b/src/utils.test.ts @@ -1,6 +1,7 @@ -import { isNumberFrom1To100, isObjectEmpty } from '@/utils.ts'; import { assertEquals } from '@std/assert'; +import { isNumberFrom1To100 } from '@/utils.ts'; + Deno.test('Value is any number from 1 to 100', () => { assertEquals(isNumberFrom1To100('latvia'), false); assertEquals(isNumberFrom1To100(1.5), false); @@ -19,11 +20,3 @@ Deno.test('Value is any number from 1 to 100', () => { assertEquals(isNumberFrom1To100('1e1'), true); }); - -Deno.test('Object is empty', () => { - assertEquals(isObjectEmpty([1]), false); - assertEquals(isObjectEmpty({ 'yolo': 'no yolo' }), false); - - assertEquals(isObjectEmpty([]), true); - assertEquals(isObjectEmpty({}), true); -}); diff --git a/src/utils.ts b/src/utils.ts index de826ad6..a1c3b1d1 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -97,13 +97,6 @@ function isNumberFrom1To100(value: unknown): boolean { return z.coerce.number().int().gte(1).lte(100).safeParse(value).success; } -function isObjectEmpty(obj: object): boolean { - for (const prop in obj) { - if (Object.hasOwn(obj, prop)) return false; - } - return true; -} - export { bech32ToPubkey, dedupeEvents, @@ -111,7 +104,6 @@ export { findTag, isNostrId, isNumberFrom1To100, - isObjectEmpty, isURL, type Nip05, nostrDate, From 882f8009dc9f7ad84bcc16bedd5c654977afc141 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 25 Jul 2024 15:06:50 -0300 Subject: [PATCH 55/59] refactor: rename isNumberFrom1To100 to percentageSchema --- src/schema.test.ts | 22 ++++++++++++++++++++++ src/schema.ts | 12 +++++++++++- src/utils.test.ts | 22 ---------------------- src/utils.ts | 5 ----- src/utils/zap-split.ts | 11 ++++++----- 5 files changed, 39 insertions(+), 33 deletions(-) create mode 100644 src/schema.test.ts delete mode 100644 src/utils.test.ts diff --git a/src/schema.test.ts b/src/schema.test.ts new file mode 100644 index 00000000..c6b577de --- /dev/null +++ b/src/schema.test.ts @@ -0,0 +1,22 @@ +import { assertEquals } from '@std/assert'; + +import { percentageSchema } from '@/schema.ts'; + +Deno.test('Value is any percentage from 1 to 100', () => { + assertEquals(percentageSchema.safeParse('latvia' as unknown).success, false); + assertEquals(percentageSchema.safeParse(1.5).success, false); + assertEquals(percentageSchema.safeParse(Infinity).success, false); + assertEquals(percentageSchema.safeParse('Infinity').success, false); + assertEquals(percentageSchema.safeParse('0').success, false); + assertEquals(percentageSchema.safeParse(0).success, false); + assertEquals(percentageSchema.safeParse(-1).success, false); + assertEquals(percentageSchema.safeParse('-10').success, false); + assertEquals(percentageSchema.safeParse([]).success, false); + assertEquals(percentageSchema.safeParse(undefined).success, false); + + for (let i = 1; i < 100; i++) { + assertEquals(percentageSchema.safeParse(String(i)).success, true); + } + + assertEquals(percentageSchema.safeParse('1e1').success, true); +}); diff --git a/src/schema.ts b/src/schema.ts index d152a0d4..fc7efd01 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -38,4 +38,14 @@ const booleanParamSchema = z.enum(['true', 'false']).transform((value) => value /** Schema for `File` objects. */ const fileSchema = z.custom((value) => value instanceof File); -export { booleanParamSchema, decode64Schema, fileSchema, filteredArray, hashtagSchema, safeUrlSchema }; +const percentageSchema = z.coerce.number().int().gte(1).lte(100); + +export { + booleanParamSchema, + decode64Schema, + fileSchema, + filteredArray, + hashtagSchema, + percentageSchema, + safeUrlSchema, +}; diff --git a/src/utils.test.ts b/src/utils.test.ts deleted file mode 100644 index d89472c6..00000000 --- a/src/utils.test.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { assertEquals } from '@std/assert'; - -import { isNumberFrom1To100 } from '@/utils.ts'; - -Deno.test('Value is any number from 1 to 100', () => { - assertEquals(isNumberFrom1To100('latvia'), false); - assertEquals(isNumberFrom1To100(1.5), false); - assertEquals(isNumberFrom1To100(Infinity), false); - assertEquals(isNumberFrom1To100('Infinity'), false); - assertEquals(isNumberFrom1To100('0'), false); - assertEquals(isNumberFrom1To100(0), false); - assertEquals(isNumberFrom1To100(-1), false); - assertEquals(isNumberFrom1To100('-10'), false); - assertEquals(isNumberFrom1To100([]), false); - assertEquals(isNumberFrom1To100(undefined), false); - - for (let i = 1; i < 100; i++) { - assertEquals(isNumberFrom1To100(String(i)), true); - } - - assertEquals(isNumberFrom1To100('1e1'), true); -}); diff --git a/src/utils.ts b/src/utils.ts index a1c3b1d1..e9213ed1 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -93,17 +93,12 @@ function isURL(value: unknown): boolean { return z.string().url().safeParse(value).success; } -function isNumberFrom1To100(value: unknown): boolean { - return z.coerce.number().int().gte(1).lte(100).safeParse(value).success; -} - export { bech32ToPubkey, dedupeEvents, eventAge, findTag, isNostrId, - isNumberFrom1To100, isURL, type Nip05, nostrDate, diff --git a/src/utils/zap-split.ts b/src/utils/zap-split.ts index 54fe04a5..830c1e7c 100644 --- a/src/utils/zap-split.ts +++ b/src/utils/zap-split.ts @@ -1,9 +1,10 @@ -import { Conf } from '@/config.ts'; -import { NSchema as n, NStore } from '@nostrify/nostrify'; -import { isNumberFrom1To100, nostrNow } from '@/utils.ts'; -import { Storages } from '@/storages.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; +import { Conf } from '@/config.ts'; import { handleEvent } from '@/pipeline.ts'; +import { NSchema as n, NStore } from '@nostrify/nostrify'; +import { nostrNow } from '@/utils.ts'; +import { percentageSchema } from '@/schema.ts'; +import { Storages } from '@/storages.ts'; type Pubkey = string; type ExtraMessage = string; @@ -29,7 +30,7 @@ export async function getZapSplits(store: NStore, pubkey: string): Promise Date: Thu, 25 Jul 2024 15:10:54 -0300 Subject: [PATCH 56/59] refactor: rename createZapSplitsIfNotExists to seedZapSplits --- src/startup.ts | 4 ++-- src/utils/zap-split.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/startup.ts b/src/startup.ts index 4ec75e03..21df4d50 100644 --- a/src/startup.ts +++ b/src/startup.ts @@ -1,7 +1,7 @@ // Starts up applications required to run before the HTTP server is on. import { Conf } from '@/config.ts'; -import { createZapSplitsIfNotExists } from '@/utils/zap-split.ts'; +import { seedZapSplits } from '@/utils/zap-split.ts'; import { cron } from '@/cron.ts'; import { startFirehose } from '@/firehose.ts'; @@ -13,4 +13,4 @@ if (Conf.cronEnabled) { cron(); } -await createZapSplitsIfNotExists(); +await seedZapSplits(); diff --git a/src/utils/zap-split.ts b/src/utils/zap-split.ts index 830c1e7c..553d9c8c 100644 --- a/src/utils/zap-split.ts +++ b/src/utils/zap-split.ts @@ -39,7 +39,7 @@ export async function getZapSplits(store: NStore, pubkey: string): Promise Date: Thu, 25 Jul 2024 15:44:05 -0300 Subject: [PATCH 57/59] fix(app.ts): import startup.ts file --- src/app.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app.ts b/src/app.ts index 690862eb..66c33424 100644 --- a/src/app.ts +++ b/src/app.ts @@ -110,6 +110,7 @@ import { import { errorHandler } from '@/controllers/error.ts'; import { metricsController } from '@/controllers/metrics.ts'; import { indexController } from '@/controllers/site.ts'; +import '@/startup.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; import { nostrController } from '@/controllers/well-known/nostr.ts'; import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts'; From 9c02a2d555d6d02ea0e4ee01a5ba5c3ea4769f35 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 26 Jul 2024 18:49:31 -0300 Subject: [PATCH 58/59] refactor: remove nip05 enhancement and tests --- src/storages/EventsDB.test.ts | 21 --------------------- src/storages/EventsDB.ts | 5 +---- 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 1b4b66e0..82c4e34a 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -217,24 +217,3 @@ Deno.test("throws a RelayError when querying an event with a large 'kind'", asyn 'kind filter too far into the future', ); }); - -Deno.test( - 'query user by NIP-05 search filter', - { ignore: Conf.db.dialect !== 'postgres' }, - async () => { - await using db = await createTestDB(); - const { store } = db; - - const event0 = await eventFixture('event-0'); - await store.event(event0); - - assertEquals(await store.query([{}]), [event0]); - assertEquals(await store.query([{ search: 'sonator.dev' }]), []); - assertEquals(await store.query([{ search: 'alex' }]), [event0]); - assertEquals(await store.query([{ search: 'gleasonator' }]), [event0]); - assertEquals(await store.query([{ search: 'com' }]), [event0]); - assertEquals(await store.query([{ search: 'mostr' }]), [event0]); - assertEquals(await store.query([{ search: 'pub' }]), [event0]); - assertEquals(await store.query([{ search: 'mostr.pub' }]), [event0]); - }, -); diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index d5171bc2..abf076c7 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -240,10 +240,7 @@ class EventsDB implements NStore { /** Build search content for a user. */ static buildUserSearchContent(event: NostrEvent): string { const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(event.content); - const nip05splitted = nip05 ? nip05.split(/[_@.]/) : []; - const nip05splitted2 = nip05 ? nip05.split(/[@]/) : []; - - return [name, nip05, ...nip05splitted, ...nip05splitted2].filter(Boolean).join('\n'); + return [name, nip05].filter(Boolean).join('\n'); } /** Build search content from tag values. */ From a271340ffc6199fb2f88837f12d5d5af914cd942 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 26 Jul 2024 19:01:18 -0300 Subject: [PATCH 59/59] test: use postgres in hydrate.test.ts --- src/storages/hydrate.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/storages/hydrate.test.ts b/src/storages/hydrate.test.ts index 6730315b..3eb70bf8 100644 --- a/src/storages/hydrate.test.ts +++ b/src/storages/hydrate.test.ts @@ -7,7 +7,7 @@ import { createTestDB, eventFixture } from '@/test.ts'; Deno.test('hydrateEvents(): author --- WITHOUT stats', async () => { const relay = new MockRelay(); - await using db = await createTestDB('sqlite://:memory:'); + await using db = await createTestDB(); const event0 = await eventFixture('event-0'); const event1 = await eventFixture('event-1'); @@ -28,7 +28,7 @@ Deno.test('hydrateEvents(): author --- WITHOUT stats', async () => { Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { const relay = new MockRelay(); - await using db = await createTestDB('sqlite://:memory:'); + await using db = await createTestDB(); const event0madePost = await eventFixture('event-0-the-one-who-post-and-users-repost'); const event0madeRepost = await eventFixture('event-0-the-one-who-repost'); @@ -57,7 +57,7 @@ Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { const relay = new MockRelay(); - await using db = await createTestDB('sqlite://:memory:'); + await using db = await createTestDB(); const event0madeQuoteRepost = await eventFixture('event-0-the-one-who-quote-repost'); const event0 = await eventFixture('event-0'); @@ -87,7 +87,7 @@ Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () => { const relay = new MockRelay(); - await using db = await createTestDB('sqlite://:memory:'); + await using db = await createTestDB(); const author = await eventFixture('event-0-makes-repost-with-quote-repost'); const event1 = await eventFixture('event-1-will-be-reposted-with-quote-repost'); @@ -116,7 +116,7 @@ Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () Deno.test('hydrateEvents(): report pubkey and post // kind 1984 --- WITHOUT stats', async () => { const relay = new MockRelay(); - await using db = await createTestDB('sqlite://:memory:'); + await using db = await createTestDB(); const authorDictator = await eventFixture('kind-0-dictator'); const authorVictim = await eventFixture('kind-0-george-orwell');