From 40c187680e8580c2cec374e8dea87ca94bd17152 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 10:22:05 -0300 Subject: [PATCH 01/29] feat: create author_search table --- src/db/migrations/032_add_author_search.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/db/migrations/032_add_author_search.ts diff --git a/src/db/migrations/032_add_author_search.ts b/src/db/migrations/032_add_author_search.ts new file mode 100644 index 00000000..d5a93c06 --- /dev/null +++ b/src/db/migrations/032_add_author_search.ts @@ -0,0 +1,18 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('author_search') + .addColumn('pubkey', 'char(64)', (col) => col.primaryKey()) + .addColumn('search', 'text', (col) => col.notNull()) + .ifNotExists() + .execute(); + + await sql`CREATE EXTENSION IF NOT EXISTS pg_trgm;`.execute(db); + await sql`CREATE INDEX author_search_search_idx ON author_search USING GIN (search gin_trgm_ops);`.execute(db); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropIndex('author_search_search_idx').ifExists().execute(); + await db.schema.dropTable('author_search').execute(); +} From 8bc8712cf37d542258c9eecf4179d99e0ec8496d Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 10:23:08 -0300 Subject: [PATCH 02/29] feat: create and add author_search interface to DittoTables --- src/db/DittoTables.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index a62c485d..38f7249c 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -6,6 +6,7 @@ export interface DittoTables extends NPostgresSchema { event_stats: EventStatsRow; pubkey_domains: PubkeyDomainRow; event_zaps: EventZapRow; + author_search: AuthorSearch; } interface AuthorStatsRow { @@ -47,3 +48,8 @@ interface EventZapRow { amount_millisats: number; comment: string; } + +interface AuthorSearch { + pubkey: string; + search: string; +} From b5aefdd93e00d2921005f763013dccbbd7dde759 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 10:24:33 -0300 Subject: [PATCH 03/29] feat: add pg_trgm extension in PGlite constructor --- src/db/adapters/DittoPglite.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/db/adapters/DittoPglite.ts b/src/db/adapters/DittoPglite.ts index 4ec7d8a5..0e93075d 100644 --- a/src/db/adapters/DittoPglite.ts +++ b/src/db/adapters/DittoPglite.ts @@ -1,4 +1,5 @@ import { PGlite } from '@electric-sql/pglite'; +import { pg_trgm } from '@electric-sql/pglite/contrib/pg_trgm'; import { PgliteDialect } from '@soapbox/kysely-pglite'; import { Kysely } from 'kysely'; @@ -10,7 +11,7 @@ export class DittoPglite { static create(databaseUrl: string): DittoDatabase { const kysely = new Kysely({ dialect: new PgliteDialect({ - database: new PGlite(databaseUrl), + database: new PGlite(databaseUrl, { extensions: { pg_trgm } }), }), log: KyselyLogger, }); From c03ea07dcb253e6ff1b59c403133ee72cea2f7bf Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 10:27:12 -0300 Subject: [PATCH 04/29] feat: create getPubkeysBySearch() function and use it inside searchEvents() function --- src/controllers/api/search.ts | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 9bddc336..19e72cac 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -1,8 +1,10 @@ import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; +import { Kysely, sql } from 'kysely'; import { z } from 'zod'; import { AppController } from '@/app.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; import { booleanParamSchema } from '@/schema.ts'; import { Storages } from '@/storages.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; @@ -89,9 +91,21 @@ async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: filter.authors = [account_id]; } + const filter2: NostrFilter = { + kinds: [0], + limit, + }; + if (type === 'accounts') { + const kysely = await Storages.kysely(); + + const pubkeys = await getPubkeysBySearch(kysely, { q, limit }); + + filter2.authors = pubkeys; // if pubkeys is empty the filter 2 will be discarded + } + const store = await Storages.search(); - return store.query([filter], { signal }) + return store.query([filter, filter2], { signal }) .then((events) => hydrateEvents({ events, store, signal })); } @@ -170,4 +184,16 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort return []; } -export { searchController }; +/** Get pubkeys whose name and NIP-05 is similar to 'q' */ +async function getPubkeysBySearch(kysely: Kysely, { q, limit }: Pick) { + const pubkeys = (await sql` + SELECT *, word_similarity(${q}, search) AS sml + FROM author_search + WHERE ${q} % search + ORDER BY sml DESC, search LIMIT ${limit} + `.execute(kysely)).rows.map((row) => (row as { pubkey: string }).pubkey); + + return pubkeys; +} + +export { getPubkeysBySearch, searchController }; From 9d2667679fdac0adc956b4927ddd8417e4f08500 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 10:28:54 -0300 Subject: [PATCH 05/29] feat(pipeline.ts): create handleAuthorSearch() function --- src/pipeline.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/pipeline.ts b/src/pipeline.ts index 88e5f29b..9d1a8038 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -57,6 +57,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise, event: NostrEvent) { } } +async function handleAuthorSearch(kysely: Kysely, event: NostrEvent) { + if (event.kind !== 0) return; + const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(event.content); + const search = [name, nip05].filter(Boolean).join(' ').trim(); + + try { + await kysely.insertInto('author_search').values({ + pubkey: event.pubkey, + search, + }).onConflict( + (oc) => + oc.column('pubkey') + .doUpdateSet({ search }), + ) + .execute(); + } catch { + // do nothing + } +} + export { handleEvent, handleZaps }; From e1cd1777e3d76b8f3b67d8d415215dc7e70ba424 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 10:30:10 -0300 Subject: [PATCH 06/29] test: add author_search table in createTestDB to drop it after use --- src/test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test.ts b/src/test.ts index 45946f00..00bf4354 100644 --- a/src/test.ts +++ b/src/test.ts @@ -63,6 +63,7 @@ export async function createTestDB() { 'pubkey_domains', 'nostr_events', 'event_zaps', + 'author_search', ] ) { await kysely.schema.dropTable(table).ifExists().cascade().execute(); From a6f1098bc6fc52dee99de61983399961af5c1aa8 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 10:32:05 -0300 Subject: [PATCH 07/29] test: getPubkeysBySearch() function --- src/controllers/api/search.test.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/controllers/api/search.test.ts diff --git a/src/controllers/api/search.test.ts b/src/controllers/api/search.test.ts new file mode 100644 index 00000000..2c5e91bd --- /dev/null +++ b/src/controllers/api/search.test.ts @@ -0,0 +1,21 @@ +import { assertEquals } from '@std/assert'; + +import { createTestDB } from '@/test.ts'; +import { getPubkeysBySearch } from '@/controllers/api/search.ts'; + +Deno.test('fuzzy search works', async () => { + await using db = await createTestDB(); + + await db.kysely.insertInto('author_search').values({ + pubkey: '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', + search: 'patrickReiis patrickdosreis.com', + }).execute(); + + assertEquals(await getPubkeysBySearch(db.kysely, { q: 'pat rick', limit: 1 }), []); + assertEquals(await getPubkeysBySearch(db.kysely, { q: 'patrick dos reis', limit: 1 }), [ + '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', + ]); + assertEquals(await getPubkeysBySearch(db.kysely, { q: 'dosreis.com', limit: 1 }), [ + '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', + ]); +}); From 24d909fd28383fda2f10934e7246483fe1be52ff Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 10:34:15 -0300 Subject: [PATCH 08/29] feat: create script to populate author_search table --- deno.json | 3 ++- scripts/db-populate-search.ts | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 scripts/db-populate-search.ts diff --git a/deno.json b/deno.json index 699ab620..0070b142 100644 --- a/deno.json +++ b/deno.json @@ -18,7 +18,8 @@ "stats:recompute": "deno run -A scripts/stats-recompute.ts", "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip -o soapbox.zip && rm soapbox.zip", "trends": "deno run -A scripts/trends.ts", - "clean:deps": "deno cache --reload src/app.ts" + "clean:deps": "deno cache --reload src/app.ts", + "db:populate-search": "deno run -A scripts/db-populate-search.ts" }, "unstable": ["cron", "ffi", "kv", "worker-options"], "exclude": ["./public"], diff --git a/scripts/db-populate-search.ts b/scripts/db-populate-search.ts new file mode 100644 index 00000000..9c698c8f --- /dev/null +++ b/scripts/db-populate-search.ts @@ -0,0 +1,35 @@ +import { NSchema as n } from '@nostrify/nostrify'; +import { Storages } from '@/storages.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; + +const kysely = await Storages.kysely(); +const stream = kysely + .selectFrom('nostr_events') + .select(['pubkey', 'content']) + .where('kind', '=', 0) + .stream(); + +const values: DittoTables['author_search'][] = []; + +for await (const author of stream) { + const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(author.content); + const search = [name, nip05].filter(Boolean).join(' ').trim(); + + values.push({ + pubkey: author.pubkey, + search, + }); +} + +try { + await kysely.insertInto('author_search').values(values).onConflict( + (oc) => + oc.column('pubkey') + .doUpdateSet((eb) => ({ search: eb.ref('excluded.search') })), + ) + .execute(); +} catch { + // do nothing +} + +Deno.exit(); From 935cc7c5a574ba9bf5cb148a14f4f0db3a932377 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 14:33:34 -0300 Subject: [PATCH 09/29] refactor: remove NIP-50 search if looking for accounts, use same filter --- src/controllers/api/search.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 19e72cac..1a1bd867 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -91,21 +91,18 @@ async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: filter.authors = [account_id]; } - const filter2: NostrFilter = { - kinds: [0], - limit, - }; if (type === 'accounts') { const kysely = await Storages.kysely(); const pubkeys = await getPubkeysBySearch(kysely, { q, limit }); - filter2.authors = pubkeys; // if pubkeys is empty the filter 2 will be discarded + filter.authors = pubkeys; + filter.search = undefined; } const store = await Storages.search(); - return store.query([filter, filter2], { signal }) + return store.query([filter], { signal }) .then((events) => hydrateEvents({ events, store, signal })); } From be76197e3ac92ceae17a50db907fda4b4ee91a76 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 15:08:36 -0300 Subject: [PATCH 10/29] refactor: remove handleAuthorSearch() function and put its logic inside parseMetadata() function --- src/pipeline.ts | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index b208e9e2..aeba0150 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -58,7 +58,6 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise + oc.column('pubkey') + .doUpdateSet({ search }), + ) + .execute(); + } catch { + // do nothing + } + if (!nip05) return; // Fetch nip05. @@ -148,7 +167,6 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise, event: NostrEvent) { } } -async function handleAuthorSearch(kysely: Kysely, event: NostrEvent) { - if (event.kind !== 0) return; - const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(event.content); - const search = [name, nip05].filter(Boolean).join(' ').trim(); - - try { - await kysely.insertInto('author_search').values({ - pubkey: event.pubkey, - search, - }).onConflict( - (oc) => - oc.column('pubkey') - .doUpdateSet({ search }), - ) - .execute(); - } catch { - // do nothing - } -} - export { handleEvent, handleZaps }; From 69c21581310a720aed184fcb7f673282d7393d9f Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 16:19:38 -0300 Subject: [PATCH 11/29] refactor: return ordered accounts by similarity relevance in searchEvents() function --- src/controllers/api/search.ts | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 1a1bd867..2555109b 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -91,19 +91,35 @@ async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: filter.authors = [account_id]; } + const pubkeys: string[] = []; if (type === 'accounts') { const kysely = await Storages.kysely(); - const pubkeys = await getPubkeysBySearch(kysely, { q, limit }); + pubkeys.push(...(await getPubkeysBySearch(kysely, { q, limit }))); + + if (!filter?.authors) filter.authors = pubkeys; + else filter.authors.push(...pubkeys); - filter.authors = pubkeys; filter.search = undefined; } const store = await Storages.search(); - return store.query([filter], { signal }) + const events = await store.query([filter], { signal }) .then((events) => hydrateEvents({ events, store, signal })); + + if (type !== 'accounts') return events; + + const orderedEvents: NostrEvent[] = events.map((event, index) => { + const pubkey = pubkeys[index]; + + const orderedEvent = events.find((e) => e.pubkey === pubkey); + if (orderedEvent) return orderedEvent; + + return event; + }); + + return orderedEvents; } /** Get event kinds to search from `type` query param. */ From 6387ee440cc9d7c773088f158c45cc17cab93a3c Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 16:36:01 -0300 Subject: [PATCH 12/29] feat: return multiple accounts in searchController --- src/controllers/api/search.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 2555109b..c778bc4f 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -49,9 +49,8 @@ const searchController: AppController = async (c) => { if (event) { events = [event]; - } else { - events = await searchEvents(result.data, signal); } + events.push(...(await searchEvents(result.data, signal))); const viewerPubkey = await c.get('signer')?.getPublicKey(); From b3e56320a04ab866b9aeeb782e794d6252be1f49 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 18:41:20 -0300 Subject: [PATCH 13/29] feat(accountSearchController): return accounts in autocomplete form --- src/controllers/api/accounts.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 9c635565..a8aa5d55 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -6,6 +6,7 @@ import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { getAuthor, getFollowedPubkeys } from '@/queries.ts'; import { booleanParamSchema, fileSchema } from '@/schema.ts'; +import { getPubkeysBySearch } from '@/controllers/api/search.ts'; import { Storages } from '@/storages.ts'; import { uploadFile } from '@/utils/upload.ts'; import { nostrNow } from '@/utils.ts'; @@ -115,6 +116,7 @@ const accountSearchQuerySchema = z.object({ const accountSearchController: AppController = async (c) => { const { signal } = c.req.raw; const { limit } = c.get('pagination'); + const kysely = await Storages.kysely(); const result = accountSearchQuerySchema.safeParse(c.req.query()); @@ -133,9 +135,22 @@ const accountSearchController: AppController = async (c) => { return c.json(pubkey ? [await accountFromPubkey(pubkey)] : []); } - const events = event ? [event] : await store.query([{ kinds: [0], search: query, limit }], { signal }); + const pubkeys = await getPubkeysBySearch(kysely, { q: query, limit }); - const accounts = await hydrateEvents({ events, store, signal }).then( + const events = event ? [event] : await store.query([{ kinds: [0], authors: pubkeys, limit }], { + signal, + }); + + const orderedEvents = events.map((event, index) => { + const pubkey = pubkeys[index]; + + const orderedEvent = events.find((e) => e.pubkey === pubkey); + if (orderedEvent) return orderedEvent; + + return event; + }); + + const accounts = await hydrateEvents({ events: orderedEvents, store, signal }).then( (events) => Promise.all( events.map((event) => renderAccount(event)), From 197b2c8c8b3b62e005134e405a430c55a0bbc0c5 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 18:49:20 -0300 Subject: [PATCH 14/29] refactor(populate search script): use store.req instead of streaming --- scripts/db-populate-search.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/db-populate-search.ts b/scripts/db-populate-search.ts index 9c698c8f..beec2d52 100644 --- a/scripts/db-populate-search.ts +++ b/scripts/db-populate-search.ts @@ -2,23 +2,23 @@ import { NSchema as n } from '@nostrify/nostrify'; import { Storages } from '@/storages.ts'; import { DittoTables } from '@/db/DittoTables.ts'; +const store = await Storages.db(); const kysely = await Storages.kysely(); -const stream = kysely - .selectFrom('nostr_events') - .select(['pubkey', 'content']) - .where('kind', '=', 0) - .stream(); const values: DittoTables['author_search'][] = []; -for await (const author of stream) { - const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(author.content); - const search = [name, nip05].filter(Boolean).join(' ').trim(); +for await (const msg of store.req([{ kinds: [0] }])) { + if (msg[0] === 'EVENT') { + const { pubkey, content } = msg[2]; - values.push({ - pubkey: author.pubkey, - search, - }); + const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(content); + const search = [name, nip05].filter(Boolean).join(' ').trim(); + + values.push({ + pubkey: pubkey, + search, + }); + } } try { From f99ea7c33fa4cf385513a487df164b48ea1d3f71 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 18:57:47 -0300 Subject: [PATCH 15/29] refactor(getPubkeysBySearch): cast as string --- src/controllers/api/search.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index c778bc4f..d62c08e8 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -198,12 +198,12 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort /** Get pubkeys whose name and NIP-05 is similar to 'q' */ async function getPubkeysBySearch(kysely: Kysely, { q, limit }: Pick) { - const pubkeys = (await sql` + const pubkeys = (await sql<{ pubkey: string }>` SELECT *, word_similarity(${q}, search) AS sml FROM author_search WHERE ${q} % search ORDER BY sml DESC, search LIMIT ${limit} - `.execute(kysely)).rows.map((row) => (row as { pubkey: string }).pubkey); + `.execute(kysely)).rows.map(({ pubkey }) => pubkey); return pubkeys; } From c24d11c6f3055cc20a5b55abd1281bf6ce28020c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 15 Sep 2024 12:40:58 -0500 Subject: [PATCH 16/29] Support NIP-50 language extension --- deno.json | 1 + deno.lock | 66 +++++++++++++++++++++++++-- src/db/DittoTables.ts | 7 +++ src/db/migrations/032_add_language.ts | 11 +++++ src/pipeline.ts | 27 ++++++++++- src/storages/EventsDB.test.ts | 17 +++++++ src/storages/EventsDB.ts | 63 +++++++++++++------------ 7 files changed, 156 insertions(+), 36 deletions(-) create mode 100644 src/db/migrations/032_add_language.ts diff --git a/deno.json b/deno.json index 699ab620..8edf0e53 100644 --- a/deno.json +++ b/deno.json @@ -59,6 +59,7 @@ "isomorphic-dompurify": "npm:isomorphic-dompurify@^2.11.0", "kysely": "npm:kysely@^0.27.4", "kysely-postgres-js": "npm:kysely-postgres-js@2.0.0", + "lande": "npm:lande@^1.0.10", "light-bolt11-decoder": "npm:light-bolt11-decoder", "linkify-plugin-hashtag": "npm:linkify-plugin-hashtag@^4.1.1", "linkify-string": "npm:linkify-string@^4.1.1", diff --git a/deno.lock b/deno.lock index a275accb..3a25d8d3 100644 --- a/deno.lock +++ b/deno.lock @@ -29,13 +29,16 @@ "jsr:@soapbox/kysely-pglite@^0.0.1": "jsr:@soapbox/kysely-pglite@0.0.1", "jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0", "jsr:@std/assert@^0.213.1": "jsr:@std/assert@0.213.1", + "jsr:@std/assert@^0.223.0": "jsr:@std/assert@0.223.0", "jsr:@std/assert@^0.224.0": "jsr:@std/assert@0.224.0", "jsr:@std/assert@^0.225.1": "jsr:@std/assert@0.225.3", + "jsr:@std/bytes@^0.223.0": "jsr:@std/bytes@0.223.0", "jsr:@std/bytes@^0.224.0": "jsr:@std/bytes@0.224.0", "jsr:@std/bytes@^1.0.0-rc.3": "jsr:@std/bytes@1.0.0", "jsr:@std/bytes@^1.0.1-rc.3": "jsr:@std/bytes@1.0.2", "jsr:@std/bytes@^1.0.2": "jsr:@std/bytes@1.0.2", "jsr:@std/bytes@^1.0.2-rc.3": "jsr:@std/bytes@1.0.2", + "jsr:@std/cli@^0.223.0": "jsr:@std/cli@0.223.0", "jsr:@std/crypto@^0.224.0": "jsr:@std/crypto@0.224.0", "jsr:@std/dotenv@^0.224.0": "jsr:@std/dotenv@0.224.2", "jsr:@std/encoding@0.213.1": "jsr:@std/encoding@0.213.1", @@ -45,14 +48,17 @@ "jsr:@std/fs@0.213.1": "jsr:@std/fs@0.213.1", "jsr:@std/fs@^0.229.3": "jsr:@std/fs@0.229.3", "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.1", + "jsr:@std/io@^0.223.0": "jsr:@std/io@0.223.0", "jsr:@std/io@^0.224": "jsr:@std/io@0.224.7", "jsr:@std/json@^0.223.0": "jsr:@std/json@0.223.0", "jsr:@std/media-types@^0.224.1": "jsr:@std/media-types@0.224.1", "jsr:@std/path@0.213.1": "jsr:@std/path@0.213.1", + "jsr:@std/path@1.0.0-rc.1": "jsr:@std/path@1.0.0-rc.1", "jsr:@std/path@^0.213.1": "jsr:@std/path@0.213.1", "jsr:@std/streams@^0.223.0": "jsr:@std/streams@0.223.0", "npm:@isaacs/ttlcache@^1.4.1": "npm:@isaacs/ttlcache@1.4.1", "npm:@noble/hashes@^1.4.0": "npm:@noble/hashes@1.4.0", + "npm:@noble/secp256k1@^2.0.0": "npm:@noble/secp256k1@2.1.0", "npm:@scure/base@^1.1.6": "npm:@scure/base@1.1.6", "npm:@scure/bip32@^1.4.0": "npm:@scure/bip32@1.4.0", "npm:@scure/bip39@^1.3.0": "npm:@scure/bip39@1.3.0", @@ -72,6 +78,7 @@ "npm:kysely@^0.27.2": "npm:kysely@0.27.4", "npm:kysely@^0.27.3": "npm:kysely@0.27.4", "npm:kysely@^0.27.4": "npm:kysely@0.27.4", + "npm:lande@^1.0.10": "npm:lande@1.0.10", "npm:light-bolt11-decoder": "npm:light-bolt11-decoder@3.1.1", "npm:linkify-plugin-hashtag@^4.1.1": "npm:linkify-plugin-hashtag@4.1.3_linkifyjs@4.1.3", "npm:linkify-string@^4.1.1": "npm:linkify-string@4.1.3_linkifyjs@4.1.3", @@ -88,6 +95,7 @@ "npm:postgres@3.4.4": "npm:postgres@3.4.4", "npm:prom-client@^15.1.2": "npm:prom-client@15.1.2", "npm:tldts@^6.0.14": "npm:tldts@6.1.18", + "npm:tseep@^1.2.1": "npm:tseep@1.2.1", "npm:type-fest@^4.3.0": "npm:type-fest@4.18.2", "npm:unfurl.js@^6.4.0": "npm:unfurl.js@6.4.0", "npm:websocket-ts@^2.1.5": "npm:websocket-ts@2.1.5", @@ -303,6 +311,9 @@ "@std/assert@0.213.1": { "integrity": "24c28178b30c8e0782c18e8e94ea72b16282207569cdd10ffb9d1d26f2edebfe" }, + "@std/assert@0.223.0": { + "integrity": "eb8d6d879d76e1cc431205bd346ed4d88dc051c6366365b1af47034b0670be24" + }, "@std/assert@0.224.0": { "integrity": "8643233ec7aec38a940a8264a6e3eed9bfa44e7a71cc6b3c8874213ff401967f" }, @@ -312,6 +323,9 @@ "jsr:@std/internal@^1.0.0" ] }, + "@std/bytes@0.223.0": { + "integrity": "84b75052cd8680942c397c2631318772b295019098f40aac5c36cead4cba51a8" + }, "@std/bytes@0.224.0": { "integrity": "a2250e1d0eb7d1c5a426f21267ab9bdeac2447fa87a3d0d1a467d3f7a6058e49" }, @@ -321,6 +335,12 @@ "@std/bytes@1.0.2": { "integrity": "fbdee322bbd8c599a6af186a1603b3355e59a5fb1baa139f8f4c3c9a1b3e3d57" }, + "@std/cli@0.223.0": { + "integrity": "2feb7970f2028904c3edc22ea916ce9538113dfc170844f3eae03578c333c356", + "dependencies": [ + "jsr:@std/assert@^0.223.0" + ] + }, "@std/crypto@0.224.0": { "integrity": "154ef3ff08ef535562ef1a718718c5b2c5fc3808f0f9100daad69e829bfcdf2d", "dependencies": [ @@ -351,7 +371,10 @@ ] }, "@std/fs@0.229.3": { - "integrity": "783bca21f24da92e04c3893c9e79653227ab016c48e96b3078377ebd5222e6eb" + "integrity": "783bca21f24da92e04c3893c9e79653227ab016c48e96b3078377ebd5222e6eb", + "dependencies": [ + "jsr:@std/path@1.0.0-rc.1" + ] }, "@std/internal@1.0.0": { "integrity": "ac6a6dfebf838582c4b4f61a6907374e27e05bedb6ce276e0f1608fe84e7cd9a" @@ -359,6 +382,13 @@ "@std/internal@1.0.1": { "integrity": "6f8c7544d06a11dd256c8d6ba54b11ed870aac6c5aeafff499892662c57673e6" }, + "@std/io@0.223.0": { + "integrity": "2d8c3c2ab3a515619b90da2c6ff5ea7b75a94383259ef4d02116b228393f84f1", + "dependencies": [ + "jsr:@std/assert@^0.223.0", + "jsr:@std/bytes@^0.223.0" + ] + }, "@std/io@0.224.0": { "integrity": "0aff885d21d829c050b8a08b1d71b54aed5841aecf227f8d77e99ec529a11e8e", "dependencies": [ @@ -396,7 +426,10 @@ ] }, "@std/json@0.223.0": { - "integrity": "9a4a255931dd0397924c6b10bb6a72fe3e28ddd876b981ada2e3b8dd0764163f" + "integrity": "9a4a255931dd0397924c6b10bb6a72fe3e28ddd876b981ada2e3b8dd0764163f", + "dependencies": [ + "jsr:@std/streams@^0.223.0" + ] }, "@std/media-types@0.224.1": { "integrity": "9e69a5daed37c5b5c6d3ce4731dc191f80e67f79bed392b0957d1d03b87f11e1" @@ -407,8 +440,16 @@ "jsr:@std/assert@^0.213.1" ] }, + "@std/path@1.0.0-rc.1": { + "integrity": "b8c00ae2f19106a6bb7cbf1ab9be52aa70de1605daeb2dbdc4f87a7cbaf10ff6" + }, "@std/streams@0.223.0": { - "integrity": "d6b28e498ced3960b04dc5d251f2dcfc1df244b5ec5a48dc23a8f9b490be3b99" + "integrity": "d6b28e498ced3960b04dc5d251f2dcfc1df244b5ec5a48dc23a8f9b490be3b99", + "dependencies": [ + "jsr:@std/assert@^0.223.0", + "jsr:@std/bytes@^0.223.0", + "jsr:@std/io@^0.223.0" + ] } }, "npm": { @@ -454,6 +495,10 @@ "integrity": "sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg==", "dependencies": {} }, + "@noble/secp256k1@2.1.0": { + "integrity": "sha512-XLEQQNdablO0XZOIniFQimiXsZDNwaYgL96dZwC54Q30imSbAOFf3NKtepc+cXyuZf5Q1HCgbqgZ2UFFuHVcEw==", + "dependencies": {} + }, "@opentelemetry/api@1.9.0": { "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", "dependencies": {} @@ -864,6 +909,12 @@ "integrity": "sha512-dyNKv2KRvYOQPLCAOCjjQuCk4YFd33BvGdf/o5bC7FiW+BB6snA81Zt+2wT9QDFzKqxKa5rrOmvlK/anehCcgA==", "dependencies": {} }, + "lande@1.0.10": { + "integrity": "sha512-yT52DQh+UV2pEp08jOYrA4drDv0DbjpiRyZYgl25ak9G2cVR2AimzrqkYQWrD9a7Ud+qkAcaiDDoNH9DXfHPmw==", + "dependencies": { + "toygrad": "toygrad@2.6.0" + } + }, "light-bolt11-decoder@3.1.1": { "integrity": "sha512-sLg/KCwYkgsHWkefWd6KqpCHrLFWWaXTOX3cf6yD2hAzL0SLpX+lFcaFK2spkjbgzG6hhijKfORDc9WoUHwX0A==", "dependencies": { @@ -1213,6 +1264,10 @@ "url-parse": "url-parse@1.5.10" } }, + "toygrad@2.6.0": { + "integrity": "sha512-g4zBmlSbvzOE5FOILxYkAybTSxijKLkj1WoNqVGnbMcWDyj4wWQ+eYSr3ik7XOpIgMq/7eBcPRTJX3DM2E0YMg==", + "dependencies": {} + }, "tr46@0.0.3": { "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", "dependencies": {} @@ -1223,6 +1278,10 @@ "punycode": "punycode@2.3.1" } }, + "tseep@1.2.1": { + "integrity": "sha512-VFnsNcPGC4qFJ1nxbIPSjTmtRZOhlqLmtwRqtLVos8mbRHki8HO9cy9Z1e89EiWyxFmq6LBviI9TQjijxw/mEw==", + "dependencies": {} + }, "type-fest@3.13.1": { "integrity": "sha512-tLq3bSNx+xSpwvAJnzrK0Ep5CLNWjvFTOp71URMaAEWBfRb9nnJiBoUe0tF8bI4ZFO3omgBR6NvnbzVUT3Ly4g==", "dependencies": {} @@ -1963,6 +2022,7 @@ "npm:isomorphic-dompurify@^2.11.0", "npm:kysely-postgres-js@2.0.0", "npm:kysely@^0.27.4", + "npm:lande@^1.0.10", "npm:light-bolt11-decoder", "npm:linkify-plugin-hashtag@^4.1.1", "npm:linkify-string@^4.1.1", diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index a62c485d..48cb06cb 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -1,6 +1,9 @@ +import { Nullable } from 'kysely'; + import { NPostgresSchema } from '@nostrify/db'; export interface DittoTables extends NPostgresSchema { + nostr_events: NostrEventsRow; nip46_tokens: NIP46TokenRow; author_stats: AuthorStatsRow; event_stats: EventStatsRow; @@ -8,6 +11,10 @@ export interface DittoTables extends NPostgresSchema { event_zaps: EventZapRow; } +type NostrEventsRow = NPostgresSchema['nostr_events'] & { + language: Nullable; +}; + interface AuthorStatsRow { pubkey: string; followers_count: number; diff --git a/src/db/migrations/032_add_language.ts b/src/db/migrations/032_add_language.ts new file mode 100644 index 00000000..a0f828fe --- /dev/null +++ b/src/db/migrations/032_add_language.ts @@ -0,0 +1,11 @@ +import { Kysely } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema.alterTable('nostr_events').addColumn('language', 'char(2)').execute(); + await db.schema.createIndex('nostr_events_language_idx').on('nostr_events').column('language').execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.alterTable('nostr_events').dropColumn('language').execute(); + await db.schema.dropIndex('nostr_events_language_idx').execute(); +} diff --git a/src/pipeline.ts b/src/pipeline.ts index 85d27964..cc4975cd 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,6 +1,8 @@ import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; +import ISO6391 from 'iso-639-1'; import { Kysely, sql } from 'kysely'; +import lande from 'lande'; import { LRUCache } from 'lru-cache'; import { z } from 'zod'; @@ -55,10 +57,11 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { + const [topResult] = lande(event.content); + + if (topResult) { + const [iso6393, confidence] = topResult; + const locale = new Intl.Locale(iso6393); + + if (confidence >= 0.95 && ISO6391.validate(locale.language)) { + const kysely = await Storages.kysely(); + try { + await kysely.updateTable('nostr_events') + .set('language', locale.language) + .where('id', '=', event.id) + .execute(); + } catch { + // do nothing + } + } + } +} + /** Determine if the event is being received in a timely manner. */ function isFresh(event: NostrEvent): boolean { return eventAge(event) < Time.seconds(10); diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 7a5f7b93..b24032aa 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -54,6 +54,23 @@ Deno.test('query events with domain search filter', async () => { assertEquals(await store.query([{ kinds: [1], search: 'domain:example.com' }]), []); }); +Deno.test('query events with language search filter', async () => { + await using db = await createTestDB(); + const { store, kysely } = db; + + const en = genEvent({ kind: 1, content: 'hello world!' }); + const es = genEvent({ kind: 1, content: 'hola mundo!' }); + + await store.event(en); + await store.event(es); + + await kysely.updateTable('nostr_events').set('language', 'en').where('id', '=', en.id).execute(); + await kysely.updateTable('nostr_events').set('language', 'es').where('id', '=', es.id).execute(); + + assertEquals(await store.query([{ search: 'language:en' }]), [en]); + assertEquals(await store.query([{ search: 'language:es' }]), [es]); +}); + Deno.test('delete events', async () => { await using db = await createTestDB(); const { store } = db; diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index bedc1cac..b4dc0b9b 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -3,7 +3,7 @@ import { NPostgres, NPostgresSchema } from '@nostrify/db'; import { NIP50, NKinds, NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { Stickynotes } from '@soapbox/stickynotes'; -import { Kysely } from 'kysely'; +import { Kysely, SelectQueryBuilder } from 'kysely'; import { nip27 } from 'nostr-tools'; import { DittoTables } from '@/db/DittoTables.ts'; @@ -145,8 +145,36 @@ class EventsDB extends NPostgres { } protected getFilterQuery(trx: Kysely, filter: NostrFilter) { - const query = super.getFilterQuery(trx, filter); - return query; + if (filter.search) { + const tokens = NIP50.parseInput(filter.search); + + let query = super.getFilterQuery(trx, { + ...filter, + search: tokens.filter((t) => typeof t === 'string').join(' '), + }) as SelectQueryBuilder>; + + const data = tokens.filter((t) => typeof t === 'object').reduce( + (acc, t) => acc.set(t.key, t.value), + new Map(), + ); + + const domain = data.get('domain'); + const language = data.get('language'); + + if (domain) { + query = query + .innerJoin('pubkey_domains', 'nostr_events.pubkey', 'pubkey_domains.pubkey') + .where('pubkey_domains.domain', '=', domain); + } + + if (language) { + query = query.where('language', '=', language); + } + + return query; + } + + return super.getFilterQuery(trx, filter); } /** Get events for filters from the database. */ @@ -260,35 +288,6 @@ class EventsDB extends NPostgres { filters = structuredClone(filters); for (const filter of filters) { - if (filter.search) { - const tokens = NIP50.parseInput(filter.search); - - const domain = (tokens.find((t) => - typeof t === 'object' && t.key === 'domain' - ) as { key: 'domain'; value: string } | undefined)?.value; - - if (domain) { - const query = this.opts.kysely - .selectFrom('pubkey_domains') - .select('pubkey') - .where('domain', '=', domain); - - if (filter.authors) { - query.where('pubkey', 'in', filter.authors); - } - - const pubkeys = await query - .execute() - .then((rows) => - rows.map((row) => row.pubkey) - ); - - filter.authors = pubkeys; - } - - filter.search = tokens.filter((t) => typeof t === 'string').join(' '); - } - if (filter.kinds) { // Ephemeral events are not stored, so don't bother querying for them. // If this results in an empty kinds array, NDatabase will remove the filter before querying and return no results. From eede3909b11e4ae5abfaa16e319db29ec6d7264f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 15 Sep 2024 13:05:54 -0500 Subject: [PATCH 17/29] Upgrade @nostrify/db to fix table joins --- deno.json | 2 +- deno.lock | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/deno.json b/deno.json index 8edf0e53..ed8be157 100644 --- a/deno.json +++ b/deno.json @@ -31,7 +31,7 @@ "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/db": "jsr:@nostrify/db@^0.31.2", + "@nostrify/db": "jsr:@nostrify/db@^0.32.2", "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.30.1", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", diff --git a/deno.lock b/deno.lock index 3a25d8d3..0a60494a 100644 --- a/deno.lock +++ b/deno.lock @@ -15,7 +15,7 @@ "jsr:@gleasonator/policy@0.5.2": "jsr:@gleasonator/policy@0.5.2", "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.11", "jsr:@lambdalisue/async@^2.1.1": "jsr:@lambdalisue/async@2.1.1", - "jsr:@nostrify/db@^0.31.2": "jsr:@nostrify/db@0.31.2", + "jsr:@nostrify/db@^0.32.2": "jsr:@nostrify/db@0.32.2", "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", @@ -47,7 +47,7 @@ "jsr:@std/fmt@0.213.1": "jsr:@std/fmt@0.213.1", "jsr:@std/fs@0.213.1": "jsr:@std/fs@0.213.1", "jsr:@std/fs@^0.229.3": "jsr:@std/fs@0.229.3", - "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.1", + "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.3", "jsr:@std/io@^0.223.0": "jsr:@std/io@0.223.0", "jsr:@std/io@^0.224": "jsr:@std/io@0.224.7", "jsr:@std/json@^0.223.0": "jsr:@std/json@0.223.0", @@ -198,11 +198,11 @@ "@lambdalisue/async@2.1.1": { "integrity": "1fc9bc6f4ed50215cd2f7217842b18cea80f81c25744f88f8c5eb4be5a1c9ab4" }, - "@nostrify/db@0.31.2": { - "integrity": "a906b64edbf84a6b482cd7c9f5df2d2237c4ec42589116097d99ceb41347b1f5", + "@nostrify/db@0.32.2": { + "integrity": "265fb41e9d5810b99f1003ce56c89e4b468e6d0c04e7b9d9e3126c4efd49c1c2", "dependencies": [ - "jsr:@nostrify/nostrify@^0.30.0", - "jsr:@nostrify/types@^0.30.0", + "jsr:@nostrify/nostrify@^0.31.0", + "jsr:@nostrify/types@^0.30.1", "npm:kysely@^0.27.3", "npm:nostr-tools@^2.7.0" ] @@ -382,6 +382,9 @@ "@std/internal@1.0.1": { "integrity": "6f8c7544d06a11dd256c8d6ba54b11ed870aac6c5aeafff499892662c57673e6" }, + "@std/internal@1.0.3": { + "integrity": "208e9b94a3d5649bd880e9ca38b885ab7651ab5b5303a56ed25de4755fb7b11e" + }, "@std/io@0.223.0": { "integrity": "2d8c3c2ab3a515619b90da2c6ff5ea7b75a94383259ef4d02116b228393f84f1", "dependencies": [ @@ -1994,7 +1997,7 @@ "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "jsr:@hono/hono@^4.4.6", "jsr:@lambdalisue/async@^2.1.1", - "jsr:@nostrify/db@^0.31.2", + "jsr:@nostrify/db@^0.32.2", "jsr:@nostrify/nostrify@^0.30.1", "jsr:@soapbox/kysely-pglite@^0.0.1", "jsr:@soapbox/stickynotes@^0.4.0", From f8902760cef6a66d5dd47abfdde4dcd5226703b2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 15 Sep 2024 13:37:23 -0500 Subject: [PATCH 18/29] Make the language index a compound index --- src/db/migrations/032_add_language.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/db/migrations/032_add_language.ts b/src/db/migrations/032_add_language.ts index a0f828fe..77bfc37e 100644 --- a/src/db/migrations/032_add_language.ts +++ b/src/db/migrations/032_add_language.ts @@ -2,10 +2,14 @@ import { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.alterTable('nostr_events').addColumn('language', 'char(2)').execute(); - await db.schema.createIndex('nostr_events_language_idx').on('nostr_events').column('language').execute(); + + await db.schema.createIndex('nostr_events_language_created_idx') + .on('nostr_events') + .columns(['language', 'created_at desc', 'id asc', 'kind']) + .execute(); } export async function down(db: Kysely): Promise { await db.schema.alterTable('nostr_events').dropColumn('language').execute(); - await db.schema.dropIndex('nostr_events_language_idx').execute(); + await db.schema.dropIndex('nostr_events_language_created_idx').execute(); } From d7ae3722c8e3a4557e997ed787e52b99c449b19b Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 15 Sep 2024 16:52:49 -0300 Subject: [PATCH 19/29] refactor: insert each event per iteration in for loop - db:populate-search --- scripts/db-populate-search.ts | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/scripts/db-populate-search.ts b/scripts/db-populate-search.ts index beec2d52..e34aaa75 100644 --- a/scripts/db-populate-search.ts +++ b/scripts/db-populate-search.ts @@ -1,12 +1,9 @@ import { NSchema as n } from '@nostrify/nostrify'; import { Storages } from '@/storages.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; const store = await Storages.db(); const kysely = await Storages.kysely(); -const values: DittoTables['author_search'][] = []; - for await (const msg of store.req([{ kinds: [0] }])) { if (msg[0] === 'EVENT') { const { pubkey, content } = msg[2]; @@ -14,22 +11,22 @@ for await (const msg of store.req([{ kinds: [0] }])) { const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(content); const search = [name, nip05].filter(Boolean).join(' ').trim(); - values.push({ - pubkey: pubkey, - search, - }); + try { + await kysely.insertInto('author_search').values({ + pubkey, + search, + }).onConflict( + (oc) => + oc.column('pubkey') + .doUpdateSet((eb) => ({ search: eb.ref('excluded.search') })), + ) + .execute(); + } catch { + // do nothing + } + } else { + break; } } -try { - await kysely.insertInto('author_search').values(values).onConflict( - (oc) => - oc.column('pubkey') - .doUpdateSet((eb) => ({ search: eb.ref('excluded.search') })), - ) - .execute(); -} catch { - // do nothing -} - Deno.exit(); From 3b8a800cd2bb4bbb2659776a96b228169044b474 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 15 Sep 2024 17:27:56 -0300 Subject: [PATCH 20/29] refactor(search enchance): map over pubkeys instead of events --- src/controllers/api/accounts.ts | 15 +++++---------- src/controllers/api/search.ts | 15 +++++---------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index a8aa5d55..4b684c33 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -137,20 +137,15 @@ const accountSearchController: AppController = async (c) => { const pubkeys = await getPubkeysBySearch(kysely, { q: query, limit }); - const events = event ? [event] : await store.query([{ kinds: [0], authors: pubkeys, limit }], { + let events = event ? [event] : await store.query([{ kinds: [0], authors: pubkeys, limit }], { signal, }); - const orderedEvents = events.map((event, index) => { - const pubkey = pubkeys[index]; + events = pubkeys.map((pubkey) => { + return events.find((event) => event.pubkey === pubkey); + }).filter((event) => event !== undefined); - const orderedEvent = events.find((e) => e.pubkey === pubkey); - if (orderedEvent) return orderedEvent; - - return event; - }); - - const accounts = await hydrateEvents({ events: orderedEvents, store, signal }).then( + const accounts = await hydrateEvents({ events, store, signal }).then( (events) => Promise.all( events.map((event) => renderAccount(event)), diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index d62c08e8..30bad8e9 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -104,21 +104,16 @@ async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: const store = await Storages.search(); - const events = await store.query([filter], { signal }) + let events = await store.query([filter], { signal }) .then((events) => hydrateEvents({ events, store, signal })); if (type !== 'accounts') return events; - const orderedEvents: NostrEvent[] = events.map((event, index) => { - const pubkey = pubkeys[index]; + events = pubkeys.map((pubkey) => { + return events.find((event) => event.pubkey === pubkey); + }).filter((event) => event !== undefined); - const orderedEvent = events.find((e) => e.pubkey === pubkey); - if (orderedEvent) return orderedEvent; - - return event; - }); - - return orderedEvents; + return events; } /** Get event kinds to search from `type` query param. */ From ed74b2464a56a51aa16a7a3f2db1353a4f610e8f Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 15 Sep 2024 17:42:26 -0300 Subject: [PATCH 21/29] refactor: write it like a normal if statement --- src/controllers/api/search.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 30bad8e9..e220413b 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -96,8 +96,11 @@ async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: pubkeys.push(...(await getPubkeysBySearch(kysely, { q, limit }))); - if (!filter?.authors) filter.authors = pubkeys; - else filter.authors.push(...pubkeys); + if (!filter?.authors) { + filter.authors = pubkeys; + } else { + filter.authors.push(...pubkeys); + } filter.search = undefined; } From c5711ea07173ffbbdab42319712df2497981b0fe Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 15 Sep 2024 17:42:58 -0300 Subject: [PATCH 22/29] refactor(accountSearchController): only reassign events if event is undefined --- src/controllers/api/accounts.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 4b684c33..6e3cac5b 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -141,10 +141,11 @@ const accountSearchController: AppController = async (c) => { signal, }); - events = pubkeys.map((pubkey) => { - return events.find((event) => event.pubkey === pubkey); - }).filter((event) => event !== undefined); - + if (!event) { + events = pubkeys.map((pubkey) => { + return events.find((event) => event.pubkey === pubkey); + }).filter((event) => event !== undefined); + } const accounts = await hydrateEvents({ events, store, signal }).then( (events) => Promise.all( From dc69f21e0bc93c2befabe7a43361c626bb46e653 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 15 Sep 2024 17:46:10 -0300 Subject: [PATCH 23/29] refactor: write map function in a cleaner way --- src/controllers/api/accounts.ts | 6 +++--- src/controllers/api/search.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 6e3cac5b..c946b697 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -142,9 +142,9 @@ const accountSearchController: AppController = async (c) => { }); if (!event) { - events = pubkeys.map((pubkey) => { - return events.find((event) => event.pubkey === pubkey); - }).filter((event) => event !== undefined); + events = pubkeys + .map((pubkey) => events.find((event) => event.pubkey === pubkey)) + .filter((event) => !!event); } const accounts = await hydrateEvents({ events, store, signal }).then( (events) => diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index e220413b..01fb6665 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -112,9 +112,9 @@ async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: if (type !== 'accounts') return events; - events = pubkeys.map((pubkey) => { - return events.find((event) => event.pubkey === pubkey); - }).filter((event) => event !== undefined); + events = pubkeys + .map((pubkey) => events.find((event) => event.pubkey === pubkey)) + .filter((event) => !!event); return events; } From 642ecfd36fe4f43e017a02e4ecf0bcf2b0ea05f2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 15 Sep 2024 17:26:02 -0500 Subject: [PATCH 24/29] Rename language migration to 033 --- src/db/migrations/{032_add_language.ts => 033_add_language.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/db/migrations/{032_add_language.ts => 033_add_language.ts} (100%) diff --git a/src/db/migrations/032_add_language.ts b/src/db/migrations/033_add_language.ts similarity index 100% rename from src/db/migrations/032_add_language.ts rename to src/db/migrations/033_add_language.ts From 394021e485542ef414165a4a46095df0b01dab84 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 15 Sep 2024 17:38:16 -0500 Subject: [PATCH 25/29] Don't use semi-colons in migration --- src/db/migrations/032_add_author_search.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/db/migrations/032_add_author_search.ts b/src/db/migrations/032_add_author_search.ts index d5a93c06..4323c252 100644 --- a/src/db/migrations/032_add_author_search.ts +++ b/src/db/migrations/032_add_author_search.ts @@ -8,8 +8,8 @@ export async function up(db: Kysely): Promise { .ifNotExists() .execute(); - await sql`CREATE EXTENSION IF NOT EXISTS pg_trgm;`.execute(db); - await sql`CREATE INDEX author_search_search_idx ON author_search USING GIN (search gin_trgm_ops);`.execute(db); + await sql`CREATE EXTENSION IF NOT EXISTS pg_trgm`.execute(db); + await sql`CREATE INDEX author_search_search_idx ON author_search USING GIN (search gin_trgm_ops)`.execute(db); } export async function down(db: Kysely): Promise { From 8510f22d1bf6708dbbebcf836fdf4bf87170df85 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 15 Sep 2024 18:33:16 -0500 Subject: [PATCH 26/29] Only save the nip05 in author search if it's valid --- src/pipeline.ts | 61 +++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index 06fc462a..87249b97 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -140,48 +140,43 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise undefined) : undefined; // Populate author_search. try { - const search = [name, nip05].filter(Boolean).join(' ').trim(); + const search = result?.pubkey === event.pubkey ? [name, nip05].filter(Boolean).join(' ').trim() : name ?? ''; - await kysely.insertInto('author_search').values({ - pubkey: event.pubkey, - search, - }).onConflict( - (oc) => - oc.column('pubkey') - .doUpdateSet({ search }), - ) - .execute(); + if (search) { + await kysely.insertInto('author_search').values({ + pubkey: event.pubkey, + search, + }).onConflict( + (oc) => + oc.column('pubkey') + .doUpdateSet({ search }), + ) + .execute(); + } } catch { // do nothing } - if (!nip05) return; + if (nip05 && result && result.pubkey === event.pubkey) { + // Track pubkey domain. + try { + const { domain } = parseNip05(nip05); - // Fetch nip05. - const result = await nip05Cache.fetch(nip05, { signal }).catch(() => undefined); - if (!result) return; - - // Ensure pubkey matches event. - const { pubkey } = result; - if (pubkey !== event.pubkey) return; - - // Track pubkey domain. - try { - const { domain } = parseNip05(nip05); - - await sql` - INSERT INTO pubkey_domains (pubkey, domain, last_updated_at) - VALUES (${pubkey}, ${domain}, ${event.created_at}) - ON CONFLICT(pubkey) DO UPDATE SET - domain = excluded.domain, - last_updated_at = excluded.last_updated_at - WHERE excluded.last_updated_at > pubkey_domains.last_updated_at - `.execute(kysely); - } catch (_e) { - // do nothing + await sql` + INSERT INTO pubkey_domains (pubkey, domain, last_updated_at) + VALUES (${event.pubkey}, ${domain}, ${event.created_at}) + ON CONFLICT(pubkey) DO UPDATE SET + domain = excluded.domain, + last_updated_at = excluded.last_updated_at + WHERE excluded.last_updated_at > pubkey_domains.last_updated_at + `.execute(kysely); + } catch (_e) { + // do nothing + } } } From 5a98ba86bfe38fdb2f5ca0df3066afc269e72e4b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 15 Sep 2024 18:58:34 -0500 Subject: [PATCH 27/29] Fix ambiguous postgres.js errors --- deno.json | 2 +- deno.lock | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/deno.json b/deno.json index fc0a6268..80c72b20 100644 --- a/deno.json +++ b/deno.json @@ -70,7 +70,7 @@ "nostr-tools": "npm:nostr-tools@2.5.1", "nostr-wasm": "npm:nostr-wasm@^0.1.0", "path-to-regexp": "npm:path-to-regexp@^7.1.0", - "postgres": "https://raw.githubusercontent.com/xyzshantaram/postgres.js/8a9bbce88b3f6425ecaacd99a80372338b157a53/deno/mod.js", + "postgres": "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/mod.js", "prom-client": "npm:prom-client@^15.1.2", "question-deno": "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/mod.ts", "tldts": "npm:tldts@^6.0.14", diff --git a/deno.lock b/deno.lock index 0a60494a..657162fd 100644 --- a/deno.lock +++ b/deno.lock @@ -1943,6 +1943,18 @@ "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/main/mod.ts": "662438fd3909984bb8cbaf3fd44d2121e949d11301baf21d6c3f057ccf9887de", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/main/src/PostgreSQLDriver.ts": "590c2fa248cff38e6e0f623050983039b5fde61e9c7131593d2922fb1f0eb921", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/main/src/PostgreSQLDriverDatabaseConnection.ts": "2158de426860bfd4f8e73afff0289bd40a11e273c8d883d4fd6474db01a9c2a7", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/mod.js": "cb68f17d6d90df318934deccdb469d740be0888e7a597a9e7eea7100ce36a252", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/polyfills.js": "318eb01f2b4cc33a46c59f3ddc11f22a56d6b1db8b7719b2ad7decee63a5bd47", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/bytes.js": "f2de43bdc8fa5dc4b169f2c70d5d8b053a3dea8f85ef011d7b27dec69e14ebb7", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/connection.js": "63bb06ad07cf802d295b35788261c34e82a80cec30b0dffafe05ccd74af3716f", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/errors.js": "85cfbed9a5ab0db41ab8e97b806c881af29807dfe99bc656fdf1a18c1c13b6c6", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/index.js": "4e8b09c7d0ce6e9eea386f59337867266498d5bb60ccd567d0bea5da03f6094d", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/large.js": "f3e770cdb7cc695f7b50687b4c6c4b7252129515486ec8def98b7582ee7c54ef", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/query.js": "67c45a5151032aa46b587abc15381fe4efd97c696e5c1b53082b8161309c4ee2", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/queue.js": "709624843223ea842bf095f6934080f19f1a059a51cbbf82e9827f3bb1bf2ca7", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/result.js": "001ff5e0c8d634674f483d07fbcd620a797e3101f842d6c20ca3ace936260465", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/subscribe.js": "9e4d0c3e573a6048e77ee2f15abbd5bcd17da9ca85a78c914553472c6d6c169b", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/types.js": "471f4a6c35412aa202a7c177c0a7e5a7c3bd225f01bbde67c947894c1b8bf6ed", "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/KeyCombo.ts": "a370b2dca76faa416d00e45479c8ce344971b5b86b44b4d0b213245c4bd2f8a3", "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/checkbox.ts": "e337ee7396aaefe6cc8c6349a445542fe7f0760311773369c9012b3fa278d21e", "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/config.ts": "a94a022c757f63ee7c410e29b97d3bfab1811889fb4483f56395cf376a911d1b", From a87497380a0fbbad4fb4f9de5ef6869d6bde3ba5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 15 Sep 2024 19:22:14 -0500 Subject: [PATCH 28/29] Make pipeline query easier to look at --- src/pipeline.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index 87249b97..8ca7ae5f 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -147,14 +147,9 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise - oc.column('pubkey') - .doUpdateSet({ search }), - ) + await kysely.insertInto('author_search') + .values({ pubkey: event.pubkey, search }) + .onConflict((oc) => oc.column('pubkey').doUpdateSet({ search })) .execute(); } } catch { From 0dcb1965bed7f89175128b9354b561824b32af87 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 16 Sep 2024 05:52:51 +0530 Subject: [PATCH 29/29] Change setup.ts to use password input instead of plaintext --- scripts/setup.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/setup.ts b/scripts/setup.ts index 1365fec5..32376692 100644 --- a/scripts/setup.ts +++ b/scripts/setup.ts @@ -54,7 +54,7 @@ if (DATABASE_URL) { const host = await question('input', 'Postgres host', 'localhost'); const port = await question('input', 'Postgres port', '5432'); const user = await question('input', 'Postgres user', 'ditto'); - const password = await question('input', 'Postgres password', 'ditto'); + const password = await question('password', 'Postgres password', true); const database = await question('input', 'Postgres database', 'ditto'); vars.DATABASE_URL = `postgres://${user}:${password}@${host}:${port}/${database}`; }