diff --git a/deno.json b/deno.json index ed8be157..fc0a6268 100644 --- a/deno.json +++ b/deno.json @@ -18,7 +18,8 @@ "stats:recompute": "deno run -A scripts/stats-recompute.ts", "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip -o soapbox.zip && rm soapbox.zip", "trends": "deno run -A scripts/trends.ts", - "clean:deps": "deno cache --reload src/app.ts" + "clean:deps": "deno cache --reload src/app.ts", + "db:populate-search": "deno run -A scripts/db-populate-search.ts" }, "unstable": ["cron", "ffi", "kv", "worker-options"], "exclude": ["./public"], diff --git a/scripts/db-populate-search.ts b/scripts/db-populate-search.ts new file mode 100644 index 00000000..e34aaa75 --- /dev/null +++ b/scripts/db-populate-search.ts @@ -0,0 +1,32 @@ +import { NSchema as n } from '@nostrify/nostrify'; +import { Storages } from '@/storages.ts'; + +const store = await Storages.db(); +const kysely = await Storages.kysely(); + +for await (const msg of store.req([{ kinds: [0] }])) { + if (msg[0] === 'EVENT') { + const { pubkey, content } = msg[2]; + + const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(content); + const search = [name, nip05].filter(Boolean).join(' ').trim(); + + try { + await kysely.insertInto('author_search').values({ + pubkey, + search, + }).onConflict( + (oc) => + oc.column('pubkey') + .doUpdateSet((eb) => ({ search: eb.ref('excluded.search') })), + ) + .execute(); + } catch { + // do nothing + } + } else { + break; + } +} + +Deno.exit(); diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 9c635565..c946b697 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -6,6 +6,7 @@ import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { getAuthor, getFollowedPubkeys } from '@/queries.ts'; import { booleanParamSchema, fileSchema } from '@/schema.ts'; +import { getPubkeysBySearch } from '@/controllers/api/search.ts'; import { Storages } from '@/storages.ts'; import { uploadFile } from '@/utils/upload.ts'; import { nostrNow } from '@/utils.ts'; @@ -115,6 +116,7 @@ const accountSearchQuerySchema = z.object({ const accountSearchController: AppController = async (c) => { const { signal } = c.req.raw; const { limit } = c.get('pagination'); + const kysely = await Storages.kysely(); const result = accountSearchQuerySchema.safeParse(c.req.query()); @@ -133,8 +135,17 @@ const accountSearchController: AppController = async (c) => { return c.json(pubkey ? [await accountFromPubkey(pubkey)] : []); } - const events = event ? [event] : await store.query([{ kinds: [0], search: query, limit }], { signal }); + const pubkeys = await getPubkeysBySearch(kysely, { q: query, limit }); + let events = event ? [event] : await store.query([{ kinds: [0], authors: pubkeys, limit }], { + signal, + }); + + if (!event) { + events = pubkeys + .map((pubkey) => events.find((event) => event.pubkey === pubkey)) + .filter((event) => !!event); + } const accounts = await hydrateEvents({ events, store, signal }).then( (events) => Promise.all( diff --git a/src/controllers/api/search.test.ts b/src/controllers/api/search.test.ts new file mode 100644 index 00000000..2c5e91bd --- /dev/null +++ b/src/controllers/api/search.test.ts @@ -0,0 +1,21 @@ +import { assertEquals } from '@std/assert'; + +import { createTestDB } from '@/test.ts'; +import { getPubkeysBySearch } from '@/controllers/api/search.ts'; + +Deno.test('fuzzy search works', async () => { + await using db = await createTestDB(); + + await db.kysely.insertInto('author_search').values({ + pubkey: '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', + search: 'patrickReiis patrickdosreis.com', + }).execute(); + + assertEquals(await getPubkeysBySearch(db.kysely, { q: 'pat rick', limit: 1 }), []); + assertEquals(await getPubkeysBySearch(db.kysely, { q: 'patrick dos reis', limit: 1 }), [ + '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', + ]); + assertEquals(await getPubkeysBySearch(db.kysely, { q: 'dosreis.com', limit: 1 }), [ + '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', + ]); +}); diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 9bddc336..01fb6665 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -1,8 +1,10 @@ import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; +import { Kysely, sql } from 'kysely'; import { z } from 'zod'; import { AppController } from '@/app.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; import { booleanParamSchema } from '@/schema.ts'; import { Storages } from '@/storages.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; @@ -47,9 +49,8 @@ const searchController: AppController = async (c) => { if (event) { events = [event]; - } else { - events = await searchEvents(result.data, signal); } + events.push(...(await searchEvents(result.data, signal))); const viewerPubkey = await c.get('signer')?.getPublicKey(); @@ -89,10 +90,33 @@ async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: filter.authors = [account_id]; } + const pubkeys: string[] = []; + if (type === 'accounts') { + const kysely = await Storages.kysely(); + + pubkeys.push(...(await getPubkeysBySearch(kysely, { q, limit }))); + + if (!filter?.authors) { + filter.authors = pubkeys; + } else { + filter.authors.push(...pubkeys); + } + + filter.search = undefined; + } + const store = await Storages.search(); - return store.query([filter], { signal }) + let events = await store.query([filter], { signal }) .then((events) => hydrateEvents({ events, store, signal })); + + if (type !== 'accounts') return events; + + events = pubkeys + .map((pubkey) => events.find((event) => event.pubkey === pubkey)) + .filter((event) => !!event); + + return events; } /** Get event kinds to search from `type` query param. */ @@ -170,4 +194,16 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort return []; } -export { searchController }; +/** Get pubkeys whose name and NIP-05 is similar to 'q' */ +async function getPubkeysBySearch(kysely: Kysely, { q, limit }: Pick) { + const pubkeys = (await sql<{ pubkey: string }>` + SELECT *, word_similarity(${q}, search) AS sml + FROM author_search + WHERE ${q} % search + ORDER BY sml DESC, search LIMIT ${limit} + `.execute(kysely)).rows.map(({ pubkey }) => pubkey); + + return pubkeys; +} + +export { getPubkeysBySearch, searchController }; diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index 48cb06cb..642db484 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -9,6 +9,7 @@ export interface DittoTables extends NPostgresSchema { event_stats: EventStatsRow; pubkey_domains: PubkeyDomainRow; event_zaps: EventZapRow; + author_search: AuthorSearch; } type NostrEventsRow = NPostgresSchema['nostr_events'] & { @@ -54,3 +55,8 @@ interface EventZapRow { amount_millisats: number; comment: string; } + +interface AuthorSearch { + pubkey: string; + search: string; +} diff --git a/src/db/adapters/DittoPglite.ts b/src/db/adapters/DittoPglite.ts index 4ec7d8a5..0e93075d 100644 --- a/src/db/adapters/DittoPglite.ts +++ b/src/db/adapters/DittoPglite.ts @@ -1,4 +1,5 @@ import { PGlite } from '@electric-sql/pglite'; +import { pg_trgm } from '@electric-sql/pglite/contrib/pg_trgm'; import { PgliteDialect } from '@soapbox/kysely-pglite'; import { Kysely } from 'kysely'; @@ -10,7 +11,7 @@ export class DittoPglite { static create(databaseUrl: string): DittoDatabase { const kysely = new Kysely({ dialect: new PgliteDialect({ - database: new PGlite(databaseUrl), + database: new PGlite(databaseUrl, { extensions: { pg_trgm } }), }), log: KyselyLogger, }); diff --git a/src/db/migrations/032_add_author_search.ts b/src/db/migrations/032_add_author_search.ts new file mode 100644 index 00000000..d5a93c06 --- /dev/null +++ b/src/db/migrations/032_add_author_search.ts @@ -0,0 +1,18 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('author_search') + .addColumn('pubkey', 'char(64)', (col) => col.primaryKey()) + .addColumn('search', 'text', (col) => col.notNull()) + .ifNotExists() + .execute(); + + await sql`CREATE EXTENSION IF NOT EXISTS pg_trgm;`.execute(db); + await sql`CREATE INDEX author_search_search_idx ON author_search USING GIN (search gin_trgm_ops);`.execute(db); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropIndex('author_search_search_idx').ifExists().execute(); + await db.schema.dropTable('author_search').execute(); +} diff --git a/src/pipeline.ts b/src/pipeline.ts index cc4975cd..06fc462a 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -136,8 +136,28 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise + oc.column('pubkey') + .doUpdateSet({ search }), + ) + .execute(); + } catch { + // do nothing + } + if (!nip05) return; // Fetch nip05. @@ -150,7 +170,6 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise