Merge remote-tracking branch 'origin/main' into language-detection

deno.json

@@ -18,7 +18,8 @@
     "stats:recompute": "deno run -A scripts/stats-recompute.ts",
     "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip -o soapbox.zip && rm soapbox.zip",
     "trends": "deno run -A scripts/trends.ts",
-    "clean:deps": "deno cache --reload src/app.ts"
+    "clean:deps": "deno cache --reload src/app.ts",
+    "db:populate-search": "deno run -A scripts/db-populate-search.ts"
   },
   "unstable": ["cron", "ffi", "kv", "worker-options"],
   "exclude": ["./public"],

scripts/db-populate-search.ts

@@ -0,0 +1,32 @@
+import { NSchema as n } from '@nostrify/nostrify';
+import { Storages } from '@/storages.ts';
+
+const store = await Storages.db();
+const kysely = await Storages.kysely();
+
+for await (const msg of store.req([{ kinds: [0] }])) {
+  if (msg[0] === 'EVENT') {
+    const { pubkey, content } = msg[2];
+
+    const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(content);
+    const search = [name, nip05].filter(Boolean).join(' ').trim();
+
+    try {
+      await kysely.insertInto('author_search').values({
+        pubkey,
+        search,
+      }).onConflict(
+        (oc) =>
+          oc.column('pubkey')
+            .doUpdateSet((eb) => ({ search: eb.ref('excluded.search') })),
+      )
+        .execute();
+    } catch {
+      // do nothing
+    }
+  } else {
+    break;
+  }
+}
+
+Deno.exit();

src/controllers/api/accounts.ts

@@ -6,6 +6,7 @@ import { type AppController } from '@/app.ts';
 import { Conf } from '@/config.ts';
 import { getAuthor, getFollowedPubkeys } from '@/queries.ts';
 import { booleanParamSchema, fileSchema } from '@/schema.ts';
+import { getPubkeysBySearch } from '@/controllers/api/search.ts';
 import { Storages } from '@/storages.ts';
 import { uploadFile } from '@/utils/upload.ts';
 import { nostrNow } from '@/utils.ts';
@@ -115,6 +116,7 @@ const accountSearchQuerySchema = z.object({
 const accountSearchController: AppController = async (c) => {
   const { signal } = c.req.raw;
   const { limit } = c.get('pagination');
+  const kysely = await Storages.kysely();
 
   const result = accountSearchQuerySchema.safeParse(c.req.query());
 
@@ -133,8 +135,17 @@ const accountSearchController: AppController = async (c) => {
     return c.json(pubkey ? [await accountFromPubkey(pubkey)] : []);
   }
 
-  const events = event ? [event] : await store.query([{ kinds: [0], search: query, limit }], { signal });
+  const pubkeys = await getPubkeysBySearch(kysely, { q: query, limit });
 
+  let events = event ? [event] : await store.query([{ kinds: [0], authors: pubkeys, limit }], {
+    signal,
+  });
+
+  if (!event) {
+    events = pubkeys
+      .map((pubkey) => events.find((event) => event.pubkey === pubkey))
+      .filter((event) => !!event);
+  }
   const accounts = await hydrateEvents({ events, store, signal }).then(
     (events) =>
       Promise.all(

src/controllers/api/search.test.ts

@@ -0,0 +1,21 @@
+import { assertEquals } from '@std/assert';
+
+import { createTestDB } from '@/test.ts';
+import { getPubkeysBySearch } from '@/controllers/api/search.ts';
+
+Deno.test('fuzzy search works', async () => {
+  await using db = await createTestDB();
+
+  await db.kysely.insertInto('author_search').values({
+    pubkey: '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4',
+    search: 'patrickReiis patrickdosreis.com',
+  }).execute();
+
+  assertEquals(await getPubkeysBySearch(db.kysely, { q: 'pat rick', limit: 1 }), []);
+  assertEquals(await getPubkeysBySearch(db.kysely, { q: 'patrick dos reis', limit: 1 }), [
+    '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4',
+  ]);
+  assertEquals(await getPubkeysBySearch(db.kysely, { q: 'dosreis.com', limit: 1 }), [
+    '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4',
+  ]);
+});

src/controllers/api/search.ts

@@ -1,8 +1,10 @@
 import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify';
 import { nip19 } from 'nostr-tools';
+import { Kysely, sql } from 'kysely';
 import { z } from 'zod';
 
 import { AppController } from '@/app.ts';
+import { DittoTables } from '@/db/DittoTables.ts';
 import { booleanParamSchema } from '@/schema.ts';
 import { Storages } from '@/storages.ts';
 import { hydrateEvents } from '@/storages/hydrate.ts';
@@ -47,9 +49,8 @@ const searchController: AppController = async (c) => {
 
   if (event) {
     events = [event];
-  } else {
-    events = await searchEvents(result.data, signal);
   }
+  events.push(...(await searchEvents(result.data, signal)));
 
   const viewerPubkey = await c.get('signer')?.getPublicKey();
 
@@ -89,10 +90,33 @@ async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal:
     filter.authors = [account_id];
   }
 
+  const pubkeys: string[] = [];
+  if (type === 'accounts') {
+    const kysely = await Storages.kysely();
+
+    pubkeys.push(...(await getPubkeysBySearch(kysely, { q, limit })));
+
+    if (!filter?.authors) {
+      filter.authors = pubkeys;
+    } else {
+      filter.authors.push(...pubkeys);
+    }
+
+    filter.search = undefined;
+  }
+
   const store = await Storages.search();
 
-  return store.query([filter], { signal })
+  let events = await store.query([filter], { signal })
     .then((events) => hydrateEvents({ events, store, signal }));
+
+  if (type !== 'accounts') return events;
+
+  events = pubkeys
+    .map((pubkey) => events.find((event) => event.pubkey === pubkey))
+    .filter((event) => !!event);
+
+  return events;
 }
 
 /** Get event kinds to search from `type` query param. */
@@ -170,4 +194,16 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort
   return [];
 }
 
-export { searchController };
+/** Get pubkeys whose name and NIP-05 is similar to 'q' */
+async function getPubkeysBySearch(kysely: Kysely<DittoTables>, { q, limit }: Pick<SearchQuery, 'q' | 'limit'>) {
+  const pubkeys = (await sql<{ pubkey: string }>`
+        SELECT *, word_similarity(${q}, search) AS sml
+        FROM author_search
+        WHERE ${q} % search
+        ORDER BY sml DESC, search LIMIT ${limit}
+      `.execute(kysely)).rows.map(({ pubkey }) => pubkey);
+
+  return pubkeys;
+}
+
+export { getPubkeysBySearch, searchController };

src/db/DittoTables.ts

@@ -9,6 +9,7 @@ export interface DittoTables extends NPostgresSchema {
   event_stats: EventStatsRow;
   pubkey_domains: PubkeyDomainRow;
   event_zaps: EventZapRow;
+  author_search: AuthorSearch;
 }
 
 type NostrEventsRow = NPostgresSchema['nostr_events'] & {
@@ -54,3 +55,8 @@ interface EventZapRow {
   amount_millisats: number;
   comment: string;
 }
+
+interface AuthorSearch {
+  pubkey: string;
+  search: string;
+}

src/db/adapters/DittoPglite.ts

@@ -1,4 +1,5 @@
 import { PGlite } from '@electric-sql/pglite';
+import { pg_trgm } from '@electric-sql/pglite/contrib/pg_trgm';
 import { PgliteDialect } from '@soapbox/kysely-pglite';
 import { Kysely } from 'kysely';
 
@@ -10,7 +11,7 @@ export class DittoPglite {
   static create(databaseUrl: string): DittoDatabase {
     const kysely = new Kysely<DittoTables>({
       dialect: new PgliteDialect({
-        database: new PGlite(databaseUrl),
+        database: new PGlite(databaseUrl, { extensions: { pg_trgm } }),
       }),
       log: KyselyLogger,
     });

src/db/migrations/032_add_author_search.ts

@@ -0,0 +1,18 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .createTable('author_search')
+    .addColumn('pubkey', 'char(64)', (col) => col.primaryKey())
+    .addColumn('search', 'text', (col) => col.notNull())
+    .ifNotExists()
+    .execute();
+
+  await sql`CREATE EXTENSION IF NOT EXISTS pg_trgm;`.execute(db);
+  await sql`CREATE INDEX author_search_search_idx ON author_search USING GIN (search gin_trgm_ops);`.execute(db);
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema.dropIndex('author_search_search_idx').ifExists().execute();
+  await db.schema.dropTable('author_search').execute();
+}

src/pipeline.ts

@@ -136,8 +136,28 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise<vo
   const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(event.content);
   if (!metadata.success) return;
 
+  const kysely = await Storages.kysely();
+
   // Get nip05.
-  const { nip05 } = metadata.data;
+  const { name, nip05 } = metadata.data;
+
+  // Populate author_search.
+  try {
+    const search = [name, nip05].filter(Boolean).join(' ').trim();
+
+    await kysely.insertInto('author_search').values({
+      pubkey: event.pubkey,
+      search,
+    }).onConflict(
+      (oc) =>
+        oc.column('pubkey')
+          .doUpdateSet({ search }),
+    )
+      .execute();
+  } catch {
+    // do nothing
+  }
+
   if (!nip05) return;
 
   // Fetch nip05.
@@ -150,7 +170,6 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise<vo
 
   // Track pubkey domain.
   try {
-    const kysely = await Storages.kysely();
     const { domain } = parseNip05(nip05);
 
     await sql`

src/test.ts

@@ -63,6 +63,7 @@ export async function createTestDB() {
             'pubkey_domains',
             'nostr_events',
             'event_zaps',
+            'author_search',
           ]
         ) {
           await kysely.schema.dropTable(table).ifExists().cascade().execute();