Merge branch 'main' into zap-notification-streaming

src/controllers/api/accounts.ts

@@ -6,7 +6,6 @@ import { type AppController } from '@/app.ts';
 import { Conf } from '@/config.ts';
 import { getAuthor, getFollowedPubkeys } from '@/queries.ts';
 import { booleanParamSchema, fileSchema } from '@/schema.ts';
-import { getPubkeysBySearch } from '@/controllers/api/search.ts';
 import { Storages } from '@/storages.ts';
 import { uploadFile } from '@/utils/upload.ts';
 import { nostrNow } from '@/utils.ts';
@@ -19,6 +18,7 @@ import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts';
 import { hydrateEvents } from '@/storages/hydrate.ts';
 import { bech32ToPubkey } from '@/utils.ts';
 import { addTag, deleteTag, findReplyTag, getTagSet } from '@/utils/tags.ts';
+import { getPubkeysBySearch } from '@/utils/search.ts';
 
 const usernameSchema = z
   .string().min(1).max(30)
@@ -117,6 +117,7 @@ const accountSearchController: AppController = async (c) => {
   const { signal } = c.req.raw;
   const { limit } = c.get('pagination');
   const kysely = await Storages.kysely();
+  const viewerPubkey = await c.get('signer')?.getPublicKey();
 
   const result = accountSearchQuerySchema.safeParse(c.req.query());
 
@@ -135,7 +136,8 @@ const accountSearchController: AppController = async (c) => {
     return c.json(pubkey ? [await accountFromPubkey(pubkey)] : []);
   }
 
-  const pubkeys = await getPubkeysBySearch(kysely, { q: query, limit });
+  const followedPubkeys: Set<string> = viewerPubkey ? await getFollowedPubkeys(viewerPubkey) : new Set();
+  const pubkeys = Array.from(await getPubkeysBySearch(kysely, { q: query, limit, followedPubkeys }));
 
   let events = event ? [event] : await store.query([{ kinds: [0], authors: pubkeys, limit }], {
     signal,

src/controllers/api/search.test.ts

@@ -1,21 +0,0 @@
-import { assertEquals } from '@std/assert';
-
-import { createTestDB } from '@/test.ts';
-import { getPubkeysBySearch } from '@/controllers/api/search.ts';
-
-Deno.test('fuzzy search works', async () => {
-  await using db = await createTestDB();
-
-  await db.kysely.insertInto('author_search').values({
-    pubkey: '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4',
-    search: 'patrickReiis patrickdosreis.com',
-  }).execute();
-
-  assertEquals(await getPubkeysBySearch(db.kysely, { q: 'pat rick', limit: 1 }), []);
-  assertEquals(await getPubkeysBySearch(db.kysely, { q: 'patrick dos reis', limit: 1 }), [
-    '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4',
-  ]);
-  assertEquals(await getPubkeysBySearch(db.kysely, { q: 'dosreis.com', limit: 1 }), [
-    '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4',
-  ]);
-});

src/controllers/api/search.ts

@@ -1,10 +1,8 @@
 import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify';
 import { nip19 } from 'nostr-tools';
-import { Kysely, sql } from 'kysely';
 import { z } from 'zod';
 
 import { AppController } from '@/app.ts';
-import { DittoTables } from '@/db/DittoTables.ts';
 import { booleanParamSchema } from '@/schema.ts';
 import { Storages } from '@/storages.ts';
 import { hydrateEvents } from '@/storages/hydrate.ts';
@@ -12,6 +10,8 @@ import { extractIdentifier, lookupPubkey } from '@/utils/lookup.ts';
 import { nip05Cache } from '@/utils/nip05.ts';
 import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts';
 import { renderStatus } from '@/views/mastodon/statuses.ts';
+import { getFollowedPubkeys } from '@/queries.ts';
+import { getPubkeysBySearch } from '@/utils/search.ts';
 
 const searchQuerySchema = z.object({
   q: z.string().transform(decodeURIComponent),
@@ -27,6 +27,7 @@ type SearchQuery = z.infer<typeof searchQuerySchema>;
 const searchController: AppController = async (c) => {
   const result = searchQuerySchema.safeParse(c.req.query());
   const { signal } = c.req.raw;
+  const viewerPubkey = await c.get('signer')?.getPublicKey();
 
   if (!result.success) {
     return c.json({ error: 'Bad request', schema: result.error }, 422);
@@ -50,9 +51,7 @@ const searchController: AppController = async (c) => {
   if (event) {
     events = [event];
   }
-  events.push(...(await searchEvents(result.data, signal)));
-
-  const viewerPubkey = await c.get('signer')?.getPublicKey();
+  events.push(...(await searchEvents({ ...result.data, viewerPubkey }, signal)));
 
   const [accounts, statuses] = await Promise.all([
     Promise.all(
@@ -77,8 +76,16 @@ const searchController: AppController = async (c) => {
 };
 
 /** Get events for the search params. */
-async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: AbortSignal): Promise<NostrEvent[]> {
-  if (type === 'hashtags') return Promise.resolve([]);
+async function searchEvents(
+  { q, type, limit, account_id, viewerPubkey }: SearchQuery & { viewerPubkey?: string },
+  signal: AbortSignal,
+): Promise<NostrEvent[]> {
+  // Hashtag search is not supported.
+  if (type === 'hashtags') {
+    return Promise.resolve([]);
+  }
+
+  const store = await Storages.search();
 
   const filter: NostrFilter = {
     kinds: typeToKinds(type),
@@ -86,35 +93,33 @@ async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal:
     limit,
   };
 
+  // For account search, use a special index, and prioritize followed accounts.
+  if (type === 'accounts') {
+    const kysely = await Storages.kysely();
+
+    const followedPubkeys = viewerPubkey ? await getFollowedPubkeys(viewerPubkey) : new Set<string>();
+    const searchPubkeys = await getPubkeysBySearch(kysely, { q, limit, followedPubkeys });
+
+    filter.authors = [...searchPubkeys];
+    filter.search = undefined;
+  }
+
+  // Results should only be shown from one author.
   if (account_id) {
     filter.authors = [account_id];
   }
 
-  const pubkeys: string[] = [];
-  if (type === 'accounts') {
-    const kysely = await Storages.kysely();
-
-    pubkeys.push(...(await getPubkeysBySearch(kysely, { q, limit })));
-
-    if (!filter?.authors) {
-      filter.authors = pubkeys;
-    } else {
-      filter.authors.push(...pubkeys);
-    }
-
-    filter.search = undefined;
-  }
-
-  const store = await Storages.search();
-
-  let events = await store.query([filter], { signal })
+  // Query the events.
+  let events = await store
+    .query([filter], { signal })
     .then((events) => hydrateEvents({ events, store, signal }));
 
-  if (type !== 'accounts') return events;
-
-  events = pubkeys
-    .map((pubkey) => events.find((event) => event.pubkey === pubkey))
-    .filter((event) => !!event);
+  // When using an authors filter, return the events in the same order as the filter.
+  if (filter.authors) {
+    events = filter.authors
+      .map((pubkey) => events.find((event) => event.pubkey === pubkey))
+      .filter((event) => !!event);
+  }
 
   return events;
 }
@@ -194,16 +199,4 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort
   return [];
 }
 
-/** Get pubkeys whose name and NIP-05 is similar to 'q' */
-async function getPubkeysBySearch(kysely: Kysely<DittoTables>, { q, limit }: Pick<SearchQuery, 'q' | 'limit'>) {
-  const pubkeys = (await sql<{ pubkey: string }>`
-        SELECT *, word_similarity(${q}, search) AS sml
-        FROM author_search
-        WHERE ${q} % search
-        ORDER BY sml DESC, search LIMIT ${limit}
-      `.execute(kysely)).rows.map(({ pubkey }) => pubkey);
-
-  return pubkeys;
-}
-
-export { getPubkeysBySearch, searchController };
+export { searchController };

src/storages/EventsDB.ts

@@ -153,22 +153,16 @@ class EventsDB extends NPostgres {
         search: tokens.filter((t) => typeof t === 'string').join(' '),
       }) as SelectQueryBuilder<DittoTables, 'nostr_events', Pick<DittoTables['nostr_events'], keyof NostrEvent>>;
 
-      const data = tokens.filter((t) => typeof t === 'object').reduce(
-        (acc, t) => acc.set(t.key, t.value),
-        new Map<string, string>(),
-      );
+      const languages = new Set<string>();
 
-      const domain = data.get('domain');
-      const language = data.get('language');
-
-      if (domain) {
-        query = query
-          .innerJoin('pubkey_domains', 'nostr_events.pubkey', 'pubkey_domains.pubkey')
-          .where('pubkey_domains.domain', '=', domain);
+      for (const token of tokens) {
+        if (typeof token === 'object' && token.key === 'language') {
+          languages.add(token.value);
+        }
       }
 
-      if (language) {
-        query = query.where('language', '=', language);
+      if (languages.size) {
+        query = query.where('language', 'in', [...languages]);
       }
 
       return query;
@@ -288,6 +282,39 @@ class EventsDB extends NPostgres {
     filters = structuredClone(filters);
 
     for (const filter of filters) {
+      if (filter.search) {
+        const tokens = NIP50.parseInput(filter.search);
+
+        const domains = new Set<string>();
+
+        for (const token of tokens) {
+          if (typeof token === 'object' && token.key === 'domain') {
+            domains.add(token.value);
+          }
+        }
+
+        if (domains.size) {
+          const query = this.opts.kysely
+            .selectFrom('pubkey_domains')
+            .select('pubkey')
+            .where('domain', 'in', [...domains]);
+
+          if (filter.authors) {
+            query.where('pubkey', 'in', filter.authors);
+          }
+
+          const pubkeys = await query.execute().then((rows) => rows.map((row) => row.pubkey));
+
+          filter.authors = pubkeys;
+        }
+
+        // Re-serialize the search string without the domain key. :facepalm:
+        filter.search = tokens
+          .filter((t) => typeof t === 'object' && t.key !== 'domain')
+          .map((t) => typeof t === 'object' ? `${t.key}:${t.value}` : t)
+          .join(' ');
+      }
+
       if (filter.kinds) {
         // Ephemeral events are not stored, so don't bother querying for them.
         // If this results in an empty kinds array, NDatabase will remove the filter before querying and return no results.

src/utils/search.test.ts

@@ -0,0 +1,27 @@
+import { assertEquals } from '@std/assert';
+
+import { createTestDB } from '@/test.ts';
+import { getPubkeysBySearch } from '@/utils/search.ts';
+
+Deno.test('fuzzy search works', async () => {
+  await using db = await createTestDB();
+
+  await db.kysely.insertInto('author_search').values({
+    pubkey: '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4',
+    search: 'patrickReiis patrickdosreis.com',
+  }).execute();
+
+  assertEquals(await getPubkeysBySearch(db.kysely, { q: 'pat rick', limit: 1, followedPubkeys: new Set() }), new Set());
+  assertEquals(
+    await getPubkeysBySearch(db.kysely, { q: 'patrick dos reis', limit: 1, followedPubkeys: new Set() }),
+    new Set([
+      '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4',
+    ]),
+  );
+  assertEquals(
+    await getPubkeysBySearch(db.kysely, { q: 'dosreis.com', limit: 1, followedPubkeys: new Set() }),
+    new Set([
+      '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4',
+    ]),
+  );
+});

src/utils/search.ts

@@ -0,0 +1,32 @@
+import { Kysely, sql } from 'kysely';
+
+import { DittoTables } from '@/db/DittoTables.ts';
+
+/** Get pubkeys whose name and NIP-05 is similar to 'q' */
+export async function getPubkeysBySearch(
+  kysely: Kysely<DittoTables>,
+  opts: { q: string; limit: number; followedPubkeys: Set<string> },
+): Promise<Set<string>> {
+  const { q, limit, followedPubkeys } = opts;
+
+  let query = kysely
+    .selectFrom('author_search')
+    .select((eb) => [
+      'pubkey',
+      'search',
+      eb.fn('word_similarity', [sql`${q}`, 'search']).as('sml'),
+    ])
+    .where(() => sql`${q} % search`)
+    .orderBy(['sml desc', 'search'])
+    .limit(limit);
+
+  const pubkeys = new Set((await query.execute()).map(({ pubkey }) => pubkey));
+
+  if (followedPubkeys.size > 0) {
+    query = query.where('pubkey', 'in', [...followedPubkeys]);
+  }
+
+  const followingPubkeys = new Set((await query.execute()).map(({ pubkey }) => pubkey));
+
+  return new Set(Array.from(followingPubkeys.union(pubkeys)).slice(0, limit));
+}