Merge branch 'search-fixes' into 'main'

search: parse bech32 ids from pasted URLs, improve Mastodon API compat

See merge request soapbox-pub/ditto!450

deno.json

@@ -65,6 +65,7 @@
     "nostr-relaypool": "npm:nostr-relaypool2@0.6.34",
     "nostr-tools": "npm:nostr-tools@2.5.1",
     "nostr-wasm": "npm:nostr-wasm@^0.1.0",
+    "path-to-regexp": "npm:path-to-regexp@^7.1.0",
     "postgres": "https://raw.githubusercontent.com/xyzshantaram/postgres.js/8a9bbce88b3f6425ecaacd99a80372338b157a53/deno/mod.js",
     "prom-client": "npm:prom-client@^15.1.2",
     "question-deno": "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/mod.ts",

deno.lock

@@ -73,6 +73,7 @@
       "npm:nostr-tools@^2.5.0": "npm:nostr-tools@2.5.1",
       "npm:nostr-tools@^2.7.0": "npm:nostr-tools@2.7.0",
       "npm:nostr-wasm@^0.1.0": "npm:nostr-wasm@0.1.0",
+      "npm:path-to-regexp@^7.1.0": "npm:path-to-regexp@7.1.0",
       "npm:postgres@3.4.4": "npm:postgres@3.4.4",
       "npm:prom-client@^15.1.2": "npm:prom-client@15.1.2",
       "npm:tldts@^6.0.14": "npm:tldts@6.1.18",
@@ -947,6 +948,10 @@
         "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==",
         "dependencies": {}
       },
+      "path-to-regexp@7.1.0": {
+        "integrity": "sha512-ZToe+MbUF4lBqk6dV8GKot4DKfzrxXsplOddH8zN3YK+qw9/McvP7+4ICjZvOne0jQhN4eJwHsX6tT0Ns19fvw==",
+        "dependencies": {}
+      },
       "picomatch@2.3.1": {
         "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
         "dependencies": {}
@@ -1856,6 +1861,7 @@
       "npm:nostr-relaypool2@0.6.34",
       "npm:nostr-tools@2.5.1",
       "npm:nostr-wasm@^0.1.0",
+      "npm:path-to-regexp@^7.1.0",
       "npm:prom-client@^15.1.2",
       "npm:tldts@^6.0.14",
       "npm:tseep@^1.2.1",

src/controllers/api/accounts.ts

@@ -10,7 +10,7 @@ import { Storages } from '@/storages.ts';
 import { uploadFile } from '@/utils/upload.ts';
 import { nostrNow } from '@/utils.ts';
 import { createEvent, paginated, parseBody, updateListEvent } from '@/utils/api.ts';
-import { lookupAccount } from '@/utils/lookup.ts';
+import { extractIdentifier, lookupAccount, lookupPubkey } from '@/utils/lookup.ts';
 import { renderAccounts, renderEventAccounts, renderStatuses } from '@/views.ts';
 import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts';
 import { renderRelationship } from '@/views/mastodon/relationships.ts';
@@ -110,45 +110,38 @@ const accountSearchQuerySchema = z.object({
   q: z.string().transform(decodeURIComponent),
   resolve: booleanParamSchema.optional().transform(Boolean),
   following: z.boolean().default(false),
-  limit: z.coerce.number().catch(20).transform((value) => Math.min(Math.max(value, 0), 40)),
 });
 
 const accountSearchController: AppController = async (c) => {
-  const result = accountSearchQuerySchema.safeParse(c.req.query());
   const { signal } = c.req.raw;
+  const { limit } = c.get('pagination');
+
+  const result = accountSearchQuerySchema.safeParse(c.req.query());
 
   if (!result.success) {
     return c.json({ error: 'Bad request', schema: result.error }, 422);
   }
 
-  const { q, limit } = result.data;
+  const query = decodeURIComponent(result.data.q);
-
-  const query = decodeURIComponent(q);
   const store = await Storages.search();
 
-  const [event, events] = await Promise.all([
+  const lookup = extractIdentifier(query);
-    lookupAccount(query),
+  const event = await lookupAccount(lookup ?? query);
-    store.query([{ kinds: [0], search: query, limit }], { signal }),
-  ]);
 
-  const results = await hydrateEvents({
+  if (!event && lookup) {
-    events: event ? [event, ...events] : events,
+    const pubkey = await lookupPubkey(lookup);
-    store,
+    return c.json(pubkey ? [await accountFromPubkey(pubkey)] : []);
-    signal,
-  });
-
-  if ((results.length < 1) && query.match(/npub1\w+/)) {
-    const possibleNpub = query;
-    try {
-      const npubHex = nip19.decode(possibleNpub);
-      return c.json([await accountFromPubkey(String(npubHex.data))]);
-    } catch (e) {
-      console.log(e);
-      return c.json([]);
-    }
   }
 
-  const accounts = await Promise.all(results.map((event) => renderAccount(event)));
+  const events = event ? [event] : await store.query([{ kinds: [0], search: query, limit }], { signal });
+
+  const accounts = await hydrateEvents({ events, store, signal }).then(
+    (events) =>
+      Promise.all(
+        events.map((event) => renderAccount(event)),
+      ),
+  );
+
   return c.json(accounts);
 };
 

src/controllers/api/search.ts

@@ -5,14 +5,11 @@ import { z } from 'zod';
 import { AppController } from '@/app.ts';
 import { booleanParamSchema } from '@/schema.ts';
 import { Storages } from '@/storages.ts';
-import { dedupeEvents } from '@/utils.ts';
+import { hydrateEvents } from '@/storages/hydrate.ts';
+import { extractIdentifier, lookupPubkey } from '@/utils/lookup.ts';
 import { nip05Cache } from '@/utils/nip05.ts';
 import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts';
 import { renderStatus } from '@/views/mastodon/statuses.ts';
-import { hydrateEvents } from '@/storages/hydrate.ts';
-
-/** Matches NIP-05 names with or without an @ in front. */
-const ACCT_REGEX = /^@?(?:([\w.+-]+)@)?([\w.-]+)$/;
 
 const searchQuerySchema = z.object({
   q: z.string().transform(decodeURIComponent),
@@ -33,43 +30,44 @@ const searchController: AppController = async (c) => {
     return c.json({ error: 'Bad request', schema: result.error }, 422);
   }
 
-  const [event, events] = await Promise.all([
+  const event = await lookupEvent(result.data, signal);
-    lookupEvent(result.data, signal),
+  const lookup = extractIdentifier(result.data.q);
-    searchEvents(result.data, signal),
-  ]);
 
-  if (event) {
+  // Render account from pubkey.
-    events.push(event);
+  if (!event && lookup) {
+    const pubkey = await lookupPubkey(lookup);
+    return c.json({
+      accounts: pubkey ? [await accountFromPubkey(pubkey)] : [],
+      statuses: [],
+      hashtags: [],
+    });
+  }
+
+  let events: NostrEvent[] = [];
+
+  if (event) {
+    events = [event];
+  } else {
+    events = await searchEvents(result.data, signal);
   }
 
-  const results = dedupeEvents(events);
   const viewerPubkey = await c.get('signer')?.getPublicKey();
 
   const [accounts, statuses] = await Promise.all([
     Promise.all(
-      results
+      events
         .filter((event) => event.kind === 0)
         .map((event) => renderAccount(event))
         .filter(Boolean),
     ),
     Promise.all(
-      results
+      events
         .filter((event) => event.kind === 1)
         .map((event) => renderStatus(event, { viewerPubkey }))
         .filter(Boolean),
     ),
   ]);
 
-  if ((result.data.type === 'accounts') && (accounts.length < 1) && (result.data.q.match(/npub1\w+/))) {
-    const possibleNpub = result.data.q;
-    try {
-      const npubHex = nip19.decode(possibleNpub);
-      accounts.push(await accountFromPubkey(String(npubHex.data)));
-    } catch (e) {
-      console.log(e);
-    }
-  }
-
   return c.json({
     accounts,
     statuses,
@@ -121,18 +119,26 @@ async function lookupEvent(query: SearchQuery, signal: AbortSignal): Promise<Nos
 
 /** Get filters to lookup the input value. */
 async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: AbortSignal): Promise<NostrFilter[]> {
-  const filters: NostrFilter[] = [];
-
   const accounts = !type || type === 'accounts';
   const statuses = !type || type === 'statuses';
 
   if (!resolve || type === 'hashtags') {
+    return [];
+  }
+
+  if (n.id().safeParse(q).success) {
+    const filters: NostrFilter[] = [];
+    if (accounts) filters.push({ kinds: [0], authors: [q] });
+    if (statuses) filters.push({ kinds: [1], ids: [q] });
     return filters;
   }
 
-  if (new RegExp(`^${nip19.BECH32_REGEX.source}$`).test(q)) {
+  const lookup = extractIdentifier(q);
+  if (!lookup) return [];
+
   try {
-      const result = nip19.decode(q);
+    const result = nip19.decode(lookup);
+    const filters: NostrFilter[] = [];
     switch (result.type) {
       case 'npub':
         if (accounts) filters.push({ kinds: [0], authors: [result.data] });
@@ -141,34 +147,27 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort
         if (accounts) filters.push({ kinds: [0], authors: [result.data.pubkey] });
         break;
       case 'note':
-          if (statuses) {
+        if (statuses) filters.push({ kinds: [1], ids: [result.data] });
-            filters.push({ kinds: [1], ids: [result.data] });
-          }
         break;
       case 'nevent':
-          if (statuses) {
+        if (statuses) filters.push({ kinds: [1], ids: [result.data.id] });
-            filters.push({ kinds: [1], ids: [result.data.id] });
-          }
         break;
     }
-    } catch (_e) {
+    return filters;
+  } catch {
     // do nothing
   }
-  } else if (/^[0-9a-f]{64}$/.test(q)) {
-    if (accounts) filters.push({ kinds: [0], authors: [q] });
-    if (statuses) filters.push({ kinds: [1], ids: [q] });
-  } else if (accounts && ACCT_REGEX.test(q)) {
-    try {
-      const { pubkey } = await nip05Cache.fetch(q, { signal });
-      if (pubkey) {
-        filters.push({ kinds: [0], authors: [pubkey] });
-      }
-    } catch (_e) {
-      // do nothing
-    }
-  }
 
-  return filters;
+  try {
+    const { pubkey } = await nip05Cache.fetch(lookup, { signal });
+    if (pubkey) {
+      return [{ kinds: [0], authors: [pubkey] }];
+    }
+  } catch {
+    // do nothing
+  }
+
+  return [];
 }
 
 export { searchController };

src/entities/MastodonAccount.ts

@@ -36,6 +36,7 @@ export interface MastodonAccount {
     };
   };
   statuses_count: number;
+  uri: string;
   url: string;
   username: string;
   ditto: {

src/utils.ts

@@ -19,7 +19,7 @@ function bech32ToPubkey(bech32: string): string | undefined {
       case 'npub':
         return decoded.data;
     }
-  } catch (_) {
+  } catch {
     //
   }
 }
@@ -78,11 +78,6 @@ async function sha256(message: string): Promise<string> {
   return hashHex;
 }
 
-/** Deduplicate events by ID. */
-function dedupeEvents(events: NostrEvent[]): NostrEvent[] {
-  return [...new Map(events.map((event) => [event.id, event])).values()];
-}
-
 /** Test whether the value is a Nostr ID. */
 function isNostrId(value: unknown): boolean {
   return n.id().safeParse(value).success;
@@ -93,18 +88,6 @@ function isURL(value: unknown): boolean {
   return z.string().url().safeParse(value).success;
 }
 
-export {
+export { bech32ToPubkey, eventAge, findTag, isNostrId, isURL, type Nip05, nostrDate, nostrNow, parseNip05, sha256 };
-  bech32ToPubkey,
-  dedupeEvents,
-  eventAge,
-  findTag,
-  isNostrId,
-  isURL,
-  type Nip05,
-  nostrDate,
-  nostrNow,
-  parseNip05,
-  sha256,
-};
 
 export { Time } from '@/utils/time.ts';

src/utils/lookup.ts

@@ -1,4 +1,6 @@
 import { NIP05, NostrEvent, NSchema as n } from '@nostrify/nostrify';
+import { nip19 } from 'nostr-tools';
+import { match } from 'path-to-regexp';
 
 import { getAuthor } from '@/queries.ts';
 import { bech32ToPubkey } from '@/utils.ts';
@@ -35,3 +37,54 @@ export async function lookupPubkey(value: string, signal?: AbortSignal): Promise
     }
   }
 }
+
+/** Extract an acct or bech32 identifier out of a URL or of itself. */
+export function extractIdentifier(value: string): string | undefined {
+  value = value.trim();
+
+  try {
+    const uri = new URL(value);
+    switch (uri.protocol) {
+      // Extract from NIP-19 URI, eg `nostr:npub1q3sle0kvfsehgsuexttt3ugjd8xdklxfwwkh559wxckmzddywnws6cd26p`.
+      case 'nostr:':
+        value = uri.pathname;
+        break;
+      // Extract from URL, eg `https://njump.me/npub1q3sle0kvfsehgsuexttt3ugjd8xdklxfwwkh559wxckmzddywnws6cd26p`.
+      case 'http:':
+      case 'https:': {
+        const accountUriMatch = match<{ acct: string }>('/users/:acct')(uri.pathname);
+        const accountUrlMatch = match<{ acct: string }>('/\\@:acct')(uri.pathname);
+        const statusUriMatch = match<{ acct: string; id: string }>('/users/:acct/statuses/:id')(uri.pathname);
+        const statusUrlMatch = match<{ acct: string; id: string }>('/\\@:acct/:id')(uri.pathname);
+        const soapboxMatch = match<{ acct: string; id: string }>('/\\@:acct/posts/:id')(uri.pathname);
+        const nostrMatch = match<{ bech32: string }>('/:bech32')(uri.pathname);
+        if (accountUriMatch) {
+          value = accountUriMatch.params.acct;
+        } else if (accountUrlMatch) {
+          value = accountUrlMatch.params.acct;
+        } else if (statusUriMatch) {
+          value = nip19.noteEncode(statusUriMatch.params.id);
+        } else if (statusUrlMatch) {
+          value = nip19.noteEncode(statusUrlMatch.params.id);
+        } else if (soapboxMatch) {
+          value = nip19.noteEncode(soapboxMatch.params.id);
+        } else if (nostrMatch) {
+          value = nostrMatch.params.bech32;
+        }
+        break;
+      }
+    }
+  } catch {
+    // do nothing
+  }
+
+  value = value.replace(/^@/, '');
+
+  if (n.bech32().safeParse(value).success) {
+    return value;
+  }
+
+  if (NIP05.regex().test(value)) {
+    return value;
+  }
+}

src/utils/note.test.ts

@@ -4,7 +4,7 @@ import { eventFixture } from '@/test.ts';
 import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts';
 
 Deno.test('parseNoteContent', () => {
-  const { html, links, firstUrl } = parseNoteContent('Hello, world!');
+  const { html, links, firstUrl } = parseNoteContent('Hello, world!', []);
   assertEquals(html, 'Hello, world!');
   assertEquals(links, []);
   assertEquals(firstUrl, undefined);

src/utils/note.ts

@@ -4,12 +4,27 @@ import linkify from 'linkifyjs';
 import { nip19, nip21, nip27 } from 'nostr-tools';
 
 import { Conf } from '@/config.ts';
+import { MastodonMention } from '@/entities/MastodonMention.ts';
 import { getUrlMediaType, isPermittedMediaType } from '@/utils/media.ts';
 
 linkify.registerCustomProtocol('nostr', true);
 linkify.registerCustomProtocol('wss');
 
-const linkifyOpts: linkify.Opts = {
+type Link = ReturnType<typeof linkify.find>[0];
+
+interface ParsedNoteContent {
+  html: string;
+  links: Link[];
+  /** First non-media URL - eligible for a preview card. */
+  firstUrl: string | undefined;
+}
+
+/** Convert Nostr content to Mastodon API HTML. Also return parsed data. */
+function parseNoteContent(content: string, mentions: MastodonMention[]): ParsedNoteContent {
+  const links = linkify.find(content).filter(isLinkURL);
+  const firstUrl = links.find(isNonMediaLink)?.href;
+
+  const html = linkifyStr(content, {
     render: {
       hashtag: ({ content }) => {
         const tag = content.replace(/^#/, '');
@@ -21,8 +36,11 @@ const linkifyOpts: linkify.Opts = {
           const { decoded } = nip21.parse(content);
           const pubkey = getDecodedPubkey(decoded);
           if (pubkey) {
-          const name = pubkey.substring(0, 8);
+            const mention = mentions.find((m) => m.id === pubkey);
-          const href = Conf.local(`/users/${pubkey}`);
+            const npub = nip19.npubEncode(pubkey);
+            const acct = mention?.acct ?? npub;
+            const name = mention?.acct ?? npub.substring(0, 8);
+            const href = mention?.url ?? Conf.local(`/@${acct}`);
             return `<span class="h-card"><a class="u-url mention" href="${href}" rel="ugc">@<span>${name}</span></a></span>`;
           } else {
             return '';
@@ -36,23 +54,7 @@ const linkifyOpts: linkify.Opts = {
         }
       },
     },
-};
+  }).replace(/\n+$/, '');
-
-type Link = ReturnType<typeof linkify.find>[0];
-
-interface ParsedNoteContent {
-  html: string;
-  links: Link[];
-  /** First non-media URL - eligible for a preview card. */
-  firstUrl: string | undefined;
-}
-
-/** Convert Nostr content to Mastodon API HTML. Also return parsed data. */
-function parseNoteContent(content: string): ParsedNoteContent {
-  // Parsing twice is ineffecient, but I don't know how to do only once.
-  const html = linkifyStr(content, linkifyOpts).replace(/\n+$/, '');
-  const links = linkify.find(content).filter(isLinkURL);
-  const firstUrl = links.find(isNonMediaLink)?.href;
 
   return {
     html,

src/views/mastodon/accounts.ts

@@ -42,10 +42,11 @@ async function renderAccount(
 
   const npub = nip19.npubEncode(pubkey);
   const parsed05 = await parseAndVerifyNip05(nip05, pubkey);
+  const acct = parsed05?.handle || npub;
 
   return {
     id: pubkey,
-    acct: parsed05?.handle || npub,
+    acct,
     avatar: picture,
     avatar_static: picture,
     bot: false,
@@ -78,7 +79,8 @@ async function renderAccount(
       }
       : undefined,
     statuses_count: event.author_stats?.notes_count ?? 0,
-    url: Conf.local(`/users/${pubkey}`),
+    uri: Conf.local(`/users/${acct}`),
+    url: Conf.local(`/@${acct}`),
     username: parsed05?.nickname || npub.substring(0, 8),
     ditto: {
       accepts_zaps: Boolean(getLnurl({ lud06, lud16 })),

src/views/mastodon/statuses.ts

@@ -46,13 +46,14 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
     [{ kinds: [0], authors: mentionedPubkeys, limit: mentionedPubkeys.length }],
   );
 
-  const { html, links, firstUrl } = parseNoteContent(stripimeta(event.content, event.tags));
+  const mentions = await Promise.all(
+    mentionedPubkeys.map((pubkey) => renderMention(pubkey, mentionedProfiles.find((event) => event.pubkey === pubkey))),
+  );
 
-  const [mentions, card, relatedEvents] = await Promise
+  const { html, links, firstUrl } = parseNoteContent(stripimeta(event.content, event.tags), mentions);
+
+  const [card, relatedEvents] = await Promise
     .all([
-      Promise.all(
-        mentionedPubkeys.map((pubkey) => toMention(pubkey, mentionedProfiles.find((event) => event.pubkey === pubkey))),
-      ),
       firstUrl ? unfurlCardCached(firstUrl) : null,
       viewerPubkey
         ? await store.query([
@@ -71,7 +72,11 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
   const bookmarkEvent = relatedEvents.find((event) => event.kind === 10003);
   const zapEvent = relatedEvents.find((event) => event.kind === 9734);
 
-  const content = buildInlineRecipients(mentions) + html;
+  const compatMentions = buildInlineRecipients(mentions.filter((m) => {
+    if (m.id === account.id) return false;
+    if (html.includes(m.url)) return false;
+    return true;
+  }));
 
   const cw = event.tags.find(([name]) => name === 'content-warning');
   const subject = event.tags.find(([name]) => name === 'subject');
@@ -95,7 +100,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
     id: event.id,
     account,
     card,
-    content,
+    content: compatMentions + html,
     created_at: nostrDate(event.created_at).toISOString(),
     in_reply_to_id: replyId ?? null,
     in_reply_to_account_id: null,
@@ -121,8 +126,8 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
     poll: null,
     quote: !event.quote ? null : await renderStatus(event.quote, { depth: depth + 1 }),
     quote_id: event.quote?.id ?? null,
-    uri: Conf.local(`/${note}`),
+    uri: Conf.local(`/users/${account.acct}/statuses/${event.id}`),
-    url: Conf.local(`/${note}`),
+    url: Conf.local(`/@${account.acct}/${event.id}`),
     zapped: Boolean(zapEvent),
     ditto: {
       external_url: Conf.external(note),
@@ -152,25 +157,14 @@ async function renderReblog(event: DittoEvent, opts: RenderStatusOpts): Promise<
   };
 }
 
-async function toMention(pubkey: string, event?: NostrEvent): Promise<MastodonMention> {
+async function renderMention(pubkey: string, event?: NostrEvent): Promise<MastodonMention> {
-  const account = event ? await renderAccount(event) : undefined;
+  const account = event ? await renderAccount(event) : await accountFromPubkey(pubkey);
-
-  if (account) {
   return {
     id: account.id,
     acct: account.acct,
     username: account.username,
     url: account.url,
   };
-  } else {
-    const npub = nip19.npubEncode(pubkey);
-    return {
-      id: pubkey,
-      acct: npub,
-      username: npub.substring(0, 8),
-      url: Conf.local(`/users/${pubkey}`),
-    };
-  }
 }
 
 function buildInlineRecipients(mentions: MastodonMention[]): string {
@@ -178,7 +172,7 @@ function buildInlineRecipients(mentions: MastodonMention[]): string {
 
   const elements = mentions.reduce<string[]>((acc, { url, username }) => {
     const name = nip19.BECH32_REGEX.test(username) ? username.substring(0, 8) : username;
-    acc.push(`<a href="${url}" class="u-url mention" rel="ugc">@<span>${name}</span></a>`);
+    acc.push(`<span class="h-card"><a class="u-url mention" href="${url}" rel="ugc">@<span>${name}</span></a></span>`);
     return acc;
   }, []);