Merge branch 'nip05-db' into 'main'

Store NIP-05 in the database

See merge request soapbox-pub/ditto!641

deno.json

@@ -21,6 +21,7 @@
     "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip -o soapbox.zip && rm soapbox.zip",
     "trends": "deno run -A --env-file --deny-read=.env scripts/trends.ts",
     "clean:deps": "deno cache --reload src/app.ts",
+    "db:populate:nip05": "deno run -A --env-file --deny-read=.env scripts/db-populate-nip05.ts",
     "db:populate-search": "deno run -A --env-file --deny-read=.env scripts/db-populate-search.ts",
     "db:populate-extensions": "deno run -A --env-file --deny-read=.env scripts/db-populate-extensions.ts",
     "db:streak:recompute": "deno run -A --env-file --deny-read=.env scripts/db-streak-recompute.ts",

scripts/db-populate-nip05.ts

@@ -0,0 +1,26 @@
+import { Semaphore } from '@core/asyncutil';
+
+import { updateAuthorData } from '@/pipeline.ts';
+import { Storages } from '@/storages.ts';
+import { NostrEvent } from '@nostrify/nostrify';
+
+const kysely = await Storages.kysely();
+const sem = new Semaphore(5);
+
+const query = kysely
+  .selectFrom('nostr_events')
+  .select(['id', 'kind', 'content', 'pubkey', 'tags', 'created_at', 'sig'])
+  .where('kind', '=', 0);
+
+for await (const row of query.stream(100)) {
+  while (sem.locked) {
+    await new Promise((resolve) => setTimeout(resolve, 0));
+  }
+
+  sem.lock(async () => {
+    const event: NostrEvent = { ...row, created_at: Number(row.created_at) };
+    await updateAuthorData(event, AbortSignal.timeout(3000));
+  });
+}
+
+Deno.exit();

src/db/DittoTables.ts

@@ -5,6 +5,7 @@ import { NPostgresSchema } from '@nostrify/db';
 export interface DittoTables extends NPostgresSchema {
   auth_tokens: AuthTokenRow;
   author_stats: AuthorStatsRow;
+  domain_favicons: DomainFaviconRow;
   event_stats: EventStatsRow;
   pubkey_domains: PubkeyDomainRow;
   event_zaps: EventZapRow;
@@ -19,6 +20,10 @@ interface AuthorStatsRow {
   search: string;
   streak_start: number | null;
   streak_end: number | null;
+  nip05: string | null;
+  nip05_domain: string | null;
+  nip05_hostname: string | null;
+  nip05_last_verified_at: number | null;
 }
 
 interface EventStatsRow {
@@ -46,6 +51,12 @@ interface PubkeyDomainRow {
   last_updated_at: number;
 }
 
+interface DomainFaviconRow {
+  domain: string;
+  favicon: string;
+  last_updated_at: number;
+}
+
 interface EventZapRow {
   receipt_id: string;
   target_event_id: string;

src/db/migrations/046_author_stats_nip05.ts

@@ -0,0 +1,48 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .alterTable('author_stats')
+    .addColumn('nip05', 'varchar(320)')
+    .addColumn('nip05_domain', 'varchar(253)')
+    .addColumn('nip05_hostname', 'varchar(253)')
+    .addColumn('nip05_last_verified_at', 'integer')
+    .execute();
+
+  await db.schema
+    .alterTable('author_stats')
+    .addCheckConstraint('author_stats_nip05_domain_lowercase_chk', sql`nip05_domain = lower(nip05_domain)`)
+    .execute();
+
+  await db.schema
+    .alterTable('author_stats')
+    .addCheckConstraint('author_stats_nip05_hostname_lowercase_chk', sql`nip05_hostname = lower(nip05_hostname)`)
+    .execute();
+
+  await db.schema
+    .alterTable('author_stats')
+    .addCheckConstraint('author_stats_nip05_hostname_domain_chk', sql`nip05_hostname like '%' || nip05_domain`)
+    .execute();
+
+  await db.schema
+    .createIndex('author_stats_nip05_domain_idx')
+    .on('author_stats')
+    .column('nip05_domain')
+    .execute();
+
+  await db.schema
+    .createIndex('author_stats_nip05_hostname_idx')
+    .on('author_stats')
+    .column('nip05_hostname')
+    .execute();
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .alterTable('author_stats')
+    .dropColumn('nip05')
+    .dropColumn('nip05_domain')
+    .dropColumn('nip05_hostname')
+    .dropColumn('nip05_last_verified_at')
+    .execute();
+}

src/db/migrations/047_add_domain_favicons.ts

@@ -0,0 +1,15 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createTable('domain_favicons')
+    .addColumn('domain', 'varchar(253)', (col) => col.primaryKey())
+    .addColumn('favicon', 'varchar(2048)', (col) => col.notNull())
+    .addColumn('last_updated_at', 'integer', (col) => col.notNull())
+    .addCheckConstraint('domain_favicons_https_chk', sql`favicon ~* '^https:\\/\\/'`)
+    .execute();
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropTable('domain_favicons').execute();
+}

src/interfaces/DittoEvent.ts

@@ -8,6 +8,11 @@ export interface AuthorStats {
   notes_count: number;
   streak_start?: number;
   streak_end?: number;
+  nip05?: string;
+  nip05_domain?: string;
+  nip05_hostname?: string;
+  nip05_last_verified_at?: number;
+  favicon?: string;
 }
 
 /** Ditto internal stats for the event. */
@@ -25,6 +30,7 @@ export interface DittoEvent extends NostrEvent {
   author_domain?: string;
   author_stats?: AuthorStats;
   event_stats?: EventStats;
+  mentions?: DittoEvent[];
   user?: DittoEvent;
   repost?: DittoEvent;
   quote?: DittoEvent;

src/pipeline.ts

@@ -1,6 +1,7 @@
 import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify';
 import { logi } from '@soapbox/logi';
-import { Kysely, sql } from 'kysely';
+import { Kysely, UpdateObject } from 'kysely';
+import tldts from 'tldts';
 import { z } from 'zod';
 
 import { pipelineEncounters } from '@/caches/pipelineEncounters.ts';
@@ -13,8 +14,9 @@ import { RelayError } from '@/RelayError.ts';
 import { AdminSigner } from '@/signers/AdminSigner.ts';
 import { hydrateEvents } from '@/storages/hydrate.ts';
 import { Storages } from '@/storages.ts';
-import { eventAge, parseNip05, Time } from '@/utils.ts';
+import { eventAge, Time } from '@/utils.ts';
 import { getAmount } from '@/utils/bolt11.ts';
+import { faviconCache } from '@/utils/favicon.ts';
 import { errorJson } from '@/utils/log.ts';
 import { nip05Cache } from '@/utils/nip05.ts';
 import { purifyEvent } from '@/utils/purify.ts';
@@ -119,7 +121,7 @@ async function handleEvent(event: DittoEvent, opts: PipelineOpts): Promise<void>
     // This needs to run in steps, and should not block the API from responding.
     Promise.allSettled([
       handleZaps(kysely, event),
-      parseMetadata(event, opts.signal),
+      updateAuthorData(event, opts.signal),
       generateSetEvents(event),
     ])
       .then(() =>
@@ -189,50 +191,81 @@ async function storeEvent(event: NostrEvent, signal?: AbortSignal): Promise<unde
 }
 
 /** Parse kind 0 metadata and track indexes in the database. */
-async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise<void> {
+async function updateAuthorData(event: NostrEvent, signal: AbortSignal): Promise<void> {
   if (event.kind !== 0) return;
 
   // Parse metadata.
   const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(event.content);
   if (!metadata.success) return;
 
+  const { name, nip05 } = metadata.data;
+
   const kysely = await Storages.kysely();
 
-  // Get nip05.
-  const { name, nip05 } = metadata.data;
-  const result = nip05 ? await nip05Cache.fetch(nip05, { signal }).catch(() => undefined) : undefined;
+  const updates: UpdateObject<DittoTables, 'author_stats'> = {};
+
+  const authorStats = await kysely
+    .selectFrom('author_stats')
+    .selectAll()
+    .where('pubkey', '=', event.pubkey)
+    .executeTakeFirst();
+
+  const lastVerified = authorStats?.nip05_last_verified_at;
+  const eventNewer = !lastVerified || event.created_at > lastVerified;
 
-  // Populate author_search.
   try {
-    const search = result?.pubkey === event.pubkey ? [name, nip05].filter(Boolean).join(' ').trim() : name ?? '';
-
-    if (search) {
-      await kysely.insertInto('author_stats')
-        .values({ pubkey: event.pubkey, search, followers_count: 0, following_count: 0, notes_count: 0 })
-        .onConflict((oc) => oc.column('pubkey').doUpdateSet({ search }))
-        .execute();
+    if (nip05 !== authorStats?.nip05 && eventNewer || !lastVerified) {
+      if (nip05) {
+        const tld = tldts.parse(nip05);
+        if (tld.isIcann && !tld.isIp && !tld.isPrivate) {
+          const pointer = await nip05Cache.fetch(nip05.toLowerCase(), { signal });
+          if (pointer.pubkey === event.pubkey) {
+            updates.nip05 = nip05;
+            updates.nip05_domain = tld.domain;
+            updates.nip05_hostname = tld.hostname;
+            updates.nip05_last_verified_at = event.created_at;
+          }
+        }
+      } else {
+        updates.nip05 = null;
+        updates.nip05_domain = null;
+        updates.nip05_hostname = null;
+        updates.nip05_last_verified_at = event.created_at;
+      }
     }
   } catch {
-    // do nothing
+    // Fallthrough.
   }
 
-  if (nip05 && result && result.pubkey === event.pubkey) {
-    // Track pubkey domain.
+  // Fetch favicon.
+  const domain = nip05?.split('@')[1].toLowerCase();
+  if (domain) {
     try {
-      const { domain } = parseNip05(nip05);
-
-      await sql`
-      INSERT INTO pubkey_domains (pubkey, domain, last_updated_at)
-      VALUES (${event.pubkey}, ${domain}, ${event.created_at})
-      ON CONFLICT(pubkey) DO UPDATE SET
-        domain = excluded.domain,
-        last_updated_at = excluded.last_updated_at
-      WHERE excluded.last_updated_at > pubkey_domains.last_updated_at
-      `.execute(kysely);
-    } catch (_e) {
-      // do nothing
+      await faviconCache.fetch(domain, { signal });
+    } catch {
+      // Fallthrough.
     }
   }
+
+  const search = [name, nip05].filter(Boolean).join(' ').trim();
+
+  if (search !== authorStats?.search) {
+    updates.search = search;
+  }
+
+  if (Object.keys(updates).length) {
+    await kysely.insertInto('author_stats')
+      .values({
+        pubkey: event.pubkey,
+        followers_count: 0,
+        following_count: 0,
+        notes_count: 0,
+        search,
+        ...updates,
+      })
+      .onConflict((oc) => oc.column('pubkey').doUpdateSet(updates))
+      .execute();
+  }
 }
 
 /** Determine if the event is being received in a timely manner. */
@@ -364,4 +397,4 @@ async function handleZaps(kysely: Kysely<DittoTables>, event: NostrEvent) {
   }
 }
 
-export { handleEvent, handleZaps };
+export { handleEvent, handleZaps, updateAuthorData };

src/storages/hydrate.ts

@@ -42,6 +42,10 @@ async function hydrateEvents(opts: HydrateOpts): Promise<DittoEvent[]> {
     cache.push(event);
   }
 
+  for (const event of await gatherMentions({ events: cache, store, signal })) {
+    cache.push(event);
+  }
+
   for (const event of await gatherAuthors({ events: cache, store, signal })) {
     cache.push(event);
   }
@@ -66,9 +70,30 @@ async function hydrateEvents(opts: HydrateOpts): Promise<DittoEvent[]> {
     cache.push(event);
   }
 
+  const authorStats = await gatherAuthorStats(cache, kysely as Kysely<DittoTables>);
+  const eventStats = await gatherEventStats(cache, kysely as Kysely<DittoTables>);
+
+  const domains = authorStats.reduce((result, { nip05_hostname }) => {
+    if (nip05_hostname) result.add(nip05_hostname);
+    return result;
+  }, new Set<string>());
+
+  const favicons = (
+    await kysely
+      .selectFrom('domain_favicons')
+      .select(['domain', 'favicon'])
+      .where('domain', 'in', [...domains])
+      .execute()
+  )
+    .reduce((result, { domain, favicon }) => {
+      result[domain] = favicon;
+      return result;
+    }, {} as Record<string, string>);
+
   const stats = {
-    authors: await gatherAuthorStats(cache, kysely as Kysely<DittoTables>),
-    events: await gatherEventStats(cache, kysely as Kysely<DittoTables>),
+    authors: authorStats,
+    events: eventStats,
+    favicons,
   };
 
   // Dedupe events.
@@ -85,7 +110,11 @@ async function hydrateEvents(opts: HydrateOpts): Promise<DittoEvent[]> {
 export function assembleEvents(
   a: DittoEvent[],
   b: DittoEvent[],
-  stats: { authors: DittoTables['author_stats'][]; events: DittoTables['event_stats'][] },
+  stats: {
+    authors: DittoTables['author_stats'][];
+    events: DittoTables['event_stats'][];
+    favicons: Record<string, string>;
+  },
 ): DittoEvent[] {
   const admin = Conf.pubkey;
 
@@ -94,6 +123,11 @@ export function assembleEvents(
       ...stat,
       streak_start: stat.streak_start ?? undefined,
       streak_end: stat.streak_end ?? undefined,
+      nip05: stat.nip05 ?? undefined,
+      nip05_domain: stat.nip05_domain ?? undefined,
+      nip05_hostname: stat.nip05_hostname ?? undefined,
+      nip05_last_verified_at: stat.nip05_last_verified_at ?? undefined,
+      favicon: stats.favicons[stat.nip05_hostname!],
     };
     return result;
   }, {} as Record<string, DittoEvent['author_stats']>);
@@ -116,6 +150,9 @@ export function assembleEvents(
       if (id) {
         event.quote = b.find((e) => matchFilter({ kinds: [1, 20], ids: [id] }, e));
       }
+
+      const pubkeys = event.tags.filter(([name]) => name === 'p').map(([_name, value]) => value);
+      event.mentions = b.filter((e) => matchFilter({ kinds: [0], authors: pubkeys }, e));
     }
 
     if (event.kind === 6) {
@@ -237,6 +274,36 @@ function gatherQuotes({ events, store, signal }: HydrateOpts): Promise<DittoEven
   );
 }
 
+/** Collect mentioned profiles from notes. */
+async function gatherMentions({ events, store, signal }: HydrateOpts): Promise<DittoEvent[]> {
+  const pubkeys = new Set<string>();
+
+  for (const event of events) {
+    if (event.kind === 1) {
+      for (const [name, value] of event.tags) {
+        if (name === 'p') {
+          pubkeys.add(value);
+        }
+      }
+    }
+  }
+
+  const authors = await store.query(
+    [{ kinds: [0], authors: [...pubkeys], limit: pubkeys.size }],
+    { signal },
+  );
+
+  for (const pubkey of pubkeys) {
+    const author = authors.find((e) => matchFilter({ kinds: [0], authors: [pubkey] }, e));
+    if (!author) {
+      const fallback = fallbackAuthor(pubkey);
+      authors.push(fallback);
+    }
+  }
+
+  return authors;
+}
+
 /** Collect authors from the events. */
 async function gatherAuthors({ events, store, signal }: HydrateOpts): Promise<DittoEvent[]> {
   const pubkeys = new Set<string>();
@@ -267,7 +334,7 @@ async function gatherAuthors({ events, store, signal }: HydrateOpts): Promise<Di
 
   for (const pubkey of pubkeys) {
     const author = authors.find((e) => matchFilter({ kinds: [0], authors: [pubkey] }, e));
-    if (author) {
+    if (!author) {
       const fallback = fallbackAuthor(pubkey);
       authors.push(fallback);
     }
@@ -390,13 +457,10 @@ async function gatherAuthorStats(
     .execute();
 
   return rows.map((row) => ({
-    pubkey: row.pubkey,
+    ...row,
     followers_count: Math.max(0, row.followers_count),
     following_count: Math.max(0, row.following_count),
     notes_count: Math.max(0, row.notes_count),
-    search: row.search,
-    streak_start: row.streak_start,
-    streak_end: row.streak_end,
   }));
 }
 

src/utils/SimpleLRU.test.ts

@@ -4,7 +4,7 @@ import { assertEquals, assertRejects } from '@std/assert';
 Deno.test("SimpleLRU doesn't repeat failed calls", async () => {
   let calls = 0;
 
-  const cache = new SimpleLRU(
+  using cache = new SimpleLRU(
     // deno-lint-ignore require-await
     async () => {
       calls++;

src/utils/SimpleLRU.ts

@@ -3,50 +3,55 @@
 import { LRUCache } from 'lru-cache';
 import { type Gauge } from 'prom-client';
 
-type FetchFn<K extends {}, V extends {}, O extends {}> = (key: K, opts: O) => Promise<V>;
-
-interface FetchFnOpts {
-  signal?: AbortSignal | null;
-}
+type FetchFn<K extends {}, V extends {}> = (key: K, opts: { signal?: AbortSignal }) => Promise<V>;
 
 type SimpleLRUOpts<K extends {}, V extends {}> = LRUCache.Options<K, V, void> & {
   gauge?: Gauge;
+  errorRefresh?: number;
 };
 
 export class SimpleLRU<
   K extends {},
   V extends {},
-  O extends {} = FetchFnOpts,
 > {
-  protected cache: LRUCache<K, V, void>;
+  protected cache: LRUCache<K, Promise<V>, void>;
+  private tids = new Set<number>();
 
-  constructor(fetchFn: FetchFn<K, V, { signal: AbortSignal }>, private opts: SimpleLRUOpts<K, V>) {
-    this.cache = new LRUCache({
-      async fetchMethod(key, _staleValue, { signal }) {
-        try {
-          return await fetchFn(key, { signal: signal as unknown as AbortSignal });
-        } catch {
-          return null as unknown as V;
-        }
-      },
-      ...opts,
-    });
+  constructor(private fetchFn: FetchFn<K, V>, private opts: SimpleLRUOpts<K, Promise<V>>) {
+    this.cache = new LRUCache({ ...opts });
   }
 
-  async fetch(key: K, opts?: O): Promise<V> {
-    const result = await this.cache.fetch(key, opts);
+  async fetch(key: K, opts?: { signal?: AbortSignal }): Promise<V> {
+    if (opts?.signal?.aborted) {
+      throw new DOMException('The signal has been aborted', 'AbortError');
+    }
 
+    const cached = await this.cache.get(key);
+
+    if (cached) {
+      return cached;
+    }
+
+    const promise = this.fetchFn(key, { signal: opts?.signal });
+
+    this.cache.set(key, promise);
+
+    promise.then(() => {
       this.opts.gauge?.set(this.cache.size);
+    }).catch(() => {
+      const tid = setTimeout(() => {
+        this.cache.delete(key);
+        this.tids.delete(tid);
+      }, this.opts.errorRefresh ?? 10_000);
+      this.tids.add(tid);
+    });
 
-    if (result === undefined || result === null) {
-      throw new Error('SimpleLRU: fetch failed');
+    return promise;
   }
 
-    return result;
-  }
-
-  put(key: K, value: V): Promise<void> {
-    this.cache.set(key, value);
-    return Promise.resolve();
+  [Symbol.dispose](): void {
+    for (const tid of this.tids) {
+      clearTimeout(tid);
+    }
   }
 }

src/utils/favicon.ts

@@ -1,14 +1,55 @@
 import { DOMParser } from '@b-fuze/deno-dom';
 import { logi } from '@soapbox/logi';
 import { safeFetch } from '@soapbox/safe-fetch';
+import { Kysely } from 'kysely';
 import tldts from 'tldts';
 
 import { Conf } from '@/config.ts';
+import { DittoTables } from '@/db/DittoTables.ts';
 import { cachedFaviconsSizeGauge } from '@/metrics.ts';
+import { Storages } from '@/storages.ts';
+import { nostrNow } from '@/utils.ts';
 import { SimpleLRU } from '@/utils/SimpleLRU.ts';
 
-const faviconCache = new SimpleLRU<string, URL>(
+export const faviconCache = new SimpleLRU<string, URL>(
   async (domain, { signal }) => {
+    const kysely = await Storages.kysely();
+
+    const row = await queryFavicon(kysely, domain);
+
+    if (row && (nostrNow() - row.last_updated_at) < (Conf.caches.favicon.ttl / 1000)) {
+      return new URL(row.favicon);
+    }
+
+    const url = await fetchFavicon(domain, signal);
+
+    await insertFavicon(kysely, domain, url.href);
+
+    return url;
+  },
+  { ...Conf.caches.favicon, gauge: cachedFaviconsSizeGauge },
+);
+
+async function queryFavicon(
+  kysely: Kysely<DittoTables>,
+  domain: string,
+): Promise<DittoTables['domain_favicons'] | undefined> {
+  return await kysely
+    .selectFrom('domain_favicons')
+    .selectAll()
+    .where('domain', '=', domain)
+    .executeTakeFirst();
+}
+
+async function insertFavicon(kysely: Kysely<DittoTables>, domain: string, favicon: string): Promise<void> {
+  await kysely
+    .insertInto('domain_favicons')
+    .values({ domain, favicon, last_updated_at: nostrNow() })
+    .onConflict((oc) => oc.column('domain').doUpdateSet({ favicon, last_updated_at: nostrNow() }))
+    .execute();
+}
+
+async function fetchFavicon(domain: string, signal?: AbortSignal): Promise<URL> {
   logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'started' });
   const tld = tldts.parse(domain);
 
@@ -45,11 +86,17 @@ const faviconCache = new SimpleLRU<string, URL>(
     }
   }
 
+  // Fallback to checking `/favicon.ico` of the domain.
+  const url = new URL('/favicon.ico', `https://${domain}/`);
+  const fallback = await safeFetch(url, { method: 'HEAD', signal });
+  const contentType = fallback.headers.get('content-type');
+
+  if (fallback.ok && contentType === 'image/vnd.microsoft.icon') {
+    logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'found', url });
+    return url;
+  }
+
   logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'failed' });
 
   throw new Error(`Favicon not found: ${domain}`);
-  },
-  { ...Conf.caches.favicon, gauge: cachedFaviconsSizeGauge },
-);
-
-export { faviconCache };
+}

src/utils/nip05.ts

@@ -9,44 +9,51 @@ import { cachedNip05sSizeGauge } from '@/metrics.ts';
 import { Storages } from '@/storages.ts';
 import { errorJson } from '@/utils/log.ts';
 import { SimpleLRU } from '@/utils/SimpleLRU.ts';
-import { Nip05, parseNip05 } from '@/utils.ts';
 
-const nip05Cache = new SimpleLRU<string, nip19.ProfilePointer>(
+export const nip05Cache = new SimpleLRU<string, nip19.ProfilePointer>(
   async (nip05, { signal }) => {
+    const store = await Storages.db();
+    return getNip05(store, nip05, signal);
+  },
+  { ...Conf.caches.nip05, gauge: cachedNip05sSizeGauge },
+);
+
+async function getNip05(
+  store: NStore,
+  nip05: string,
+  signal?: AbortSignal,
+): Promise<nip19.ProfilePointer> {
   const tld = tldts.parse(nip05);
 
   if (!tld.isIcann || tld.isIp || tld.isPrivate) {
     throw new Error(`Invalid NIP-05: ${nip05}`);
   }
 
-    const [name, domain] = nip05.split('@');
-
   logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'started' });
 
+  const [name, domain] = nip05.split('@');
+
   try {
     if (domain === Conf.url.host) {
-        const store = await Storages.db();
       const pointer = await localNip05Lookup(store, name);
       if (pointer) {
-          logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', pubkey: pointer.pubkey });
+        logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'local', pubkey: pointer.pubkey });
         return pointer;
       } else {
         throw new Error(`Not found: ${nip05}`);
       }
     } else {
-        const result = await NIP05.lookup(nip05, { fetch: safeFetch, signal });
-        logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', pubkey: result.pubkey });
-        return result;
+      const pointer = await NIP05.lookup(nip05, { fetch: safeFetch, signal });
+      logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'fetch', pubkey: pointer.pubkey });
+      return pointer;
     }
   } catch (e) {
     logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'failed', error: errorJson(e) });
     throw e;
   }
-  },
-  { ...Conf.caches.nip05, gauge: cachedNip05sSizeGauge },
-);
+}
 
-async function localNip05Lookup(store: NStore, localpart: string): Promise<nip19.ProfilePointer | undefined> {
+export async function localNip05Lookup(store: NStore, localpart: string): Promise<nip19.ProfilePointer | undefined> {
   const [grant] = await store.query([{
     kinds: [30360],
     '#d': [`${localpart}@${Conf.url.host}`],
@@ -60,21 +67,3 @@ async function localNip05Lookup(store: NStore, localpart: string): Promise<nip19
     return { pubkey, relays: [Conf.relay] };
   }
 }
-
-export async function parseAndVerifyNip05(
-  nip05: string | undefined,
-  pubkey: string,
-  signal = AbortSignal.timeout(3000),
-): Promise<Nip05 | undefined> {
-  if (!nip05) return;
-  try {
-    const result = await nip05Cache.fetch(nip05, { signal });
-    if (result.pubkey === pubkey) {
-      return parseNip05(nip05);
-    }
-  } catch (_e) {
-    // do nothing
-  }
-}
-
-export { localNip05Lookup, nip05Cache };

src/utils/stats.ts

@@ -323,6 +323,10 @@ export async function countAuthorStats(
     search,
     streak_start: null,
     streak_end: null,
+    nip05: null,
+    nip05_domain: null,
+    nip05_hostname: null,
+    nip05_last_verified_at: null,
   };
 }
 

src/views/mastodon/accounts.ts

@@ -6,11 +6,9 @@ import { MastodonAccount } from '@/entities/MastodonAccount.ts';
 import { type DittoEvent } from '@/interfaces/DittoEvent.ts';
 import { metadataSchema } from '@/schemas/nostr.ts';
 import { getLnurl } from '@/utils/lnurl.ts';
-import { parseAndVerifyNip05 } from '@/utils/nip05.ts';
 import { parseNoteContent } from '@/utils/note.ts';
 import { getTagSet } from '@/utils/tags.ts';
-import { faviconCache } from '@/utils/favicon.ts';
-import { nostrDate, nostrNow } from '@/utils.ts';
+import { nostrDate, nostrNow, parseNip05 } from '@/utils.ts';
 import { renderEmojis } from '@/views/mastodon/emojis.ts';
 
 type ToAccountOpts = {
@@ -20,16 +18,14 @@ type ToAccountOpts = {
   withSource?: false;
 };
 
-async function renderAccount(
-  event: Omit<DittoEvent, 'id' | 'sig'>,
-  opts: ToAccountOpts = {},
-  signal = AbortSignal.timeout(3000),
-): Promise<MastodonAccount> {
+function renderAccount(event: Omit<DittoEvent, 'id' | 'sig'>, opts: ToAccountOpts = {}): MastodonAccount {
   const { pubkey } = event;
 
+  const stats = event.author_stats;
   const names = getTagSet(event.user?.tags ?? [], 'n');
+
   if (names.has('disabled')) {
-    const account = await accountFromPubkey(pubkey, opts);
+    const account = accountFromPubkey(pubkey, opts);
     account.pleroma.deactivated = true;
     return account;
   }
@@ -48,17 +44,9 @@ async function renderAccount(
 
   const npub = nip19.npubEncode(pubkey);
   const nprofile = nip19.nprofileEncode({ pubkey, relays: [Conf.relay] });
-  const parsed05 = await parseAndVerifyNip05(nip05, pubkey, signal);
+  const parsed05 = stats?.nip05 ? parseNip05(stats.nip05) : undefined;
   const acct = parsed05?.handle || npub;
 
-  let favicon: URL | undefined;
-  if (parsed05?.domain) {
-    try {
-      favicon = await faviconCache.fetch(parsed05.domain, { signal });
-    } catch {
-      favicon = new URL('/favicon.ico', `https://${parsed05.domain}/`);
-    }
-  }
   const { html } = parseNoteContent(about || '', []);
 
   const fields = _fields
@@ -70,8 +58,8 @@ async function renderAccount(
     })) ?? [];
 
   let streakDays = 0;
-  let streakStart = event.author_stats?.streak_start ?? null;
-  let streakEnd = event.author_stats?.streak_end ?? null;
+  let streakStart = stats?.streak_start ?? null;
+  let streakEnd = stats?.streak_end ?? null;
   const { streakWindow } = Conf;
 
   if (streakStart && streakEnd) {
@@ -97,8 +85,8 @@ async function renderAccount(
     emojis: renderEmojis(event),
     fields: fields.map((field) => ({ ...field, value: parseNoteContent(field.value, []).html })),
     follow_requests_count: 0,
-    followers_count: event.author_stats?.followers_count ?? 0,
-    following_count: event.author_stats?.following_count ?? 0,
+    followers_count: stats?.followers_count ?? 0,
+    following_count: stats?.following_count ?? 0,
     fqn: parsed05?.handle || npub,
     header: banner,
     header_static: banner,
@@ -122,7 +110,7 @@ async function renderAccount(
         },
       }
       : undefined,
-    statuses_count: event.author_stats?.notes_count ?? 0,
+    statuses_count: stats?.notes_count ?? 0,
     uri: Conf.local(`/users/${acct}`),
     url: Conf.local(`/@${acct}`),
     username: parsed05?.nickname || npub.substring(0, 8),
@@ -144,7 +132,7 @@ async function renderAccount(
       is_local: parsed05?.domain === Conf.url.host,
       settings_store: opts.withSource ? opts.settingsStore : undefined,
       tags: [...getTagSet(event.user?.tags ?? [], 't')],
-      favicon: favicon?.toString(),
+      favicon: stats?.favicon,
     },
     nostr: {
       pubkey,
@@ -154,7 +142,7 @@ async function renderAccount(
   };
 }
 
-function accountFromPubkey(pubkey: string, opts: ToAccountOpts = {}): Promise<MastodonAccount> {
+function accountFromPubkey(pubkey: string, opts: ToAccountOpts = {}): MastodonAccount {
   const event: UnsignedEvent = {
     kind: 0,
     pubkey,

src/views/mastodon/statuses.ts

@@ -7,7 +7,7 @@ import { MastodonMention } from '@/entities/MastodonMention.ts';
 import { MastodonStatus } from '@/entities/MastodonStatus.ts';
 import { type DittoEvent } from '@/interfaces/DittoEvent.ts';
 import { Storages } from '@/storages.ts';
-import { isNostrId, nostrDate } from '@/utils.ts';
+import { nostrDate } from '@/utils.ts';
 import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts';
 import { findReplyTag } from '@/utils/tags.ts';
 import { unfurlCardCached } from '@/utils/unfurl.ts';
@@ -33,28 +33,14 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
   });
 
   const account = event.author
-    ? await renderAccount({ ...event.author, author_stats: event.author_stats })
-    : await accountFromPubkey(event.pubkey);
+    ? renderAccount({ ...event.author, author_stats: event.author_stats })
+    : accountFromPubkey(event.pubkey);
 
   const replyId = findReplyTag(event.tags)?.[1];
 
-  const mentionedPubkeys = [
-    ...new Set(
-      event.tags
-        .filter(([name, value]) => name === 'p' && isNostrId(value))
-        .map(([, value]) => value),
-    ),
-  ];
-
   const store = await Storages.db();
 
-  const mentionedProfiles = await store.query(
-    [{ kinds: [0], authors: mentionedPubkeys, limit: mentionedPubkeys.length }],
-  );
-
-  const mentions = await Promise.all(
-    mentionedPubkeys.map((pubkey) => renderMention(pubkey, mentionedProfiles.find((event) => event.pubkey === pubkey))),
-  );
+  const mentions = event.mentions?.map((event) => renderMention(event)) ?? [];
 
   const { html, links, firstUrl } = parseNoteContent(stripimeta(event.content, event.tags), mentions);
 
@@ -170,8 +156,8 @@ async function renderReblog(event: DittoEvent, opts: RenderStatusOpts): Promise<
   };
 }
 
-async function renderMention(pubkey: string, event?: NostrEvent): Promise<MastodonMention> {
-  const account = event ? await renderAccount(event) : await accountFromPubkey(pubkey);
+function renderMention(event: NostrEvent): MastodonMention {
+  const account = renderAccount(event);
   return {
     id: account.id,
     acct: account.acct,