Hook everything up? (In a messy way)

src/db/DittoTables.ts

@@ -23,6 +23,7 @@ interface AuthorStatsRow {
   nip05: string | null;
   nip05_domain: string | null;
   nip05_hostname: string | null;
+  nip05_last_verified_at: number | null;
 }
 
 interface EventStatsRow {

src/db/migrations/047_add_domain_favicons.ts

@@ -6,7 +6,7 @@ export async function up(db: Kysely<unknown>): Promise<void> {
     .addColumn('domain', 'varchar(253)', (col) => col.primaryKey())
     .addColumn('favicon', 'varchar(2048)', (col) => col.notNull())
     .addColumn('last_updated_at', 'integer', (col) => col.notNull())
-    .addCheckConstraint('domain_favicons_https_chk', sql`url ~* '^https:\\/\\/'`)
+    .addCheckConstraint('domain_favicons_https_chk', sql`favicon ~* '^https:\\/\\/'`)
     .execute();
 }
 

src/pipeline.ts

@@ -1,6 +1,6 @@
 import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify';
 import { logi } from '@soapbox/logi';
-import { Kysely, sql } from 'kysely';
+import { Kysely } from 'kysely';
 import { z } from 'zod';
 
 import { pipelineEncounters } from '@/caches/pipelineEncounters.ts';
@@ -13,8 +13,9 @@ import { RelayError } from '@/RelayError.ts';
 import { AdminSigner } from '@/signers/AdminSigner.ts';
 import { hydrateEvents } from '@/storages/hydrate.ts';
 import { Storages } from '@/storages.ts';
-import { eventAge, parseNip05, Time } from '@/utils.ts';
+import { eventAge, Time } from '@/utils.ts';
 import { getAmount } from '@/utils/bolt11.ts';
+import { faviconCache } from '@/utils/favicon.ts';
 import { errorJson } from '@/utils/log.ts';
 import { nip05Cache } from '@/utils/nip05.ts';
 import { purifyEvent } from '@/utils/purify.ts';
@@ -202,6 +203,12 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise<vo
   const { name, nip05 } = metadata.data;
   const result = nip05 ? await nip05Cache.fetch(nip05, { signal }).catch(() => undefined) : undefined;
 
+  // Fetch favicon.
+  const domain = nip05?.split('@')[1].toLowerCase();
+  if (domain) {
+    await faviconCache.fetch(domain, { signal });
+  }
+
   // Populate author_search.
   try {
     const search = result?.pubkey === event.pubkey ? [name, nip05].filter(Boolean).join(' ').trim() : name ?? '';
@@ -215,24 +222,6 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise<vo
   } catch {
     // do nothing
   }
-
-  if (nip05 && result && result.pubkey === event.pubkey) {
-    // Track pubkey domain.
-    try {
-      const { domain } = parseNip05(nip05);
-
-      await sql`
-      INSERT INTO pubkey_domains (pubkey, domain, last_updated_at)
-      VALUES (${event.pubkey}, ${domain}, ${event.created_at})
-      ON CONFLICT(pubkey) DO UPDATE SET
-        domain = excluded.domain,
-        last_updated_at = excluded.last_updated_at
-      WHERE excluded.last_updated_at > pubkey_domains.last_updated_at
-      `.execute(kysely);
-    } catch (_e) {
-      // do nothing
-    }
-  }
 }
 
 /** Determine if the event is being received in a timely manner. */

src/storages/hydrate.ts

@@ -122,6 +122,7 @@ export function assembleEvents(
       nip05: stat.nip05 ?? undefined,
       nip05_domain: stat.nip05_domain ?? undefined,
       nip05_hostname: stat.nip05_hostname ?? undefined,
+      nip05_last_verified_at: stat.nip05_last_verified_at ?? undefined,
       favicon: stats.favicons[stat.nip05_hostname!],
     };
     return result;

src/utils/favicon.ts

@@ -1,55 +1,91 @@
 import { DOMParser } from '@b-fuze/deno-dom';
 import { logi } from '@soapbox/logi';
+import { Kysely } from 'kysely';
 import tldts from 'tldts';
 
 import { Conf } from '@/config.ts';
+import { DittoTables } from '@/db/DittoTables.ts';
 import { cachedFaviconsSizeGauge } from '@/metrics.ts';
+import { Storages } from '@/storages.ts';
+import { nostrNow } from '@/utils.ts';
 import { SimpleLRU } from '@/utils/SimpleLRU.ts';
 import { fetchWorker } from '@/workers/fetch.ts';
 
-const faviconCache = new SimpleLRU<string, URL>(
+export const faviconCache = new SimpleLRU<string, URL>(
   async (domain, { signal }) => {
-    logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'started' });
-    const tld = tldts.parse(domain);
+    const kysely = await Storages.kysely();
 
-    if (!tld.isIcann || tld.isIp || tld.isPrivate) {
-      throw new Error(`Invalid favicon domain: ${domain}`);
+    const row = await queryFavicon(kysely, domain);
+
+    if (row && (nostrNow() - row.last_updated_at) < (Conf.caches.favicon.ttl / 1000)) {
+      return new URL(row.favicon);
     }
 
-    const rootUrl = new URL('/', `https://${domain}/`);
-    const response = await fetchWorker(rootUrl, { signal });
-    const html = await response.text();
+    const url = await fetchFavicon(domain, signal);
 
-    const doc = new DOMParser().parseFromString(html, 'text/html');
-    const link = doc.querySelector('link[rel="icon"], link[rel="shortcut icon"]');
+    insertFavicon(kysely, domain, url.href).catch(() => {});
 
-    if (link) {
-      const href = link.getAttribute('href');
-      if (href) {
-        let url: URL | undefined;
-
-        try {
-          url = new URL(href);
-        } catch {
-          try {
-            url = new URL(href, rootUrl);
-          } catch {
-            // fall through
-          }
-        }
-
-        if (url) {
-          logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'found', url });
-          return url;
-        }
-      }
-    }
-
-    logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'failed' });
-
-    throw new Error(`Favicon not found: ${domain}`);
+    return url;
   },
   { ...Conf.caches.favicon, gauge: cachedFaviconsSizeGauge },
 );
 
-export { faviconCache };
+async function queryFavicon(
+  kysely: Kysely<DittoTables>,
+  domain: string,
+): Promise<DittoTables['domain_favicons'] | undefined> {
+  return await kysely
+    .selectFrom('domain_favicons')
+    .selectAll()
+    .where('domain', '=', domain)
+    .executeTakeFirst();
+}
+
+async function insertFavicon(kysely: Kysely<DittoTables>, domain: string, favicon: string): Promise<void> {
+  await kysely
+    .insertInto('domain_favicons')
+    .values({ domain, favicon, last_updated_at: nostrNow() })
+    .execute();
+}
+
+async function fetchFavicon(domain: string, signal?: AbortSignal): Promise<URL> {
+  logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'started' });
+  const tld = tldts.parse(domain);
+
+  if (!tld.isIcann || tld.isIp || tld.isPrivate) {
+    throw new Error(`Invalid favicon domain: ${domain}`);
+  }
+
+  const rootUrl = new URL('/', `https://${domain}/`);
+  const response = await fetchWorker(rootUrl, { signal });
+  const html = await response.text();
+
+  const doc = new DOMParser().parseFromString(html, 'text/html');
+  const link = doc.querySelector('link[rel="icon"], link[rel="shortcut icon"]');
+
+  if (link) {
+    const href = link.getAttribute('href');
+    if (href) {
+      let url: URL | undefined;
+
+      try {
+        url = new URL(href);
+      } catch {
+        try {
+          url = new URL(href, rootUrl);
+        } catch {
+          // fall through
+        }
+      }
+
+      if (url) {
+        logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'found', url });
+        return url;
+      }
+    }
+  }
+
+  logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'failed' });
+
+  throw new Error(`Favicon not found: ${domain}`);
+}

src/utils/nip05.ts

@@ -1,51 +1,120 @@
 import { nip19 } from 'nostr-tools';
 import { NIP05, NStore } from '@nostrify/nostrify';
 import { logi } from '@soapbox/logi';
+import { Kysely } from 'kysely';
 import tldts from 'tldts';
 
 import { Conf } from '@/config.ts';
+import { DittoTables } from '@/db/DittoTables.ts';
 import { cachedNip05sSizeGauge } from '@/metrics.ts';
 import { Storages } from '@/storages.ts';
 import { errorJson } from '@/utils/log.ts';
 import { SimpleLRU } from '@/utils/SimpleLRU.ts';
-import { Nip05, parseNip05 } from '@/utils.ts';
+import { Nip05, nostrNow, parseNip05 } from '@/utils.ts';
 import { fetchWorker } from '@/workers/fetch.ts';
 
 export const nip05Cache = new SimpleLRU<string, nip19.ProfilePointer>(
   async (nip05, { signal }) => {
-    const tld = tldts.parse(nip05);
-
-    if (!tld.isIcann || tld.isIp || tld.isPrivate) {
-      throw new Error(`Invalid NIP-05: ${nip05}`);
-    }
-
-    const [name, domain] = nip05.split('@');
-
-    logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'started' });
-
-    try {
-      if (domain === Conf.url.host) {
-        const store = await Storages.db();
-        const pointer = await localNip05Lookup(store, name);
-        if (pointer) {
-          logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', pubkey: pointer.pubkey });
-          return pointer;
-        } else {
-          throw new Error(`Not found: ${nip05}`);
-        }
-      } else {
-        const result = await NIP05.lookup(nip05, { fetch: fetchWorker, signal });
-        logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', pubkey: result.pubkey });
-        return result;
-      }
-    } catch (e) {
-      logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'failed', error: errorJson(e) });
-      throw e;
-    }
+    const store = await Storages.db();
+    const kysely = await Storages.kysely();
+    return getNip05(kysely, store, nip05, signal);
   },
   { ...Conf.caches.nip05, gauge: cachedNip05sSizeGauge },
 );
 
+async function getNip05(
+  kysely: Kysely<DittoTables>,
+  store: NStore,
+  nip05: string,
+  signal?: AbortSignal,
+): Promise<nip19.ProfilePointer> {
+  const tld = tldts.parse(nip05);
+
+  if (!tld.isIcann || tld.isIp || tld.isPrivate) {
+    throw new Error(`Invalid NIP-05: ${nip05}`);
+  }
+
+  logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'started' });
+
+  let pointer: nip19.ProfilePointer | undefined = await queryNip05(kysely, nip05);
+
+  if (pointer) {
+    logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'db', pubkey: pointer.pubkey });
+    return pointer;
+  }
+
+  const [name, domain] = nip05.split('@');
+
+  try {
+    if (domain === Conf.url.host) {
+      pointer = await localNip05Lookup(store, name);
+      if (pointer) {
+        logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'local', pubkey: pointer.pubkey });
+      } else {
+        throw new Error(`Not found: ${nip05}`);
+      }
+    } else {
+      pointer = await NIP05.lookup(nip05, { fetch: fetchWorker, signal });
+      logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'fetch', pubkey: pointer.pubkey });
+    }
+  } catch (e) {
+    logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'failed', error: errorJson(e) });
+    throw e;
+  }
+
+  insertNip05(kysely, nip05, pointer.pubkey).catch((e) => {
+    logi({ level: 'error', ns: 'ditto.nip05', nip05, state: 'insert_failed', error: errorJson(e) });
+  });
+
+  return pointer;
+}
+
+async function queryNip05(kysely: Kysely<DittoTables>, nip05: string): Promise<nip19.ProfilePointer | undefined> {
+  const row = await kysely
+    .selectFrom('author_stats')
+    .select('pubkey')
+    .where('nip05', '=', nip05)
+    .executeTakeFirst();
+
+  if (row) {
+    return { pubkey: row.pubkey };
+  }
+}
+
+async function insertNip05(kysely: Kysely<DittoTables>, nip05: string, pubkey: string, ts = nostrNow()): Promise<void> {
+  const tld = tldts.parse(nip05);
+
+  if (!tld.isIcann || tld.isIp || tld.isPrivate) {
+    throw new Error(`Invalid NIP-05: ${nip05}`);
+  }
+
+  await kysely
+    .insertInto('author_stats')
+    .values({
+      pubkey,
+      nip05,
+      nip05_domain: tld.domain,
+      nip05_hostname: tld.hostname,
+      nip05_last_verified_at: ts,
+      followers_count: 0, // TODO: fix `author_stats` types so setting these aren't required
+      following_count: 0,
+      notes_count: 0,
+      search: nip05,
+    })
+    .onConflict((oc) =>
+      oc
+        .column('pubkey')
+        .doUpdateSet({
+          nip05,
+          nip05_domain: tld.domain,
+          nip05_hostname: tld.hostname,
+          nip05_last_verified_at: ts,
+        })
+        .where('nip05_last_verified_at', '<', ts)
+    )
+    .execute();
+}
+
 export async function localNip05Lookup(store: NStore, localpart: string): Promise<nip19.ProfilePointer | undefined> {
   const [grant] = await store.query([{
     kinds: [30360],

src/utils/stats.ts

@@ -326,6 +326,7 @@ export async function countAuthorStats(
     nip05: null,
     nip05_domain: null,
     nip05_hostname: null,
+    nip05_last_verified_at: null,
   };
 }