Merge branch 'main' into cashu

Conflicts:
	deno.json

deno.json

@@ -21,6 +21,7 @@
     "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip -o soapbox.zip && rm soapbox.zip",
     "trends": "deno run -A --env-file --deny-read=.env scripts/trends.ts",
     "clean:deps": "deno cache --reload src/app.ts",
+    "db:populate:nip05": "deno run -A --env-file --deny-read=.env scripts/db-populate-nip05.ts",
     "db:populate-search": "deno run -A --env-file --deny-read=.env scripts/db-populate-search.ts",
     "db:populate-extensions": "deno run -A --env-file --deny-read=.env scripts/db-populate-extensions.ts",
     "db:streak:recompute": "deno run -A --env-file --deny-read=.env scripts/db-streak-recompute.ts",
@@ -40,12 +41,12 @@
     "@b-fuze/deno-dom": "jsr:@b-fuze/deno-dom@^0.1.47",
     "@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/s3-lite-client@^0.7.4",
     "@cashu/cashu-ts": "npm:@cashu/cashu-ts@^2.2.0",
+    "@core/asyncutil": "jsr:@core/asyncutil@^1.2.0",
     "@electric-sql/pglite": "npm:@electric-sql/pglite@^0.2.8",
     "@esroyo/scoped-performance": "jsr:@esroyo/scoped-performance@^3.1.0",
     "@gfx/canvas-wasm": "jsr:@gfx/canvas-wasm@^0.4.2",
     "@hono/hono": "jsr:@hono/hono@^4.4.6",
     "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1",
-    "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1",
     "@negrel/webpush": "jsr:@negrel/webpush@^0.3.0",
     "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0",
     "@nostrify/db": "jsr:@nostrify/db@^0.38.0",

deno.lock

@@ -3,6 +3,7 @@
   "specifiers": {
     "jsr:@b-fuze/deno-dom@~0.1.47": "0.1.48",
     "jsr:@bradenmacdonald/s3-lite-client@~0.7.4": "0.7.6",
+    "jsr:@core/asyncutil@^1.2.0": "1.2.0",
     "jsr:@denosaurs/plug@1.0.3": "1.0.3",
     "jsr:@esroyo/scoped-performance@^3.1.0": "3.1.0",
     "jsr:@gfx/canvas-wasm@~0.4.2": "0.4.2",
@@ -28,7 +29,6 @@
     "jsr:@gleasonator/policy@0.9.3": "0.9.3",
     "jsr:@gleasonator/policy@0.9.4": "0.9.4",
     "jsr:@hono/hono@^4.4.6": "4.6.15",
-    "jsr:@lambdalisue/async@^2.1.1": "2.1.1",
     "jsr:@negrel/http-ece@0.6.0": "0.6.0",
     "jsr:@negrel/webpush@0.3": "0.3.0",
     "jsr:@nostrify/db@0.38": "0.38.0",
@@ -154,6 +154,9 @@
         "jsr:@std/io@0.224"
       ]
     },
+    "@core/asyncutil@1.2.0": {
+      "integrity": "9967f15190c60df032c13f72ce5ac73d185c34f31c53dc918d8800025854c118"
+    },
     "@denosaurs/plug@1.0.3": {
       "integrity": "b010544e386bea0ff3a1d05e0c88f704ea28cbd4d753439c2f1ee021a85d4640",
       "dependencies": [
@@ -338,9 +341,6 @@
     "@hono/hono@4.6.15": {
       "integrity": "935b3b12e98e4b22bcd1aa4dbe6587321e431c79829eba61f535b4ede39fd8b1"
     },
-    "@lambdalisue/async@2.1.1": {
-      "integrity": "1fc9bc6f4ed50215cd2f7217842b18cea80f81c25744f88f8c5eb4be5a1c9ab4"
-    },
     "@negrel/http-ece@0.6.0": {
       "integrity": "7afdd81b86ea5b21a9677b323c01c3338705e11cc2bfed250870f5349d8f86f7",
       "dependencies": [
@@ -2427,10 +2427,10 @@
     "dependencies": [
       "jsr:@b-fuze/deno-dom@~0.1.47",
       "jsr:@bradenmacdonald/s3-lite-client@~0.7.4",
+      "jsr:@core/asyncutil@^1.2.0",
       "jsr:@esroyo/scoped-performance@^3.1.0",
       "jsr:@gfx/canvas-wasm@~0.4.2",
       "jsr:@hono/hono@^4.4.6",
-      "jsr:@lambdalisue/async@^2.1.1",
       "jsr:@negrel/webpush@0.3",
       "jsr:@nostrify/db@0.38",
       "jsr:@nostrify/nostrify@~0.38.1",

scripts/db-import.ts

@@ -1,4 +1,4 @@
-import { Semaphore } from '@lambdalisue/async';
+import { Semaphore } from '@core/asyncutil';
 import { NostrEvent } from '@nostrify/nostrify';
 import { JsonParseStream } from '@std/json/json-parse-stream';
 import { TextLineStream } from '@std/streams/text-line-stream';

scripts/db-populate-nip05.ts

@@ -0,0 +1,26 @@
+import { Semaphore } from '@core/asyncutil';
+
+import { updateAuthorData } from '@/pipeline.ts';
+import { Storages } from '@/storages.ts';
+import { NostrEvent } from '@nostrify/nostrify';
+
+const kysely = await Storages.kysely();
+const sem = new Semaphore(5);
+
+const query = kysely
+  .selectFrom('nostr_events')
+  .select(['id', 'kind', 'content', 'pubkey', 'tags', 'created_at', 'sig'])
+  .where('kind', '=', 0);
+
+for await (const row of query.stream(100)) {
+  while (sem.locked) {
+    await new Promise((resolve) => setTimeout(resolve, 0));
+  }
+
+  sem.lock(async () => {
+    const event: NostrEvent = { ...row, created_at: Number(row.created_at) };
+    await updateAuthorData(event, AbortSignal.timeout(3000));
+  });
+}
+
+Deno.exit();

src/db/DittoTables.ts

@@ -5,6 +5,7 @@ import { NPostgresSchema } from '@nostrify/db';
 export interface DittoTables extends NPostgresSchema {
   auth_tokens: AuthTokenRow;
   author_stats: AuthorStatsRow;
+  domain_favicons: DomainFaviconRow;
   event_stats: EventStatsRow;
   pubkey_domains: PubkeyDomainRow;
   event_zaps: EventZapRow;
@@ -19,6 +20,10 @@ interface AuthorStatsRow {
   search: string;
   streak_start: number | null;
   streak_end: number | null;
+  nip05: string | null;
+  nip05_domain: string | null;
+  nip05_hostname: string | null;
+  nip05_last_verified_at: number | null;
 }
 
 interface EventStatsRow {
@@ -46,6 +51,12 @@ interface PubkeyDomainRow {
   last_updated_at: number;
 }
 
+interface DomainFaviconRow {
+  domain: string;
+  favicon: string;
+  last_updated_at: number;
+}
+
 interface EventZapRow {
   receipt_id: string;
   target_event_id: string;

src/db/migrations/046_author_stats_nip05.ts

@@ -0,0 +1,48 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .alterTable('author_stats')
+    .addColumn('nip05', 'varchar(320)')
+    .addColumn('nip05_domain', 'varchar(253)')
+    .addColumn('nip05_hostname', 'varchar(253)')
+    .addColumn('nip05_last_verified_at', 'integer')
+    .execute();
+
+  await db.schema
+    .alterTable('author_stats')
+    .addCheckConstraint('author_stats_nip05_domain_lowercase_chk', sql`nip05_domain = lower(nip05_domain)`)
+    .execute();
+
+  await db.schema
+    .alterTable('author_stats')
+    .addCheckConstraint('author_stats_nip05_hostname_lowercase_chk', sql`nip05_hostname = lower(nip05_hostname)`)
+    .execute();
+
+  await db.schema
+    .alterTable('author_stats')
+    .addCheckConstraint('author_stats_nip05_hostname_domain_chk', sql`nip05_hostname like '%' || nip05_domain`)
+    .execute();
+
+  await db.schema
+    .createIndex('author_stats_nip05_domain_idx')
+    .on('author_stats')
+    .column('nip05_domain')
+    .execute();
+
+  await db.schema
+    .createIndex('author_stats_nip05_hostname_idx')
+    .on('author_stats')
+    .column('nip05_hostname')
+    .execute();
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .alterTable('author_stats')
+    .dropColumn('nip05')
+    .dropColumn('nip05_domain')
+    .dropColumn('nip05_hostname')
+    .dropColumn('nip05_last_verified_at')
+    .execute();
+}

src/db/migrations/047_add_domain_favicons.ts

@@ -0,0 +1,15 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createTable('domain_favicons')
+    .addColumn('domain', 'varchar(253)', (col) => col.primaryKey())
+    .addColumn('favicon', 'varchar(2048)', (col) => col.notNull())
+    .addColumn('last_updated_at', 'integer', (col) => col.notNull())
+    .addCheckConstraint('domain_favicons_https_chk', sql`favicon ~* '^https:\\/\\/'`)
+    .execute();
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropTable('domain_favicons').execute();
+}

src/firehose.ts

@@ -1,4 +1,4 @@
-import { Semaphore } from '@lambdalisue/async';
+import { Semaphore } from '@core/asyncutil';
 import { logi } from '@soapbox/logi';
 
 import { Conf } from '@/config.ts';

src/interfaces/DittoEvent.ts

@@ -8,6 +8,11 @@ export interface AuthorStats {
   notes_count: number;
   streak_start?: number;
   streak_end?: number;
+  nip05?: string;
+  nip05_domain?: string;
+  nip05_hostname?: string;
+  nip05_last_verified_at?: number;
+  favicon?: string;
 }
 
 /** Ditto internal stats for the event. */
@@ -25,6 +30,7 @@ export interface DittoEvent extends NostrEvent {
   author_domain?: string;
   author_stats?: AuthorStats;
   event_stats?: EventStats;
+  mentions?: DittoEvent[];
   user?: DittoEvent;
   repost?: DittoEvent;
   quote?: DittoEvent;

src/middleware/translatorMiddleware.ts

@@ -1,6 +1,7 @@
+import { safeFetch } from '@soapbox/safe-fetch';
+
 import { AppMiddleware } from '@/app.ts';
 import { Conf } from '@/config.ts';
-import { fetchWorker } from '@/workers/fetch.ts';
 import { DeepLTranslator } from '@/translators/DeepLTranslator.ts';
 import { LibreTranslateTranslator } from '@/translators/LibreTranslateTranslator.ts';
 
@@ -10,7 +11,7 @@ export const translatorMiddleware: AppMiddleware = async (c, next) => {
     case 'deepl': {
       const { deeplApiKey: apiKey, deeplBaseUrl: baseUrl } = Conf;
       if (apiKey) {
-        c.set('translator', new DeepLTranslator({ baseUrl, apiKey, fetch: fetchWorker }));
+        c.set('translator', new DeepLTranslator({ baseUrl, apiKey, fetch: safeFetch }));
       }
       break;
     }
@@ -18,7 +19,7 @@ export const translatorMiddleware: AppMiddleware = async (c, next) => {
     case 'libretranslate': {
       const { libretranslateApiKey: apiKey, libretranslateBaseUrl: baseUrl } = Conf;
       if (apiKey) {
-        c.set('translator', new LibreTranslateTranslator({ baseUrl, apiKey, fetch: fetchWorker }));
+        c.set('translator', new LibreTranslateTranslator({ baseUrl, apiKey, fetch: safeFetch }));
       }
       break;
     }

src/middleware/uploaderMiddleware.ts

@@ -1,11 +1,11 @@
 import { BlossomUploader, NostrBuildUploader } from '@nostrify/nostrify/uploaders';
+import { safeFetch } from '@soapbox/safe-fetch';
 
 import { AppMiddleware } from '@/app.ts';
 import { Conf } from '@/config.ts';
 import { DenoUploader } from '@/uploaders/DenoUploader.ts';
 import { IPFSUploader } from '@/uploaders/IPFSUploader.ts';
 import { S3Uploader } from '@/uploaders/S3Uploader.ts';
-import { fetchWorker } from '@/workers/fetch.ts';
 
 /** Set an uploader for the user. */
 export const uploaderMiddleware: AppMiddleware = async (c, next) => {
@@ -29,17 +29,17 @@ export const uploaderMiddleware: AppMiddleware = async (c, next) => {
       );
       break;
     case 'ipfs':
-      c.set('uploader', new IPFSUploader({ baseUrl: Conf.mediaDomain, apiUrl: Conf.ipfs.apiUrl, fetch: fetchWorker }));
+      c.set('uploader', new IPFSUploader({ baseUrl: Conf.mediaDomain, apiUrl: Conf.ipfs.apiUrl, fetch: safeFetch }));
       break;
     case 'local':
       c.set('uploader', new DenoUploader({ baseUrl: Conf.mediaDomain, dir: Conf.uploadsDir }));
       break;
     case 'nostrbuild':
-      c.set('uploader', new NostrBuildUploader({ endpoint: Conf.nostrbuildEndpoint, signer, fetch: fetchWorker }));
+      c.set('uploader', new NostrBuildUploader({ endpoint: Conf.nostrbuildEndpoint, signer, fetch: safeFetch }));
       break;
     case 'blossom':
       if (signer) {
-        c.set('uploader', new BlossomUploader({ servers: Conf.blossomServers, signer, fetch: fetchWorker }));
+        c.set('uploader', new BlossomUploader({ servers: Conf.blossomServers, signer, fetch: safeFetch }));
       }
       break;
   }

src/notify.ts

@@ -1,4 +1,4 @@
-import { Semaphore } from '@lambdalisue/async';
+import { Semaphore } from '@core/asyncutil';
 
 import { pipelineEncounters } from '@/caches/pipelineEncounters.ts';
 import { Conf } from '@/config.ts';

src/pipeline.ts

@@ -1,6 +1,7 @@
 import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify';
 import { logi } from '@soapbox/logi';
-import { Kysely, sql } from 'kysely';
+import { Kysely, UpdateObject } from 'kysely';
+import tldts from 'tldts';
 import { z } from 'zod';
 
 import { pipelineEncounters } from '@/caches/pipelineEncounters.ts';
@@ -13,8 +14,9 @@ import { RelayError } from '@/RelayError.ts';
 import { AdminSigner } from '@/signers/AdminSigner.ts';
 import { hydrateEvents } from '@/storages/hydrate.ts';
 import { Storages } from '@/storages.ts';
-import { eventAge, parseNip05, Time } from '@/utils.ts';
+import { eventAge, Time } from '@/utils.ts';
 import { getAmount } from '@/utils/bolt11.ts';
+import { faviconCache } from '@/utils/favicon.ts';
 import { errorJson } from '@/utils/log.ts';
 import { nip05Cache } from '@/utils/nip05.ts';
 import { purifyEvent } from '@/utils/purify.ts';
@@ -119,7 +121,7 @@ async function handleEvent(event: DittoEvent, opts: PipelineOpts): Promise<void>
     // This needs to run in steps, and should not block the API from responding.
     Promise.allSettled([
       handleZaps(kysely, event),
-      parseMetadata(event, opts.signal),
+      updateAuthorData(event, opts.signal),
       generateSetEvents(event),
     ])
       .then(() =>
@@ -189,50 +191,81 @@ async function storeEvent(event: NostrEvent, signal?: AbortSignal): Promise<unde
 }
 
 /** Parse kind 0 metadata and track indexes in the database. */
-async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise<void> {
+async function updateAuthorData(event: NostrEvent, signal: AbortSignal): Promise<void> {
   if (event.kind !== 0) return;
 
   // Parse metadata.
   const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(event.content);
   if (!metadata.success) return;
 
+  const { name, nip05 } = metadata.data;
+
   const kysely = await Storages.kysely();
 
-  // Get nip05.
+  const updates: UpdateObject<DittoTables, 'author_stats'> = {};
-  const { name, nip05 } = metadata.data;
+
-  const result = nip05 ? await nip05Cache.fetch(nip05, { signal }).catch(() => undefined) : undefined;
+  const authorStats = await kysely
+    .selectFrom('author_stats')
+    .selectAll()
+    .where('pubkey', '=', event.pubkey)
+    .executeTakeFirst();
+
+  const lastVerified = authorStats?.nip05_last_verified_at;
+  const eventNewer = !lastVerified || event.created_at > lastVerified;
 
-  // Populate author_search.
   try {
-    const search = result?.pubkey === event.pubkey ? [name, nip05].filter(Boolean).join(' ').trim() : name ?? '';
+    if (nip05 !== authorStats?.nip05 && eventNewer || !lastVerified) {
-
+      if (nip05) {
-    if (search) {
+        const tld = tldts.parse(nip05);
-      await kysely.insertInto('author_stats')
+        if (tld.isIcann && !tld.isIp && !tld.isPrivate) {
-        .values({ pubkey: event.pubkey, search, followers_count: 0, following_count: 0, notes_count: 0 })
+          const pointer = await nip05Cache.fetch(nip05.toLowerCase(), { signal });
-        .onConflict((oc) => oc.column('pubkey').doUpdateSet({ search }))
+          if (pointer.pubkey === event.pubkey) {
-        .execute();
+            updates.nip05 = nip05;
+            updates.nip05_domain = tld.domain;
+            updates.nip05_hostname = tld.hostname;
+            updates.nip05_last_verified_at = event.created_at;
+          }
+        }
+      } else {
+        updates.nip05 = null;
+        updates.nip05_domain = null;
+        updates.nip05_hostname = null;
+        updates.nip05_last_verified_at = event.created_at;
+      }
     }
   } catch {
-    // do nothing
+    // Fallthrough.
   }
 
-  if (nip05 && result && result.pubkey === event.pubkey) {
+  // Fetch favicon.
-    // Track pubkey domain.
+  const domain = nip05?.split('@')[1].toLowerCase();
+  if (domain) {
     try {
-      const { domain } = parseNip05(nip05);
+      await faviconCache.fetch(domain, { signal });
-
+    } catch {
-      await sql`
+      // Fallthrough.
-      INSERT INTO pubkey_domains (pubkey, domain, last_updated_at)
-      VALUES (${event.pubkey}, ${domain}, ${event.created_at})
-      ON CONFLICT(pubkey) DO UPDATE SET
-        domain = excluded.domain,
-        last_updated_at = excluded.last_updated_at
-      WHERE excluded.last_updated_at > pubkey_domains.last_updated_at
-      `.execute(kysely);
-    } catch (_e) {
-      // do nothing
     }
   }
+
+  const search = [name, nip05].filter(Boolean).join(' ').trim();
+
+  if (search !== authorStats?.search) {
+    updates.search = search;
+  }
+
+  if (Object.keys(updates).length) {
+    await kysely.insertInto('author_stats')
+      .values({
+        pubkey: event.pubkey,
+        followers_count: 0,
+        following_count: 0,
+        notes_count: 0,
+        search,
+        ...updates,
+      })
+      .onConflict((oc) => oc.column('pubkey').doUpdateSet(updates))
+      .execute();
+  }
 }
 
 /** Determine if the event is being received in a timely manner. */
@@ -364,4 +397,4 @@ async function handleZaps(kysely: Kysely<DittoTables>, event: NostrEvent) {
   }
 }
 
-export { handleEvent, handleZaps };
+export { handleEvent, handleZaps, updateAuthorData };

src/storages/hydrate.ts

@@ -42,6 +42,10 @@ async function hydrateEvents(opts: HydrateOpts): Promise<DittoEvent[]> {
     cache.push(event);
   }
 
+  for (const event of await gatherMentions({ events: cache, store, signal })) {
+    cache.push(event);
+  }
+
   for (const event of await gatherAuthors({ events: cache, store, signal })) {
     cache.push(event);
   }
@@ -66,9 +70,30 @@ async function hydrateEvents(opts: HydrateOpts): Promise<DittoEvent[]> {
     cache.push(event);
   }
 
+  const authorStats = await gatherAuthorStats(cache, kysely as Kysely<DittoTables>);
+  const eventStats = await gatherEventStats(cache, kysely as Kysely<DittoTables>);
+
+  const domains = authorStats.reduce((result, { nip05_hostname }) => {
+    if (nip05_hostname) result.add(nip05_hostname);
+    return result;
+  }, new Set<string>());
+
+  const favicons = (
+    await kysely
+      .selectFrom('domain_favicons')
+      .select(['domain', 'favicon'])
+      .where('domain', 'in', [...domains])
+      .execute()
+  )
+    .reduce((result, { domain, favicon }) => {
+      result[domain] = favicon;
+      return result;
+    }, {} as Record<string, string>);
+
   const stats = {
-    authors: await gatherAuthorStats(cache, kysely as Kysely<DittoTables>),
+    authors: authorStats,
-    events: await gatherEventStats(cache, kysely as Kysely<DittoTables>),
+    events: eventStats,
+    favicons,
   };
 
   // Dedupe events.
@@ -85,7 +110,11 @@ async function hydrateEvents(opts: HydrateOpts): Promise<DittoEvent[]> {
 export function assembleEvents(
   a: DittoEvent[],
   b: DittoEvent[],
-  stats: { authors: DittoTables['author_stats'][]; events: DittoTables['event_stats'][] },
+  stats: {
+    authors: DittoTables['author_stats'][];
+    events: DittoTables['event_stats'][];
+    favicons: Record<string, string>;
+  },
 ): DittoEvent[] {
   const admin = Conf.pubkey;
 
@@ -94,6 +123,11 @@ export function assembleEvents(
       ...stat,
       streak_start: stat.streak_start ?? undefined,
       streak_end: stat.streak_end ?? undefined,
+      nip05: stat.nip05 ?? undefined,
+      nip05_domain: stat.nip05_domain ?? undefined,
+      nip05_hostname: stat.nip05_hostname ?? undefined,
+      nip05_last_verified_at: stat.nip05_last_verified_at ?? undefined,
+      favicon: stats.favicons[stat.nip05_hostname!],
     };
     return result;
   }, {} as Record<string, DittoEvent['author_stats']>);
@@ -116,6 +150,9 @@ export function assembleEvents(
       if (id) {
         event.quote = b.find((e) => matchFilter({ kinds: [1, 20], ids: [id] }, e));
       }
+
+      const pubkeys = event.tags.filter(([name]) => name === 'p').map(([_name, value]) => value);
+      event.mentions = b.filter((e) => matchFilter({ kinds: [0], authors: pubkeys }, e));
     }
 
     if (event.kind === 6) {
@@ -237,6 +274,36 @@ function gatherQuotes({ events, store, signal }: HydrateOpts): Promise<DittoEven
   );
 }
 
+/** Collect mentioned profiles from notes. */
+async function gatherMentions({ events, store, signal }: HydrateOpts): Promise<DittoEvent[]> {
+  const pubkeys = new Set<string>();
+
+  for (const event of events) {
+    if (event.kind === 1) {
+      for (const [name, value] of event.tags) {
+        if (name === 'p') {
+          pubkeys.add(value);
+        }
+      }
+    }
+  }
+
+  const authors = await store.query(
+    [{ kinds: [0], authors: [...pubkeys], limit: pubkeys.size }],
+    { signal },
+  );
+
+  for (const pubkey of pubkeys) {
+    const author = authors.find((e) => matchFilter({ kinds: [0], authors: [pubkey] }, e));
+    if (!author) {
+      const fallback = fallbackAuthor(pubkey);
+      authors.push(fallback);
+    }
+  }
+
+  return authors;
+}
+
 /** Collect authors from the events. */
 async function gatherAuthors({ events, store, signal }: HydrateOpts): Promise<DittoEvent[]> {
   const pubkeys = new Set<string>();
@@ -267,7 +334,7 @@ async function gatherAuthors({ events, store, signal }: HydrateOpts): Promise<Di
 
   for (const pubkey of pubkeys) {
     const author = authors.find((e) => matchFilter({ kinds: [0], authors: [pubkey] }, e));
-    if (author) {
+    if (!author) {
       const fallback = fallbackAuthor(pubkey);
       authors.push(fallback);
     }
@@ -390,13 +457,10 @@ async function gatherAuthorStats(
     .execute();
 
   return rows.map((row) => ({
-    pubkey: row.pubkey,
+    ...row,
     followers_count: Math.max(0, row.followers_count),
     following_count: Math.max(0, row.following_count),
     notes_count: Math.max(0, row.notes_count),
-    search: row.search,
-    streak_start: row.streak_start,
-    streak_end: row.streak_end,
   }));
 }
 

src/utils/SimpleLRU.test.ts

@@ -4,7 +4,7 @@ import { assertEquals, assertRejects } from '@std/assert';
 Deno.test("SimpleLRU doesn't repeat failed calls", async () => {
   let calls = 0;
 
-  const cache = new SimpleLRU(
+  using cache = new SimpleLRU(
     // deno-lint-ignore require-await
     async () => {
       calls++;

src/utils/SimpleLRU.ts

@@ -3,50 +3,55 @@
 import { LRUCache } from 'lru-cache';
 import { type Gauge } from 'prom-client';
 
-type FetchFn<K extends {}, V extends {}, O extends {}> = (key: K, opts: O) => Promise<V>;
+type FetchFn<K extends {}, V extends {}> = (key: K, opts: { signal?: AbortSignal }) => Promise<V>;
-
-interface FetchFnOpts {
-  signal?: AbortSignal | null;
-}
 
 type SimpleLRUOpts<K extends {}, V extends {}> = LRUCache.Options<K, V, void> & {
   gauge?: Gauge;
+  errorRefresh?: number;
 };
 
 export class SimpleLRU<
   K extends {},
   V extends {},
-  O extends {} = FetchFnOpts,
 > {
-  protected cache: LRUCache<K, V, void>;
+  protected cache: LRUCache<K, Promise<V>, void>;
+  private tids = new Set<number>();
 
-  constructor(fetchFn: FetchFn<K, V, { signal: AbortSignal }>, private opts: SimpleLRUOpts<K, V>) {
+  constructor(private fetchFn: FetchFn<K, V>, private opts: SimpleLRUOpts<K, Promise<V>>) {
-    this.cache = new LRUCache({
+    this.cache = new LRUCache({ ...opts });
-      async fetchMethod(key, _staleValue, { signal }) {
-        try {
-          return await fetchFn(key, { signal: signal as unknown as AbortSignal });
-        } catch {
-          return null as unknown as V;
-        }
-      },
-      ...opts,
-    });
   }
 
-  async fetch(key: K, opts?: O): Promise<V> {
+  async fetch(key: K, opts?: { signal?: AbortSignal }): Promise<V> {
-    const result = await this.cache.fetch(key, opts);
+    if (opts?.signal?.aborted) {
+      throw new DOMException('The signal has been aborted', 'AbortError');
+    }
 
+    const cached = await this.cache.get(key);
+
+    if (cached) {
+      return cached;
+    }
+
+    const promise = this.fetchFn(key, { signal: opts?.signal });
+
+    this.cache.set(key, promise);
+
+    promise.then(() => {
       this.opts.gauge?.set(this.cache.size);
+    }).catch(() => {
+      const tid = setTimeout(() => {
+        this.cache.delete(key);
+        this.tids.delete(tid);
+      }, this.opts.errorRefresh ?? 10_000);
+      this.tids.add(tid);
+    });
 
-    if (result === undefined || result === null) {
+    return promise;
-      throw new Error('SimpleLRU: fetch failed');
   }
 
-    return result;
+  [Symbol.dispose](): void {
+    for (const tid of this.tids) {
+      clearTimeout(tid);
     }
-
-  put(key: K, value: V): Promise<void> {
-    this.cache.set(key, value);
-    return Promise.resolve();
   }
 }

src/utils/favicon.ts

@@ -1,14 +1,55 @@
 import { DOMParser } from '@b-fuze/deno-dom';
 import { logi } from '@soapbox/logi';
+import { safeFetch } from '@soapbox/safe-fetch';
+import { Kysely } from 'kysely';
 import tldts from 'tldts';
 
 import { Conf } from '@/config.ts';
+import { DittoTables } from '@/db/DittoTables.ts';
 import { cachedFaviconsSizeGauge } from '@/metrics.ts';
+import { Storages } from '@/storages.ts';
+import { nostrNow } from '@/utils.ts';
 import { SimpleLRU } from '@/utils/SimpleLRU.ts';
-import { fetchWorker } from '@/workers/fetch.ts';
 
-const faviconCache = new SimpleLRU<string, URL>(
+export const faviconCache = new SimpleLRU<string, URL>(
   async (domain, { signal }) => {
+    const kysely = await Storages.kysely();
+
+    const row = await queryFavicon(kysely, domain);
+
+    if (row && (nostrNow() - row.last_updated_at) < (Conf.caches.favicon.ttl / 1000)) {
+      return new URL(row.favicon);
+    }
+
+    const url = await fetchFavicon(domain, signal);
+
+    await insertFavicon(kysely, domain, url.href);
+
+    return url;
+  },
+  { ...Conf.caches.favicon, gauge: cachedFaviconsSizeGauge },
+);
+
+async function queryFavicon(
+  kysely: Kysely<DittoTables>,
+  domain: string,
+): Promise<DittoTables['domain_favicons'] | undefined> {
+  return await kysely
+    .selectFrom('domain_favicons')
+    .selectAll()
+    .where('domain', '=', domain)
+    .executeTakeFirst();
+}
+
+async function insertFavicon(kysely: Kysely<DittoTables>, domain: string, favicon: string): Promise<void> {
+  await kysely
+    .insertInto('domain_favicons')
+    .values({ domain, favicon, last_updated_at: nostrNow() })
+    .onConflict((oc) => oc.column('domain').doUpdateSet({ favicon, last_updated_at: nostrNow() }))
+    .execute();
+}
+
+async function fetchFavicon(domain: string, signal?: AbortSignal): Promise<URL> {
   logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'started' });
   const tld = tldts.parse(domain);
 
@@ -17,7 +58,7 @@ const faviconCache = new SimpleLRU<string, URL>(
   }
 
   const rootUrl = new URL('/', `https://${domain}/`);
-    const response = await fetchWorker(rootUrl, { signal });
+  const response = await safeFetch(rootUrl, { signal });
   const html = await response.text();
 
   const doc = new DOMParser().parseFromString(html, 'text/html');
@@ -45,11 +86,17 @@ const faviconCache = new SimpleLRU<string, URL>(
     }
   }
 
+  // Fallback to checking `/favicon.ico` of the domain.
+  const url = new URL('/favicon.ico', `https://${domain}/`);
+  const fallback = await safeFetch(url, { method: 'HEAD', signal });
+  const contentType = fallback.headers.get('content-type');
+
+  if (fallback.ok && contentType === 'image/vnd.microsoft.icon') {
+    logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'found', url });
+    return url;
+  }
+
   logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'failed' });
 
   throw new Error(`Favicon not found: ${domain}`);
-  },
+}
-  { ...Conf.caches.favicon, gauge: cachedFaviconsSizeGauge },
-);
-
-export { faviconCache };

src/utils/lnurl.ts

@@ -1,19 +1,19 @@
 import { NostrEvent } from '@nostrify/nostrify';
 import { LNURL, LNURLDetails } from '@nostrify/nostrify/ln';
 import { logi } from '@soapbox/logi';
+import { safeFetch } from '@soapbox/safe-fetch';
 import { JsonValue } from '@std/json';
 
 import { cachedLnurlsSizeGauge } from '@/metrics.ts';
 import { SimpleLRU } from '@/utils/SimpleLRU.ts';
 import { errorJson } from '@/utils/log.ts';
 import { Time } from '@/utils/time.ts';
-import { fetchWorker } from '@/workers/fetch.ts';
 
 const lnurlCache = new SimpleLRU<string, LNURLDetails>(
   async (lnurl, { signal }) => {
     logi({ level: 'info', ns: 'ditto.lnurl', lnurl, state: 'started' });
     try {
-      const details = await LNURL.lookup(lnurl, { fetch: fetchWorker, signal });
+      const details = await LNURL.lookup(lnurl, { fetch: safeFetch, signal });
       logi({ level: 'info', ns: 'ditto.lnurl', lnurl, state: 'found', details: details as unknown as JsonValue });
       return details;
     } catch (e) {
@@ -62,7 +62,7 @@ async function getInvoice(params: CallbackParams, signal?: AbortSignal): Promise
   const { pr } = await LNURL.callback(
     details.callback,
     params,
-    { fetch: fetchWorker, signal },
+    { fetch: safeFetch, signal },
   );
 
   return pr;

src/utils/nip05.ts

@@ -1,6 +1,7 @@
 import { nip19 } from 'nostr-tools';
 import { NIP05, NStore } from '@nostrify/nostrify';
 import { logi } from '@soapbox/logi';
+import { safeFetch } from '@soapbox/safe-fetch';
 import tldts from 'tldts';
 
 import { Conf } from '@/config.ts';
@@ -8,45 +9,51 @@ import { cachedNip05sSizeGauge } from '@/metrics.ts';
 import { Storages } from '@/storages.ts';
 import { errorJson } from '@/utils/log.ts';
 import { SimpleLRU } from '@/utils/SimpleLRU.ts';
-import { Nip05, parseNip05 } from '@/utils.ts';
-import { fetchWorker } from '@/workers/fetch.ts';
 
-const nip05Cache = new SimpleLRU<string, nip19.ProfilePointer>(
+export const nip05Cache = new SimpleLRU<string, nip19.ProfilePointer>(
   async (nip05, { signal }) => {
+    const store = await Storages.db();
+    return getNip05(store, nip05, signal);
+  },
+  { ...Conf.caches.nip05, gauge: cachedNip05sSizeGauge },
+);
+
+async function getNip05(
+  store: NStore,
+  nip05: string,
+  signal?: AbortSignal,
+): Promise<nip19.ProfilePointer> {
   const tld = tldts.parse(nip05);
 
   if (!tld.isIcann || tld.isIp || tld.isPrivate) {
     throw new Error(`Invalid NIP-05: ${nip05}`);
   }
 
-    const [name, domain] = nip05.split('@');
-
   logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'started' });
 
+  const [name, domain] = nip05.split('@');
+
   try {
     if (domain === Conf.url.host) {
-        const store = await Storages.db();
       const pointer = await localNip05Lookup(store, name);
       if (pointer) {
-          logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', pubkey: pointer.pubkey });
+        logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'local', pubkey: pointer.pubkey });
         return pointer;
       } else {
         throw new Error(`Not found: ${nip05}`);
       }
     } else {
-        const result = await NIP05.lookup(nip05, { fetch: fetchWorker, signal });
+      const pointer = await NIP05.lookup(nip05, { fetch: safeFetch, signal });
-        logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', pubkey: result.pubkey });
+      logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'fetch', pubkey: pointer.pubkey });
-        return result;
+      return pointer;
     }
   } catch (e) {
     logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'failed', error: errorJson(e) });
     throw e;
   }
-  },
+}
-  { ...Conf.caches.nip05, gauge: cachedNip05sSizeGauge },
-);
 
-async function localNip05Lookup(store: NStore, localpart: string): Promise<nip19.ProfilePointer | undefined> {
+export async function localNip05Lookup(store: NStore, localpart: string): Promise<nip19.ProfilePointer | undefined> {
   const [grant] = await store.query([{
     kinds: [30360],
     '#d': [`${localpart}@${Conf.url.host}`],
@@ -60,21 +67,3 @@ async function localNip05Lookup(store: NStore, localpart: string): Promise<nip19
     return { pubkey, relays: [Conf.relay] };
   }
 }
-
-export async function parseAndVerifyNip05(
-  nip05: string | undefined,
-  pubkey: string,
-  signal = AbortSignal.timeout(3000),
-): Promise<Nip05 | undefined> {
-  if (!nip05) return;
-  try {
-    const result = await nip05Cache.fetch(nip05, { signal });
-    if (result.pubkey === pubkey) {
-      return parseNip05(nip05);
-    }
-  } catch (_e) {
-    // do nothing
-  }
-}
-
-export { localNip05Lookup, nip05Cache };

src/utils/stats.ts

@@ -323,6 +323,10 @@ export async function countAuthorStats(
     search,
     streak_start: null,
     streak_end: null,
+    nip05: null,
+    nip05_domain: null,
+    nip05_hostname: null,
+    nip05_last_verified_at: null,
   };
 }
 

src/utils/unfurl.ts

@@ -1,5 +1,6 @@
 import TTLCache from '@isaacs/ttlcache';
 import { logi } from '@soapbox/logi';
+import { safeFetch } from '@soapbox/safe-fetch';
 import DOMPurify from 'isomorphic-dompurify';
 import { unfurl } from 'unfurl.js';
 
@@ -7,13 +8,12 @@ import { Conf } from '@/config.ts';
 import { PreviewCard } from '@/entities/PreviewCard.ts';
 import { cachedLinkPreviewSizeGauge } from '@/metrics.ts';
 import { errorJson } from '@/utils/log.ts';
-import { fetchWorker } from '@/workers/fetch.ts';
 
 async function unfurlCard(url: string, signal: AbortSignal): Promise<PreviewCard | null> {
   try {
     const result = await unfurl(url, {
       fetch: (url) =>
-        fetchWorker(url, {
+        safeFetch(url, {
           headers: {
             'Accept': 'text/html, application/xhtml+xml',
             'User-Agent': Conf.fetchUserAgent,

src/views/mastodon/accounts.ts

@@ -6,11 +6,9 @@ import { MastodonAccount } from '@/entities/MastodonAccount.ts';
 import { type DittoEvent } from '@/interfaces/DittoEvent.ts';
 import { metadataSchema } from '@/schemas/nostr.ts';
 import { getLnurl } from '@/utils/lnurl.ts';
-import { parseAndVerifyNip05 } from '@/utils/nip05.ts';
 import { parseNoteContent } from '@/utils/note.ts';
 import { getTagSet } from '@/utils/tags.ts';
-import { faviconCache } from '@/utils/favicon.ts';
+import { nostrDate, nostrNow, parseNip05 } from '@/utils.ts';
-import { nostrDate, nostrNow } from '@/utils.ts';
 import { renderEmojis } from '@/views/mastodon/emojis.ts';
 
 type ToAccountOpts = {
@@ -20,16 +18,14 @@ type ToAccountOpts = {
   withSource?: false;
 };
 
-async function renderAccount(
+function renderAccount(event: Omit<DittoEvent, 'id' | 'sig'>, opts: ToAccountOpts = {}): MastodonAccount {
-  event: Omit<DittoEvent, 'id' | 'sig'>,
-  opts: ToAccountOpts = {},
-  signal = AbortSignal.timeout(3000),
-): Promise<MastodonAccount> {
   const { pubkey } = event;
 
+  const stats = event.author_stats;
   const names = getTagSet(event.user?.tags ?? [], 'n');
+
   if (names.has('disabled')) {
-    const account = await accountFromPubkey(pubkey, opts);
+    const account = accountFromPubkey(pubkey, opts);
     account.pleroma.deactivated = true;
     return account;
   }
@@ -48,17 +44,9 @@ async function renderAccount(
 
   const npub = nip19.npubEncode(pubkey);
   const nprofile = nip19.nprofileEncode({ pubkey, relays: [Conf.relay] });
-  const parsed05 = await parseAndVerifyNip05(nip05, pubkey, signal);
+  const parsed05 = stats?.nip05 ? parseNip05(stats.nip05) : undefined;
   const acct = parsed05?.handle || npub;
 
-  let favicon: URL | undefined;
-  if (parsed05?.domain) {
-    try {
-      favicon = await faviconCache.fetch(parsed05.domain, { signal });
-    } catch {
-      favicon = new URL('/favicon.ico', `https://${parsed05.domain}/`);
-    }
-  }
   const { html } = parseNoteContent(about || '', []);
 
   const fields = _fields
@@ -70,8 +58,8 @@ async function renderAccount(
     })) ?? [];
 
   let streakDays = 0;
-  let streakStart = event.author_stats?.streak_start ?? null;
+  let streakStart = stats?.streak_start ?? null;
-  let streakEnd = event.author_stats?.streak_end ?? null;
+  let streakEnd = stats?.streak_end ?? null;
   const { streakWindow } = Conf;
 
   if (streakStart && streakEnd) {
@@ -97,8 +85,8 @@ async function renderAccount(
     emojis: renderEmojis(event),
     fields: fields.map((field) => ({ ...field, value: parseNoteContent(field.value, []).html })),
     follow_requests_count: 0,
-    followers_count: event.author_stats?.followers_count ?? 0,
+    followers_count: stats?.followers_count ?? 0,
-    following_count: event.author_stats?.following_count ?? 0,
+    following_count: stats?.following_count ?? 0,
     fqn: parsed05?.handle || npub,
     header: banner,
     header_static: banner,
@@ -122,7 +110,7 @@ async function renderAccount(
         },
       }
       : undefined,
-    statuses_count: event.author_stats?.notes_count ?? 0,
+    statuses_count: stats?.notes_count ?? 0,
     uri: Conf.local(`/users/${acct}`),
     url: Conf.local(`/@${acct}`),
     username: parsed05?.nickname || npub.substring(0, 8),
@@ -144,7 +132,7 @@ async function renderAccount(
       is_local: parsed05?.domain === Conf.url.host,
       settings_store: opts.withSource ? opts.settingsStore : undefined,
       tags: [...getTagSet(event.user?.tags ?? [], 't')],
-      favicon: favicon?.toString(),
+      favicon: stats?.favicon,
     },
     nostr: {
       pubkey,
@@ -154,7 +142,7 @@ async function renderAccount(
   };
 }
 
-function accountFromPubkey(pubkey: string, opts: ToAccountOpts = {}): Promise<MastodonAccount> {
+function accountFromPubkey(pubkey: string, opts: ToAccountOpts = {}): MastodonAccount {
   const event: UnsignedEvent = {
     kind: 0,
     pubkey,

src/views/mastodon/statuses.ts

@@ -7,7 +7,7 @@ import { MastodonMention } from '@/entities/MastodonMention.ts';
 import { MastodonStatus } from '@/entities/MastodonStatus.ts';
 import { type DittoEvent } from '@/interfaces/DittoEvent.ts';
 import { Storages } from '@/storages.ts';
-import { isNostrId, nostrDate } from '@/utils.ts';
+import { nostrDate } from '@/utils.ts';
 import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts';
 import { findReplyTag } from '@/utils/tags.ts';
 import { unfurlCardCached } from '@/utils/unfurl.ts';
@@ -33,28 +33,14 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
   });
 
   const account = event.author
-    ? await renderAccount({ ...event.author, author_stats: event.author_stats })
+    ? renderAccount({ ...event.author, author_stats: event.author_stats })
-    : await accountFromPubkey(event.pubkey);
+    : accountFromPubkey(event.pubkey);
 
   const replyId = findReplyTag(event.tags)?.[1];
 
-  const mentionedPubkeys = [
-    ...new Set(
-      event.tags
-        .filter(([name, value]) => name === 'p' && isNostrId(value))
-        .map(([, value]) => value),
-    ),
-  ];
-
   const store = await Storages.db();
 
-  const mentionedProfiles = await store.query(
+  const mentions = event.mentions?.map((event) => renderMention(event)) ?? [];
-    [{ kinds: [0], authors: mentionedPubkeys, limit: mentionedPubkeys.length }],
-  );
-
-  const mentions = await Promise.all(
-    mentionedPubkeys.map((pubkey) => renderMention(pubkey, mentionedProfiles.find((event) => event.pubkey === pubkey))),
-  );
 
   const { html, links, firstUrl } = parseNoteContent(stripimeta(event.content, event.tags), mentions);
 
@@ -170,8 +156,8 @@ async function renderReblog(event: DittoEvent, opts: RenderStatusOpts): Promise<
   };
 }
 
-async function renderMention(pubkey: string, event?: NostrEvent): Promise<MastodonMention> {
+function renderMention(event: NostrEvent): MastodonMention {
-  const account = event ? await renderAccount(event) : await accountFromPubkey(pubkey);
+  const account = renderAccount(event);
   return {
     id: account.id,
     acct: account.acct,

src/workers/fetch.test.ts

@@ -1,29 +0,0 @@
-import { assertEquals, assertRejects } from '@std/assert';
-
-import { fetchWorker } from '@/workers/fetch.ts';
-
-Deno.test({
-  name: 'fetchWorker',
-  async fn() {
-    const response = await fetchWorker('https://httpbingo.org/get');
-    const json = await response.json();
-    assertEquals(json.headers.Host, ['httpbingo.org']);
-  },
-  sanitizeResources: false,
-});
-
-Deno.test({
-  name: 'fetchWorker with AbortSignal',
-  async fn() {
-    const controller = new AbortController();
-    const signal = controller.signal;
-
-    setTimeout(() => controller.abort(), 100);
-    assertRejects(() => fetchWorker('https://httpbingo.org/delay/10', { signal }));
-
-    await new Promise<void>((resolve) => {
-      signal.addEventListener('abort', () => resolve(), { once: true });
-    });
-  },
-  sanitizeResources: false,
-});

src/workers/fetch.ts

@@ -1,86 +0,0 @@
-import * as Comlink from 'comlink';
-
-import { FetchWorker } from './fetch.worker.ts';
-import './handlers/abortsignal.ts';
-
-import { fetchResponsesCounter } from '@/metrics.ts';
-
-const worker = new Worker(new URL('./fetch.worker.ts', import.meta.url), { type: 'module', name: 'fetchWorker' });
-const client = Comlink.wrap<typeof FetchWorker>(worker);
-
-// Wait for the worker to be ready before we start using it.
-const ready = new Promise<void>((resolve) => {
-  const handleEvent = () => {
-    self.removeEventListener('message', handleEvent);
-    resolve();
-  };
-  worker.addEventListener('message', handleEvent);
-});
-
-/**
- * Fetch implementation with a Web Worker.
- * Calling this performs the fetch in a separate CPU thread so it doesn't block the main thread.
- */
-const fetchWorker: typeof fetch = async (...args) => {
-  await ready;
-
-  const [url, init] = serializeFetchArgs(args);
-  const { body, signal, ...rest } = init;
-
-  const result = await client.fetch(url, { ...rest, body: await prepareBodyForWorker(body) }, signal);
-  const response = new Response(...result);
-
-  const { method } = init;
-  const { status } = response;
-  fetchResponsesCounter.inc({ method, status });
-
-  return response;
-};
-
-/** Take arguments to `fetch`, and turn them into something we can send over Comlink. */
-function serializeFetchArgs(args: Parameters<typeof fetch>): [string, RequestInit] {
-  const request = normalizeRequest(args);
-  const init = requestToInit(request);
-  return [request.url, init];
-}
-
-/** Get a `Request` object from arguments to `fetch`. */
-function normalizeRequest(args: Parameters<typeof fetch>): Request {
-  return new Request(...args);
-}
-
-/** Get the body as a type we can transfer over Web Workers. */
-async function prepareBodyForWorker(
-  body: BodyInit | undefined | null,
-): Promise<ArrayBuffer | Blob | string | undefined | null> {
-  if (!body || typeof body === 'string' || body instanceof ArrayBuffer || body instanceof Blob) {
-    return body;
-  } else {
-    const response = new Response(body);
-    return await response.arrayBuffer();
-  }
-}
-
-/**
- * Convert a `Request` object into its serialized `RequestInit` format.
- * `RequestInit` is a subset of `Request`, just lacking helper methods like `json()`,
- * making it easier to serialize (exceptions: `body` and `signal`).
- */
-function requestToInit(request: Request): RequestInit {
-  return {
-    method: request.method,
-    headers: [...request.headers.entries()],
-    body: request.body,
-    referrer: request.referrer,
-    referrerPolicy: request.referrerPolicy,
-    mode: request.mode,
-    credentials: request.credentials,
-    cache: request.cache,
-    redirect: request.redirect,
-    integrity: request.integrity,
-    keepalive: request.keepalive,
-    signal: request.signal,
-  };
-}
-
-export { fetchWorker };

src/workers/fetch.worker.ts

@@ -1,33 +0,0 @@
-/// <reference lib="webworker" />
-
-import { safeFetch } from '@soapbox/safe-fetch';
-import { logi } from '@soapbox/logi';
-import * as Comlink from 'comlink';
-
-import '@/workers/handlers/abortsignal.ts';
-import '@/sentry.ts';
-
-export const FetchWorker = {
-  async fetch(
-    url: string,
-    init: Omit<RequestInit, 'signal'>,
-    signal: AbortSignal | null | undefined,
-  ): Promise<[BodyInit, ResponseInit]> {
-    logi({ level: 'debug', ns: 'ditto.fetch', method: init.method ?? 'GET', url });
-
-    const response = await safeFetch(url, { ...init, signal });
-
-    return [
-      await response.arrayBuffer(),
-      {
-        status: response.status,
-        statusText: response.statusText,
-        headers: [...response.headers.entries()],
-      },
-    ];
-  },
-};
-
-Comlink.expose(FetchWorker);
-
-self.postMessage('ready');

src/workers/handlers/abortsignal.ts

@@ -1,46 +0,0 @@
-import * as Comlink from 'comlink';
-
-const signalFinalizers = new FinalizationRegistry((port: MessagePort) => {
-  port.postMessage(null);
-  port.close();
-});
-
-Comlink.transferHandlers.set('abortsignal', {
-  canHandle(value) {
-    return value instanceof AbortSignal || value?.constructor?.name === 'AbortSignal';
-  },
-  serialize(signal) {
-    if (signal.aborted) {
-      return [{ aborted: true }];
-    }
-
-    const { port1, port2 } = new MessageChannel();
-    signal.addEventListener(
-      'abort',
-      () => port1.postMessage({ reason: signal.reason }),
-      { once: true },
-    );
-
-    signalFinalizers?.register(signal, port1);
-
-    return [{ aborted: false, port: port2 }, [port2]];
-  },
-  deserialize({ aborted, port }) {
-    if (aborted || !port) {
-      return AbortSignal.abort();
-    }
-
-    const ctrl = new AbortController();
-
-    port.addEventListener('message', (ev) => {
-      if (ev.data && 'reason' in ev.data) {
-        ctrl.abort(ev.data.reason);
-      }
-      port.close();
-    }, { once: true });
-
-    port.start();
-
-    return ctrl.signal;
-  },
-} as Comlink.TransferHandler<AbortSignal, { aborted: boolean; port?: MessagePort }>);

src/workers/policy.ts

@@ -5,8 +5,6 @@ import * as Comlink from 'comlink';
 import { Conf } from '@/config.ts';
 import type { CustomPolicy } from '@/workers/policy.worker.ts';
 
-import '@/workers/handlers/abortsignal.ts';
-
 class PolicyWorker implements NPolicy {
   private worker: Comlink.Remote<CustomPolicy>;
   private ready: Promise<void>;

src/workers/policy.worker.ts

@@ -6,8 +6,6 @@ import * as Comlink from 'comlink';
 import { DittoDB } from '@/db/DittoDB.ts';
 import { EventsDB } from '@/storages/EventsDB.ts';
 
-import '@/workers/handlers/abortsignal.ts';
-
 // @ts-ignore Don't try to access the env from this worker.
 Deno.env = new Map<string, string>();