From d9b0bc1437570c36c7ad73600297dbe0bd993b0f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 7 Feb 2025 13:35:37 -0600 Subject: [PATCH 01/22] Add nip05 and favicon results to the database, make renderAccount synchronous --- src/db/DittoTables.ts | 10 ++++ src/db/migrations/046_author_stats_nip05.ts | 48 ++++++++++++++++++++ src/db/migrations/047_add_domain_favicons.ts | 15 ++++++ src/interfaces/DittoEvent.ts | 5 ++ src/storages/hydrate.ts | 40 +++++++++++++--- src/utils/nip05.ts | 6 +-- src/utils/stats.ts | 3 ++ src/views/mastodon/accounts.ts | 41 +++++++---------- 8 files changed, 133 insertions(+), 35 deletions(-) create mode 100644 src/db/migrations/046_author_stats_nip05.ts create mode 100644 src/db/migrations/047_add_domain_favicons.ts diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index 7baaa42c..e07e7002 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -5,6 +5,7 @@ import { NPostgresSchema } from '@nostrify/db'; export interface DittoTables extends NPostgresSchema { auth_tokens: AuthTokenRow; author_stats: AuthorStatsRow; + domain_favicons: DomainFaviconRow; event_stats: EventStatsRow; pubkey_domains: PubkeyDomainRow; event_zaps: EventZapRow; @@ -19,6 +20,9 @@ interface AuthorStatsRow { search: string; streak_start: number | null; streak_end: number | null; + nip05: string | null; + nip05_domain: string | null; + nip05_hostname: string | null; } interface EventStatsRow { @@ -46,6 +50,12 @@ interface PubkeyDomainRow { last_updated_at: number; } +interface DomainFaviconRow { + domain: string; + favicon: string; + last_updated_at: number; +} + interface EventZapRow { receipt_id: string; target_event_id: string; diff --git a/src/db/migrations/046_author_stats_nip05.ts b/src/db/migrations/046_author_stats_nip05.ts new file mode 100644 index 00000000..12c23773 --- /dev/null +++ b/src/db/migrations/046_author_stats_nip05.ts @@ -0,0 +1,48 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .alterTable('author_stats') + .addColumn('nip05', 'varchar(320)') + .addColumn('nip05_domain', 'varchar(253)') + .addColumn('nip05_hostname', 'varchar(253)') + .addColumn('nip05_last_verified_at', 'integer') + .execute(); + + await db.schema + .alterTable('author_stats') + .addCheckConstraint('author_stats_nip05_domain_lowercase_chk', sql`nip05_domain = lower(nip05_domain)`) + .execute(); + + await db.schema + .alterTable('author_stats') + .addCheckConstraint('author_stats_nip05_hostname_lowercase_chk', sql`nip05_hostname = lower(nip05_hostname)`) + .execute(); + + await db.schema + .alterTable('author_stats') + .addCheckConstraint('author_stats_nip05_hostname_domain_chk', sql`nip05_hostname like '%' || nip05_domain`) + .execute(); + + await db.schema + .createIndex('author_stats_nip05_domain_idx') + .on('author_stats') + .column('nip05_domain') + .execute(); + + await db.schema + .createIndex('author_stats_nip05_hostname_idx') + .on('author_stats') + .column('nip05_hostname') + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema + .alterTable('author_stats') + .dropColumn('nip05') + .dropColumn('nip05_domain') + .dropColumn('nip05_hostname') + .dropColumn('nip05_last_verified_at') + .execute(); +} diff --git a/src/db/migrations/047_add_domain_favicons.ts b/src/db/migrations/047_add_domain_favicons.ts new file mode 100644 index 00000000..38bda03d --- /dev/null +++ b/src/db/migrations/047_add_domain_favicons.ts @@ -0,0 +1,15 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('domain_favicons') + .addColumn('domain', 'varchar(253)', (col) => col.primaryKey()) + .addColumn('favicon', 'varchar(2048)', (col) => col.notNull()) + .addColumn('last_updated_at', 'integer', (col) => col.notNull()) + .addCheckConstraint('domain_favicons_https_chk', sql`url ~* '^https:\\/\\/'`) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable('domain_favicons').execute(); +} diff --git a/src/interfaces/DittoEvent.ts b/src/interfaces/DittoEvent.ts index 293a7ab4..75db7f73 100644 --- a/src/interfaces/DittoEvent.ts +++ b/src/interfaces/DittoEvent.ts @@ -8,6 +8,11 @@ export interface AuthorStats { notes_count: number; streak_start?: number; streak_end?: number; + nip05?: string; + nip05_domain?: string; + nip05_hostname?: string; + nip05_last_verified_at?: number; + favicon?: string; } /** Ditto internal stats for the event. */ diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 160dd1cc..b9f0fad0 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -66,9 +66,30 @@ async function hydrateEvents(opts: HydrateOpts): Promise { cache.push(event); } + const authorStats = await gatherAuthorStats(cache, kysely as Kysely); + const eventStats = await gatherEventStats(cache, kysely as Kysely); + + const domains = authorStats.reduce((result, { nip05_hostname }) => { + if (nip05_hostname) result.add(nip05_hostname); + return result; + }, new Set()); + + const favicons = ( + await kysely + .selectFrom('domain_favicons') + .select(['domain', 'favicon']) + .where('domain', 'in', [...domains]) + .execute() + ) + .reduce((result, { domain, favicon }) => { + result[domain] = favicon; + return result; + }, {} as Record); + const stats = { - authors: await gatherAuthorStats(cache, kysely as Kysely), - events: await gatherEventStats(cache, kysely as Kysely), + authors: authorStats, + events: eventStats, + favicons, }; // Dedupe events. @@ -85,7 +106,11 @@ async function hydrateEvents(opts: HydrateOpts): Promise { export function assembleEvents( a: DittoEvent[], b: DittoEvent[], - stats: { authors: DittoTables['author_stats'][]; events: DittoTables['event_stats'][] }, + stats: { + authors: DittoTables['author_stats'][]; + events: DittoTables['event_stats'][]; + favicons: Record; + }, ): DittoEvent[] { const admin = Conf.pubkey; @@ -94,6 +119,10 @@ export function assembleEvents( ...stat, streak_start: stat.streak_start ?? undefined, streak_end: stat.streak_end ?? undefined, + nip05: stat.nip05 ?? undefined, + nip05_domain: stat.nip05_domain ?? undefined, + nip05_hostname: stat.nip05_hostname ?? undefined, + favicon: stats.favicons[stat.nip05_hostname!], }; return result; }, {} as Record); @@ -390,13 +419,10 @@ async function gatherAuthorStats( .execute(); return rows.map((row) => ({ - pubkey: row.pubkey, + ...row, followers_count: Math.max(0, row.followers_count), following_count: Math.max(0, row.following_count), notes_count: Math.max(0, row.notes_count), - search: row.search, - streak_start: row.streak_start, - streak_end: row.streak_end, })); } diff --git a/src/utils/nip05.ts b/src/utils/nip05.ts index 65f425a3..3860a6cb 100644 --- a/src/utils/nip05.ts +++ b/src/utils/nip05.ts @@ -11,7 +11,7 @@ import { SimpleLRU } from '@/utils/SimpleLRU.ts'; import { Nip05, parseNip05 } from '@/utils.ts'; import { fetchWorker } from '@/workers/fetch.ts'; -const nip05Cache = new SimpleLRU( +export const nip05Cache = new SimpleLRU( async (nip05, { signal }) => { const tld = tldts.parse(nip05); @@ -46,7 +46,7 @@ const nip05Cache = new SimpleLRU( { ...Conf.caches.nip05, gauge: cachedNip05sSizeGauge }, ); -async function localNip05Lookup(store: NStore, localpart: string): Promise { +export async function localNip05Lookup(store: NStore, localpart: string): Promise { const [grant] = await store.query([{ kinds: [30360], '#d': [`${localpart}@${Conf.url.host}`], @@ -76,5 +76,3 @@ export async function parseAndVerifyNip05( // do nothing } } - -export { localNip05Lookup, nip05Cache }; diff --git a/src/utils/stats.ts b/src/utils/stats.ts index 0821fed2..0a675aee 100644 --- a/src/utils/stats.ts +++ b/src/utils/stats.ts @@ -323,6 +323,9 @@ export async function countAuthorStats( search, streak_start: null, streak_end: null, + nip05: null, + nip05_domain: null, + nip05_hostname: null, }; } diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index 0c2d1dcc..3940b905 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -6,11 +6,9 @@ import { MastodonAccount } from '@/entities/MastodonAccount.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { metadataSchema } from '@/schemas/nostr.ts'; import { getLnurl } from '@/utils/lnurl.ts'; -import { parseAndVerifyNip05 } from '@/utils/nip05.ts'; import { parseNoteContent } from '@/utils/note.ts'; import { getTagSet } from '@/utils/tags.ts'; -import { faviconCache } from '@/utils/favicon.ts'; -import { nostrDate, nostrNow } from '@/utils.ts'; +import { nostrDate, nostrNow, parseNip05 } from '@/utils.ts'; import { renderEmojis } from '@/views/mastodon/emojis.ts'; type ToAccountOpts = { @@ -20,16 +18,14 @@ type ToAccountOpts = { withSource?: false; }; -async function renderAccount( - event: Omit, - opts: ToAccountOpts = {}, - signal = AbortSignal.timeout(3000), -): Promise { +function renderAccount(event: Omit, opts: ToAccountOpts = {}): MastodonAccount { const { pubkey } = event; + const stats = event.author_stats; const names = getTagSet(event.user?.tags ?? [], 'n'); + if (names.has('disabled')) { - const account = await accountFromPubkey(pubkey, opts); + const account = accountFromPubkey(pubkey, opts); account.pleroma.deactivated = true; return account; } @@ -48,17 +44,14 @@ async function renderAccount( const npub = nip19.npubEncode(pubkey); const nprofile = nip19.nprofileEncode({ pubkey, relays: [Conf.relay] }); - const parsed05 = await parseAndVerifyNip05(nip05, pubkey, signal); + const parsed05 = stats?.nip05 ? parseNip05(stats.nip05) : undefined; const acct = parsed05?.handle || npub; - let favicon: URL | undefined; - if (parsed05?.domain) { - try { - favicon = await faviconCache.fetch(parsed05.domain, { signal }); - } catch { - favicon = new URL('/favicon.ico', `https://${parsed05.domain}/`); - } + let favicon: string | undefined = stats?.favicon; + if (!favicon && parsed05) { + favicon = new URL('/favicon.ico', `https://${parsed05.domain}/`).toString(); } + const { html } = parseNoteContent(about || '', []); const fields = _fields @@ -70,8 +63,8 @@ async function renderAccount( })) ?? []; let streakDays = 0; - let streakStart = event.author_stats?.streak_start ?? null; - let streakEnd = event.author_stats?.streak_end ?? null; + let streakStart = stats?.streak_start ?? null; + let streakEnd = stats?.streak_end ?? null; const { streakWindow } = Conf; if (streakStart && streakEnd) { @@ -97,8 +90,8 @@ async function renderAccount( emojis: renderEmojis(event), fields: fields.map((field) => ({ ...field, value: parseNoteContent(field.value, []).html })), follow_requests_count: 0, - followers_count: event.author_stats?.followers_count ?? 0, - following_count: event.author_stats?.following_count ?? 0, + followers_count: stats?.followers_count ?? 0, + following_count: stats?.following_count ?? 0, fqn: parsed05?.handle || npub, header: banner, header_static: banner, @@ -122,7 +115,7 @@ async function renderAccount( }, } : undefined, - statuses_count: event.author_stats?.notes_count ?? 0, + statuses_count: stats?.notes_count ?? 0, uri: Conf.local(`/users/${acct}`), url: Conf.local(`/@${acct}`), username: parsed05?.nickname || npub.substring(0, 8), @@ -144,7 +137,7 @@ async function renderAccount( is_local: parsed05?.domain === Conf.url.host, settings_store: opts.withSource ? opts.settingsStore : undefined, tags: [...getTagSet(event.user?.tags ?? [], 't')], - favicon: favicon?.toString(), + favicon, }, nostr: { pubkey, @@ -154,7 +147,7 @@ async function renderAccount( }; } -function accountFromPubkey(pubkey: string, opts: ToAccountOpts = {}): Promise { +function accountFromPubkey(pubkey: string, opts: ToAccountOpts = {}): MastodonAccount { const event: UnsignedEvent = { kind: 0, pubkey, From 93141c1db188f244c929f119d3a737eebcd448f6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 7 Feb 2025 15:39:25 -0600 Subject: [PATCH 02/22] Hook everything up? (In a messy way) --- src/db/DittoTables.ts | 1 + src/db/migrations/047_add_domain_favicons.ts | 2 +- src/pipeline.ts | 29 ++--- src/storages/hydrate.ts | 1 + src/utils/favicon.ts | 108 ++++++++++------ src/utils/nip05.ts | 129 ++++++++++++++----- src/utils/stats.ts | 1 + 7 files changed, 184 insertions(+), 87 deletions(-) diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index e07e7002..19ea6e1b 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -23,6 +23,7 @@ interface AuthorStatsRow { nip05: string | null; nip05_domain: string | null; nip05_hostname: string | null; + nip05_last_verified_at: number | null; } interface EventStatsRow { diff --git a/src/db/migrations/047_add_domain_favicons.ts b/src/db/migrations/047_add_domain_favicons.ts index 38bda03d..b8d7af77 100644 --- a/src/db/migrations/047_add_domain_favicons.ts +++ b/src/db/migrations/047_add_domain_favicons.ts @@ -6,7 +6,7 @@ export async function up(db: Kysely): Promise { .addColumn('domain', 'varchar(253)', (col) => col.primaryKey()) .addColumn('favicon', 'varchar(2048)', (col) => col.notNull()) .addColumn('last_updated_at', 'integer', (col) => col.notNull()) - .addCheckConstraint('domain_favicons_https_chk', sql`url ~* '^https:\\/\\/'`) + .addCheckConstraint('domain_favicons_https_chk', sql`favicon ~* '^https:\\/\\/'`) .execute(); } diff --git a/src/pipeline.ts b/src/pipeline.ts index a4161233..9f9b8365 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,6 +1,6 @@ import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; -import { Kysely, sql } from 'kysely'; +import { Kysely } from 'kysely'; import { z } from 'zod'; import { pipelineEncounters } from '@/caches/pipelineEncounters.ts'; @@ -13,8 +13,9 @@ import { RelayError } from '@/RelayError.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; -import { eventAge, parseNip05, Time } from '@/utils.ts'; +import { eventAge, Time } from '@/utils.ts'; import { getAmount } from '@/utils/bolt11.ts'; +import { faviconCache } from '@/utils/favicon.ts'; import { errorJson } from '@/utils/log.ts'; import { nip05Cache } from '@/utils/nip05.ts'; import { purifyEvent } from '@/utils/purify.ts'; @@ -202,6 +203,12 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise undefined) : undefined; + // Fetch favicon. + const domain = nip05?.split('@')[1].toLowerCase(); + if (domain) { + await faviconCache.fetch(domain, { signal }); + } + // Populate author_search. try { const search = result?.pubkey === event.pubkey ? [name, nip05].filter(Boolean).join(' ').trim() : name ?? ''; @@ -215,24 +222,6 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise pubkey_domains.last_updated_at - `.execute(kysely); - } catch (_e) { - // do nothing - } - } } /** Determine if the event is being received in a timely manner. */ diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index b9f0fad0..aff68f39 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -122,6 +122,7 @@ export function assembleEvents( nip05: stat.nip05 ?? undefined, nip05_domain: stat.nip05_domain ?? undefined, nip05_hostname: stat.nip05_hostname ?? undefined, + nip05_last_verified_at: stat.nip05_last_verified_at ?? undefined, favicon: stats.favicons[stat.nip05_hostname!], }; return result; diff --git a/src/utils/favicon.ts b/src/utils/favicon.ts index 9833de1c..2ee7c7f7 100644 --- a/src/utils/favicon.ts +++ b/src/utils/favicon.ts @@ -1,55 +1,91 @@ import { DOMParser } from '@b-fuze/deno-dom'; import { logi } from '@soapbox/logi'; +import { Kysely } from 'kysely'; import tldts from 'tldts'; import { Conf } from '@/config.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; import { cachedFaviconsSizeGauge } from '@/metrics.ts'; +import { Storages } from '@/storages.ts'; +import { nostrNow } from '@/utils.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; import { fetchWorker } from '@/workers/fetch.ts'; -const faviconCache = new SimpleLRU( +export const faviconCache = new SimpleLRU( async (domain, { signal }) => { - logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'started' }); - const tld = tldts.parse(domain); + const kysely = await Storages.kysely(); - if (!tld.isIcann || tld.isIp || tld.isPrivate) { - throw new Error(`Invalid favicon domain: ${domain}`); + const row = await queryFavicon(kysely, domain); + + if (row && (nostrNow() - row.last_updated_at) < (Conf.caches.favicon.ttl / 1000)) { + return new URL(row.favicon); } - const rootUrl = new URL('/', `https://${domain}/`); - const response = await fetchWorker(rootUrl, { signal }); - const html = await response.text(); + const url = await fetchFavicon(domain, signal); - const doc = new DOMParser().parseFromString(html, 'text/html'); - const link = doc.querySelector('link[rel="icon"], link[rel="shortcut icon"]'); + insertFavicon(kysely, domain, url.href).catch(() => {}); - if (link) { - const href = link.getAttribute('href'); - if (href) { - let url: URL | undefined; - - try { - url = new URL(href); - } catch { - try { - url = new URL(href, rootUrl); - } catch { - // fall through - } - } - - if (url) { - logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'found', url }); - return url; - } - } - } - - logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'failed' }); - - throw new Error(`Favicon not found: ${domain}`); + return url; }, { ...Conf.caches.favicon, gauge: cachedFaviconsSizeGauge }, ); -export { faviconCache }; +async function queryFavicon( + kysely: Kysely, + domain: string, +): Promise { + return await kysely + .selectFrom('domain_favicons') + .selectAll() + .where('domain', '=', domain) + .executeTakeFirst(); +} + +async function insertFavicon(kysely: Kysely, domain: string, favicon: string): Promise { + await kysely + .insertInto('domain_favicons') + .values({ domain, favicon, last_updated_at: nostrNow() }) + .execute(); +} + +async function fetchFavicon(domain: string, signal?: AbortSignal): Promise { + logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'started' }); + const tld = tldts.parse(domain); + + if (!tld.isIcann || tld.isIp || tld.isPrivate) { + throw new Error(`Invalid favicon domain: ${domain}`); + } + + const rootUrl = new URL('/', `https://${domain}/`); + const response = await fetchWorker(rootUrl, { signal }); + const html = await response.text(); + + const doc = new DOMParser().parseFromString(html, 'text/html'); + const link = doc.querySelector('link[rel="icon"], link[rel="shortcut icon"]'); + + if (link) { + const href = link.getAttribute('href'); + if (href) { + let url: URL | undefined; + + try { + url = new URL(href); + } catch { + try { + url = new URL(href, rootUrl); + } catch { + // fall through + } + } + + if (url) { + logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'found', url }); + return url; + } + } + } + + logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'failed' }); + + throw new Error(`Favicon not found: ${domain}`); +} diff --git a/src/utils/nip05.ts b/src/utils/nip05.ts index 3860a6cb..d180d610 100644 --- a/src/utils/nip05.ts +++ b/src/utils/nip05.ts @@ -1,51 +1,120 @@ import { nip19 } from 'nostr-tools'; import { NIP05, NStore } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; +import { Kysely } from 'kysely'; import tldts from 'tldts'; import { Conf } from '@/config.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; import { cachedNip05sSizeGauge } from '@/metrics.ts'; import { Storages } from '@/storages.ts'; import { errorJson } from '@/utils/log.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; -import { Nip05, parseNip05 } from '@/utils.ts'; +import { Nip05, nostrNow, parseNip05 } from '@/utils.ts'; import { fetchWorker } from '@/workers/fetch.ts'; export const nip05Cache = new SimpleLRU( async (nip05, { signal }) => { - const tld = tldts.parse(nip05); - - if (!tld.isIcann || tld.isIp || tld.isPrivate) { - throw new Error(`Invalid NIP-05: ${nip05}`); - } - - const [name, domain] = nip05.split('@'); - - logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'started' }); - - try { - if (domain === Conf.url.host) { - const store = await Storages.db(); - const pointer = await localNip05Lookup(store, name); - if (pointer) { - logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', pubkey: pointer.pubkey }); - return pointer; - } else { - throw new Error(`Not found: ${nip05}`); - } - } else { - const result = await NIP05.lookup(nip05, { fetch: fetchWorker, signal }); - logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', pubkey: result.pubkey }); - return result; - } - } catch (e) { - logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'failed', error: errorJson(e) }); - throw e; - } + const store = await Storages.db(); + const kysely = await Storages.kysely(); + return getNip05(kysely, store, nip05, signal); }, { ...Conf.caches.nip05, gauge: cachedNip05sSizeGauge }, ); +async function getNip05( + kysely: Kysely, + store: NStore, + nip05: string, + signal?: AbortSignal, +): Promise { + const tld = tldts.parse(nip05); + + if (!tld.isIcann || tld.isIp || tld.isPrivate) { + throw new Error(`Invalid NIP-05: ${nip05}`); + } + + logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'started' }); + + let pointer: nip19.ProfilePointer | undefined = await queryNip05(kysely, nip05); + + if (pointer) { + logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'db', pubkey: pointer.pubkey }); + return pointer; + } + + const [name, domain] = nip05.split('@'); + + try { + if (domain === Conf.url.host) { + pointer = await localNip05Lookup(store, name); + if (pointer) { + logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'local', pubkey: pointer.pubkey }); + } else { + throw new Error(`Not found: ${nip05}`); + } + } else { + pointer = await NIP05.lookup(nip05, { fetch: fetchWorker, signal }); + logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'fetch', pubkey: pointer.pubkey }); + } + } catch (e) { + logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'failed', error: errorJson(e) }); + throw e; + } + + insertNip05(kysely, nip05, pointer.pubkey).catch((e) => { + logi({ level: 'error', ns: 'ditto.nip05', nip05, state: 'insert_failed', error: errorJson(e) }); + }); + + return pointer; +} + +async function queryNip05(kysely: Kysely, nip05: string): Promise { + const row = await kysely + .selectFrom('author_stats') + .select('pubkey') + .where('nip05', '=', nip05) + .executeTakeFirst(); + + if (row) { + return { pubkey: row.pubkey }; + } +} + +async function insertNip05(kysely: Kysely, nip05: string, pubkey: string, ts = nostrNow()): Promise { + const tld = tldts.parse(nip05); + + if (!tld.isIcann || tld.isIp || tld.isPrivate) { + throw new Error(`Invalid NIP-05: ${nip05}`); + } + + await kysely + .insertInto('author_stats') + .values({ + pubkey, + nip05, + nip05_domain: tld.domain, + nip05_hostname: tld.hostname, + nip05_last_verified_at: ts, + followers_count: 0, // TODO: fix `author_stats` types so setting these aren't required + following_count: 0, + notes_count: 0, + search: nip05, + }) + .onConflict((oc) => + oc + .column('pubkey') + .doUpdateSet({ + nip05, + nip05_domain: tld.domain, + nip05_hostname: tld.hostname, + nip05_last_verified_at: ts, + }) + .where('nip05_last_verified_at', '<', ts) + ) + .execute(); +} + export async function localNip05Lookup(store: NStore, localpart: string): Promise { const [grant] = await store.query([{ kinds: [30360], diff --git a/src/utils/stats.ts b/src/utils/stats.ts index 0a675aee..972541d3 100644 --- a/src/utils/stats.ts +++ b/src/utils/stats.ts @@ -326,6 +326,7 @@ export async function countAuthorStats( nip05: null, nip05_domain: null, nip05_hostname: null, + nip05_last_verified_at: null, }; } From 5157a90b63af402eb6dde6ad8adb4ad609aecf09 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 7 Feb 2025 16:03:22 -0600 Subject: [PATCH 03/22] Add populate:nip05 script --- deno.json | 1 + scripts/db-populate-nip05.ts | 45 ++++++++++++++++++++++++++++++++++++ src/utils/favicon.ts | 2 +- src/utils/nip05.ts | 7 +++++- 4 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 scripts/db-populate-nip05.ts diff --git a/deno.json b/deno.json index 562aab51..52d57926 100644 --- a/deno.json +++ b/deno.json @@ -21,6 +21,7 @@ "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip -o soapbox.zip && rm soapbox.zip", "trends": "deno run -A --env-file --deny-read=.env scripts/trends.ts", "clean:deps": "deno cache --reload src/app.ts", + "db:populate:nip05": "deno run -A --env-file --deny-read=.env scripts/db-populate-nip05.ts", "db:populate-search": "deno run -A --env-file --deny-read=.env scripts/db-populate-search.ts", "db:populate-extensions": "deno run -A --env-file --deny-read=.env scripts/db-populate-extensions.ts", "db:streak:recompute": "deno run -A --env-file --deny-read=.env scripts/db-streak-recompute.ts", diff --git a/scripts/db-populate-nip05.ts b/scripts/db-populate-nip05.ts new file mode 100644 index 00000000..b2b05651 --- /dev/null +++ b/scripts/db-populate-nip05.ts @@ -0,0 +1,45 @@ +import { NSchema as n } from '@nostrify/nostrify'; + +import { Storages } from '@/storages.ts'; +import { faviconCache } from '@/utils/favicon.ts'; +import { nip05Cache } from '@/utils/nip05.ts'; + +const store = await Storages.db(); +const kysely = await Storages.kysely(); +const statsQuery = kysely.selectFrom('author_stats').select('pubkey'); + +for await (const { pubkey } of statsQuery.stream(10)) { + const signal = AbortSignal.timeout(30_000); // generous timeout + + try { + const [author] = await store.query([{ kinds: [0], authors: [pubkey], limit: 1 }]); + + if (!author) { + continue; + } + + // Parse metadata. + const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(author.content); + if (!metadata.success) continue; + + // Update nip05. + const { nip05 } = metadata.data; + if (nip05) { + try { + await nip05Cache.fetch(nip05, { signal }); + } catch { + // Ignore. + } + } + + // Update favicon. + const domain = nip05?.split('@')[1].toLowerCase(); + if (domain) { + await faviconCache.fetch(domain, { signal }); + } + } catch { + continue; + } +} + +Deno.exit(); diff --git a/src/utils/favicon.ts b/src/utils/favicon.ts index 2ee7c7f7..549ae8df 100644 --- a/src/utils/favicon.ts +++ b/src/utils/favicon.ts @@ -23,7 +23,7 @@ export const faviconCache = new SimpleLRU( const url = await fetchFavicon(domain, signal); - insertFavicon(kysely, domain, url.href).catch(() => {}); + await insertFavicon(kysely, domain, url.href); return url; }, diff --git a/src/utils/nip05.ts b/src/utils/nip05.ts index d180d610..6bed4c6d 100644 --- a/src/utils/nip05.ts +++ b/src/utils/nip05.ts @@ -110,7 +110,12 @@ async function insertNip05(kysely: Kysely, nip05: string, pubkey: s nip05_hostname: tld.hostname, nip05_last_verified_at: ts, }) - .where('nip05_last_verified_at', '<', ts) + .where((eb) => + eb.or([ + eb('author_stats.nip05_last_verified_at', '<', ts), + eb('author_stats.nip05_last_verified_at', 'is', null), + ]) + ) ) .execute(); } From a4a26d7575c8ce371b27e5f417bd03f5d6c89cb2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 7 Feb 2025 17:48:07 -0600 Subject: [PATCH 04/22] Make db:populate:nip05 script more efficient --- scripts/db-populate-nip05.ts | 53 ++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/scripts/db-populate-nip05.ts b/scripts/db-populate-nip05.ts index b2b05651..aaa97e46 100644 --- a/scripts/db-populate-nip05.ts +++ b/scripts/db-populate-nip05.ts @@ -5,40 +5,35 @@ import { faviconCache } from '@/utils/favicon.ts'; import { nip05Cache } from '@/utils/nip05.ts'; const store = await Storages.db(); -const kysely = await Storages.kysely(); -const statsQuery = kysely.selectFrom('author_stats').select('pubkey'); -for await (const { pubkey } of statsQuery.stream(10)) { - const signal = AbortSignal.timeout(30_000); // generous timeout +for await (const msg of store.req([{ kinds: [0] }])) { + if (msg[0] === 'EVENT') { + const signal = AbortSignal.timeout(30_000); // generous timeout + const event = msg[2]; - try { - const [author] = await store.query([{ kinds: [0], authors: [pubkey], limit: 1 }]); + try { + // Parse metadata. + const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(event.content); + if (!metadata.success) continue; - if (!author) { + // Update nip05. + const { nip05 } = metadata.data; + if (nip05) { + try { + await nip05Cache.fetch(nip05, { signal }); + } catch { + // Ignore. + } + } + + // Update favicon. + const domain = nip05?.split('@')[1].toLowerCase(); + if (domain) { + await faviconCache.fetch(domain, { signal }); + } + } catch { continue; } - - // Parse metadata. - const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(author.content); - if (!metadata.success) continue; - - // Update nip05. - const { nip05 } = metadata.data; - if (nip05) { - try { - await nip05Cache.fetch(nip05, { signal }); - } catch { - // Ignore. - } - } - - // Update favicon. - const domain = nip05?.split('@')[1].toLowerCase(); - if (domain) { - await faviconCache.fetch(domain, { signal }); - } - } catch { - continue; } } From b0dc7faaffcd82241979bde766c8a4937f5f1c73 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 7 Feb 2025 17:49:39 -0600 Subject: [PATCH 05/22] Simplify db:populate:nip05 script --- scripts/db-populate-nip05.ts | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/scripts/db-populate-nip05.ts b/scripts/db-populate-nip05.ts index aaa97e46..6e4dfa48 100644 --- a/scripts/db-populate-nip05.ts +++ b/scripts/db-populate-nip05.ts @@ -11,28 +11,28 @@ for await (const msg of store.req([{ kinds: [0] }])) { const signal = AbortSignal.timeout(30_000); // generous timeout const event = msg[2]; - try { - // Parse metadata. - const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(event.content); - if (!metadata.success) continue; + // Parse metadata. + const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(event.content); + if (!metadata.success) continue; - // Update nip05. - const { nip05 } = metadata.data; - if (nip05) { - try { - await nip05Cache.fetch(nip05, { signal }); - } catch { - // Ignore. - } + // Update nip05. + const { nip05 } = metadata.data; + if (nip05) { + try { + await nip05Cache.fetch(nip05, { signal }); + } catch { + // Ignore. } + } - // Update favicon. - const domain = nip05?.split('@')[1].toLowerCase(); - if (domain) { + // Update favicon. + const domain = nip05?.split('@')[1].toLowerCase(); + if (domain) { + try { await faviconCache.fetch(domain, { signal }); + } catch { + // Ignore. } - } catch { - continue; } } } From b902abc7ccc4ad031924f5232abbac8a2b0f9f7e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 7 Feb 2025 18:13:39 -0600 Subject: [PATCH 06/22] Use an even simpler query for db:populate:nip05 script --- scripts/db-populate-nip05.ts | 50 +++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/scripts/db-populate-nip05.ts b/scripts/db-populate-nip05.ts index 6e4dfa48..c22c0c12 100644 --- a/scripts/db-populate-nip05.ts +++ b/scripts/db-populate-nip05.ts @@ -4,35 +4,37 @@ import { Storages } from '@/storages.ts'; import { faviconCache } from '@/utils/favicon.ts'; import { nip05Cache } from '@/utils/nip05.ts'; -const store = await Storages.db(); +const kysely = await Storages.kysely(); -for await (const msg of store.req([{ kinds: [0] }])) { - if (msg[0] === 'EVENT') { - const signal = AbortSignal.timeout(30_000); // generous timeout - const event = msg[2]; +const query = kysely + .selectFrom('nostr_events') + .select('content') + .where('kind', '=', 0); - // Parse metadata. - const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(event.content); - if (!metadata.success) continue; +for await (const { content } of query.stream(100)) { + const signal = AbortSignal.timeout(30_000); // generous timeout - // Update nip05. - const { nip05 } = metadata.data; - if (nip05) { - try { - await nip05Cache.fetch(nip05, { signal }); - } catch { - // Ignore. - } + // Parse metadata. + const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(content); + if (!metadata.success) continue; + + // Update nip05. + const { nip05 } = metadata.data; + if (nip05) { + try { + await nip05Cache.fetch(nip05, { signal }); + } catch { + // Ignore. } + } - // Update favicon. - const domain = nip05?.split('@')[1].toLowerCase(); - if (domain) { - try { - await faviconCache.fetch(domain, { signal }); - } catch { - // Ignore. - } + // Update favicon. + const domain = nip05?.split('@')[1].toLowerCase(); + if (domain) { + try { + await faviconCache.fetch(domain, { signal }); + } catch { + // Ignore. } } } From 7780507a150cc476fb0c4d0b9f2b0f522a9e8573 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 7 Feb 2025 18:17:50 -0600 Subject: [PATCH 07/22] Add semaphore to nip05 script --- scripts/db-populate-nip05.ts | 52 +++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/scripts/db-populate-nip05.ts b/scripts/db-populate-nip05.ts index c22c0c12..48e792f6 100644 --- a/scripts/db-populate-nip05.ts +++ b/scripts/db-populate-nip05.ts @@ -1,3 +1,4 @@ +import { Semaphore } from '@lambdalisue/async'; import { NSchema as n } from '@nostrify/nostrify'; import { Storages } from '@/storages.ts'; @@ -5,6 +6,7 @@ import { faviconCache } from '@/utils/favicon.ts'; import { nip05Cache } from '@/utils/nip05.ts'; const kysely = await Storages.kysely(); +const sem = new Semaphore(5); const query = kysely .selectFrom('nostr_events') @@ -12,31 +14,37 @@ const query = kysely .where('kind', '=', 0); for await (const { content } of query.stream(100)) { - const signal = AbortSignal.timeout(30_000); // generous timeout - - // Parse metadata. - const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(content); - if (!metadata.success) continue; - - // Update nip05. - const { nip05 } = metadata.data; - if (nip05) { - try { - await nip05Cache.fetch(nip05, { signal }); - } catch { - // Ignore. - } + while (sem.locked) { + await new Promise((resolve) => setTimeout(resolve, 0)); } - // Update favicon. - const domain = nip05?.split('@')[1].toLowerCase(); - if (domain) { - try { - await faviconCache.fetch(domain, { signal }); - } catch { - // Ignore. + sem.lock(async () => { + const signal = AbortSignal.timeout(30_000); // generous timeout + + // Parse metadata. + const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(content); + if (!metadata.success) return; + + // Update nip05. + const { nip05 } = metadata.data; + if (nip05) { + try { + await nip05Cache.fetch(nip05, { signal }); + } catch { + // Ignore. + } } - } + + // Update favicon. + const domain = nip05?.split('@')[1].toLowerCase(); + if (domain) { + try { + await faviconCache.fetch(domain, { signal }); + } catch { + // Ignore. + } + } + }); } Deno.exit(); From 5811a1915146457189e818dee4834e2e4930b4ed Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 7 Feb 2025 19:05:37 -0600 Subject: [PATCH 08/22] Fix mentions in statuses? --- src/interfaces/DittoEvent.ts | 1 + src/storages/hydrate.ts | 38 +++++++++++++++++++++++++++++++++- src/views/mastodon/statuses.ts | 26 ++++++----------------- 3 files changed, 44 insertions(+), 21 deletions(-) diff --git a/src/interfaces/DittoEvent.ts b/src/interfaces/DittoEvent.ts index 75db7f73..bca65856 100644 --- a/src/interfaces/DittoEvent.ts +++ b/src/interfaces/DittoEvent.ts @@ -30,6 +30,7 @@ export interface DittoEvent extends NostrEvent { author_domain?: string; author_stats?: AuthorStats; event_stats?: EventStats; + mentions?: DittoEvent[]; user?: DittoEvent; repost?: DittoEvent; quote?: DittoEvent; diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index aff68f39..77b64fdc 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -30,6 +30,10 @@ async function hydrateEvents(opts: HydrateOpts): Promise { const cache = [...events]; + for (const event of await gatherMentions({ events: cache, store, signal })) { + cache.push(event); + } + for (const event of await gatherReposts({ events: cache, store, signal })) { cache.push(event); } @@ -146,6 +150,9 @@ export function assembleEvents( if (id) { event.quote = b.find((e) => matchFilter({ kinds: [1, 20], ids: [id] }, e)); } + + const pubkeys = event.tags.filter(([name]) => name === 'p').map(([_name, value]) => value); + event.mentions = b.filter((e) => matchFilter({ kinds: [0], authors: pubkeys }, e)); } if (event.kind === 6) { @@ -267,6 +274,35 @@ function gatherQuotes({ events, store, signal }: HydrateOpts): Promise { + const pubkeys = new Set(); + + for (const event of events) { + if (event.kind === 1) { + const pubkey = event.tags.find(([name]) => name === 'p')?.[1]; + if (pubkey) { + pubkeys.add(pubkey); + } + } + } + + const authors = await store.query( + [{ kinds: [0], authors: [...pubkeys], limit: pubkeys.size }], + { signal }, + ); + + for (const pubkey of pubkeys) { + const author = authors.find((e) => matchFilter({ kinds: [0], authors: [pubkey] }, e)); + if (!author) { + const fallback = fallbackAuthor(pubkey); + authors.push(fallback); + } + } + + return authors; +} + /** Collect authors from the events. */ async function gatherAuthors({ events, store, signal }: HydrateOpts): Promise { const pubkeys = new Set(); @@ -297,7 +333,7 @@ async function gatherAuthors({ events, store, signal }: HydrateOpts): Promise matchFilter({ kinds: [0], authors: [pubkey] }, e)); - if (author) { + if (!author) { const fallback = fallbackAuthor(pubkey); authors.push(fallback); } diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 265cf442..0c0eb9f2 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -7,7 +7,7 @@ import { MastodonMention } from '@/entities/MastodonMention.ts'; import { MastodonStatus } from '@/entities/MastodonStatus.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { Storages } from '@/storages.ts'; -import { isNostrId, nostrDate } from '@/utils.ts'; +import { nostrDate } from '@/utils.ts'; import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts'; import { findReplyTag } from '@/utils/tags.ts'; import { unfurlCardCached } from '@/utils/unfurl.ts'; @@ -33,28 +33,14 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< }); const account = event.author - ? await renderAccount({ ...event.author, author_stats: event.author_stats }) - : await accountFromPubkey(event.pubkey); + ? renderAccount({ ...event.author, author_stats: event.author_stats }) + : accountFromPubkey(event.pubkey); const replyId = findReplyTag(event.tags)?.[1]; - const mentionedPubkeys = [ - ...new Set( - event.tags - .filter(([name, value]) => name === 'p' && isNostrId(value)) - .map(([, value]) => value), - ), - ]; - const store = await Storages.db(); - const mentionedProfiles = await store.query( - [{ kinds: [0], authors: mentionedPubkeys, limit: mentionedPubkeys.length }], - ); - - const mentions = await Promise.all( - mentionedPubkeys.map((pubkey) => renderMention(pubkey, mentionedProfiles.find((event) => event.pubkey === pubkey))), - ); + const mentions = event.mentions?.map((event) => renderMention(event)) ?? []; const { html, links, firstUrl } = parseNoteContent(stripimeta(event.content, event.tags), mentions); @@ -170,8 +156,8 @@ async function renderReblog(event: DittoEvent, opts: RenderStatusOpts): Promise< }; } -async function renderMention(pubkey: string, event?: NostrEvent): Promise { - const account = event ? await renderAccount(event) : await accountFromPubkey(pubkey); +function renderMention(event: NostrEvent): MastodonMention { + const account = renderAccount(event); return { id: account.id, acct: account.acct, From b8c67a85d0167bb657f269286339f64455167cf3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 8 Feb 2025 09:58:02 -0600 Subject: [PATCH 09/22] hydrate: move gatherMentions down --- src/storages/hydrate.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 77b64fdc..76919a31 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -30,10 +30,6 @@ async function hydrateEvents(opts: HydrateOpts): Promise { const cache = [...events]; - for (const event of await gatherMentions({ events: cache, store, signal })) { - cache.push(event); - } - for (const event of await gatherReposts({ events: cache, store, signal })) { cache.push(event); } @@ -46,6 +42,10 @@ async function hydrateEvents(opts: HydrateOpts): Promise { cache.push(event); } + for (const event of await gatherMentions({ events: cache, store, signal })) { + cache.push(event); + } + for (const event of await gatherAuthors({ events: cache, store, signal })) { cache.push(event); } From e7027af1ae24263668c959d72abc53444dad1523 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 8 Feb 2025 10:13:38 -0600 Subject: [PATCH 10/22] Fix hydrating mentions --- src/storages/hydrate.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 76919a31..a162571a 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -280,9 +280,10 @@ async function gatherMentions({ events, store, signal }: HydrateOpts): Promise name === 'p')?.[1]; - if (pubkey) { - pubkeys.add(pubkey); + for (const [name, value] of event.tags) { + if (name === 'p') { + pubkeys.add(value); + } } } } From 48507b7505bf295d4ee4a57afb5a2ca5b9bfba2f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Feb 2025 11:57:09 -0600 Subject: [PATCH 11/22] faviconCache: check favicon.ico explicitly --- src/utils/favicon.ts | 10 ++++++++++ src/views/mastodon/accounts.ts | 7 +------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/utils/favicon.ts b/src/utils/favicon.ts index 549ae8df..b81b50cd 100644 --- a/src/utils/favicon.ts +++ b/src/utils/favicon.ts @@ -85,6 +85,16 @@ async function fetchFavicon(domain: string, signal?: AbortSignal): Promise } } + // Fallback to checking `/favicon.ico` of the domain. + const url = new URL('/favicon.ico', `https://${domain}/`); + const fallback = await fetchWorker(url, { method: 'HEAD', signal }); + const contentType = fallback.headers.get('content-type'); + + if (fallback.ok && contentType === 'image/vnd.microsoft.icon') { + logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'found', url }); + return url; + } + logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'failed' }); throw new Error(`Favicon not found: ${domain}`); diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index 3940b905..4e29e388 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -47,11 +47,6 @@ function renderAccount(event: Omit, opts: ToAccountOpt const parsed05 = stats?.nip05 ? parseNip05(stats.nip05) : undefined; const acct = parsed05?.handle || npub; - let favicon: string | undefined = stats?.favicon; - if (!favicon && parsed05) { - favicon = new URL('/favicon.ico', `https://${parsed05.domain}/`).toString(); - } - const { html } = parseNoteContent(about || '', []); const fields = _fields @@ -137,7 +132,7 @@ function renderAccount(event: Omit, opts: ToAccountOpt is_local: parsed05?.domain === Conf.url.host, settings_store: opts.withSource ? opts.settingsStore : undefined, tags: [...getTagSet(event.user?.tags ?? [], 't')], - favicon, + favicon: stats?.favicon, }, nostr: { pubkey, From 8c60a4842b6cdf819a494f04ad990334ad5fe864 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Feb 2025 13:27:05 -0600 Subject: [PATCH 12/22] Fix NIP05 verification --- scripts/db-populate-nip05.ts | 36 +++----------- src/pipeline.ts | 72 +++++++++++++++++++++------- src/utils/nip05.ts | 93 ++---------------------------------- 3 files changed, 65 insertions(+), 136 deletions(-) diff --git a/scripts/db-populate-nip05.ts b/scripts/db-populate-nip05.ts index 48e792f6..ec74847b 100644 --- a/scripts/db-populate-nip05.ts +++ b/scripts/db-populate-nip05.ts @@ -1,49 +1,25 @@ import { Semaphore } from '@lambdalisue/async'; -import { NSchema as n } from '@nostrify/nostrify'; +import { updateAuthorData } from '@/pipeline.ts'; import { Storages } from '@/storages.ts'; -import { faviconCache } from '@/utils/favicon.ts'; -import { nip05Cache } from '@/utils/nip05.ts'; +import { NostrEvent } from '@nostrify/nostrify'; const kysely = await Storages.kysely(); const sem = new Semaphore(5); const query = kysely .selectFrom('nostr_events') - .select('content') + .select(['id', 'kind', 'content', 'pubkey', 'tags', 'created_at', 'sig']) .where('kind', '=', 0); -for await (const { content } of query.stream(100)) { +for await (const row of query.stream(100)) { while (sem.locked) { await new Promise((resolve) => setTimeout(resolve, 0)); } sem.lock(async () => { - const signal = AbortSignal.timeout(30_000); // generous timeout - - // Parse metadata. - const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(content); - if (!metadata.success) return; - - // Update nip05. - const { nip05 } = metadata.data; - if (nip05) { - try { - await nip05Cache.fetch(nip05, { signal }); - } catch { - // Ignore. - } - } - - // Update favicon. - const domain = nip05?.split('@')[1].toLowerCase(); - if (domain) { - try { - await faviconCache.fetch(domain, { signal }); - } catch { - // Ignore. - } - } + const event: NostrEvent = { ...row, created_at: Number(row.created_at) }; + await updateAuthorData(event, AbortSignal.timeout(3000)); }); } diff --git a/src/pipeline.ts b/src/pipeline.ts index 9f9b8365..32fe353e 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,6 +1,7 @@ import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; -import { Kysely } from 'kysely'; +import { Kysely, UpdateObject } from 'kysely'; +import tldts from 'tldts'; import { z } from 'zod'; import { pipelineEncounters } from '@/caches/pipelineEncounters.ts'; @@ -120,7 +121,7 @@ async function handleEvent(event: DittoEvent, opts: PipelineOpts): Promise // This needs to run in steps, and should not block the API from responding. Promise.allSettled([ handleZaps(kysely, event), - parseMetadata(event, opts.signal), + updateAuthorData(event, opts.signal), generateSetEvents(event), ]) .then(() => @@ -190,18 +191,47 @@ async function storeEvent(event: NostrEvent, signal?: AbortSignal): Promise { +async function updateAuthorData(event: NostrEvent, signal: AbortSignal): Promise { if (event.kind !== 0) return; // Parse metadata. const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(event.content); if (!metadata.success) return; + const { name, nip05 } = metadata.data; + const kysely = await Storages.kysely(); - // Get nip05. - const { name, nip05 } = metadata.data; - const result = nip05 ? await nip05Cache.fetch(nip05, { signal }).catch(() => undefined) : undefined; + const updates: UpdateObject = {}; + + const authorStats = await kysely + .selectFrom('author_stats') + .selectAll() + .where('pubkey', '=', event.pubkey) + .executeTakeFirst(); + + const lastVerified = authorStats?.nip05_last_verified_at; + const eventNewer = !lastVerified || event.created_at > lastVerified; + + if (nip05 !== authorStats?.nip05 && eventNewer) { + if (nip05) { + const tld = tldts.parse(nip05); + if (tld.isIcann && !tld.isIp && !tld.isPrivate) { + const pointer = await nip05Cache.fetch(nip05.toLowerCase(), { signal }); + if (pointer.pubkey === event.pubkey) { + updates.nip05 = nip05; + updates.nip05_domain = tld.domain; + updates.nip05_hostname = tld.hostname; + updates.nip05_last_verified_at = event.created_at; + } + } + } else { + updates.nip05 = null; + updates.nip05_domain = null; + updates.nip05_hostname = null; + updates.nip05_last_verified_at = event.created_at; + } + } // Fetch favicon. const domain = nip05?.split('@')[1].toLowerCase(); @@ -209,18 +239,24 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise oc.column('pubkey').doUpdateSet({ search })) - .execute(); - } - } catch { - // do nothing + if (search !== authorStats?.search) { + updates.search = search; + } + + if (Object.keys(updates).length) { + await kysely.insertInto('author_stats') + .values({ + pubkey: event.pubkey, + followers_count: 0, + following_count: 0, + notes_count: 0, + search, + ...updates, + }) + .onConflict((oc) => oc.column('pubkey').doUpdateSet(updates)) + .execute(); } } @@ -353,4 +389,4 @@ async function handleZaps(kysely: Kysely, event: NostrEvent) { } } -export { handleEvent, handleZaps }; +export { handleEvent, handleZaps, updateAuthorData }; diff --git a/src/utils/nip05.ts b/src/utils/nip05.ts index 6bed4c6d..66ccb16e 100644 --- a/src/utils/nip05.ts +++ b/src/utils/nip05.ts @@ -1,29 +1,24 @@ import { nip19 } from 'nostr-tools'; import { NIP05, NStore } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; -import { Kysely } from 'kysely'; import tldts from 'tldts'; import { Conf } from '@/config.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; import { cachedNip05sSizeGauge } from '@/metrics.ts'; import { Storages } from '@/storages.ts'; import { errorJson } from '@/utils/log.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; -import { Nip05, nostrNow, parseNip05 } from '@/utils.ts'; import { fetchWorker } from '@/workers/fetch.ts'; export const nip05Cache = new SimpleLRU( async (nip05, { signal }) => { const store = await Storages.db(); - const kysely = await Storages.kysely(); - return getNip05(kysely, store, nip05, signal); + return getNip05(store, nip05, signal); }, { ...Conf.caches.nip05, gauge: cachedNip05sSizeGauge }, ); async function getNip05( - kysely: Kysely, store: NStore, nip05: string, signal?: AbortSignal, @@ -36,88 +31,26 @@ async function getNip05( logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'started' }); - let pointer: nip19.ProfilePointer | undefined = await queryNip05(kysely, nip05); - - if (pointer) { - logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'db', pubkey: pointer.pubkey }); - return pointer; - } - const [name, domain] = nip05.split('@'); try { if (domain === Conf.url.host) { - pointer = await localNip05Lookup(store, name); + const pointer = await localNip05Lookup(store, name); if (pointer) { logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'local', pubkey: pointer.pubkey }); + return pointer; } else { throw new Error(`Not found: ${nip05}`); } } else { - pointer = await NIP05.lookup(nip05, { fetch: fetchWorker, signal }); + const pointer = await NIP05.lookup(nip05, { fetch: fetchWorker, signal }); logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'fetch', pubkey: pointer.pubkey }); + return pointer; } } catch (e) { logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'failed', error: errorJson(e) }); throw e; } - - insertNip05(kysely, nip05, pointer.pubkey).catch((e) => { - logi({ level: 'error', ns: 'ditto.nip05', nip05, state: 'insert_failed', error: errorJson(e) }); - }); - - return pointer; -} - -async function queryNip05(kysely: Kysely, nip05: string): Promise { - const row = await kysely - .selectFrom('author_stats') - .select('pubkey') - .where('nip05', '=', nip05) - .executeTakeFirst(); - - if (row) { - return { pubkey: row.pubkey }; - } -} - -async function insertNip05(kysely: Kysely, nip05: string, pubkey: string, ts = nostrNow()): Promise { - const tld = tldts.parse(nip05); - - if (!tld.isIcann || tld.isIp || tld.isPrivate) { - throw new Error(`Invalid NIP-05: ${nip05}`); - } - - await kysely - .insertInto('author_stats') - .values({ - pubkey, - nip05, - nip05_domain: tld.domain, - nip05_hostname: tld.hostname, - nip05_last_verified_at: ts, - followers_count: 0, // TODO: fix `author_stats` types so setting these aren't required - following_count: 0, - notes_count: 0, - search: nip05, - }) - .onConflict((oc) => - oc - .column('pubkey') - .doUpdateSet({ - nip05, - nip05_domain: tld.domain, - nip05_hostname: tld.hostname, - nip05_last_verified_at: ts, - }) - .where((eb) => - eb.or([ - eb('author_stats.nip05_last_verified_at', '<', ts), - eb('author_stats.nip05_last_verified_at', 'is', null), - ]) - ) - ) - .execute(); } export async function localNip05Lookup(store: NStore, localpart: string): Promise { @@ -134,19 +67,3 @@ export async function localNip05Lookup(store: NStore, localpart: string): Promis return { pubkey, relays: [Conf.relay] }; } } - -export async function parseAndVerifyNip05( - nip05: string | undefined, - pubkey: string, - signal = AbortSignal.timeout(3000), -): Promise { - if (!nip05) return; - try { - const result = await nip05Cache.fetch(nip05, { signal }); - if (result.pubkey === pubkey) { - return parseNip05(nip05); - } - } catch (_e) { - // do nothing - } -} From dd009de5be3585be29f95ca6542107ce153374a1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Feb 2025 13:31:17 -0600 Subject: [PATCH 13/22] Wrap nip05 updates in a try-catch --- src/pipeline.ts | 34 +++++++++++++++++++--------------- src/utils/favicon.ts | 1 + 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index 32fe353e..3f78e9c5 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -213,24 +213,28 @@ async function updateAuthorData(event: NostrEvent, signal: AbortSignal): Promise const lastVerified = authorStats?.nip05_last_verified_at; const eventNewer = !lastVerified || event.created_at > lastVerified; - if (nip05 !== authorStats?.nip05 && eventNewer) { - if (nip05) { - const tld = tldts.parse(nip05); - if (tld.isIcann && !tld.isIp && !tld.isPrivate) { - const pointer = await nip05Cache.fetch(nip05.toLowerCase(), { signal }); - if (pointer.pubkey === event.pubkey) { - updates.nip05 = nip05; - updates.nip05_domain = tld.domain; - updates.nip05_hostname = tld.hostname; - updates.nip05_last_verified_at = event.created_at; + try { + if (nip05 !== authorStats?.nip05 && eventNewer) { + if (nip05) { + const tld = tldts.parse(nip05); + if (tld.isIcann && !tld.isIp && !tld.isPrivate) { + const pointer = await nip05Cache.fetch(nip05.toLowerCase(), { signal }); + if (pointer.pubkey === event.pubkey) { + updates.nip05 = nip05; + updates.nip05_domain = tld.domain; + updates.nip05_hostname = tld.hostname; + updates.nip05_last_verified_at = event.created_at; + } } + } else { + updates.nip05 = null; + updates.nip05_domain = null; + updates.nip05_hostname = null; + updates.nip05_last_verified_at = event.created_at; } - } else { - updates.nip05 = null; - updates.nip05_domain = null; - updates.nip05_hostname = null; - updates.nip05_last_verified_at = event.created_at; } + } catch { + // Fallthrough. } // Fetch favicon. diff --git a/src/utils/favicon.ts b/src/utils/favicon.ts index b81b50cd..c4e9a8c3 100644 --- a/src/utils/favicon.ts +++ b/src/utils/favicon.ts @@ -45,6 +45,7 @@ async function insertFavicon(kysely: Kysely, domain: string, favico await kysely .insertInto('domain_favicons') .values({ domain, favicon, last_updated_at: nostrNow() }) + .onConflict((oc) => oc.column('domain').doUpdateSet({ favicon, last_updated_at: nostrNow() })) .execute(); } From 8386fe7609c4d0c83df3e2f92b22255c6f930693 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Feb 2025 13:32:47 -0600 Subject: [PATCH 14/22] try-catch favicon fetch --- src/pipeline.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index 3f78e9c5..f2d4ad5c 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -240,7 +240,11 @@ async function updateAuthorData(event: NostrEvent, signal: AbortSignal): Promise // Fetch favicon. const domain = nip05?.split('@')[1].toLowerCase(); if (domain) { - await faviconCache.fetch(domain, { signal }); + try { + await faviconCache.fetch(domain, { signal }); + } catch { + // Fallthrough. + } } const search = [name, nip05].filter(Boolean).join(' ').trim(); From 41419e84dc46119d6feb67213d93755b9b411a6d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Feb 2025 14:20:32 -0600 Subject: [PATCH 15/22] Refetch nip05 if last_verified is null --- src/pipeline.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index f2d4ad5c..7540bc82 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -214,7 +214,7 @@ async function updateAuthorData(event: NostrEvent, signal: AbortSignal): Promise const eventNewer = !lastVerified || event.created_at > lastVerified; try { - if (nip05 !== authorStats?.nip05 && eventNewer) { + if (nip05 !== authorStats?.nip05 && eventNewer || !lastVerified) { if (nip05) { const tld = tldts.parse(nip05); if (tld.isIcann && !tld.isIp && !tld.isPrivate) { From ebbde66824380accff2243810d8812b6c3f68c12 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Feb 2025 14:33:57 -0600 Subject: [PATCH 16/22] Add @core/asyncutil --- deno.json | 1 + deno.lock | 1 + 2 files changed, 2 insertions(+) diff --git a/deno.json b/deno.json index 52d57926..c82f3f1d 100644 --- a/deno.json +++ b/deno.json @@ -40,6 +40,7 @@ "@/": "./src/", "@b-fuze/deno-dom": "jsr:@b-fuze/deno-dom@^0.1.47", "@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", + "@core/asyncutil": "jsr:@core/asyncutil@^1.2.0", "@electric-sql/pglite": "npm:@electric-sql/pglite@^0.2.8", "@esroyo/scoped-performance": "jsr:@esroyo/scoped-performance@^3.1.0", "@gfx/canvas-wasm": "jsr:@gfx/canvas-wasm@^0.4.2", diff --git a/deno.lock b/deno.lock index 7235d7dd..9179e478 100644 --- a/deno.lock +++ b/deno.lock @@ -2367,6 +2367,7 @@ "dependencies": [ "jsr:@b-fuze/deno-dom@~0.1.47", "jsr:@bradenmacdonald/s3-lite-client@~0.7.4", + "jsr:@core/asyncutil@^1.2.0", "jsr:@esroyo/scoped-performance@^3.1.0", "jsr:@gfx/canvas-wasm@~0.4.2", "jsr:@hono/hono@^4.4.6", From 576a66460ff2f3d25a013890d28f4bb568e6ad82 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Feb 2025 14:45:26 -0600 Subject: [PATCH 17/22] fetchWorker: preemptively throw if signal is aborted --- src/workers/fetch.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/workers/fetch.ts b/src/workers/fetch.ts index bb5588ed..e1915fb4 100644 --- a/src/workers/fetch.ts +++ b/src/workers/fetch.ts @@ -27,6 +27,10 @@ const fetchWorker: typeof fetch = async (...args) => { const [url, init] = serializeFetchArgs(args); const { body, signal, ...rest } = init; + if (signal?.aborted) { + throw new DOMException('The signal has been aborted', 'AbortError'); + } + const result = await client.fetch(url, { ...rest, body: await prepareBodyForWorker(body) }, signal); const response = new Response(...result); From 93874df06397d3b11291fd8f7a1f10d4a7cd5935 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Feb 2025 14:50:28 -0600 Subject: [PATCH 18/22] fetchWorker: log responses --- src/workers/fetch.worker.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/workers/fetch.worker.ts b/src/workers/fetch.worker.ts index 4a67c6b8..d0cd1f44 100644 --- a/src/workers/fetch.worker.ts +++ b/src/workers/fetch.worker.ts @@ -13,10 +13,19 @@ export const FetchWorker = { init: Omit, signal: AbortSignal | null | undefined, ): Promise<[BodyInit, ResponseInit]> { - logi({ level: 'debug', ns: 'ditto.fetch', method: init.method ?? 'GET', url }); + logi({ level: 'debug', ns: 'ditto.fetch', state: 'started', method: init.method ?? 'GET', url }); const response = await safeFetch(url, { ...init, signal }); + logi({ + level: 'debug', + ns: 'ditto.fetch', + state: 'finished', + method: init.method ?? 'GET', + url, + status: response.status, + }); + return [ await response.arrayBuffer(), { From a98bfdd0c6e6717ca809bcb9e62b0315c49c0a17 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Feb 2025 14:52:43 -0600 Subject: [PATCH 19/22] fetchWorker: try throwing a preemptive AbortError inside the worker itself --- src/workers/fetch.worker.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/workers/fetch.worker.ts b/src/workers/fetch.worker.ts index d0cd1f44..7d0d1fa9 100644 --- a/src/workers/fetch.worker.ts +++ b/src/workers/fetch.worker.ts @@ -13,6 +13,10 @@ export const FetchWorker = { init: Omit, signal: AbortSignal | null | undefined, ): Promise<[BodyInit, ResponseInit]> { + if (signal?.aborted) { + throw new DOMException('The signal has been aborted', 'AbortError'); + } + logi({ level: 'debug', ns: 'ditto.fetch', state: 'started', method: init.method ?? 'GET', url }); const response = await safeFetch(url, { ...init, signal }); From 838f773b846b25425b4110726a0c293aea0a6246 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Feb 2025 15:01:25 -0600 Subject: [PATCH 20/22] Remove fetchWorker --- src/middleware/translatorMiddleware.ts | 7 ++- src/middleware/uploaderMiddleware.ts | 8 +-- src/utils/favicon.ts | 4 +- src/utils/lnurl.ts | 6 +- src/utils/nip05.ts | 4 +- src/utils/unfurl.ts | 4 +- src/workers/fetch.test.ts | 29 --------- src/workers/fetch.ts | 86 -------------------------- src/workers/fetch.worker.ts | 33 ---------- src/workers/handlers/abortsignal.ts | 46 -------------- src/workers/policy.ts | 2 - src/workers/policy.worker.ts | 2 - 12 files changed, 17 insertions(+), 214 deletions(-) delete mode 100644 src/workers/fetch.test.ts delete mode 100644 src/workers/fetch.ts delete mode 100644 src/workers/fetch.worker.ts delete mode 100644 src/workers/handlers/abortsignal.ts diff --git a/src/middleware/translatorMiddleware.ts b/src/middleware/translatorMiddleware.ts index f5a6baa2..ef123dab 100644 --- a/src/middleware/translatorMiddleware.ts +++ b/src/middleware/translatorMiddleware.ts @@ -1,6 +1,7 @@ +import { safeFetch } from '@soapbox/safe-fetch'; + import { AppMiddleware } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { fetchWorker } from '@/workers/fetch.ts'; import { DeepLTranslator } from '@/translators/DeepLTranslator.ts'; import { LibreTranslateTranslator } from '@/translators/LibreTranslateTranslator.ts'; @@ -10,7 +11,7 @@ export const translatorMiddleware: AppMiddleware = async (c, next) => { case 'deepl': { const { deeplApiKey: apiKey, deeplBaseUrl: baseUrl } = Conf; if (apiKey) { - c.set('translator', new DeepLTranslator({ baseUrl, apiKey, fetch: fetchWorker })); + c.set('translator', new DeepLTranslator({ baseUrl, apiKey, fetch: safeFetch })); } break; } @@ -18,7 +19,7 @@ export const translatorMiddleware: AppMiddleware = async (c, next) => { case 'libretranslate': { const { libretranslateApiKey: apiKey, libretranslateBaseUrl: baseUrl } = Conf; if (apiKey) { - c.set('translator', new LibreTranslateTranslator({ baseUrl, apiKey, fetch: fetchWorker })); + c.set('translator', new LibreTranslateTranslator({ baseUrl, apiKey, fetch: safeFetch })); } break; } diff --git a/src/middleware/uploaderMiddleware.ts b/src/middleware/uploaderMiddleware.ts index 96a47336..6866b883 100644 --- a/src/middleware/uploaderMiddleware.ts +++ b/src/middleware/uploaderMiddleware.ts @@ -1,11 +1,11 @@ import { BlossomUploader, NostrBuildUploader } from '@nostrify/nostrify/uploaders'; +import { safeFetch } from '@soapbox/safe-fetch'; import { AppMiddleware } from '@/app.ts'; import { Conf } from '@/config.ts'; import { DenoUploader } from '@/uploaders/DenoUploader.ts'; import { IPFSUploader } from '@/uploaders/IPFSUploader.ts'; import { S3Uploader } from '@/uploaders/S3Uploader.ts'; -import { fetchWorker } from '@/workers/fetch.ts'; /** Set an uploader for the user. */ export const uploaderMiddleware: AppMiddleware = async (c, next) => { @@ -29,17 +29,17 @@ export const uploaderMiddleware: AppMiddleware = async (c, next) => { ); break; case 'ipfs': - c.set('uploader', new IPFSUploader({ baseUrl: Conf.mediaDomain, apiUrl: Conf.ipfs.apiUrl, fetch: fetchWorker })); + c.set('uploader', new IPFSUploader({ baseUrl: Conf.mediaDomain, apiUrl: Conf.ipfs.apiUrl, fetch: safeFetch })); break; case 'local': c.set('uploader', new DenoUploader({ baseUrl: Conf.mediaDomain, dir: Conf.uploadsDir })); break; case 'nostrbuild': - c.set('uploader', new NostrBuildUploader({ endpoint: Conf.nostrbuildEndpoint, signer, fetch: fetchWorker })); + c.set('uploader', new NostrBuildUploader({ endpoint: Conf.nostrbuildEndpoint, signer, fetch: safeFetch })); break; case 'blossom': if (signer) { - c.set('uploader', new BlossomUploader({ servers: Conf.blossomServers, signer, fetch: fetchWorker })); + c.set('uploader', new BlossomUploader({ servers: Conf.blossomServers, signer, fetch: safeFetch })); } break; } diff --git a/src/utils/favicon.ts b/src/utils/favicon.ts index 9833de1c..70d59de8 100644 --- a/src/utils/favicon.ts +++ b/src/utils/favicon.ts @@ -1,11 +1,11 @@ import { DOMParser } from '@b-fuze/deno-dom'; import { logi } from '@soapbox/logi'; +import { safeFetch } from '@soapbox/safe-fetch'; import tldts from 'tldts'; import { Conf } from '@/config.ts'; import { cachedFaviconsSizeGauge } from '@/metrics.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; -import { fetchWorker } from '@/workers/fetch.ts'; const faviconCache = new SimpleLRU( async (domain, { signal }) => { @@ -17,7 +17,7 @@ const faviconCache = new SimpleLRU( } const rootUrl = new URL('/', `https://${domain}/`); - const response = await fetchWorker(rootUrl, { signal }); + const response = await safeFetch(rootUrl, { signal }); const html = await response.text(); const doc = new DOMParser().parseFromString(html, 'text/html'); diff --git a/src/utils/lnurl.ts b/src/utils/lnurl.ts index c70f5751..4fd44988 100644 --- a/src/utils/lnurl.ts +++ b/src/utils/lnurl.ts @@ -1,19 +1,19 @@ import { NostrEvent } from '@nostrify/nostrify'; import { LNURL, LNURLDetails } from '@nostrify/nostrify/ln'; import { logi } from '@soapbox/logi'; +import { safeFetch } from '@soapbox/safe-fetch'; import { JsonValue } from '@std/json'; import { cachedLnurlsSizeGauge } from '@/metrics.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; import { errorJson } from '@/utils/log.ts'; import { Time } from '@/utils/time.ts'; -import { fetchWorker } from '@/workers/fetch.ts'; const lnurlCache = new SimpleLRU( async (lnurl, { signal }) => { logi({ level: 'info', ns: 'ditto.lnurl', lnurl, state: 'started' }); try { - const details = await LNURL.lookup(lnurl, { fetch: fetchWorker, signal }); + const details = await LNURL.lookup(lnurl, { fetch: safeFetch, signal }); logi({ level: 'info', ns: 'ditto.lnurl', lnurl, state: 'found', details: details as unknown as JsonValue }); return details; } catch (e) { @@ -62,7 +62,7 @@ async function getInvoice(params: CallbackParams, signal?: AbortSignal): Promise const { pr } = await LNURL.callback( details.callback, params, - { fetch: fetchWorker, signal }, + { fetch: safeFetch, signal }, ); return pr; diff --git a/src/utils/nip05.ts b/src/utils/nip05.ts index 65f425a3..ccb08bf2 100644 --- a/src/utils/nip05.ts +++ b/src/utils/nip05.ts @@ -1,6 +1,7 @@ import { nip19 } from 'nostr-tools'; import { NIP05, NStore } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; +import { safeFetch } from '@soapbox/safe-fetch'; import tldts from 'tldts'; import { Conf } from '@/config.ts'; @@ -9,7 +10,6 @@ import { Storages } from '@/storages.ts'; import { errorJson } from '@/utils/log.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; import { Nip05, parseNip05 } from '@/utils.ts'; -import { fetchWorker } from '@/workers/fetch.ts'; const nip05Cache = new SimpleLRU( async (nip05, { signal }) => { @@ -34,7 +34,7 @@ const nip05Cache = new SimpleLRU( throw new Error(`Not found: ${nip05}`); } } else { - const result = await NIP05.lookup(nip05, { fetch: fetchWorker, signal }); + const result = await NIP05.lookup(nip05, { fetch: safeFetch, signal }); logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', pubkey: result.pubkey }); return result; } diff --git a/src/utils/unfurl.ts b/src/utils/unfurl.ts index 731b586e..f895b71f 100644 --- a/src/utils/unfurl.ts +++ b/src/utils/unfurl.ts @@ -1,5 +1,6 @@ import TTLCache from '@isaacs/ttlcache'; import { logi } from '@soapbox/logi'; +import { safeFetch } from '@soapbox/safe-fetch'; import DOMPurify from 'isomorphic-dompurify'; import { unfurl } from 'unfurl.js'; @@ -7,13 +8,12 @@ import { Conf } from '@/config.ts'; import { PreviewCard } from '@/entities/PreviewCard.ts'; import { cachedLinkPreviewSizeGauge } from '@/metrics.ts'; import { errorJson } from '@/utils/log.ts'; -import { fetchWorker } from '@/workers/fetch.ts'; async function unfurlCard(url: string, signal: AbortSignal): Promise { try { const result = await unfurl(url, { fetch: (url) => - fetchWorker(url, { + safeFetch(url, { headers: { 'Accept': 'text/html, application/xhtml+xml', 'User-Agent': Conf.fetchUserAgent, diff --git a/src/workers/fetch.test.ts b/src/workers/fetch.test.ts deleted file mode 100644 index e4c698d4..00000000 --- a/src/workers/fetch.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { assertEquals, assertRejects } from '@std/assert'; - -import { fetchWorker } from '@/workers/fetch.ts'; - -Deno.test({ - name: 'fetchWorker', - async fn() { - const response = await fetchWorker('https://httpbingo.org/get'); - const json = await response.json(); - assertEquals(json.headers.Host, ['httpbingo.org']); - }, - sanitizeResources: false, -}); - -Deno.test({ - name: 'fetchWorker with AbortSignal', - async fn() { - const controller = new AbortController(); - const signal = controller.signal; - - setTimeout(() => controller.abort(), 100); - assertRejects(() => fetchWorker('https://httpbingo.org/delay/10', { signal })); - - await new Promise((resolve) => { - signal.addEventListener('abort', () => resolve(), { once: true }); - }); - }, - sanitizeResources: false, -}); diff --git a/src/workers/fetch.ts b/src/workers/fetch.ts deleted file mode 100644 index bb5588ed..00000000 --- a/src/workers/fetch.ts +++ /dev/null @@ -1,86 +0,0 @@ -import * as Comlink from 'comlink'; - -import { FetchWorker } from './fetch.worker.ts'; -import './handlers/abortsignal.ts'; - -import { fetchResponsesCounter } from '@/metrics.ts'; - -const worker = new Worker(new URL('./fetch.worker.ts', import.meta.url), { type: 'module', name: 'fetchWorker' }); -const client = Comlink.wrap(worker); - -// Wait for the worker to be ready before we start using it. -const ready = new Promise((resolve) => { - const handleEvent = () => { - self.removeEventListener('message', handleEvent); - resolve(); - }; - worker.addEventListener('message', handleEvent); -}); - -/** - * Fetch implementation with a Web Worker. - * Calling this performs the fetch in a separate CPU thread so it doesn't block the main thread. - */ -const fetchWorker: typeof fetch = async (...args) => { - await ready; - - const [url, init] = serializeFetchArgs(args); - const { body, signal, ...rest } = init; - - const result = await client.fetch(url, { ...rest, body: await prepareBodyForWorker(body) }, signal); - const response = new Response(...result); - - const { method } = init; - const { status } = response; - fetchResponsesCounter.inc({ method, status }); - - return response; -}; - -/** Take arguments to `fetch`, and turn them into something we can send over Comlink. */ -function serializeFetchArgs(args: Parameters): [string, RequestInit] { - const request = normalizeRequest(args); - const init = requestToInit(request); - return [request.url, init]; -} - -/** Get a `Request` object from arguments to `fetch`. */ -function normalizeRequest(args: Parameters): Request { - return new Request(...args); -} - -/** Get the body as a type we can transfer over Web Workers. */ -async function prepareBodyForWorker( - body: BodyInit | undefined | null, -): Promise { - if (!body || typeof body === 'string' || body instanceof ArrayBuffer || body instanceof Blob) { - return body; - } else { - const response = new Response(body); - return await response.arrayBuffer(); - } -} - -/** - * Convert a `Request` object into its serialized `RequestInit` format. - * `RequestInit` is a subset of `Request`, just lacking helper methods like `json()`, - * making it easier to serialize (exceptions: `body` and `signal`). - */ -function requestToInit(request: Request): RequestInit { - return { - method: request.method, - headers: [...request.headers.entries()], - body: request.body, - referrer: request.referrer, - referrerPolicy: request.referrerPolicy, - mode: request.mode, - credentials: request.credentials, - cache: request.cache, - redirect: request.redirect, - integrity: request.integrity, - keepalive: request.keepalive, - signal: request.signal, - }; -} - -export { fetchWorker }; diff --git a/src/workers/fetch.worker.ts b/src/workers/fetch.worker.ts deleted file mode 100644 index 4a67c6b8..00000000 --- a/src/workers/fetch.worker.ts +++ /dev/null @@ -1,33 +0,0 @@ -/// - -import { safeFetch } from '@soapbox/safe-fetch'; -import { logi } from '@soapbox/logi'; -import * as Comlink from 'comlink'; - -import '@/workers/handlers/abortsignal.ts'; -import '@/sentry.ts'; - -export const FetchWorker = { - async fetch( - url: string, - init: Omit, - signal: AbortSignal | null | undefined, - ): Promise<[BodyInit, ResponseInit]> { - logi({ level: 'debug', ns: 'ditto.fetch', method: init.method ?? 'GET', url }); - - const response = await safeFetch(url, { ...init, signal }); - - return [ - await response.arrayBuffer(), - { - status: response.status, - statusText: response.statusText, - headers: [...response.headers.entries()], - }, - ]; - }, -}; - -Comlink.expose(FetchWorker); - -self.postMessage('ready'); diff --git a/src/workers/handlers/abortsignal.ts b/src/workers/handlers/abortsignal.ts deleted file mode 100644 index 14cf9f41..00000000 --- a/src/workers/handlers/abortsignal.ts +++ /dev/null @@ -1,46 +0,0 @@ -import * as Comlink from 'comlink'; - -const signalFinalizers = new FinalizationRegistry((port: MessagePort) => { - port.postMessage(null); - port.close(); -}); - -Comlink.transferHandlers.set('abortsignal', { - canHandle(value) { - return value instanceof AbortSignal || value?.constructor?.name === 'AbortSignal'; - }, - serialize(signal) { - if (signal.aborted) { - return [{ aborted: true }]; - } - - const { port1, port2 } = new MessageChannel(); - signal.addEventListener( - 'abort', - () => port1.postMessage({ reason: signal.reason }), - { once: true }, - ); - - signalFinalizers?.register(signal, port1); - - return [{ aborted: false, port: port2 }, [port2]]; - }, - deserialize({ aborted, port }) { - if (aborted || !port) { - return AbortSignal.abort(); - } - - const ctrl = new AbortController(); - - port.addEventListener('message', (ev) => { - if (ev.data && 'reason' in ev.data) { - ctrl.abort(ev.data.reason); - } - port.close(); - }, { once: true }); - - port.start(); - - return ctrl.signal; - }, -} as Comlink.TransferHandler); diff --git a/src/workers/policy.ts b/src/workers/policy.ts index fdc33698..7b3d23b0 100644 --- a/src/workers/policy.ts +++ b/src/workers/policy.ts @@ -5,8 +5,6 @@ import * as Comlink from 'comlink'; import { Conf } from '@/config.ts'; import type { CustomPolicy } from '@/workers/policy.worker.ts'; -import '@/workers/handlers/abortsignal.ts'; - class PolicyWorker implements NPolicy { private worker: Comlink.Remote; private ready: Promise; diff --git a/src/workers/policy.worker.ts b/src/workers/policy.worker.ts index 5e9d4d4a..00540b03 100644 --- a/src/workers/policy.worker.ts +++ b/src/workers/policy.worker.ts @@ -6,8 +6,6 @@ import * as Comlink from 'comlink'; import { DittoDB } from '@/db/DittoDB.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; -import '@/workers/handlers/abortsignal.ts'; - // @ts-ignore Don't try to access the env from this worker. Deno.env = new Map(); From 433c2a4347190db2e5ee13c73a58503fccd750fa Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Feb 2025 15:06:13 -0600 Subject: [PATCH 21/22] @lambdalisue/async -> @core/asyncutil --- deno.json | 2 +- deno.lock | 10 +++++----- scripts/db-import.ts | 2 +- src/firehose.ts | 2 +- src/notify.ts | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/deno.json b/deno.json index 562aab51..b591b43c 100644 --- a/deno.json +++ b/deno.json @@ -39,12 +39,12 @@ "@/": "./src/", "@b-fuze/deno-dom": "jsr:@b-fuze/deno-dom@^0.1.47", "@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", + "@core/asyncutil": "jsr:@core/asyncutil@^1.2.0", "@electric-sql/pglite": "npm:@electric-sql/pglite@^0.2.8", "@esroyo/scoped-performance": "jsr:@esroyo/scoped-performance@^3.1.0", "@gfx/canvas-wasm": "jsr:@gfx/canvas-wasm@^0.4.2", "@hono/hono": "jsr:@hono/hono@^4.4.6", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", - "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1", "@negrel/webpush": "jsr:@negrel/webpush@^0.3.0", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", "@nostrify/db": "jsr:@nostrify/db@^0.38.0", diff --git a/deno.lock b/deno.lock index 7235d7dd..874085e8 100644 --- a/deno.lock +++ b/deno.lock @@ -3,6 +3,7 @@ "specifiers": { "jsr:@b-fuze/deno-dom@~0.1.47": "0.1.48", "jsr:@bradenmacdonald/s3-lite-client@~0.7.4": "0.7.6", + "jsr:@core/asyncutil@^1.2.0": "1.2.0", "jsr:@denosaurs/plug@1.0.3": "1.0.3", "jsr:@esroyo/scoped-performance@^3.1.0": "3.1.0", "jsr:@gfx/canvas-wasm@~0.4.2": "0.4.2", @@ -28,7 +29,6 @@ "jsr:@gleasonator/policy@0.9.3": "0.9.3", "jsr:@gleasonator/policy@0.9.4": "0.9.4", "jsr:@hono/hono@^4.4.6": "4.6.15", - "jsr:@lambdalisue/async@^2.1.1": "2.1.1", "jsr:@negrel/http-ece@0.6.0": "0.6.0", "jsr:@negrel/webpush@0.3": "0.3.0", "jsr:@nostrify/db@0.38": "0.38.0", @@ -153,6 +153,9 @@ "jsr:@std/io@0.224" ] }, + "@core/asyncutil@1.2.0": { + "integrity": "9967f15190c60df032c13f72ce5ac73d185c34f31c53dc918d8800025854c118" + }, "@denosaurs/plug@1.0.3": { "integrity": "b010544e386bea0ff3a1d05e0c88f704ea28cbd4d753439c2f1ee021a85d4640", "dependencies": [ @@ -337,9 +340,6 @@ "@hono/hono@4.6.15": { "integrity": "935b3b12e98e4b22bcd1aa4dbe6587321e431c79829eba61f535b4ede39fd8b1" }, - "@lambdalisue/async@2.1.1": { - "integrity": "1fc9bc6f4ed50215cd2f7217842b18cea80f81c25744f88f8c5eb4be5a1c9ab4" - }, "@negrel/http-ece@0.6.0": { "integrity": "7afdd81b86ea5b21a9677b323c01c3338705e11cc2bfed250870f5349d8f86f7", "dependencies": [ @@ -2367,10 +2367,10 @@ "dependencies": [ "jsr:@b-fuze/deno-dom@~0.1.47", "jsr:@bradenmacdonald/s3-lite-client@~0.7.4", + "jsr:@core/asyncutil@^1.2.0", "jsr:@esroyo/scoped-performance@^3.1.0", "jsr:@gfx/canvas-wasm@~0.4.2", "jsr:@hono/hono@^4.4.6", - "jsr:@lambdalisue/async@^2.1.1", "jsr:@negrel/webpush@0.3", "jsr:@nostrify/db@0.38", "jsr:@nostrify/nostrify@~0.38.1", diff --git a/scripts/db-import.ts b/scripts/db-import.ts index c34384bf..ed884453 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -1,4 +1,4 @@ -import { Semaphore } from '@lambdalisue/async'; +import { Semaphore } from '@core/asyncutil'; import { NostrEvent } from '@nostrify/nostrify'; import { JsonParseStream } from '@std/json/json-parse-stream'; import { TextLineStream } from '@std/streams/text-line-stream'; diff --git a/src/firehose.ts b/src/firehose.ts index fca2e079..f04752b2 100644 --- a/src/firehose.ts +++ b/src/firehose.ts @@ -1,4 +1,4 @@ -import { Semaphore } from '@lambdalisue/async'; +import { Semaphore } from '@core/asyncutil'; import { logi } from '@soapbox/logi'; import { Conf } from '@/config.ts'; diff --git a/src/notify.ts b/src/notify.ts index b1ee3517..44ed5619 100644 --- a/src/notify.ts +++ b/src/notify.ts @@ -1,4 +1,4 @@ -import { Semaphore } from '@lambdalisue/async'; +import { Semaphore } from '@core/asyncutil'; import { pipelineEncounters } from '@/caches/pipelineEncounters.ts'; import { Conf } from '@/config.ts'; From 16f3a13364e34409d6925d50731680d04e4b4311 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Feb 2025 17:22:53 -0600 Subject: [PATCH 22/22] SimpleLRU: respect AbortSignal --- src/utils/SimpleLRU.test.ts | 2 +- src/utils/SimpleLRU.ts | 63 ++++++++++++++++++++----------------- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/src/utils/SimpleLRU.test.ts b/src/utils/SimpleLRU.test.ts index a73e4f36..03fbfe8a 100644 --- a/src/utils/SimpleLRU.test.ts +++ b/src/utils/SimpleLRU.test.ts @@ -4,7 +4,7 @@ import { assertEquals, assertRejects } from '@std/assert'; Deno.test("SimpleLRU doesn't repeat failed calls", async () => { let calls = 0; - const cache = new SimpleLRU( + using cache = new SimpleLRU( // deno-lint-ignore require-await async () => { calls++; diff --git a/src/utils/SimpleLRU.ts b/src/utils/SimpleLRU.ts index f18a6211..4d8780b7 100644 --- a/src/utils/SimpleLRU.ts +++ b/src/utils/SimpleLRU.ts @@ -3,50 +3,55 @@ import { LRUCache } from 'lru-cache'; import { type Gauge } from 'prom-client'; -type FetchFn = (key: K, opts: O) => Promise; - -interface FetchFnOpts { - signal?: AbortSignal | null; -} +type FetchFn = (key: K, opts: { signal?: AbortSignal }) => Promise; type SimpleLRUOpts = LRUCache.Options & { gauge?: Gauge; + errorRefresh?: number; }; export class SimpleLRU< K extends {}, V extends {}, - O extends {} = FetchFnOpts, > { - protected cache: LRUCache; + protected cache: LRUCache, void>; + private tids = new Set(); - constructor(fetchFn: FetchFn, private opts: SimpleLRUOpts) { - this.cache = new LRUCache({ - async fetchMethod(key, _staleValue, { signal }) { - try { - return await fetchFn(key, { signal: signal as unknown as AbortSignal }); - } catch { - return null as unknown as V; - } - }, - ...opts, - }); + constructor(private fetchFn: FetchFn, private opts: SimpleLRUOpts>) { + this.cache = new LRUCache({ ...opts }); } - async fetch(key: K, opts?: O): Promise { - const result = await this.cache.fetch(key, opts); - - this.opts.gauge?.set(this.cache.size); - - if (result === undefined || result === null) { - throw new Error('SimpleLRU: fetch failed'); + async fetch(key: K, opts?: { signal?: AbortSignal }): Promise { + if (opts?.signal?.aborted) { + throw new DOMException('The signal has been aborted', 'AbortError'); } - return result; + const cached = await this.cache.get(key); + + if (cached) { + return cached; + } + + const promise = this.fetchFn(key, { signal: opts?.signal }); + + this.cache.set(key, promise); + + promise.then(() => { + this.opts.gauge?.set(this.cache.size); + }).catch(() => { + const tid = setTimeout(() => { + this.cache.delete(key); + this.tids.delete(tid); + }, this.opts.errorRefresh ?? 10_000); + this.tids.add(tid); + }); + + return promise; } - put(key: K, value: V): Promise { - this.cache.set(key, value); - return Promise.resolve(); + [Symbol.dispose](): void { + for (const tid of this.tids) { + clearTimeout(tid); + } } }