diff --git a/deno.json b/deno.json index b591b43c..dabb1ac9 100644 --- a/deno.json +++ b/deno.json @@ -21,6 +21,7 @@ "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip -o soapbox.zip && rm soapbox.zip", "trends": "deno run -A --env-file --deny-read=.env scripts/trends.ts", "clean:deps": "deno cache --reload src/app.ts", + "db:populate:nip05": "deno run -A --env-file --deny-read=.env scripts/db-populate-nip05.ts", "db:populate-search": "deno run -A --env-file --deny-read=.env scripts/db-populate-search.ts", "db:populate-extensions": "deno run -A --env-file --deny-read=.env scripts/db-populate-extensions.ts", "db:streak:recompute": "deno run -A --env-file --deny-read=.env scripts/db-streak-recompute.ts", diff --git a/scripts/db-populate-nip05.ts b/scripts/db-populate-nip05.ts new file mode 100644 index 00000000..df11f007 --- /dev/null +++ b/scripts/db-populate-nip05.ts @@ -0,0 +1,26 @@ +import { Semaphore } from '@core/asyncutil'; + +import { updateAuthorData } from '@/pipeline.ts'; +import { Storages } from '@/storages.ts'; +import { NostrEvent } from '@nostrify/nostrify'; + +const kysely = await Storages.kysely(); +const sem = new Semaphore(5); + +const query = kysely + .selectFrom('nostr_events') + .select(['id', 'kind', 'content', 'pubkey', 'tags', 'created_at', 'sig']) + .where('kind', '=', 0); + +for await (const row of query.stream(100)) { + while (sem.locked) { + await new Promise((resolve) => setTimeout(resolve, 0)); + } + + sem.lock(async () => { + const event: NostrEvent = { ...row, created_at: Number(row.created_at) }; + await updateAuthorData(event, AbortSignal.timeout(3000)); + }); +} + +Deno.exit(); diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index 7baaa42c..19ea6e1b 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -5,6 +5,7 @@ import { NPostgresSchema } from '@nostrify/db'; export interface DittoTables extends NPostgresSchema { auth_tokens: AuthTokenRow; author_stats: AuthorStatsRow; + domain_favicons: DomainFaviconRow; event_stats: EventStatsRow; pubkey_domains: PubkeyDomainRow; event_zaps: EventZapRow; @@ -19,6 +20,10 @@ interface AuthorStatsRow { search: string; streak_start: number | null; streak_end: number | null; + nip05: string | null; + nip05_domain: string | null; + nip05_hostname: string | null; + nip05_last_verified_at: number | null; } interface EventStatsRow { @@ -46,6 +51,12 @@ interface PubkeyDomainRow { last_updated_at: number; } +interface DomainFaviconRow { + domain: string; + favicon: string; + last_updated_at: number; +} + interface EventZapRow { receipt_id: string; target_event_id: string; diff --git a/src/db/migrations/046_author_stats_nip05.ts b/src/db/migrations/046_author_stats_nip05.ts new file mode 100644 index 00000000..12c23773 --- /dev/null +++ b/src/db/migrations/046_author_stats_nip05.ts @@ -0,0 +1,48 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .alterTable('author_stats') + .addColumn('nip05', 'varchar(320)') + .addColumn('nip05_domain', 'varchar(253)') + .addColumn('nip05_hostname', 'varchar(253)') + .addColumn('nip05_last_verified_at', 'integer') + .execute(); + + await db.schema + .alterTable('author_stats') + .addCheckConstraint('author_stats_nip05_domain_lowercase_chk', sql`nip05_domain = lower(nip05_domain)`) + .execute(); + + await db.schema + .alterTable('author_stats') + .addCheckConstraint('author_stats_nip05_hostname_lowercase_chk', sql`nip05_hostname = lower(nip05_hostname)`) + .execute(); + + await db.schema + .alterTable('author_stats') + .addCheckConstraint('author_stats_nip05_hostname_domain_chk', sql`nip05_hostname like '%' || nip05_domain`) + .execute(); + + await db.schema + .createIndex('author_stats_nip05_domain_idx') + .on('author_stats') + .column('nip05_domain') + .execute(); + + await db.schema + .createIndex('author_stats_nip05_hostname_idx') + .on('author_stats') + .column('nip05_hostname') + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema + .alterTable('author_stats') + .dropColumn('nip05') + .dropColumn('nip05_domain') + .dropColumn('nip05_hostname') + .dropColumn('nip05_last_verified_at') + .execute(); +} diff --git a/src/db/migrations/047_add_domain_favicons.ts b/src/db/migrations/047_add_domain_favicons.ts new file mode 100644 index 00000000..b8d7af77 --- /dev/null +++ b/src/db/migrations/047_add_domain_favicons.ts @@ -0,0 +1,15 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('domain_favicons') + .addColumn('domain', 'varchar(253)', (col) => col.primaryKey()) + .addColumn('favicon', 'varchar(2048)', (col) => col.notNull()) + .addColumn('last_updated_at', 'integer', (col) => col.notNull()) + .addCheckConstraint('domain_favicons_https_chk', sql`favicon ~* '^https:\\/\\/'`) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable('domain_favicons').execute(); +} diff --git a/src/interfaces/DittoEvent.ts b/src/interfaces/DittoEvent.ts index 293a7ab4..bca65856 100644 --- a/src/interfaces/DittoEvent.ts +++ b/src/interfaces/DittoEvent.ts @@ -8,6 +8,11 @@ export interface AuthorStats { notes_count: number; streak_start?: number; streak_end?: number; + nip05?: string; + nip05_domain?: string; + nip05_hostname?: string; + nip05_last_verified_at?: number; + favicon?: string; } /** Ditto internal stats for the event. */ @@ -25,6 +30,7 @@ export interface DittoEvent extends NostrEvent { author_domain?: string; author_stats?: AuthorStats; event_stats?: EventStats; + mentions?: DittoEvent[]; user?: DittoEvent; repost?: DittoEvent; quote?: DittoEvent; diff --git a/src/pipeline.ts b/src/pipeline.ts index a4161233..7540bc82 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,6 +1,7 @@ import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; -import { Kysely, sql } from 'kysely'; +import { Kysely, UpdateObject } from 'kysely'; +import tldts from 'tldts'; import { z } from 'zod'; import { pipelineEncounters } from '@/caches/pipelineEncounters.ts'; @@ -13,8 +14,9 @@ import { RelayError } from '@/RelayError.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; -import { eventAge, parseNip05, Time } from '@/utils.ts'; +import { eventAge, Time } from '@/utils.ts'; import { getAmount } from '@/utils/bolt11.ts'; +import { faviconCache } from '@/utils/favicon.ts'; import { errorJson } from '@/utils/log.ts'; import { nip05Cache } from '@/utils/nip05.ts'; import { purifyEvent } from '@/utils/purify.ts'; @@ -119,7 +121,7 @@ async function handleEvent(event: DittoEvent, opts: PipelineOpts): Promise // This needs to run in steps, and should not block the API from responding. Promise.allSettled([ handleZaps(kysely, event), - parseMetadata(event, opts.signal), + updateAuthorData(event, opts.signal), generateSetEvents(event), ]) .then(() => @@ -189,50 +191,81 @@ async function storeEvent(event: NostrEvent, signal?: AbortSignal): Promise { +async function updateAuthorData(event: NostrEvent, signal: AbortSignal): Promise { if (event.kind !== 0) return; // Parse metadata. const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(event.content); if (!metadata.success) return; + const { name, nip05 } = metadata.data; + const kysely = await Storages.kysely(); - // Get nip05. - const { name, nip05 } = metadata.data; - const result = nip05 ? await nip05Cache.fetch(nip05, { signal }).catch(() => undefined) : undefined; + const updates: UpdateObject = {}; + + const authorStats = await kysely + .selectFrom('author_stats') + .selectAll() + .where('pubkey', '=', event.pubkey) + .executeTakeFirst(); + + const lastVerified = authorStats?.nip05_last_verified_at; + const eventNewer = !lastVerified || event.created_at > lastVerified; - // Populate author_search. try { - const search = result?.pubkey === event.pubkey ? [name, nip05].filter(Boolean).join(' ').trim() : name ?? ''; - - if (search) { - await kysely.insertInto('author_stats') - .values({ pubkey: event.pubkey, search, followers_count: 0, following_count: 0, notes_count: 0 }) - .onConflict((oc) => oc.column('pubkey').doUpdateSet({ search })) - .execute(); + if (nip05 !== authorStats?.nip05 && eventNewer || !lastVerified) { + if (nip05) { + const tld = tldts.parse(nip05); + if (tld.isIcann && !tld.isIp && !tld.isPrivate) { + const pointer = await nip05Cache.fetch(nip05.toLowerCase(), { signal }); + if (pointer.pubkey === event.pubkey) { + updates.nip05 = nip05; + updates.nip05_domain = tld.domain; + updates.nip05_hostname = tld.hostname; + updates.nip05_last_verified_at = event.created_at; + } + } + } else { + updates.nip05 = null; + updates.nip05_domain = null; + updates.nip05_hostname = null; + updates.nip05_last_verified_at = event.created_at; + } } } catch { - // do nothing + // Fallthrough. } - if (nip05 && result && result.pubkey === event.pubkey) { - // Track pubkey domain. + // Fetch favicon. + const domain = nip05?.split('@')[1].toLowerCase(); + if (domain) { try { - const { domain } = parseNip05(nip05); - - await sql` - INSERT INTO pubkey_domains (pubkey, domain, last_updated_at) - VALUES (${event.pubkey}, ${domain}, ${event.created_at}) - ON CONFLICT(pubkey) DO UPDATE SET - domain = excluded.domain, - last_updated_at = excluded.last_updated_at - WHERE excluded.last_updated_at > pubkey_domains.last_updated_at - `.execute(kysely); - } catch (_e) { - // do nothing + await faviconCache.fetch(domain, { signal }); + } catch { + // Fallthrough. } } + + const search = [name, nip05].filter(Boolean).join(' ').trim(); + + if (search !== authorStats?.search) { + updates.search = search; + } + + if (Object.keys(updates).length) { + await kysely.insertInto('author_stats') + .values({ + pubkey: event.pubkey, + followers_count: 0, + following_count: 0, + notes_count: 0, + search, + ...updates, + }) + .onConflict((oc) => oc.column('pubkey').doUpdateSet(updates)) + .execute(); + } } /** Determine if the event is being received in a timely manner. */ @@ -364,4 +397,4 @@ async function handleZaps(kysely: Kysely, event: NostrEvent) { } } -export { handleEvent, handleZaps }; +export { handleEvent, handleZaps, updateAuthorData }; diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 160dd1cc..a162571a 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -42,6 +42,10 @@ async function hydrateEvents(opts: HydrateOpts): Promise { cache.push(event); } + for (const event of await gatherMentions({ events: cache, store, signal })) { + cache.push(event); + } + for (const event of await gatherAuthors({ events: cache, store, signal })) { cache.push(event); } @@ -66,9 +70,30 @@ async function hydrateEvents(opts: HydrateOpts): Promise { cache.push(event); } + const authorStats = await gatherAuthorStats(cache, kysely as Kysely); + const eventStats = await gatherEventStats(cache, kysely as Kysely); + + const domains = authorStats.reduce((result, { nip05_hostname }) => { + if (nip05_hostname) result.add(nip05_hostname); + return result; + }, new Set()); + + const favicons = ( + await kysely + .selectFrom('domain_favicons') + .select(['domain', 'favicon']) + .where('domain', 'in', [...domains]) + .execute() + ) + .reduce((result, { domain, favicon }) => { + result[domain] = favicon; + return result; + }, {} as Record); + const stats = { - authors: await gatherAuthorStats(cache, kysely as Kysely), - events: await gatherEventStats(cache, kysely as Kysely), + authors: authorStats, + events: eventStats, + favicons, }; // Dedupe events. @@ -85,7 +110,11 @@ async function hydrateEvents(opts: HydrateOpts): Promise { export function assembleEvents( a: DittoEvent[], b: DittoEvent[], - stats: { authors: DittoTables['author_stats'][]; events: DittoTables['event_stats'][] }, + stats: { + authors: DittoTables['author_stats'][]; + events: DittoTables['event_stats'][]; + favicons: Record; + }, ): DittoEvent[] { const admin = Conf.pubkey; @@ -94,6 +123,11 @@ export function assembleEvents( ...stat, streak_start: stat.streak_start ?? undefined, streak_end: stat.streak_end ?? undefined, + nip05: stat.nip05 ?? undefined, + nip05_domain: stat.nip05_domain ?? undefined, + nip05_hostname: stat.nip05_hostname ?? undefined, + nip05_last_verified_at: stat.nip05_last_verified_at ?? undefined, + favicon: stats.favicons[stat.nip05_hostname!], }; return result; }, {} as Record); @@ -116,6 +150,9 @@ export function assembleEvents( if (id) { event.quote = b.find((e) => matchFilter({ kinds: [1, 20], ids: [id] }, e)); } + + const pubkeys = event.tags.filter(([name]) => name === 'p').map(([_name, value]) => value); + event.mentions = b.filter((e) => matchFilter({ kinds: [0], authors: pubkeys }, e)); } if (event.kind === 6) { @@ -237,6 +274,36 @@ function gatherQuotes({ events, store, signal }: HydrateOpts): Promise { + const pubkeys = new Set(); + + for (const event of events) { + if (event.kind === 1) { + for (const [name, value] of event.tags) { + if (name === 'p') { + pubkeys.add(value); + } + } + } + } + + const authors = await store.query( + [{ kinds: [0], authors: [...pubkeys], limit: pubkeys.size }], + { signal }, + ); + + for (const pubkey of pubkeys) { + const author = authors.find((e) => matchFilter({ kinds: [0], authors: [pubkey] }, e)); + if (!author) { + const fallback = fallbackAuthor(pubkey); + authors.push(fallback); + } + } + + return authors; +} + /** Collect authors from the events. */ async function gatherAuthors({ events, store, signal }: HydrateOpts): Promise { const pubkeys = new Set(); @@ -267,7 +334,7 @@ async function gatherAuthors({ events, store, signal }: HydrateOpts): Promise matchFilter({ kinds: [0], authors: [pubkey] }, e)); - if (author) { + if (!author) { const fallback = fallbackAuthor(pubkey); authors.push(fallback); } @@ -390,13 +457,10 @@ async function gatherAuthorStats( .execute(); return rows.map((row) => ({ - pubkey: row.pubkey, + ...row, followers_count: Math.max(0, row.followers_count), following_count: Math.max(0, row.following_count), notes_count: Math.max(0, row.notes_count), - search: row.search, - streak_start: row.streak_start, - streak_end: row.streak_end, })); } diff --git a/src/utils/SimpleLRU.test.ts b/src/utils/SimpleLRU.test.ts index a73e4f36..03fbfe8a 100644 --- a/src/utils/SimpleLRU.test.ts +++ b/src/utils/SimpleLRU.test.ts @@ -4,7 +4,7 @@ import { assertEquals, assertRejects } from '@std/assert'; Deno.test("SimpleLRU doesn't repeat failed calls", async () => { let calls = 0; - const cache = new SimpleLRU( + using cache = new SimpleLRU( // deno-lint-ignore require-await async () => { calls++; diff --git a/src/utils/SimpleLRU.ts b/src/utils/SimpleLRU.ts index f18a6211..4d8780b7 100644 --- a/src/utils/SimpleLRU.ts +++ b/src/utils/SimpleLRU.ts @@ -3,50 +3,55 @@ import { LRUCache } from 'lru-cache'; import { type Gauge } from 'prom-client'; -type FetchFn = (key: K, opts: O) => Promise; - -interface FetchFnOpts { - signal?: AbortSignal | null; -} +type FetchFn = (key: K, opts: { signal?: AbortSignal }) => Promise; type SimpleLRUOpts = LRUCache.Options & { gauge?: Gauge; + errorRefresh?: number; }; export class SimpleLRU< K extends {}, V extends {}, - O extends {} = FetchFnOpts, > { - protected cache: LRUCache; + protected cache: LRUCache, void>; + private tids = new Set(); - constructor(fetchFn: FetchFn, private opts: SimpleLRUOpts) { - this.cache = new LRUCache({ - async fetchMethod(key, _staleValue, { signal }) { - try { - return await fetchFn(key, { signal: signal as unknown as AbortSignal }); - } catch { - return null as unknown as V; - } - }, - ...opts, - }); + constructor(private fetchFn: FetchFn, private opts: SimpleLRUOpts>) { + this.cache = new LRUCache({ ...opts }); } - async fetch(key: K, opts?: O): Promise { - const result = await this.cache.fetch(key, opts); - - this.opts.gauge?.set(this.cache.size); - - if (result === undefined || result === null) { - throw new Error('SimpleLRU: fetch failed'); + async fetch(key: K, opts?: { signal?: AbortSignal }): Promise { + if (opts?.signal?.aborted) { + throw new DOMException('The signal has been aborted', 'AbortError'); } - return result; + const cached = await this.cache.get(key); + + if (cached) { + return cached; + } + + const promise = this.fetchFn(key, { signal: opts?.signal }); + + this.cache.set(key, promise); + + promise.then(() => { + this.opts.gauge?.set(this.cache.size); + }).catch(() => { + const tid = setTimeout(() => { + this.cache.delete(key); + this.tids.delete(tid); + }, this.opts.errorRefresh ?? 10_000); + this.tids.add(tid); + }); + + return promise; } - put(key: K, value: V): Promise { - this.cache.set(key, value); - return Promise.resolve(); + [Symbol.dispose](): void { + for (const tid of this.tids) { + clearTimeout(tid); + } } } diff --git a/src/utils/favicon.ts b/src/utils/favicon.ts index 70d59de8..fc49c75d 100644 --- a/src/utils/favicon.ts +++ b/src/utils/favicon.ts @@ -1,55 +1,102 @@ import { DOMParser } from '@b-fuze/deno-dom'; import { logi } from '@soapbox/logi'; import { safeFetch } from '@soapbox/safe-fetch'; +import { Kysely } from 'kysely'; import tldts from 'tldts'; import { Conf } from '@/config.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; import { cachedFaviconsSizeGauge } from '@/metrics.ts'; +import { Storages } from '@/storages.ts'; +import { nostrNow } from '@/utils.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; -const faviconCache = new SimpleLRU( +export const faviconCache = new SimpleLRU( async (domain, { signal }) => { - logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'started' }); - const tld = tldts.parse(domain); + const kysely = await Storages.kysely(); - if (!tld.isIcann || tld.isIp || tld.isPrivate) { - throw new Error(`Invalid favicon domain: ${domain}`); + const row = await queryFavicon(kysely, domain); + + if (row && (nostrNow() - row.last_updated_at) < (Conf.caches.favicon.ttl / 1000)) { + return new URL(row.favicon); } - const rootUrl = new URL('/', `https://${domain}/`); - const response = await safeFetch(rootUrl, { signal }); - const html = await response.text(); + const url = await fetchFavicon(domain, signal); - const doc = new DOMParser().parseFromString(html, 'text/html'); - const link = doc.querySelector('link[rel="icon"], link[rel="shortcut icon"]'); + await insertFavicon(kysely, domain, url.href); - if (link) { - const href = link.getAttribute('href'); - if (href) { - let url: URL | undefined; - - try { - url = new URL(href); - } catch { - try { - url = new URL(href, rootUrl); - } catch { - // fall through - } - } - - if (url) { - logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'found', url }); - return url; - } - } - } - - logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'failed' }); - - throw new Error(`Favicon not found: ${domain}`); + return url; }, { ...Conf.caches.favicon, gauge: cachedFaviconsSizeGauge }, ); -export { faviconCache }; +async function queryFavicon( + kysely: Kysely, + domain: string, +): Promise { + return await kysely + .selectFrom('domain_favicons') + .selectAll() + .where('domain', '=', domain) + .executeTakeFirst(); +} + +async function insertFavicon(kysely: Kysely, domain: string, favicon: string): Promise { + await kysely + .insertInto('domain_favicons') + .values({ domain, favicon, last_updated_at: nostrNow() }) + .onConflict((oc) => oc.column('domain').doUpdateSet({ favicon, last_updated_at: nostrNow() })) + .execute(); +} + +async function fetchFavicon(domain: string, signal?: AbortSignal): Promise { + logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'started' }); + const tld = tldts.parse(domain); + + if (!tld.isIcann || tld.isIp || tld.isPrivate) { + throw new Error(`Invalid favicon domain: ${domain}`); + } + + const rootUrl = new URL('/', `https://${domain}/`); + const response = await safeFetch(rootUrl, { signal }); + const html = await response.text(); + + const doc = new DOMParser().parseFromString(html, 'text/html'); + const link = doc.querySelector('link[rel="icon"], link[rel="shortcut icon"]'); + + if (link) { + const href = link.getAttribute('href'); + if (href) { + let url: URL | undefined; + + try { + url = new URL(href); + } catch { + try { + url = new URL(href, rootUrl); + } catch { + // fall through + } + } + + if (url) { + logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'found', url }); + return url; + } + } + } + + // Fallback to checking `/favicon.ico` of the domain. + const url = new URL('/favicon.ico', `https://${domain}/`); + const fallback = await safeFetch(url, { method: 'HEAD', signal }); + const contentType = fallback.headers.get('content-type'); + + if (fallback.ok && contentType === 'image/vnd.microsoft.icon') { + logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'found', url }); + return url; + } + + logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'failed' }); + + throw new Error(`Favicon not found: ${domain}`); +} diff --git a/src/utils/nip05.ts b/src/utils/nip05.ts index ccb08bf2..1cc991b9 100644 --- a/src/utils/nip05.ts +++ b/src/utils/nip05.ts @@ -9,44 +9,51 @@ import { cachedNip05sSizeGauge } from '@/metrics.ts'; import { Storages } from '@/storages.ts'; import { errorJson } from '@/utils/log.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; -import { Nip05, parseNip05 } from '@/utils.ts'; -const nip05Cache = new SimpleLRU( +export const nip05Cache = new SimpleLRU( async (nip05, { signal }) => { - const tld = tldts.parse(nip05); - - if (!tld.isIcann || tld.isIp || tld.isPrivate) { - throw new Error(`Invalid NIP-05: ${nip05}`); - } - - const [name, domain] = nip05.split('@'); - - logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'started' }); - - try { - if (domain === Conf.url.host) { - const store = await Storages.db(); - const pointer = await localNip05Lookup(store, name); - if (pointer) { - logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', pubkey: pointer.pubkey }); - return pointer; - } else { - throw new Error(`Not found: ${nip05}`); - } - } else { - const result = await NIP05.lookup(nip05, { fetch: safeFetch, signal }); - logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', pubkey: result.pubkey }); - return result; - } - } catch (e) { - logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'failed', error: errorJson(e) }); - throw e; - } + const store = await Storages.db(); + return getNip05(store, nip05, signal); }, { ...Conf.caches.nip05, gauge: cachedNip05sSizeGauge }, ); -async function localNip05Lookup(store: NStore, localpart: string): Promise { +async function getNip05( + store: NStore, + nip05: string, + signal?: AbortSignal, +): Promise { + const tld = tldts.parse(nip05); + + if (!tld.isIcann || tld.isIp || tld.isPrivate) { + throw new Error(`Invalid NIP-05: ${nip05}`); + } + + logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'started' }); + + const [name, domain] = nip05.split('@'); + + try { + if (domain === Conf.url.host) { + const pointer = await localNip05Lookup(store, name); + if (pointer) { + logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'local', pubkey: pointer.pubkey }); + return pointer; + } else { + throw new Error(`Not found: ${nip05}`); + } + } else { + const pointer = await NIP05.lookup(nip05, { fetch: safeFetch, signal }); + logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'fetch', pubkey: pointer.pubkey }); + return pointer; + } + } catch (e) { + logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'failed', error: errorJson(e) }); + throw e; + } +} + +export async function localNip05Lookup(store: NStore, localpart: string): Promise { const [grant] = await store.query([{ kinds: [30360], '#d': [`${localpart}@${Conf.url.host}`], @@ -60,21 +67,3 @@ async function localNip05Lookup(store: NStore, localpart: string): Promise { - if (!nip05) return; - try { - const result = await nip05Cache.fetch(nip05, { signal }); - if (result.pubkey === pubkey) { - return parseNip05(nip05); - } - } catch (_e) { - // do nothing - } -} - -export { localNip05Lookup, nip05Cache }; diff --git a/src/utils/stats.ts b/src/utils/stats.ts index 0821fed2..972541d3 100644 --- a/src/utils/stats.ts +++ b/src/utils/stats.ts @@ -323,6 +323,10 @@ export async function countAuthorStats( search, streak_start: null, streak_end: null, + nip05: null, + nip05_domain: null, + nip05_hostname: null, + nip05_last_verified_at: null, }; } diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index 99dd3523..c2cf41ca 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -6,11 +6,9 @@ import { MastodonAccount } from '@/entities/MastodonAccount.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { metadataSchema } from '@/schemas/nostr.ts'; import { getLnurl } from '@/utils/lnurl.ts'; -import { parseAndVerifyNip05 } from '@/utils/nip05.ts'; import { parseNoteContent } from '@/utils/note.ts'; import { getTagSet } from '@/utils/tags.ts'; -import { faviconCache } from '@/utils/favicon.ts'; -import { nostrDate, nostrNow } from '@/utils.ts'; +import { nostrDate, nostrNow, parseNip05 } from '@/utils.ts'; import { renderEmojis } from '@/views/mastodon/emojis.ts'; type ToAccountOpts = { @@ -20,16 +18,14 @@ type ToAccountOpts = { withSource?: false; }; -async function renderAccount( - event: Omit, - opts: ToAccountOpts = {}, - signal = AbortSignal.timeout(3000), -): Promise { +function renderAccount(event: Omit, opts: ToAccountOpts = {}): MastodonAccount { const { pubkey } = event; + const stats = event.author_stats; const names = getTagSet(event.user?.tags ?? [], 'n'); + if (names.has('disabled')) { - const account = await accountFromPubkey(pubkey, opts); + const account = accountFromPubkey(pubkey, opts); account.pleroma.deactivated = true; return account; } @@ -48,17 +44,9 @@ async function renderAccount( const npub = nip19.npubEncode(pubkey); const nprofile = nip19.nprofileEncode({ pubkey, relays: [Conf.relay] }); - const parsed05 = await parseAndVerifyNip05(nip05, pubkey, signal); + const parsed05 = stats?.nip05 ? parseNip05(stats.nip05) : undefined; const acct = parsed05?.handle || npub; - let favicon: URL | undefined; - if (parsed05?.domain) { - try { - favicon = await faviconCache.fetch(parsed05.domain, { signal }); - } catch { - favicon = new URL('/favicon.ico', `https://${parsed05.domain}/`); - } - } const { html } = parseNoteContent(about || '', []); const fields = _fields @@ -70,8 +58,8 @@ async function renderAccount( })) ?? []; let streakDays = 0; - let streakStart = event.author_stats?.streak_start ?? null; - let streakEnd = event.author_stats?.streak_end ?? null; + let streakStart = stats?.streak_start ?? null; + let streakEnd = stats?.streak_end ?? null; const { streakWindow } = Conf; if (streakStart && streakEnd) { @@ -97,8 +85,8 @@ async function renderAccount( emojis: renderEmojis(event), fields: fields.map((field) => ({ ...field, value: parseNoteContent(field.value, []).html })), follow_requests_count: 0, - followers_count: event.author_stats?.followers_count ?? 0, - following_count: event.author_stats?.following_count ?? 0, + followers_count: stats?.followers_count ?? 0, + following_count: stats?.following_count ?? 0, fqn: parsed05?.handle || npub, header: banner, header_static: banner, @@ -122,7 +110,7 @@ async function renderAccount( }, } : undefined, - statuses_count: event.author_stats?.notes_count ?? 0, + statuses_count: stats?.notes_count ?? 0, uri: Conf.local(`/users/${acct}`), url: Conf.local(`/@${acct}`), username: parsed05?.nickname || npub.substring(0, 8), @@ -144,7 +132,7 @@ async function renderAccount( is_local: parsed05?.domain === Conf.url.host, settings_store: opts.withSource ? opts.settingsStore : undefined, tags: [...getTagSet(event.user?.tags ?? [], 't')], - favicon: favicon?.toString(), + favicon: stats?.favicon, }, nostr: { pubkey, @@ -154,7 +142,7 @@ async function renderAccount( }; } -function accountFromPubkey(pubkey: string, opts: ToAccountOpts = {}): Promise { +function accountFromPubkey(pubkey: string, opts: ToAccountOpts = {}): MastodonAccount { const event: UnsignedEvent = { kind: 0, pubkey, diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 265cf442..0c0eb9f2 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -7,7 +7,7 @@ import { MastodonMention } from '@/entities/MastodonMention.ts'; import { MastodonStatus } from '@/entities/MastodonStatus.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { Storages } from '@/storages.ts'; -import { isNostrId, nostrDate } from '@/utils.ts'; +import { nostrDate } from '@/utils.ts'; import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts'; import { findReplyTag } from '@/utils/tags.ts'; import { unfurlCardCached } from '@/utils/unfurl.ts'; @@ -33,28 +33,14 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< }); const account = event.author - ? await renderAccount({ ...event.author, author_stats: event.author_stats }) - : await accountFromPubkey(event.pubkey); + ? renderAccount({ ...event.author, author_stats: event.author_stats }) + : accountFromPubkey(event.pubkey); const replyId = findReplyTag(event.tags)?.[1]; - const mentionedPubkeys = [ - ...new Set( - event.tags - .filter(([name, value]) => name === 'p' && isNostrId(value)) - .map(([, value]) => value), - ), - ]; - const store = await Storages.db(); - const mentionedProfiles = await store.query( - [{ kinds: [0], authors: mentionedPubkeys, limit: mentionedPubkeys.length }], - ); - - const mentions = await Promise.all( - mentionedPubkeys.map((pubkey) => renderMention(pubkey, mentionedProfiles.find((event) => event.pubkey === pubkey))), - ); + const mentions = event.mentions?.map((event) => renderMention(event)) ?? []; const { html, links, firstUrl } = parseNoteContent(stripimeta(event.content, event.tags), mentions); @@ -170,8 +156,8 @@ async function renderReblog(event: DittoEvent, opts: RenderStatusOpts): Promise< }; } -async function renderMention(pubkey: string, event?: NostrEvent): Promise { - const account = event ? await renderAccount(event) : await accountFromPubkey(pubkey); +function renderMention(event: NostrEvent): MastodonMention { + const account = renderAccount(event); return { id: account.id, acct: account.acct,