From 93141c1db188f244c929f119d3a737eebcd448f6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 7 Feb 2025 15:39:25 -0600 Subject: [PATCH] Hook everything up? (In a messy way) --- src/db/DittoTables.ts | 1 + src/db/migrations/047_add_domain_favicons.ts | 2 +- src/pipeline.ts | 29 ++--- src/storages/hydrate.ts | 1 + src/utils/favicon.ts | 108 ++++++++++------ src/utils/nip05.ts | 129 ++++++++++++++----- src/utils/stats.ts | 1 + 7 files changed, 184 insertions(+), 87 deletions(-) diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index e07e7002..19ea6e1b 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -23,6 +23,7 @@ interface AuthorStatsRow { nip05: string | null; nip05_domain: string | null; nip05_hostname: string | null; + nip05_last_verified_at: number | null; } interface EventStatsRow { diff --git a/src/db/migrations/047_add_domain_favicons.ts b/src/db/migrations/047_add_domain_favicons.ts index 38bda03d..b8d7af77 100644 --- a/src/db/migrations/047_add_domain_favicons.ts +++ b/src/db/migrations/047_add_domain_favicons.ts @@ -6,7 +6,7 @@ export async function up(db: Kysely): Promise { .addColumn('domain', 'varchar(253)', (col) => col.primaryKey()) .addColumn('favicon', 'varchar(2048)', (col) => col.notNull()) .addColumn('last_updated_at', 'integer', (col) => col.notNull()) - .addCheckConstraint('domain_favicons_https_chk', sql`url ~* '^https:\\/\\/'`) + .addCheckConstraint('domain_favicons_https_chk', sql`favicon ~* '^https:\\/\\/'`) .execute(); } diff --git a/src/pipeline.ts b/src/pipeline.ts index a4161233..9f9b8365 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,6 +1,6 @@ import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; -import { Kysely, sql } from 'kysely'; +import { Kysely } from 'kysely'; import { z } from 'zod'; import { pipelineEncounters } from '@/caches/pipelineEncounters.ts'; @@ -13,8 +13,9 @@ import { RelayError } from '@/RelayError.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; -import { eventAge, parseNip05, Time } from '@/utils.ts'; +import { eventAge, Time } from '@/utils.ts'; import { getAmount } from '@/utils/bolt11.ts'; +import { faviconCache } from '@/utils/favicon.ts'; import { errorJson } from '@/utils/log.ts'; import { nip05Cache } from '@/utils/nip05.ts'; import { purifyEvent } from '@/utils/purify.ts'; @@ -202,6 +203,12 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise undefined) : undefined; + // Fetch favicon. + const domain = nip05?.split('@')[1].toLowerCase(); + if (domain) { + await faviconCache.fetch(domain, { signal }); + } + // Populate author_search. try { const search = result?.pubkey === event.pubkey ? [name, nip05].filter(Boolean).join(' ').trim() : name ?? ''; @@ -215,24 +222,6 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise pubkey_domains.last_updated_at - `.execute(kysely); - } catch (_e) { - // do nothing - } - } } /** Determine if the event is being received in a timely manner. */ diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index b9f0fad0..aff68f39 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -122,6 +122,7 @@ export function assembleEvents( nip05: stat.nip05 ?? undefined, nip05_domain: stat.nip05_domain ?? undefined, nip05_hostname: stat.nip05_hostname ?? undefined, + nip05_last_verified_at: stat.nip05_last_verified_at ?? undefined, favicon: stats.favicons[stat.nip05_hostname!], }; return result; diff --git a/src/utils/favicon.ts b/src/utils/favicon.ts index 9833de1c..2ee7c7f7 100644 --- a/src/utils/favicon.ts +++ b/src/utils/favicon.ts @@ -1,55 +1,91 @@ import { DOMParser } from '@b-fuze/deno-dom'; import { logi } from '@soapbox/logi'; +import { Kysely } from 'kysely'; import tldts from 'tldts'; import { Conf } from '@/config.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; import { cachedFaviconsSizeGauge } from '@/metrics.ts'; +import { Storages } from '@/storages.ts'; +import { nostrNow } from '@/utils.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; import { fetchWorker } from '@/workers/fetch.ts'; -const faviconCache = new SimpleLRU( +export const faviconCache = new SimpleLRU( async (domain, { signal }) => { - logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'started' }); - const tld = tldts.parse(domain); + const kysely = await Storages.kysely(); - if (!tld.isIcann || tld.isIp || tld.isPrivate) { - throw new Error(`Invalid favicon domain: ${domain}`); + const row = await queryFavicon(kysely, domain); + + if (row && (nostrNow() - row.last_updated_at) < (Conf.caches.favicon.ttl / 1000)) { + return new URL(row.favicon); } - const rootUrl = new URL('/', `https://${domain}/`); - const response = await fetchWorker(rootUrl, { signal }); - const html = await response.text(); + const url = await fetchFavicon(domain, signal); - const doc = new DOMParser().parseFromString(html, 'text/html'); - const link = doc.querySelector('link[rel="icon"], link[rel="shortcut icon"]'); + insertFavicon(kysely, domain, url.href).catch(() => {}); - if (link) { - const href = link.getAttribute('href'); - if (href) { - let url: URL | undefined; - - try { - url = new URL(href); - } catch { - try { - url = new URL(href, rootUrl); - } catch { - // fall through - } - } - - if (url) { - logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'found', url }); - return url; - } - } - } - - logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'failed' }); - - throw new Error(`Favicon not found: ${domain}`); + return url; }, { ...Conf.caches.favicon, gauge: cachedFaviconsSizeGauge }, ); -export { faviconCache }; +async function queryFavicon( + kysely: Kysely, + domain: string, +): Promise { + return await kysely + .selectFrom('domain_favicons') + .selectAll() + .where('domain', '=', domain) + .executeTakeFirst(); +} + +async function insertFavicon(kysely: Kysely, domain: string, favicon: string): Promise { + await kysely + .insertInto('domain_favicons') + .values({ domain, favicon, last_updated_at: nostrNow() }) + .execute(); +} + +async function fetchFavicon(domain: string, signal?: AbortSignal): Promise { + logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'started' }); + const tld = tldts.parse(domain); + + if (!tld.isIcann || tld.isIp || tld.isPrivate) { + throw new Error(`Invalid favicon domain: ${domain}`); + } + + const rootUrl = new URL('/', `https://${domain}/`); + const response = await fetchWorker(rootUrl, { signal }); + const html = await response.text(); + + const doc = new DOMParser().parseFromString(html, 'text/html'); + const link = doc.querySelector('link[rel="icon"], link[rel="shortcut icon"]'); + + if (link) { + const href = link.getAttribute('href'); + if (href) { + let url: URL | undefined; + + try { + url = new URL(href); + } catch { + try { + url = new URL(href, rootUrl); + } catch { + // fall through + } + } + + if (url) { + logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'found', url }); + return url; + } + } + } + + logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'failed' }); + + throw new Error(`Favicon not found: ${domain}`); +} diff --git a/src/utils/nip05.ts b/src/utils/nip05.ts index 3860a6cb..d180d610 100644 --- a/src/utils/nip05.ts +++ b/src/utils/nip05.ts @@ -1,51 +1,120 @@ import { nip19 } from 'nostr-tools'; import { NIP05, NStore } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; +import { Kysely } from 'kysely'; import tldts from 'tldts'; import { Conf } from '@/config.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; import { cachedNip05sSizeGauge } from '@/metrics.ts'; import { Storages } from '@/storages.ts'; import { errorJson } from '@/utils/log.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; -import { Nip05, parseNip05 } from '@/utils.ts'; +import { Nip05, nostrNow, parseNip05 } from '@/utils.ts'; import { fetchWorker } from '@/workers/fetch.ts'; export const nip05Cache = new SimpleLRU( async (nip05, { signal }) => { - const tld = tldts.parse(nip05); - - if (!tld.isIcann || tld.isIp || tld.isPrivate) { - throw new Error(`Invalid NIP-05: ${nip05}`); - } - - const [name, domain] = nip05.split('@'); - - logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'started' }); - - try { - if (domain === Conf.url.host) { - const store = await Storages.db(); - const pointer = await localNip05Lookup(store, name); - if (pointer) { - logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', pubkey: pointer.pubkey }); - return pointer; - } else { - throw new Error(`Not found: ${nip05}`); - } - } else { - const result = await NIP05.lookup(nip05, { fetch: fetchWorker, signal }); - logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', pubkey: result.pubkey }); - return result; - } - } catch (e) { - logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'failed', error: errorJson(e) }); - throw e; - } + const store = await Storages.db(); + const kysely = await Storages.kysely(); + return getNip05(kysely, store, nip05, signal); }, { ...Conf.caches.nip05, gauge: cachedNip05sSizeGauge }, ); +async function getNip05( + kysely: Kysely, + store: NStore, + nip05: string, + signal?: AbortSignal, +): Promise { + const tld = tldts.parse(nip05); + + if (!tld.isIcann || tld.isIp || tld.isPrivate) { + throw new Error(`Invalid NIP-05: ${nip05}`); + } + + logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'started' }); + + let pointer: nip19.ProfilePointer | undefined = await queryNip05(kysely, nip05); + + if (pointer) { + logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'db', pubkey: pointer.pubkey }); + return pointer; + } + + const [name, domain] = nip05.split('@'); + + try { + if (domain === Conf.url.host) { + pointer = await localNip05Lookup(store, name); + if (pointer) { + logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'local', pubkey: pointer.pubkey }); + } else { + throw new Error(`Not found: ${nip05}`); + } + } else { + pointer = await NIP05.lookup(nip05, { fetch: fetchWorker, signal }); + logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', source: 'fetch', pubkey: pointer.pubkey }); + } + } catch (e) { + logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'failed', error: errorJson(e) }); + throw e; + } + + insertNip05(kysely, nip05, pointer.pubkey).catch((e) => { + logi({ level: 'error', ns: 'ditto.nip05', nip05, state: 'insert_failed', error: errorJson(e) }); + }); + + return pointer; +} + +async function queryNip05(kysely: Kysely, nip05: string): Promise { + const row = await kysely + .selectFrom('author_stats') + .select('pubkey') + .where('nip05', '=', nip05) + .executeTakeFirst(); + + if (row) { + return { pubkey: row.pubkey }; + } +} + +async function insertNip05(kysely: Kysely, nip05: string, pubkey: string, ts = nostrNow()): Promise { + const tld = tldts.parse(nip05); + + if (!tld.isIcann || tld.isIp || tld.isPrivate) { + throw new Error(`Invalid NIP-05: ${nip05}`); + } + + await kysely + .insertInto('author_stats') + .values({ + pubkey, + nip05, + nip05_domain: tld.domain, + nip05_hostname: tld.hostname, + nip05_last_verified_at: ts, + followers_count: 0, // TODO: fix `author_stats` types so setting these aren't required + following_count: 0, + notes_count: 0, + search: nip05, + }) + .onConflict((oc) => + oc + .column('pubkey') + .doUpdateSet({ + nip05, + nip05_domain: tld.domain, + nip05_hostname: tld.hostname, + nip05_last_verified_at: ts, + }) + .where('nip05_last_verified_at', '<', ts) + ) + .execute(); +} + export async function localNip05Lookup(store: NStore, localpart: string): Promise { const [grant] = await store.query([{ kinds: [30360], diff --git a/src/utils/stats.ts b/src/utils/stats.ts index 0a675aee..972541d3 100644 --- a/src/utils/stats.ts +++ b/src/utils/stats.ts @@ -326,6 +326,7 @@ export async function countAuthorStats( nip05: null, nip05_domain: null, nip05_hostname: null, + nip05_last_verified_at: null, }; }