From 8eccdafa6470705498da7fe918fb6facbac407be Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 21 Jan 2024 20:22:11 -0600 Subject: [PATCH 1/4] Improve the NIP-05 cache --- src/controllers/api/search.ts | 16 ++++--- src/deps.ts | 4 ++ src/utils.ts | 7 +-- src/utils/SimpleLRU.ts | 37 +++++++++++++++ src/utils/nip05.ts | 83 +++++++--------------------------- src/views/mastodon/accounts.ts | 18 ++++++-- 6 files changed, 85 insertions(+), 80 deletions(-) create mode 100644 src/utils/SimpleLRU.ts diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 5feeef67..349f51e3 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -5,7 +5,7 @@ import { booleanParamSchema } from '@/schema.ts'; import { nostrIdSchema } from '@/schemas/nostr.ts'; import { searchStore } from '@/storages.ts'; import { dedupeEvents } from '@/utils.ts'; -import { lookupNip05Cached } from '@/utils/nip05.ts'; +import { nip05Cache } from '@/utils/nip05.ts'; import { renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; @@ -95,13 +95,13 @@ function typeToKinds(type: SearchQuery['type']): number[] { /** Resolve a searched value into an event, if applicable. */ async function lookupEvent(query: SearchQuery, signal: AbortSignal): Promise { - const filters = await getLookupFilters(query); + const filters = await getLookupFilters(query, signal); const [event] = await searchStore.filter(filters, { limit: 1, signal }); return event; } /** Get filters to lookup the input value. */ -async function getLookupFilters({ q, type, resolve }: SearchQuery): Promise { +async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: AbortSignal): Promise { const filters: DittoFilter[] = []; const accounts = !type || type === 'accounts'; @@ -139,9 +139,13 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery): Promise | undefined> { +async function lookupAccount(value: string, signal = AbortSignal.timeout(3000)): Promise | undefined> { console.log(`Looking up ${value}`); - const pubkey = bech32ToPubkey(value) || await lookupNip05Cached(value); + const pubkey = bech32ToPubkey(value) || + await nip05Cache.fetch(value, { signal }).then(({ pubkey }) => pubkey).catch(() => undefined); if (pubkey) { return getAuthor(pubkey); diff --git a/src/utils/SimpleLRU.ts b/src/utils/SimpleLRU.ts new file mode 100644 index 00000000..c8af3472 --- /dev/null +++ b/src/utils/SimpleLRU.ts @@ -0,0 +1,37 @@ +// deno-lint-ignore-file ban-types + +import { LRUCache, type MapCache } from '@/deps.ts'; + +type FetchFn = (key: K, opts: O) => Promise; + +interface FetchFnOpts { + signal?: AbortSignal | null; +} + +export class SimpleLRU< + K extends {}, + V extends {}, + O extends {} = FetchFnOpts, +> implements MapCache { + protected cache: LRUCache; + + constructor(fetchFn: FetchFn, opts: LRUCache.Options) { + this.cache = new LRUCache({ + fetchMethod: (key, _staleValue, { signal }) => fetchFn(key, { signal: signal as AbortSignal }), + ...opts, + }); + } + + async fetch(key: K, opts?: O): Promise { + const result = await this.cache.fetch(key, opts); + if (result === undefined) { + throw new Error('SimpleLRU: fetch failed'); + } + return result; + } + + put(key: K, value: V): Promise { + this.cache.set(key, value); + return Promise.resolve(); + } +} diff --git a/src/utils/nip05.ts b/src/utils/nip05.ts index f885641e..e94b36ce 100644 --- a/src/utils/nip05.ts +++ b/src/utils/nip05.ts @@ -1,73 +1,22 @@ -import { Debug, TTLCache, z } from '@/deps.ts'; +import { Debug, NIP05, nip19 } from '@/deps.ts'; +import { SimpleLRU } from '@/utils/SimpleLRU.ts'; import { Time } from '@/utils/time.ts'; -import { fetchWorker } from '@/workers/fetch.ts'; const debug = Debug('ditto:nip05'); -const nip05Cache = new TTLCache>({ ttl: Time.hours(1), max: 5000 }); - -const NIP05_REGEX = /^(?:([\w.+-]+)@)?([\w.-]+)$/; - -interface LookupOpts { - signal?: AbortSignal; -} - -/** Get pubkey from NIP-05. */ -async function lookup(value: string, opts: LookupOpts = {}): Promise { - const { signal = AbortSignal.timeout(2000) } = opts; - - const match = value.match(NIP05_REGEX); - if (!match) return null; - - const [_, name = '_', domain] = match; - - try { - const res = await fetchWorker(`https://${domain}/.well-known/nostr.json?name=${name}`, { - signal, - }); - - const { names } = nostrJsonSchema.parse(await res.json()); - - return names[name] || null; - } catch (e) { - debug(e); - return null; - } -} - -/** nostr.json schema. */ -const nostrJsonSchema = z.object({ - names: z.record(z.string(), z.string()), - relays: z.record(z.string(), z.array(z.string())).optional().catch(undefined), -}); - -/** - * Lookup the NIP-05 and serve from cache first. - * To prevent race conditions we put the promise in the cache instead of the result. - */ -function lookupNip05Cached(value: string): Promise { - const cached = nip05Cache.get(value); - if (cached !== undefined) return cached; - - debug(`Lookup ${value}`); - const result = lookup(value); - nip05Cache.set(value, result); - - result.then((result) => { - if (result) { - debug(`Found: ${value} is ${result}`); - } else { - debug(`Not found: ${value} is ${result}`); +const nip05Cache = new SimpleLRU( + async (key, { signal }) => { + debug(`Lookup ${key}`); + try { + const result = await NIP05.lookup(key, { fetch, signal }); + debug(`Found: ${key} is ${result.pubkey}`); + return result; + } catch (e) { + debug(`Not found: ${key}`); + throw e; } - }); + }, + { max: 5000, ttl: Time.hours(1) }, +); - return result; -} - -/** Verify the NIP-05 matches the pubkey, with cache. */ -async function verifyNip05Cached(value: string, pubkey: string): Promise { - const result = await lookupNip05Cached(value); - return result === pubkey; -} - -export { lookupNip05Cached, verifyNip05Cached }; +export { nip05Cache }; diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index 0030f3bf..574a1c41 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -3,7 +3,7 @@ import { findUser } from '@/db/users.ts'; import { lodash, nip19, type UnsignedEvent } from '@/deps.ts'; import { jsonMetaContentSchema } from '@/schemas/nostr.ts'; import { type DittoEvent } from '@/storages/types.ts'; -import { verifyNip05Cached } from '@/utils/nip05.ts'; +import { nip05Cache } from '@/utils/nip05.ts'; import { Nip05, nostrDate, nostrNow, parseNip05 } from '@/utils.ts'; import { renderEmojis } from '@/views/mastodon/emojis.ts'; @@ -86,9 +86,19 @@ function accountFromPubkey(pubkey: string, opts: ToAccountOpts = {}) { return renderAccount(event, opts); } -async function parseAndVerifyNip05(nip05: string | undefined, pubkey: string): Promise { - if (nip05 && await verifyNip05Cached(nip05, pubkey)) { - return parseNip05(nip05); +async function parseAndVerifyNip05( + nip05: string | undefined, + pubkey: string, + signal = AbortSignal.timeout(3000), +): Promise { + if (!nip05) return; + try { + const result = await nip05Cache.fetch(nip05, { signal }); + if (result.pubkey === pubkey) { + return parseNip05(nip05); + } + } catch (_e) { + // do nothing } } From 4bec5f6f7808a259a1cb01e7c5fd91e6d1136695 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 22 Jan 2024 11:43:46 -0600 Subject: [PATCH 2/4] Try using httpbin in tests, cuz CI runner is hanging on example.com --- src/workers/fetch.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/workers/fetch.test.ts b/src/workers/fetch.test.ts index c985ee62..0d6e0500 100644 --- a/src/workers/fetch.test.ts +++ b/src/workers/fetch.test.ts @@ -1,4 +1,4 @@ -import { assert, assertRejects } from '@/deps-test.ts'; +import { assertEquals, assertRejects } from '@/deps-test.ts'; import { fetchWorker } from './fetch.ts'; @@ -7,9 +7,9 @@ await sleep(2000); Deno.test({ name: 'fetchWorker', async fn() { - const response = await fetchWorker('https://example.com'); - const text = await response.text(); - assert(text.includes('Example Domain')); + const response = await fetchWorker('http://httpbin.org/get'); + const json = await response.json(); + assertEquals(json.headers.Host, 'httpbin.org'); }, sanitizeResources: false, }); From dc6a6ccb5f04373c67530be2e4e13fcf4fab0475 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 22 Jan 2024 11:48:52 -0600 Subject: [PATCH 3/4] fetch.test: don't sleep at the beginning? I'm really confused why it's not working --- src/workers/fetch.test.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/workers/fetch.test.ts b/src/workers/fetch.test.ts index 0d6e0500..d9a6102f 100644 --- a/src/workers/fetch.test.ts +++ b/src/workers/fetch.test.ts @@ -2,8 +2,6 @@ import { assertEquals, assertRejects } from '@/deps-test.ts'; import { fetchWorker } from './fetch.ts'; -await sleep(2000); - Deno.test({ name: 'fetchWorker', async fn() { @@ -29,7 +27,3 @@ Deno.test({ }, sanitizeResources: false, }); - -function sleep(ms: number) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} From fc3934fa90e119a52243f87d34661463ccb3e21d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 22 Jan 2024 11:55:34 -0600 Subject: [PATCH 4/4] fetchWorker: wait for the worker to be ready before using it --- src/workers/fetch.ts | 20 +++++++++++++------- src/workers/fetch.worker.ts | 2 ++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/workers/fetch.ts b/src/workers/fetch.ts index 3246e78e..510d806f 100644 --- a/src/workers/fetch.ts +++ b/src/workers/fetch.ts @@ -4,21 +4,27 @@ import './handlers/abortsignal.ts'; import type { FetchWorker } from './fetch.worker.ts'; -const _worker = Comlink.wrap( - new Worker( - new URL('./fetch.worker.ts', import.meta.url), - { type: 'module' }, - ), -); +const worker = new Worker(new URL('./fetch.worker.ts', import.meta.url), { type: 'module' }); +const client = Comlink.wrap(worker); + +// Wait for the worker to be ready before we start using it. +const ready = new Promise((resolve) => { + const handleEvent = () => { + self.removeEventListener('message', handleEvent); + resolve(); + }; + worker.addEventListener('message', handleEvent); +}); /** * Fetch implementation with a Web Worker. * Calling this performs the fetch in a separate CPU thread so it doesn't block the main thread. */ const fetchWorker: typeof fetch = async (...args) => { + await ready; const [url, init] = serializeFetchArgs(args); const { body, signal, ...rest } = init; - const result = await _worker.fetch(url, { ...rest, body: await prepareBodyForWorker(body) }, signal); + const result = await client.fetch(url, { ...rest, body: await prepareBodyForWorker(body) }, signal); return new Response(...result); }; diff --git a/src/workers/fetch.worker.ts b/src/workers/fetch.worker.ts index 8a2c0b11..8e79465f 100644 --- a/src/workers/fetch.worker.ts +++ b/src/workers/fetch.worker.ts @@ -24,3 +24,5 @@ export const FetchWorker = { }; Comlink.expose(FetchWorker); + +self.postMessage('ready');