From af262b5d524993da1b733acdd235ccb1e47f27d1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 7 Feb 2025 12:06:34 -0600 Subject: [PATCH 1/3] Whoops, fix streak days calculation --- src/views/mastodon/accounts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index 0c2d1dcc..99dd3523 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -81,7 +81,7 @@ async function renderAccount( streakEnd = null; } else { const delta = streakEnd - streakStart; - streakDays = Math.max(Math.ceil(delta / streakWindow), 1); + streakDays = Math.max(Math.ceil(delta / 86400), 1); } } From 838f773b846b25425b4110726a0c293aea0a6246 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Feb 2025 15:01:25 -0600 Subject: [PATCH 2/3] Remove fetchWorker --- src/middleware/translatorMiddleware.ts | 7 ++- src/middleware/uploaderMiddleware.ts | 8 +-- src/utils/favicon.ts | 4 +- src/utils/lnurl.ts | 6 +- src/utils/nip05.ts | 4 +- src/utils/unfurl.ts | 4 +- src/workers/fetch.test.ts | 29 --------- src/workers/fetch.ts | 86 -------------------------- src/workers/fetch.worker.ts | 33 ---------- src/workers/handlers/abortsignal.ts | 46 -------------- src/workers/policy.ts | 2 - src/workers/policy.worker.ts | 2 - 12 files changed, 17 insertions(+), 214 deletions(-) delete mode 100644 src/workers/fetch.test.ts delete mode 100644 src/workers/fetch.ts delete mode 100644 src/workers/fetch.worker.ts delete mode 100644 src/workers/handlers/abortsignal.ts diff --git a/src/middleware/translatorMiddleware.ts b/src/middleware/translatorMiddleware.ts index f5a6baa2..ef123dab 100644 --- a/src/middleware/translatorMiddleware.ts +++ b/src/middleware/translatorMiddleware.ts @@ -1,6 +1,7 @@ +import { safeFetch } from '@soapbox/safe-fetch'; + import { AppMiddleware } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { fetchWorker } from '@/workers/fetch.ts'; import { DeepLTranslator } from '@/translators/DeepLTranslator.ts'; import { LibreTranslateTranslator } from '@/translators/LibreTranslateTranslator.ts'; @@ -10,7 +11,7 @@ export const translatorMiddleware: AppMiddleware = async (c, next) => { case 'deepl': { const { deeplApiKey: apiKey, deeplBaseUrl: baseUrl } = Conf; if (apiKey) { - c.set('translator', new DeepLTranslator({ baseUrl, apiKey, fetch: fetchWorker })); + c.set('translator', new DeepLTranslator({ baseUrl, apiKey, fetch: safeFetch })); } break; } @@ -18,7 +19,7 @@ export const translatorMiddleware: AppMiddleware = async (c, next) => { case 'libretranslate': { const { libretranslateApiKey: apiKey, libretranslateBaseUrl: baseUrl } = Conf; if (apiKey) { - c.set('translator', new LibreTranslateTranslator({ baseUrl, apiKey, fetch: fetchWorker })); + c.set('translator', new LibreTranslateTranslator({ baseUrl, apiKey, fetch: safeFetch })); } break; } diff --git a/src/middleware/uploaderMiddleware.ts b/src/middleware/uploaderMiddleware.ts index 96a47336..6866b883 100644 --- a/src/middleware/uploaderMiddleware.ts +++ b/src/middleware/uploaderMiddleware.ts @@ -1,11 +1,11 @@ import { BlossomUploader, NostrBuildUploader } from '@nostrify/nostrify/uploaders'; +import { safeFetch } from '@soapbox/safe-fetch'; import { AppMiddleware } from '@/app.ts'; import { Conf } from '@/config.ts'; import { DenoUploader } from '@/uploaders/DenoUploader.ts'; import { IPFSUploader } from '@/uploaders/IPFSUploader.ts'; import { S3Uploader } from '@/uploaders/S3Uploader.ts'; -import { fetchWorker } from '@/workers/fetch.ts'; /** Set an uploader for the user. */ export const uploaderMiddleware: AppMiddleware = async (c, next) => { @@ -29,17 +29,17 @@ export const uploaderMiddleware: AppMiddleware = async (c, next) => { ); break; case 'ipfs': - c.set('uploader', new IPFSUploader({ baseUrl: Conf.mediaDomain, apiUrl: Conf.ipfs.apiUrl, fetch: fetchWorker })); + c.set('uploader', new IPFSUploader({ baseUrl: Conf.mediaDomain, apiUrl: Conf.ipfs.apiUrl, fetch: safeFetch })); break; case 'local': c.set('uploader', new DenoUploader({ baseUrl: Conf.mediaDomain, dir: Conf.uploadsDir })); break; case 'nostrbuild': - c.set('uploader', new NostrBuildUploader({ endpoint: Conf.nostrbuildEndpoint, signer, fetch: fetchWorker })); + c.set('uploader', new NostrBuildUploader({ endpoint: Conf.nostrbuildEndpoint, signer, fetch: safeFetch })); break; case 'blossom': if (signer) { - c.set('uploader', new BlossomUploader({ servers: Conf.blossomServers, signer, fetch: fetchWorker })); + c.set('uploader', new BlossomUploader({ servers: Conf.blossomServers, signer, fetch: safeFetch })); } break; } diff --git a/src/utils/favicon.ts b/src/utils/favicon.ts index 9833de1c..70d59de8 100644 --- a/src/utils/favicon.ts +++ b/src/utils/favicon.ts @@ -1,11 +1,11 @@ import { DOMParser } from '@b-fuze/deno-dom'; import { logi } from '@soapbox/logi'; +import { safeFetch } from '@soapbox/safe-fetch'; import tldts from 'tldts'; import { Conf } from '@/config.ts'; import { cachedFaviconsSizeGauge } from '@/metrics.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; -import { fetchWorker } from '@/workers/fetch.ts'; const faviconCache = new SimpleLRU( async (domain, { signal }) => { @@ -17,7 +17,7 @@ const faviconCache = new SimpleLRU( } const rootUrl = new URL('/', `https://${domain}/`); - const response = await fetchWorker(rootUrl, { signal }); + const response = await safeFetch(rootUrl, { signal }); const html = await response.text(); const doc = new DOMParser().parseFromString(html, 'text/html'); diff --git a/src/utils/lnurl.ts b/src/utils/lnurl.ts index c70f5751..4fd44988 100644 --- a/src/utils/lnurl.ts +++ b/src/utils/lnurl.ts @@ -1,19 +1,19 @@ import { NostrEvent } from '@nostrify/nostrify'; import { LNURL, LNURLDetails } from '@nostrify/nostrify/ln'; import { logi } from '@soapbox/logi'; +import { safeFetch } from '@soapbox/safe-fetch'; import { JsonValue } from '@std/json'; import { cachedLnurlsSizeGauge } from '@/metrics.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; import { errorJson } from '@/utils/log.ts'; import { Time } from '@/utils/time.ts'; -import { fetchWorker } from '@/workers/fetch.ts'; const lnurlCache = new SimpleLRU( async (lnurl, { signal }) => { logi({ level: 'info', ns: 'ditto.lnurl', lnurl, state: 'started' }); try { - const details = await LNURL.lookup(lnurl, { fetch: fetchWorker, signal }); + const details = await LNURL.lookup(lnurl, { fetch: safeFetch, signal }); logi({ level: 'info', ns: 'ditto.lnurl', lnurl, state: 'found', details: details as unknown as JsonValue }); return details; } catch (e) { @@ -62,7 +62,7 @@ async function getInvoice(params: CallbackParams, signal?: AbortSignal): Promise const { pr } = await LNURL.callback( details.callback, params, - { fetch: fetchWorker, signal }, + { fetch: safeFetch, signal }, ); return pr; diff --git a/src/utils/nip05.ts b/src/utils/nip05.ts index 65f425a3..ccb08bf2 100644 --- a/src/utils/nip05.ts +++ b/src/utils/nip05.ts @@ -1,6 +1,7 @@ import { nip19 } from 'nostr-tools'; import { NIP05, NStore } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; +import { safeFetch } from '@soapbox/safe-fetch'; import tldts from 'tldts'; import { Conf } from '@/config.ts'; @@ -9,7 +10,6 @@ import { Storages } from '@/storages.ts'; import { errorJson } from '@/utils/log.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; import { Nip05, parseNip05 } from '@/utils.ts'; -import { fetchWorker } from '@/workers/fetch.ts'; const nip05Cache = new SimpleLRU( async (nip05, { signal }) => { @@ -34,7 +34,7 @@ const nip05Cache = new SimpleLRU( throw new Error(`Not found: ${nip05}`); } } else { - const result = await NIP05.lookup(nip05, { fetch: fetchWorker, signal }); + const result = await NIP05.lookup(nip05, { fetch: safeFetch, signal }); logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', pubkey: result.pubkey }); return result; } diff --git a/src/utils/unfurl.ts b/src/utils/unfurl.ts index 731b586e..f895b71f 100644 --- a/src/utils/unfurl.ts +++ b/src/utils/unfurl.ts @@ -1,5 +1,6 @@ import TTLCache from '@isaacs/ttlcache'; import { logi } from '@soapbox/logi'; +import { safeFetch } from '@soapbox/safe-fetch'; import DOMPurify from 'isomorphic-dompurify'; import { unfurl } from 'unfurl.js'; @@ -7,13 +8,12 @@ import { Conf } from '@/config.ts'; import { PreviewCard } from '@/entities/PreviewCard.ts'; import { cachedLinkPreviewSizeGauge } from '@/metrics.ts'; import { errorJson } from '@/utils/log.ts'; -import { fetchWorker } from '@/workers/fetch.ts'; async function unfurlCard(url: string, signal: AbortSignal): Promise { try { const result = await unfurl(url, { fetch: (url) => - fetchWorker(url, { + safeFetch(url, { headers: { 'Accept': 'text/html, application/xhtml+xml', 'User-Agent': Conf.fetchUserAgent, diff --git a/src/workers/fetch.test.ts b/src/workers/fetch.test.ts deleted file mode 100644 index e4c698d4..00000000 --- a/src/workers/fetch.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { assertEquals, assertRejects } from '@std/assert'; - -import { fetchWorker } from '@/workers/fetch.ts'; - -Deno.test({ - name: 'fetchWorker', - async fn() { - const response = await fetchWorker('https://httpbingo.org/get'); - const json = await response.json(); - assertEquals(json.headers.Host, ['httpbingo.org']); - }, - sanitizeResources: false, -}); - -Deno.test({ - name: 'fetchWorker with AbortSignal', - async fn() { - const controller = new AbortController(); - const signal = controller.signal; - - setTimeout(() => controller.abort(), 100); - assertRejects(() => fetchWorker('https://httpbingo.org/delay/10', { signal })); - - await new Promise((resolve) => { - signal.addEventListener('abort', () => resolve(), { once: true }); - }); - }, - sanitizeResources: false, -}); diff --git a/src/workers/fetch.ts b/src/workers/fetch.ts deleted file mode 100644 index bb5588ed..00000000 --- a/src/workers/fetch.ts +++ /dev/null @@ -1,86 +0,0 @@ -import * as Comlink from 'comlink'; - -import { FetchWorker } from './fetch.worker.ts'; -import './handlers/abortsignal.ts'; - -import { fetchResponsesCounter } from '@/metrics.ts'; - -const worker = new Worker(new URL('./fetch.worker.ts', import.meta.url), { type: 'module', name: 'fetchWorker' }); -const client = Comlink.wrap(worker); - -// Wait for the worker to be ready before we start using it. -const ready = new Promise((resolve) => { - const handleEvent = () => { - self.removeEventListener('message', handleEvent); - resolve(); - }; - worker.addEventListener('message', handleEvent); -}); - -/** - * Fetch implementation with a Web Worker. - * Calling this performs the fetch in a separate CPU thread so it doesn't block the main thread. - */ -const fetchWorker: typeof fetch = async (...args) => { - await ready; - - const [url, init] = serializeFetchArgs(args); - const { body, signal, ...rest } = init; - - const result = await client.fetch(url, { ...rest, body: await prepareBodyForWorker(body) }, signal); - const response = new Response(...result); - - const { method } = init; - const { status } = response; - fetchResponsesCounter.inc({ method, status }); - - return response; -}; - -/** Take arguments to `fetch`, and turn them into something we can send over Comlink. */ -function serializeFetchArgs(args: Parameters): [string, RequestInit] { - const request = normalizeRequest(args); - const init = requestToInit(request); - return [request.url, init]; -} - -/** Get a `Request` object from arguments to `fetch`. */ -function normalizeRequest(args: Parameters): Request { - return new Request(...args); -} - -/** Get the body as a type we can transfer over Web Workers. */ -async function prepareBodyForWorker( - body: BodyInit | undefined | null, -): Promise { - if (!body || typeof body === 'string' || body instanceof ArrayBuffer || body instanceof Blob) { - return body; - } else { - const response = new Response(body); - return await response.arrayBuffer(); - } -} - -/** - * Convert a `Request` object into its serialized `RequestInit` format. - * `RequestInit` is a subset of `Request`, just lacking helper methods like `json()`, - * making it easier to serialize (exceptions: `body` and `signal`). - */ -function requestToInit(request: Request): RequestInit { - return { - method: request.method, - headers: [...request.headers.entries()], - body: request.body, - referrer: request.referrer, - referrerPolicy: request.referrerPolicy, - mode: request.mode, - credentials: request.credentials, - cache: request.cache, - redirect: request.redirect, - integrity: request.integrity, - keepalive: request.keepalive, - signal: request.signal, - }; -} - -export { fetchWorker }; diff --git a/src/workers/fetch.worker.ts b/src/workers/fetch.worker.ts deleted file mode 100644 index 4a67c6b8..00000000 --- a/src/workers/fetch.worker.ts +++ /dev/null @@ -1,33 +0,0 @@ -/// - -import { safeFetch } from '@soapbox/safe-fetch'; -import { logi } from '@soapbox/logi'; -import * as Comlink from 'comlink'; - -import '@/workers/handlers/abortsignal.ts'; -import '@/sentry.ts'; - -export const FetchWorker = { - async fetch( - url: string, - init: Omit, - signal: AbortSignal | null | undefined, - ): Promise<[BodyInit, ResponseInit]> { - logi({ level: 'debug', ns: 'ditto.fetch', method: init.method ?? 'GET', url }); - - const response = await safeFetch(url, { ...init, signal }); - - return [ - await response.arrayBuffer(), - { - status: response.status, - statusText: response.statusText, - headers: [...response.headers.entries()], - }, - ]; - }, -}; - -Comlink.expose(FetchWorker); - -self.postMessage('ready'); diff --git a/src/workers/handlers/abortsignal.ts b/src/workers/handlers/abortsignal.ts deleted file mode 100644 index 14cf9f41..00000000 --- a/src/workers/handlers/abortsignal.ts +++ /dev/null @@ -1,46 +0,0 @@ -import * as Comlink from 'comlink'; - -const signalFinalizers = new FinalizationRegistry((port: MessagePort) => { - port.postMessage(null); - port.close(); -}); - -Comlink.transferHandlers.set('abortsignal', { - canHandle(value) { - return value instanceof AbortSignal || value?.constructor?.name === 'AbortSignal'; - }, - serialize(signal) { - if (signal.aborted) { - return [{ aborted: true }]; - } - - const { port1, port2 } = new MessageChannel(); - signal.addEventListener( - 'abort', - () => port1.postMessage({ reason: signal.reason }), - { once: true }, - ); - - signalFinalizers?.register(signal, port1); - - return [{ aborted: false, port: port2 }, [port2]]; - }, - deserialize({ aborted, port }) { - if (aborted || !port) { - return AbortSignal.abort(); - } - - const ctrl = new AbortController(); - - port.addEventListener('message', (ev) => { - if (ev.data && 'reason' in ev.data) { - ctrl.abort(ev.data.reason); - } - port.close(); - }, { once: true }); - - port.start(); - - return ctrl.signal; - }, -} as Comlink.TransferHandler); diff --git a/src/workers/policy.ts b/src/workers/policy.ts index fdc33698..7b3d23b0 100644 --- a/src/workers/policy.ts +++ b/src/workers/policy.ts @@ -5,8 +5,6 @@ import * as Comlink from 'comlink'; import { Conf } from '@/config.ts'; import type { CustomPolicy } from '@/workers/policy.worker.ts'; -import '@/workers/handlers/abortsignal.ts'; - class PolicyWorker implements NPolicy { private worker: Comlink.Remote; private ready: Promise; diff --git a/src/workers/policy.worker.ts b/src/workers/policy.worker.ts index 5e9d4d4a..00540b03 100644 --- a/src/workers/policy.worker.ts +++ b/src/workers/policy.worker.ts @@ -6,8 +6,6 @@ import * as Comlink from 'comlink'; import { DittoDB } from '@/db/DittoDB.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; -import '@/workers/handlers/abortsignal.ts'; - // @ts-ignore Don't try to access the env from this worker. Deno.env = new Map(); From 433c2a4347190db2e5ee13c73a58503fccd750fa Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Feb 2025 15:06:13 -0600 Subject: [PATCH 3/3] @lambdalisue/async -> @core/asyncutil --- deno.json | 2 +- deno.lock | 10 +++++----- scripts/db-import.ts | 2 +- src/firehose.ts | 2 +- src/notify.ts | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/deno.json b/deno.json index 562aab51..b591b43c 100644 --- a/deno.json +++ b/deno.json @@ -39,12 +39,12 @@ "@/": "./src/", "@b-fuze/deno-dom": "jsr:@b-fuze/deno-dom@^0.1.47", "@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", + "@core/asyncutil": "jsr:@core/asyncutil@^1.2.0", "@electric-sql/pglite": "npm:@electric-sql/pglite@^0.2.8", "@esroyo/scoped-performance": "jsr:@esroyo/scoped-performance@^3.1.0", "@gfx/canvas-wasm": "jsr:@gfx/canvas-wasm@^0.4.2", "@hono/hono": "jsr:@hono/hono@^4.4.6", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", - "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1", "@negrel/webpush": "jsr:@negrel/webpush@^0.3.0", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", "@nostrify/db": "jsr:@nostrify/db@^0.38.0", diff --git a/deno.lock b/deno.lock index 7235d7dd..874085e8 100644 --- a/deno.lock +++ b/deno.lock @@ -3,6 +3,7 @@ "specifiers": { "jsr:@b-fuze/deno-dom@~0.1.47": "0.1.48", "jsr:@bradenmacdonald/s3-lite-client@~0.7.4": "0.7.6", + "jsr:@core/asyncutil@^1.2.0": "1.2.0", "jsr:@denosaurs/plug@1.0.3": "1.0.3", "jsr:@esroyo/scoped-performance@^3.1.0": "3.1.0", "jsr:@gfx/canvas-wasm@~0.4.2": "0.4.2", @@ -28,7 +29,6 @@ "jsr:@gleasonator/policy@0.9.3": "0.9.3", "jsr:@gleasonator/policy@0.9.4": "0.9.4", "jsr:@hono/hono@^4.4.6": "4.6.15", - "jsr:@lambdalisue/async@^2.1.1": "2.1.1", "jsr:@negrel/http-ece@0.6.0": "0.6.0", "jsr:@negrel/webpush@0.3": "0.3.0", "jsr:@nostrify/db@0.38": "0.38.0", @@ -153,6 +153,9 @@ "jsr:@std/io@0.224" ] }, + "@core/asyncutil@1.2.0": { + "integrity": "9967f15190c60df032c13f72ce5ac73d185c34f31c53dc918d8800025854c118" + }, "@denosaurs/plug@1.0.3": { "integrity": "b010544e386bea0ff3a1d05e0c88f704ea28cbd4d753439c2f1ee021a85d4640", "dependencies": [ @@ -337,9 +340,6 @@ "@hono/hono@4.6.15": { "integrity": "935b3b12e98e4b22bcd1aa4dbe6587321e431c79829eba61f535b4ede39fd8b1" }, - "@lambdalisue/async@2.1.1": { - "integrity": "1fc9bc6f4ed50215cd2f7217842b18cea80f81c25744f88f8c5eb4be5a1c9ab4" - }, "@negrel/http-ece@0.6.0": { "integrity": "7afdd81b86ea5b21a9677b323c01c3338705e11cc2bfed250870f5349d8f86f7", "dependencies": [ @@ -2367,10 +2367,10 @@ "dependencies": [ "jsr:@b-fuze/deno-dom@~0.1.47", "jsr:@bradenmacdonald/s3-lite-client@~0.7.4", + "jsr:@core/asyncutil@^1.2.0", "jsr:@esroyo/scoped-performance@^3.1.0", "jsr:@gfx/canvas-wasm@~0.4.2", "jsr:@hono/hono@^4.4.6", - "jsr:@lambdalisue/async@^2.1.1", "jsr:@negrel/webpush@0.3", "jsr:@nostrify/db@0.38", "jsr:@nostrify/nostrify@~0.38.1", diff --git a/scripts/db-import.ts b/scripts/db-import.ts index c34384bf..ed884453 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -1,4 +1,4 @@ -import { Semaphore } from '@lambdalisue/async'; +import { Semaphore } from '@core/asyncutil'; import { NostrEvent } from '@nostrify/nostrify'; import { JsonParseStream } from '@std/json/json-parse-stream'; import { TextLineStream } from '@std/streams/text-line-stream'; diff --git a/src/firehose.ts b/src/firehose.ts index fca2e079..f04752b2 100644 --- a/src/firehose.ts +++ b/src/firehose.ts @@ -1,4 +1,4 @@ -import { Semaphore } from '@lambdalisue/async'; +import { Semaphore } from '@core/asyncutil'; import { logi } from '@soapbox/logi'; import { Conf } from '@/config.ts'; diff --git a/src/notify.ts b/src/notify.ts index b1ee3517..44ed5619 100644 --- a/src/notify.ts +++ b/src/notify.ts @@ -1,4 +1,4 @@ -import { Semaphore } from '@lambdalisue/async'; +import { Semaphore } from '@core/asyncutil'; import { pipelineEncounters } from '@/caches/pipelineEncounters.ts'; import { Conf } from '@/config.ts';