From e523da9d19578fb213a7e9c28ed8e81b068948a3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 30 Dec 2024 14:29:45 -0600 Subject: [PATCH 01/12] Upgrade Nostrify --- deno.json | 5 ++-- deno.lock | 75 ++++++++++++++++++++++++++++++++++++++----------------- 2 files changed, 55 insertions(+), 25 deletions(-) diff --git a/deno.json b/deno.json index b37f7e8c..3c9eef1a 100644 --- a/deno.json +++ b/deno.json @@ -46,8 +46,9 @@ "@negrel/webpush": "jsr:@negrel/webpush@^0.3.0", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", "@nostrify/db": "jsr:@nostrify/db@^0.36.1", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.36.0", - "@nostrify/policies": "jsr:@nostrify/policies@^0.35.0", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.37.0", + "@nostrify/policies": "jsr:@nostrify/policies@^0.36.1", + "@nostrify/types": "jsr:@nostrify/types@^0.36.0", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-pglite": "jsr:@soapbox/kysely-pglite@^1.0.0", diff --git a/deno.lock b/deno.lock index f9169200..2781f233 100644 --- a/deno.lock +++ b/deno.lock @@ -26,27 +26,27 @@ "jsr:@gleasonator/policy@0.9.1": "0.9.1", "jsr:@gleasonator/policy@0.9.2": "0.9.2", "jsr:@gleasonator/policy@0.9.3": "0.9.3", - "jsr:@hono/hono@^4.4.6": "4.6.2", + "jsr:@hono/hono@^4.4.6": "4.6.15", "jsr:@lambdalisue/async@^2.1.1": "2.1.1", "jsr:@negrel/http-ece@0.6.0": "0.6.0", "jsr:@negrel/webpush@0.3": "0.3.0", "jsr:@nostrify/db@~0.36.1": "0.36.1", "jsr:@nostrify/nostrify@0.31": "0.31.0", "jsr:@nostrify/nostrify@0.32": "0.32.0", - "jsr:@nostrify/nostrify@0.35": "0.35.0", - "jsr:@nostrify/nostrify@0.36": "0.36.0", + "jsr:@nostrify/nostrify@0.36": "0.36.2", + "jsr:@nostrify/nostrify@0.37": "0.37.0", "jsr:@nostrify/nostrify@~0.22.1": "0.22.5", "jsr:@nostrify/nostrify@~0.22.4": "0.22.4", "jsr:@nostrify/nostrify@~0.22.5": "0.22.5", "jsr:@nostrify/policies@0.33": "0.33.0", "jsr:@nostrify/policies@0.33.1": "0.33.1", "jsr:@nostrify/policies@0.34": "0.34.0", - "jsr:@nostrify/policies@0.35": "0.35.0", "jsr:@nostrify/policies@0.36": "0.36.0", "jsr:@nostrify/policies@~0.33.1": "0.33.1", "jsr:@nostrify/policies@~0.36.1": "0.36.1", "jsr:@nostrify/types@0.30": "0.30.1", "jsr:@nostrify/types@0.35": "0.35.0", + "jsr:@nostrify/types@0.36": "0.36.0", "jsr:@nostrify/types@~0.30.1": "0.30.1", "jsr:@soapbox/kysely-pglite@1": "1.0.0", "jsr:@soapbox/safe-fetch@2": "2.0.0", @@ -60,7 +60,7 @@ "jsr:@std/bytes@0.224.0": "0.224.0", "jsr:@std/bytes@^1.0.0-rc.3": "1.0.0", "jsr:@std/bytes@^1.0.1-rc.3": "1.0.2", - "jsr:@std/bytes@^1.0.2": "1.0.2", + "jsr:@std/bytes@^1.0.2": "1.0.4", "jsr:@std/bytes@^1.0.2-rc.3": "1.0.2", "jsr:@std/cli@0.223": "0.223.0", "jsr:@std/crypto@0.224": "0.224.0", @@ -72,9 +72,9 @@ "jsr:@std/fmt@0.213.1": "0.213.1", "jsr:@std/fs@0.213.1": "0.213.1", "jsr:@std/fs@~0.229.3": "0.229.3", - "jsr:@std/internal@1": "1.0.4", + "jsr:@std/internal@1": "1.0.5", "jsr:@std/io@0.223": "0.223.0", - "jsr:@std/io@0.224": "0.224.8", + "jsr:@std/io@0.224": "0.224.9", "jsr:@std/json@0.223": "0.223.0", "jsr:@std/media-types@0.224.0": "0.224.0", "jsr:@std/media-types@~0.224.1": "0.224.1", @@ -139,7 +139,10 @@ ] }, "@b-fuze/deno-dom@0.1.48": { - "integrity": "bf5b591aef2e9e9c59adfcbb93a9ecd45bab5b7c8263625beafa5c8f1662e7da" + "integrity": "bf5b591aef2e9e9c59adfcbb93a9ecd45bab5b7c8263625beafa5c8f1662e7da", + "dependencies": [ + "jsr:@denosaurs/plug" + ] }, "@bradenmacdonald/s3-lite-client@0.7.6": { "integrity": "2b5976dca95d207dc88e23f9807e3eecbc441b0cf547dcda5784afe6668404d1", @@ -321,6 +324,9 @@ "@hono/hono@4.6.2": { "integrity": "35fcf3be4687825080b01bed7bbe2ac66f8d8b8939f0bad459661bf3b46d916f" }, + "@hono/hono@4.6.15": { + "integrity": "935b3b12e98e4b22bcd1aa4dbe6587321e431c79829eba61f535b4ede39fd8b1" + }, "@lambdalisue/async@2.1.1": { "integrity": "1fc9bc6f4ed50215cd2f7217842b18cea80f81c25744f88f8c5eb4be5a1c9ab4" }, @@ -420,8 +426,23 @@ "npm:zod" ] }, - "@nostrify/nostrify@0.35.0": { - "integrity": "9bfef4883838b8b4cb2e2b28a60b72de95391ca5b789bc7206a2baea054dea55", + "@nostrify/nostrify@0.36.0": { + "integrity": "f00dbff1f02a2c496c5e85eeeb7a84101b7dd874d87456449dc71b6d037e40fc", + "dependencies": [ + "jsr:@nostrify/types@0.35", + "jsr:@std/crypto", + "jsr:@std/encoding@~0.224.1", + "npm:@scure/base", + "npm:@scure/bip32", + "npm:@scure/bip39", + "npm:lru-cache@^10.2.0", + "npm:nostr-tools@^2.7.0", + "npm:websocket-ts", + "npm:zod" + ] + }, + "@nostrify/nostrify@0.36.2": { + "integrity": "cc4787ca170b623a2e5dfed1baa4426077daa6143af728ea7dd325d58f4d04d6", "dependencies": [ "jsr:@nostrify/types@0.35", "jsr:@std/encoding@~0.224.1", @@ -433,10 +454,10 @@ "npm:zod" ] }, - "@nostrify/nostrify@0.36.0": { - "integrity": "f00dbff1f02a2c496c5e85eeeb7a84101b7dd874d87456449dc71b6d037e40fc", + "@nostrify/nostrify@0.37.0": { + "integrity": "fa1439cc5e9a74986c4fb799a38a9ed7bd8663c62ae2a9363ca9b987548e27a0", "dependencies": [ - "jsr:@nostrify/types@0.35", + "jsr:@nostrify/types@0.36", "jsr:@std/crypto", "jsr:@std/encoding@~0.224.1", "npm:@scure/base", @@ -470,14 +491,6 @@ "npm:nostr-tools@^2.7.0" ] }, - "@nostrify/policies@0.35.0": { - "integrity": "b828fac9f253e460a9587c05588b7dae6a0a32c5a9c9083e449219887b9e8e20", - "dependencies": [ - "jsr:@nostrify/nostrify@0.35", - "jsr:@nostrify/types@0.35", - "npm:nostr-tools@^2.7.0" - ] - }, "@nostrify/policies@0.36.0": { "integrity": "ad1930de48ce03cdf34da456af1563b487581d1d86683cd416ad760ae40b1fb3", "dependencies": [ @@ -503,6 +516,9 @@ "@nostrify/types@0.35.0": { "integrity": "b8d515563d467072694557d5626fa1600f74e83197eef45dd86a9a99c64f7fe6" }, + "@nostrify/types@0.36.0": { + "integrity": "b3413467debcbd298d217483df4e2aae6c335a34765c90ac7811cf7c637600e7" + }, "@soapbox/kysely-pglite@1.0.0": { "integrity": "0954b1bf3deab051c479cba966b1e6ed5a0a966aa21d1f40143ec8f5efcd475d", "dependencies": [ @@ -545,6 +561,9 @@ "@std/bytes@1.0.2": { "integrity": "fbdee322bbd8c599a6af186a1603b3355e59a5fb1baa139f8f4c3c9a1b3e3d57" }, + "@std/bytes@1.0.4": { + "integrity": "11a0debe522707c95c7b7ef89b478c13fb1583a7cfb9a85674cd2cc2e3a28abc" + }, "@std/cli@0.223.0": { "integrity": "2feb7970f2028904c3edc22ea916ce9538113dfc170844f3eae03578c333c356", "dependencies": [ @@ -601,6 +620,9 @@ "@std/internal@1.0.4": { "integrity": "62e8e4911527e5e4f307741a795c0b0a9e6958d0b3790716ae71ce085f755422" }, + "@std/internal@1.0.5": { + "integrity": "54a546004f769c1ac9e025abd15a76b6671ddc9687e2313b67376125650dc7ba" + }, "@std/io@0.223.0": { "integrity": "2d8c3c2ab3a515619b90da2c6ff5ea7b75a94383259ef4d02116b228393f84f1", "dependencies": [ @@ -650,6 +672,12 @@ "jsr:@std/bytes@^1.0.2" ] }, + "@std/io@0.224.9": { + "integrity": "4414664b6926f665102e73c969cfda06d2c4c59bd5d0c603fd4f1b1c840d6ee3", + "dependencies": [ + "jsr:@std/bytes@^1.0.2" + ] + }, "@std/json@0.223.0": { "integrity": "9a4a255931dd0397924c6b10bb6a72fe3e28ddd876b981ada2e3b8dd0764163f", "dependencies": [ @@ -2308,8 +2336,9 @@ "jsr:@lambdalisue/async@^2.1.1", "jsr:@negrel/webpush@0.3", "jsr:@nostrify/db@~0.36.1", - "jsr:@nostrify/nostrify@0.36", - "jsr:@nostrify/policies@0.35", + "jsr:@nostrify/nostrify@0.37", + "jsr:@nostrify/policies@~0.36.1", + "jsr:@nostrify/types@0.36", "jsr:@soapbox/kysely-pglite@1", "jsr:@soapbox/safe-fetch@2", "jsr:@soapbox/stickynotes@0.4", From 7a60b4b8d80101f483d4a14aa612457bb80f6c6a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 5 Jan 2025 11:23:18 -0600 Subject: [PATCH 02/12] Support kind 20 "Picture" events (NIP-68) --- scripts/nostr-pull.ts | 2 +- src/config.ts | 2 +- src/controllers/api/accounts.ts | 4 ++-- src/controllers/api/ditto.ts | 2 +- src/controllers/api/reactions.ts | 4 ++-- src/controllers/api/search.ts | 6 +++--- src/controllers/api/statuses.ts | 6 +++--- src/controllers/api/streaming.ts | 10 +++++----- src/controllers/api/timelines.ts | 8 ++++---- src/controllers/api/trends.ts | 2 +- src/storages/hydrate.ts | 12 ++++++------ src/utils/stats.ts | 2 +- src/views.ts | 2 +- 13 files changed, 31 insertions(+), 31 deletions(-) diff --git a/scripts/nostr-pull.ts b/scripts/nostr-pull.ts index 573b5f01..c7ad21d3 100644 --- a/scripts/nostr-pull.ts +++ b/scripts/nostr-pull.ts @@ -47,7 +47,7 @@ const importUsers = async ( if (!profilesOnly) { matched.push( ...await conn.query( - authors.map((author) => ({ kinds: [1], authors: [author], limit: 200 })), + authors.map((author) => ({ kinds: [1, 20], authors: [author], limit: 200 })), ), ); } diff --git a/src/config.ts b/src/config.ts index 68bf3ed8..3c1e8923 100644 --- a/src/config.ts +++ b/src/config.ts @@ -252,7 +252,7 @@ class Conf { } /** Nostr event kinds of events to listen for on the firehose. */ static get firehoseKinds(): number[] { - return (Deno.env.get('FIREHOSE_KINDS') ?? '0, 1, 3, 5, 6, 7, 9735, 10002') + return (Deno.env.get('FIREHOSE_KINDS') ?? '0, 1, 3, 5, 6, 7, 20, 9735, 10002') .split(/[, ]+/g) .map(Number); } diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 03284a15..6705924e 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -234,7 +234,7 @@ const accountStatusesController: AppController = async (c) => { const filter: NostrFilter = { authors: [pubkey], - kinds: [1, 6], + kinds: [1, 6, 20], since, until, limit, @@ -473,7 +473,7 @@ const favouritesController: AppController = async (c) => { .map((event) => event.tags.find((tag) => tag[0] === 'e')?.[1]) .filter((id): id is string => !!id); - const events1 = await store.query([{ kinds: [1], ids }], { signal }) + const events1 = await store.query([{ kinds: [1, 20], ids }], { signal }) .then((events) => hydrateEvents({ events, store, signal })); const viewerPubkey = await c.get('signer')?.getPublicKey(); diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index 765862ec..3a1ce98d 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -260,7 +260,7 @@ export const statusZapSplitsController: AppController = async (c) => { const id = c.req.param('id'); const { signal } = c.req.raw; - const [event] = await store.query([{ kinds: [1], ids: [id], limit: 1 }], { signal }); + const [event] = await store.query([{ kinds: [1, 20], ids: [id], limit: 1 }], { signal }); if (!event) { return c.json({ error: 'Event not found' }, 404); } diff --git a/src/controllers/api/reactions.ts b/src/controllers/api/reactions.ts index 7d42c197..0beb985d 100644 --- a/src/controllers/api/reactions.ts +++ b/src/controllers/api/reactions.ts @@ -20,7 +20,7 @@ const reactionController: AppController = async (c) => { } const store = await Storages.db(); - const [event] = await store.query([{ kinds: [1], ids: [id], limit: 1 }]); + const [event] = await store.query([{ kinds: [1, 20], ids: [id], limit: 1 }]); if (!event) { return c.json({ error: 'Status not found' }, 404); @@ -56,7 +56,7 @@ const deleteReactionController: AppController = async (c) => { } const [event] = await store.query([ - { kinds: [1], ids: [id], limit: 1 }, + { kinds: [1, 20], ids: [id], limit: 1 }, ]); if (!event) { diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 4c3aa75f..8bfe4ffd 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -166,7 +166,7 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort if (n.id().safeParse(q).success) { const filters: NostrFilter[] = []; if (accounts) filters.push({ kinds: [0], authors: [q] }); - if (statuses) filters.push({ kinds: [1], ids: [q] }); + if (statuses) filters.push({ kinds: [1, 20], ids: [q] }); return filters; } @@ -184,10 +184,10 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort if (accounts) filters.push({ kinds: [0], authors: [result.data.pubkey] }); break; case 'note': - if (statuses) filters.push({ kinds: [1], ids: [result.data] }); + if (statuses) filters.push({ kinds: [1, 20], ids: [result.data] }); break; case 'nevent': - if (statuses) filters.push({ kinds: [1], ids: [result.data.id] }); + if (statuses) filters.push({ kinds: [1, 20], ids: [result.data.id] }); break; } return filters; diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index c27b9a74..d466b551 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -397,7 +397,7 @@ const unreblogStatusController: AppController = async (c) => { const pubkey = await c.get('signer')?.getPublicKey()!; const store = await Storages.db(); - const [event] = await store.query([{ ids: [eventId], kinds: [1] }]); + const [event] = await store.query([{ ids: [eventId], kinds: [1, 20] }]); if (!event) { return c.json({ error: 'Record not found' }, 404); } @@ -429,13 +429,13 @@ const quotesController: AppController = async (c) => { const params = c.get('pagination'); const store = await Storages.db(); - const [event] = await store.query([{ ids: [id], kinds: [1] }]); + const [event] = await store.query([{ ids: [id], kinds: [1, 20] }]); if (!event) { return c.json({ error: 'Event not found.' }, 404); } const quotes = await store - .query([{ kinds: [1], '#q': [event.id], ...params }]) + .query([{ kinds: [1, 20], '#q': [event.id], ...params }]) .then((events) => hydrateEvents({ events, store })); const viewerPubkey = await c.get('signer')?.getPublicKey(); diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index 5e90085d..079f74cd 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -214,20 +214,20 @@ async function topicToFilter( switch (topic) { case 'public': - return { kinds: [1, 6] }; + return { kinds: [1, 6, 20] }; case 'public:local': - return { kinds: [1, 6], search: `domain:${host}` }; + return { kinds: [1, 6, 20], search: `domain:${host}` }; case 'hashtag': - if (query.tag) return { kinds: [1, 6], '#t': [query.tag] }; + if (query.tag) return { kinds: [1, 6, 20], '#t': [query.tag] }; break; case 'hashtag:local': - if (query.tag) return { kinds: [1, 6], '#t': [query.tag], search: `domain:${host}` }; + if (query.tag) return { kinds: [1, 6, 20], '#t': [query.tag], search: `domain:${host}` }; break; case 'user': // HACK: this puts the user's entire contacts list into RAM, // and then calls `matchFilters` over it. Refreshing the page // is required after following a new user. - return pubkey ? { kinds: [1, 6], authors: [...await getFeedPubkeys(pubkey)] } : undefined; + return pubkey ? { kinds: [1, 6, 20], authors: [...await getFeedPubkeys(pubkey)] } : undefined; } } diff --git a/src/controllers/api/timelines.ts b/src/controllers/api/timelines.ts index cd0d7ff1..fa5f44f6 100644 --- a/src/controllers/api/timelines.ts +++ b/src/controllers/api/timelines.ts @@ -14,7 +14,7 @@ const homeTimelineController: AppController = async (c) => { const params = c.get('pagination'); const pubkey = await c.get('signer')?.getPublicKey()!; const authors = [...await getFeedPubkeys(pubkey)]; - return renderStatuses(c, [{ authors, kinds: [1, 6], ...params }]); + return renderStatuses(c, [{ authors, kinds: [1, 6, 20], ...params }]); }; const publicQuerySchema = z.object({ @@ -33,7 +33,7 @@ const publicTimelineController: AppController = (c) => { const { local, instance, language } = result.data; - const filter: NostrFilter = { kinds: [1], ...params }; + const filter: NostrFilter = { kinds: [1, 20], ...params }; const search: `${string}:${string}`[] = []; @@ -57,7 +57,7 @@ const publicTimelineController: AppController = (c) => { const hashtagTimelineController: AppController = (c) => { const hashtag = c.req.param('hashtag')!.toLowerCase(); const params = c.get('pagination'); - return renderStatuses(c, [{ kinds: [1], '#t': [hashtag], ...params }]); + return renderStatuses(c, [{ kinds: [1, 20], '#t': [hashtag], ...params }]); }; const suggestedTimelineController: AppController = async (c) => { @@ -70,7 +70,7 @@ const suggestedTimelineController: AppController = async (c) => { const authors = [...getTagSet(follows?.tags ?? [], 'p')]; - return renderStatuses(c, [{ authors, kinds: [1], ...params }]); + return renderStatuses(c, [{ authors, kinds: [1, 20], ...params }]); }; /** Render statuses for timelines. */ diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts index 45e2d117..a7906192 100644 --- a/src/controllers/api/trends.ts +++ b/src/controllers/api/trends.ts @@ -134,7 +134,7 @@ const trendingStatusesController: AppController = async (c) => { return c.json([]); } - const results = await store.query([{ kinds: [1], ids }]) + const results = await store.query([{ kinds: [1, 20], ids }]) .then((events) => hydrateEvents({ events, store })); // Sort events in the order they appear in the label. diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 7f5c8125..28dcea47 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -102,21 +102,21 @@ export function assembleEvents( if (event.kind === 1) { const id = findQuoteTag(event.tags)?.[1] || findQuoteInContent(event.content); if (id) { - event.quote = b.find((e) => matchFilter({ kinds: [1], ids: [id] }, e)); + event.quote = b.find((e) => matchFilter({ kinds: [1, 20], ids: [id] }, e)); } } if (event.kind === 6) { const id = event.tags.find(([name]) => name === 'e')?.[1]; if (id) { - event.repost = b.find((e) => matchFilter({ kinds: [1], ids: [id] }, e)); + event.repost = b.find((e) => matchFilter({ kinds: [1, 20], ids: [id] }, e)); } } if (event.kind === 7) { const id = event.tags.findLast(([name]) => name === 'e')?.[1]; if (id) { - event.reacted = b.find((e) => matchFilter({ kinds: [1], ids: [id] }, e)); + event.reacted = b.find((e) => matchFilter({ kinds: [1, 20], ids: [id] }, e)); } } @@ -130,7 +130,7 @@ export function assembleEvents( const ids = event.tags.filter(([name]) => name === 'e').map(([_name, value]) => value); for (const id of ids) { - const reported = b.find((e) => matchFilter({ kinds: [1], ids: [id] }, e)); + const reported = b.find((e) => matchFilter({ kinds: [1, 20], ids: [id] }, e)); if (reported) { reportedEvents.push(reported); } @@ -146,7 +146,7 @@ export function assembleEvents( const id = event.tags.find(([name]) => name === 'e')?.[1]; if (id) { - event.zapped = b.find((e) => matchFilter({ kinds: [1], ids: [id] }, e)); + event.zapped = b.find((e) => matchFilter({ kinds: [1, 20], ids: [id] }, e)); } const zapRequestString = event?.tags?.find(([name]) => name === 'description')?.[1]; @@ -313,7 +313,7 @@ function gatherReportedNotes({ events, store, signal }: HydrateOpts): Promise { const [{ count: followers_count }, { count: notes_count }, [followList], [kind0]] = await Promise.all([ store.count([{ kinds: [3], '#p': [pubkey] }]), - store.count([{ kinds: [1], authors: [pubkey] }]), + store.count([{ kinds: [1, 20], authors: [pubkey] }]), store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]), store.query([{ kinds: [0], authors: [pubkey], limit: 1 }]), ]); diff --git a/src/views.ts b/src/views.ts index e333eebd..562043db 100644 --- a/src/views.ts +++ b/src/views.ts @@ -75,7 +75,7 @@ async function renderStatuses(c: AppContext, ids: string[], signal = AbortSignal const store = await Storages.db(); const { limit } = c.get('pagination'); - const events = await store.query([{ kinds: [1], ids, limit }], { signal }) + const events = await store.query([{ kinds: [1, 20], ids, limit }], { signal }) .then((events) => hydrateEvents({ events, store, signal })); if (!events.length) { From 079177ea0b7690ebb6d0839b7d90bfdf2788c39e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 5 Jan 2025 11:26:05 -0600 Subject: [PATCH 03/12] EventsDB: index kind 20 in search the same as kind 1 --- src/storages/EventsDB.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 6dccdcb2..aa2a0106 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -283,6 +283,7 @@ class EventsDB extends NPostgres { case 0: return EventsDB.buildUserSearchContent(event); case 1: + case 20: return nip27.replaceAll(event.content, () => ''); case 30009: return EventsDB.buildTagsSearchContent(event.tags.filter(([t]) => t !== 'alt')); From b61eb2ff119be8b4f00b0e08874be17bce48f72f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 5 Jan 2025 11:37:38 -0600 Subject: [PATCH 04/12] Fix favourites of kind 20 events --- src/controllers/api/statuses.ts | 7 +++++-- src/queries.ts | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index d466b551..38ed591a 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -290,7 +290,7 @@ const deleteStatusController: AppController = async (c) => { const contextController: AppController = async (c) => { const id = c.req.param('id'); const store = c.get('store'); - const event = await getEvent(id, { kind: 1, relations: ['author', 'event_stats', 'author_stats'] }); + const [event] = await store.query([{ kinds: [1, 20], ids: [id] }]); const viewerPubkey = await c.get('signer')?.getPublicKey(); async function renderStatuses(events: NostrEvent[]) { @@ -325,7 +325,8 @@ const contextController: AppController = async (c) => { const favouriteController: AppController = async (c) => { const id = c.req.param('id'); - const target = await getEvent(id, { kind: 1, relations: ['author', 'event_stats', 'author_stats'] }); + const store = await Storages.db(); + const [target] = await store.query([{ ids: [id], kinds: [1, 20] }]); if (target) { await createEvent({ @@ -337,6 +338,8 @@ const favouriteController: AppController = async (c) => { ], }, c); + await hydrateEvents({ events: [target], store }); + const status = await renderStatus(target, { viewerPubkey: await c.get('signer')?.getPublicKey() }); if (status) { diff --git a/src/queries.ts b/src/queries.ts index e93027d9..36066ce2 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -16,7 +16,7 @@ interface GetEventOpts { signal?: AbortSignal; /** Event kind. */ kind?: number; - /** Relations to include on the event. */ + /** @deprecated Relations to include on the event. */ relations?: DittoRelation[]; } From 51981009c4e1ee0b06ead91f14eb061a79a5c097 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 Jan 2025 18:19:32 -0600 Subject: [PATCH 05/12] Add cache-control headers to /packs/* --- src/app.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/app.ts b/src/app.ts index c9d51441..32061184 100644 --- a/src/app.ts +++ b/src/app.ts @@ -407,11 +407,17 @@ app.get('/notice/*', frontendController); app.get('/timeline/*', frontendController); // Known static file routes +app.get('/sw.js', publicFiles); app.get('/favicon.ico', publicFiles, staticFiles); app.get('/images/*', publicFiles, staticFiles); app.get('/instance/*', publicFiles); -app.get('/packs/*', publicFiles); -app.get('/sw.js', publicFiles); + +// Packs contains immutable static files +app.get('/packs/*', async (c, next) => { + c.header('Cache-Control', 'public, max-age=31536000, immutable'); + c.header('Strict-Transport-Security', '"max-age=31536000" always'); + await next(); +}, publicFiles); // Site index app.get('/', frontendController, indexController); From 5b9868bc3a0ff0b0071db18f34bcef5c9d12bda7 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 Jan 2025 18:59:25 -0600 Subject: [PATCH 06/12] Add a Caddyfile Related: https://gitlab.com/soapbox-pub/ditto/-/issues/265 --- installation/Caddyfile | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 installation/Caddyfile diff --git a/installation/Caddyfile b/installation/Caddyfile new file mode 100644 index 00000000..3a91c501 --- /dev/null +++ b/installation/Caddyfile @@ -0,0 +1,23 @@ +example.com { + @public path /packs/* /instance/* /images/* /favicon.ico /sw.js /sw.js.map + + handle /packs/* { + root * /opt/ditto/public + header Cache-Control "public, max-age=31536000, immutable" + header Strict-Transport-Security "max-age=31536000" + file_server + } + + handle @public { + root * /opt/ditto/public + file_server + } + + handle /metrics { + respond "Access denied" 403 + } + + handle { + reverse_proxy :4036 + } +} \ No newline at end of file From d368bf90d56f23a58ced25f0914cbd12c9f35444 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 Jan 2025 13:13:22 -0600 Subject: [PATCH 07/12] Caddyfile: enable access log, add X-Real-IP header for rate limiting --- installation/Caddyfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/installation/Caddyfile b/installation/Caddyfile index 3a91c501..79630c2a 100644 --- a/installation/Caddyfile +++ b/installation/Caddyfile @@ -1,4 +1,16 @@ +# Cloudflare real IP configuration for rate-limiting +# { +# servers { +# # https://www.cloudflare.com/ips/ +# trusted_proxies static 173.245.48.0/20 103.21.244.0/22 103.22.200.0/22 103.31.4.0/22 141.101.64.0/18 108.162.192.0/18 190.93.240.0/20 188.114.96.0/20 197.234.240.0/22 198.41.128.0/17 162.158.0.0/15 104.16.0.0/13 104.24.0.0/14 172.64.0.0/13 131.0.72.0/22 2400:cb00::/32 2606:4700::/32 2803:f800::/32 2405:b500::/32 2405:8100::/32 2a06:98c0::/29 2c0f:f248::/32 +# trusted_proxies_strict +# } +# } + example.com { + log + request_header X-Real-IP {client_ip} + @public path /packs/* /instance/* /images/* /favicon.ico /sw.js /sw.js.map handle /packs/* { From 93a035e3ff15286082b4c0df47348355231fe147 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 20 Jan 2025 22:47:21 -0600 Subject: [PATCH 08/12] Streaming: handle token errors as 401s --- src/controllers/api/streaming.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index 079f74cd..cad87e0b 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -18,6 +18,7 @@ import { getTokenHash } from '@/utils/auth.ts'; import { bech32ToPubkey, Time } from '@/utils.ts'; import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts'; import { renderNotification } from '@/views/mastodon/notifications.ts'; +import { HTTPException } from '@hono/hono/http-exception'; const console = new Stickynotes('ditto:streaming'); @@ -236,13 +237,17 @@ async function getTokenPubkey(token: string): Promise { const kysely = await Storages.kysely(); const tokenHash = await getTokenHash(token as `token1${string}`); - const { pubkey } = await kysely + const row = await kysely .selectFrom('auth_tokens') .select('pubkey') .where('token_hash', '=', tokenHash) - .executeTakeFirstOrThrow(); + .executeTakeFirst(); - return pubkey; + if (!row) { + throw new HTTPException(401, { message: 'Invalid access token' }); + } + + return row.pubkey; } else { return bech32ToPubkey(token); } From e89853c56dc496bfbe817b92b17138c5252072a8 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 21 Jan 2025 13:12:39 -0300 Subject: [PATCH 09/12] fix: mention with hyphen --- src/controllers/api/statuses.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index c27b9a74..1f4a94cd 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -151,7 +151,7 @@ const createStatusController: AppController = async (c) => { let content = await asyncReplaceAll( data.status ?? '', - /(? { const pubkey = await lookupPubkey(username); if (!pubkey) return match; From 7fdfb806f46a4182344e1bf8a293b2039944b399 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 Jan 2025 17:30:59 -0600 Subject: [PATCH 10/12] pipeline: skip refetch of encountered events --- src/caches/pipelineEncounters.ts | 3 ++ src/controllers/nostr/relay.ts | 2 +- src/firehose.ts | 2 +- src/notify.ts | 18 ++++++++--- src/pipeline.ts | 53 +++++++++++++++++++------------- src/trends.ts | 2 +- src/utils/api.ts | 2 +- 7 files changed, 51 insertions(+), 31 deletions(-) create mode 100644 src/caches/pipelineEncounters.ts diff --git a/src/caches/pipelineEncounters.ts b/src/caches/pipelineEncounters.ts new file mode 100644 index 00000000..491a416f --- /dev/null +++ b/src/caches/pipelineEncounters.ts @@ -0,0 +1,3 @@ +import { LRUCache } from 'lru-cache'; + +export const pipelineEncounters = new LRUCache({ max: 5000 }); diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 71397ee8..74bd8a56 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -138,7 +138,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) { relayEventsCounter.inc({ kind: event.kind.toString() }); try { // This will store it (if eligible) and run other side-effects. - await pipeline.handleEvent(purifyEvent(event), AbortSignal.timeout(1000)); + await pipeline.handleEvent(purifyEvent(event), { source: 'relay', signal: AbortSignal.timeout(1000) }); send(['OK', event.id, true, '']); } catch (e) { if (e instanceof RelayError) { diff --git a/src/firehose.ts b/src/firehose.ts index da8ab9c1..0dd88ba2 100644 --- a/src/firehose.ts +++ b/src/firehose.ts @@ -27,7 +27,7 @@ export async function startFirehose(): Promise { sem.lock(async () => { try { - await pipeline.handleEvent(event, AbortSignal.timeout(5000)); + await pipeline.handleEvent(event, { source: 'firehose', signal: AbortSignal.timeout(5000) }); } catch (e) { console.warn(e); } diff --git a/src/notify.ts b/src/notify.ts index 69480875..cda22718 100644 --- a/src/notify.ts +++ b/src/notify.ts @@ -1,24 +1,32 @@ import { Semaphore } from '@lambdalisue/async'; +import { Stickynotes } from '@soapbox/stickynotes'; +import { pipelineEncounters } from '@/caches/pipelineEncounters.ts'; import { Conf } from '@/config.ts'; import * as pipeline from '@/pipeline.ts'; import { Storages } from '@/storages.ts'; const sem = new Semaphore(1); +const console = new Stickynotes('ditto:notify'); export async function startNotify(): Promise { const { listen } = await Storages.database(); const store = await Storages.db(); - listen('nostr_event', (payload) => { + listen('nostr_event', (id) => { + if (pipelineEncounters.has(id)) { + console.debug(`Skip event ${id} because it was already in the pipeline`); + return; + } + sem.lock(async () => { try { - const id = payload; - const timeout = Conf.db.timeouts.default; + const signal = AbortSignal.timeout(Conf.db.timeouts.default); + + const [event] = await store.query([{ ids: [id], limit: 1 }], { signal }); - const [event] = await store.query([{ ids: [id], limit: 1 }], { signal: AbortSignal.timeout(timeout) }); if (event) { - await pipeline.handleEvent(event, AbortSignal.timeout(timeout)); + await pipeline.handleEvent(event, { source: 'notify', signal }); } } catch (e) { console.warn(e); diff --git a/src/pipeline.ts b/src/pipeline.ts index 5becff20..688fe3bb 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,7 +1,6 @@ import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify'; import { Stickynotes } from '@soapbox/stickynotes'; import { Kysely, sql } from 'kysely'; -import { LRUCache } from 'lru-cache'; import { z } from 'zod'; import { Conf } from '@/config.ts'; @@ -23,14 +22,25 @@ import { getTagSet } from '@/utils/tags.ts'; import { renderWebPushNotification } from '@/views/mastodon/push.ts'; import { policyWorker } from '@/workers/policy.ts'; import { verifyEventWorker } from '@/workers/verify.ts'; +import { pipelineEncounters } from '@/caches/pipelineEncounters.ts'; const console = new Stickynotes('ditto:pipeline'); +interface PipelineOpts { + signal: AbortSignal; + source: 'relay' | 'api' | 'firehose' | 'pipeline' | 'notify' | 'internal'; +} + /** * Common pipeline function to process (and maybe store) events. * It is idempotent, so it can be called multiple times for the same event. */ -async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { +async function handleEvent(event: DittoEvent, opts: PipelineOpts): Promise { + // Skip events that have already been encountered. + if (pipelineEncounters.get(event.id)) { + throw new RelayError('duplicate', 'already have this event'); + } + // Reject events that are too far in the future. if (eventAge(event) < -Time.minutes(1)) { throw new RelayError('invalid', 'event too far in the future'); } @@ -51,11 +61,12 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise ${event.id}`); @@ -71,12 +82,12 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise({ max: 1000 }); - -/** Encounter the event, and return whether it has already been encountered. */ -function encounterEvent(event: NostrEvent): boolean { - const encountered = !!encounters.get(event.id); - if (!encountered) { - encounters.set(event.id, true); - } - return encountered; -} - /** Check whether the event has a NIP-70 `-` tag. */ function isProtectedEvent(event: NostrEvent): boolean { return event.tags.some(([name]) => name === '-'); @@ -326,7 +335,7 @@ async function generateSetEvents(event: NostrEvent): Promise { created_at: Math.floor(Date.now() / 1000), }); - await handleEvent(rel, AbortSignal.timeout(1000)); + await handleEvent(rel, { source: 'pipeline', signal: AbortSignal.timeout(1000) }); } if (event.kind === 3036 && tagsAdmin) { @@ -344,7 +353,7 @@ async function generateSetEvents(event: NostrEvent): Promise { created_at: Math.floor(Date.now() / 1000), }); - await handleEvent(rel, AbortSignal.timeout(1000)); + await handleEvent(rel, { source: 'pipeline', signal: AbortSignal.timeout(1000) }); } } diff --git a/src/trends.ts b/src/trends.ts index aced6800..cbe85c14 100644 --- a/src/trends.ts +++ b/src/trends.ts @@ -111,7 +111,7 @@ export async function updateTrendingTags( created_at: Math.floor(Date.now() / 1000), }); - await handleEvent(label, signal); + await handleEvent(label, { source: 'internal', signal }); console.info(`Trending ${l} updated.`); } catch (e) { console.error(`Error updating trending ${l}: ${e instanceof Error ? e.message : e}`); diff --git a/src/utils/api.ts b/src/utils/api.ts index 4bbd32fc..37cbbbf9 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -161,7 +161,7 @@ async function updateNames(k: number, d: string, n: Record, c: async function publishEvent(event: NostrEvent, c: AppContext): Promise { debug('EVENT', event); try { - await pipeline.handleEvent(event, c.req.raw.signal); + await pipeline.handleEvent(event, { source: 'api', signal: c.req.raw.signal }); const client = await Storages.client(); await client.event(purifyEvent(event)); } catch (e) { From 281872b0ad951afba57d5ca0582c8c4c882bb769 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 Jan 2025 18:34:34 -0600 Subject: [PATCH 11/12] Add notActivityPub middleware to stop AP requests on /users/* --- src/app.ts | 4 ++-- src/middleware/notActivitypubMiddleware.ts | 18 ++++++++++++++++++ src/utils/api.ts | 19 ------------------- 3 files changed, 20 insertions(+), 21 deletions(-) create mode 100644 src/middleware/notActivitypubMiddleware.ts diff --git a/src/app.ts b/src/app.ts index 32061184..b18ba281 100644 --- a/src/app.ts +++ b/src/app.ts @@ -133,6 +133,7 @@ import { DittoTranslator } from '@/interfaces/DittoTranslator.ts'; import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts'; import { cspMiddleware } from '@/middleware/cspMiddleware.ts'; import { metricsMiddleware } from '@/middleware/metricsMiddleware.ts'; +import { notActivitypubMiddleware } from '@/middleware/notActivitypubMiddleware.ts'; import { paginationMiddleware } from '@/middleware/paginationMiddleware.ts'; import { rateLimitMiddleware } from '@/middleware/rateLimitMiddleware.ts'; import { requireSigner } from '@/middleware/requireSigner.ts'; @@ -179,7 +180,6 @@ app.use('*', rateLimitMiddleware(300, Time.minutes(5))); app.use('/api/*', metricsMiddleware, paginationMiddleware, logger(debug)); app.use('/.well-known/*', metricsMiddleware, logger(debug)); -app.use('/users/*', metricsMiddleware, logger(debug)); app.use('/nodeinfo/*', metricsMiddleware, logger(debug)); app.use('/oauth/*', metricsMiddleware, logger(debug)); @@ -400,7 +400,7 @@ app.use('/oauth/*', notImplementedController); app.get('/:acct{@.*}', frontendController); app.get('/:acct{@.*}/*', frontendController); app.get('/:bech32{^[\x21-\x7E]{1,83}1[023456789acdefghjklmnpqrstuvwxyz]{6,}$}', frontendController); -app.get('/users/*', frontendController); +app.get('/users/*', notActivitypubMiddleware, frontendController); app.get('/tags/*', frontendController); app.get('/statuses/*', frontendController); app.get('/notice/*', frontendController); diff --git a/src/middleware/notActivitypubMiddleware.ts b/src/middleware/notActivitypubMiddleware.ts new file mode 100644 index 00000000..b25e6f61 --- /dev/null +++ b/src/middleware/notActivitypubMiddleware.ts @@ -0,0 +1,18 @@ +import { MiddlewareHandler } from '@hono/hono'; + +const ACTIVITYPUB_TYPES = [ + 'application/activity+json', + 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', +]; + +/** Return 4xx errors on common (unsupported) ActivityPub routes to prevent AP traffic. */ +export const notActivitypubMiddleware: MiddlewareHandler = async (c, next) => { + const accept = c.req.header('accept'); + const types = accept?.split(',')?.map((type) => type.trim()) ?? []; + + if (types.every((type) => ACTIVITYPUB_TYPES.includes(type))) { + return c.text('ActivityPub is not supported', 406); + } + + await next(); +}; diff --git a/src/utils/api.ts b/src/utils/api.ts index 37cbbbf9..9e7125c6 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -261,24 +261,6 @@ function paginatedList( return c.json(results, 200, headers); } -/** JSON-LD context. */ -type LDContext = (string | Record>)[]; - -/** Add a basic JSON-LD context to ActivityStreams object, if it doesn't already exist. */ -function maybeAddContext(object: T): T & { '@context': LDContext } { - return { - '@context': ['https://www.w3.org/ns/activitystreams'], - ...object, - }; -} - -/** Like hono's `c.json()` except returns JSON-LD. */ -function activityJson(c: Context, object: T) { - const response = c.json(maybeAddContext(object)); - response.headers.set('content-type', 'application/activity+json; charset=UTF-8'); - return response; -} - /** Rewrite the URL of the request object to use the local domain. */ function localRequest(c: Context): Request { return Object.create(c.req.raw, { @@ -300,7 +282,6 @@ function assertAuthenticated(c: AppContext, author: NostrEvent): void { } export { - activityJson, assertAuthenticated, createAdminEvent, createEvent, From 6d31949944150163d9d3589337b9140d507f8104 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 Jan 2025 18:49:08 -0600 Subject: [PATCH 12/12] notActivitypubMiddleware: add bare ld+json to ACTIVITYPUB_TYPES --- src/middleware/notActivitypubMiddleware.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/middleware/notActivitypubMiddleware.ts b/src/middleware/notActivitypubMiddleware.ts index b25e6f61..1cdb9cfb 100644 --- a/src/middleware/notActivitypubMiddleware.ts +++ b/src/middleware/notActivitypubMiddleware.ts @@ -2,6 +2,7 @@ import { MiddlewareHandler } from '@hono/hono'; const ACTIVITYPUB_TYPES = [ 'application/activity+json', + 'application/ld+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', ];