From f15b6f79c0dc2ede9353bea94c21b7b4f38b4c81 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 14 Mar 2025 23:29:20 -0500 Subject: [PATCH 01/22] Implement custom emojis --- packages/ditto/app.ts | 4 +- .../ditto/routes/customEmojisRoute.test.ts | 67 +++++++++++++++ packages/ditto/routes/customEmojisRoute.ts | 86 +++++++++++++++++++ 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 packages/ditto/routes/customEmojisRoute.test.ts create mode 100644 packages/ditto/routes/customEmojisRoute.ts diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index d656644a..6432668c 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -147,6 +147,7 @@ import { rateLimitMiddleware } from '@/middleware/rateLimitMiddleware.ts'; import { uploaderMiddleware } from '@/middleware/uploaderMiddleware.ts'; import { translatorMiddleware } from '@/middleware/translatorMiddleware.ts'; import { logiMiddleware } from '@/middleware/logiMiddleware.ts'; +import customEmojisRoute from '@/routes/customEmojisRoute.ts'; import dittoNamesRoute from '@/routes/dittoNamesRoute.ts'; import pleromaAdminPermissionGroupsRoute from '@/routes/pleromaAdminPermissionGroupsRoute.ts'; import { DittoRelayStore } from '@/storages/DittoRelayStore.ts'; @@ -520,8 +521,9 @@ app.delete('/api/v1/pleroma/admin/users/tag', userMiddleware({ role: 'admin' }), app.patch('/api/v1/pleroma/admin/users/suggest', userMiddleware({ role: 'admin' }), pleromaAdminSuggestController); app.patch('/api/v1/pleroma/admin/users/unsuggest', userMiddleware({ role: 'admin' }), pleromaAdminUnsuggestController); +app.route('/api/v1/custom_emojis', customEmojisRoute); + // Not (yet) implemented. -app.get('/api/v1/custom_emojis', emptyArrayController); app.get('/api/v1/filters', emptyArrayController); app.get('/api/v1/domain_blocks', emptyArrayController); app.get('/api/v1/conversations', emptyArrayController); diff --git a/packages/ditto/routes/customEmojisRoute.test.ts b/packages/ditto/routes/customEmojisRoute.test.ts new file mode 100644 index 00000000..6feac4cc --- /dev/null +++ b/packages/ditto/routes/customEmojisRoute.test.ts @@ -0,0 +1,67 @@ +import { TestApp } from '@ditto/mastoapi/test'; +import { NSecSigner } from '@nostrify/nostrify'; +import { genEvent } from '@nostrify/nostrify/test'; +import { assertEquals } from '@std/assert'; +import { generateSecretKey } from 'nostr-tools'; + +import route from './customEmojisRoute.ts'; + +Deno.test('customEmojisRoute', async (t) => { + await using test = new TestApp(route); + const { relay } = test.var; + + const sk = generateSecretKey(); + const user = test.user({ relay, signer: new NSecSigner(sk) }); + const pubkey = await user.signer.getPublicKey(); + + await t.step('no emojis', async () => { + const response = await test.api.get('/'); + const body = await response.json(); + + assertEquals(response.status, 200); + assertEquals(body, []); + }); + + await t.step('with emoji packs', async () => { + const pack = genEvent({ + kind: 30030, + tags: [ + ['d', 'soapbox'], + ['emoji', 'soapbox', 'https://soapbox.pub/favicon.ico'], + ['emoji', 'ditto', 'https://ditto.pub/favicon.ico'], + ], + }, sk); + + const list = genEvent({ + kind: 10030, + tags: [ + ['a', `30030:${pubkey}:soapbox`], + ['emoji', 'gleasonator', 'https://gleasonator.dev/favicon.ico'], + ], + }, sk); + + await relay.event(pack); + await relay.event(list); + + const response = await test.api.get('/'); + const body = await response.json(); + + assertEquals(response.status, 200); + assertEquals(body, [{ + shortcode: 'gleasonator', + url: 'https://gleasonator.dev/favicon.ico', + static_url: 'https://gleasonator.dev/favicon.ico', + visible_in_picker: true, + }, { + shortcode: 'soapbox', + url: 'https://soapbox.pub/favicon.ico', + static_url: 'https://soapbox.pub/favicon.ico', + visible_in_picker: true, + }, { + shortcode: 'ditto', + url: 'https://ditto.pub/favicon.ico', + static_url: 'https://ditto.pub/favicon.ico', + visible_in_picker: true, + }]); + }); +}); diff --git a/packages/ditto/routes/customEmojisRoute.ts b/packages/ditto/routes/customEmojisRoute.ts new file mode 100644 index 00000000..32c7a3d0 --- /dev/null +++ b/packages/ditto/routes/customEmojisRoute.ts @@ -0,0 +1,86 @@ +import { userMiddleware } from '@ditto/mastoapi/middleware'; +import { DittoRoute } from '@ditto/mastoapi/router'; +import { NostrFilter } from '@nostrify/nostrify'; + +const route = new DittoRoute(); + +interface MastodonCustomEmoji { + shortcode: string; + url: string; + static_url: string; + visible_in_picker: boolean; + category?: string; +} + +route.get('/', userMiddleware(), async (c) => { + const { relay, user, signal } = c.var; + + const pubkey = await user.signer.getPublicKey(); + + const [emojiList] = await relay.query([{ kinds: [10030], authors: [pubkey] }], { signal }); + + if (!emojiList) { + return c.json([]); + } + + const a = new Set(); + const emojis = new Map(); + + for (const tag of emojiList.tags) { + if (tag[0] === 'emoji') { + const [, shortcode, url] = tag; + + if (!emojis.has(shortcode)) { + try { + emojis.set(shortcode, new URL(url)); + } catch { + // continue + } + } + } + + if (tag[0] === 'a') { + a.add(tag[1]); + } + } + + const filters: NostrFilter[] = []; + + for (const addr of a) { + const match = addr.match(/^30030:([0-9a-f]{64}):(.+)$/); + + if (match) { + const [, pubkey, d] = match; + filters.push({ kinds: [30030], authors: [pubkey], '#d': [d] }); + } + } + + if (!filters.length) { + return c.json([]); + } + + for (const event of await relay.query(filters, { signal })) { + for (const [t, shortcode, url] of event.tags) { + if (t === 'emoji') { + if (!emojis.has(shortcode)) { + try { + emojis.set(shortcode, new URL(url)); + } catch { + // continue + } + } + } + } + } + + return c.json([...emojis.entries()].map(([shortcode, url]): MastodonCustomEmoji => { + return { + shortcode, + url: url.toString(), + static_url: url.toString(), + visible_in_picker: true, + }; + })); +}); + +export default route; From b1a1ace0ac8675aa76aefffca84399bad040160d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 14 Mar 2025 23:58:42 -0500 Subject: [PATCH 02/22] Refactor getCustomEmojis function, support emoji categories --- .../ditto/routes/customEmojisRoute.test.ts | 2 + packages/ditto/routes/customEmojisRoute.ts | 51 +++++++++++++------ 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/packages/ditto/routes/customEmojisRoute.test.ts b/packages/ditto/routes/customEmojisRoute.test.ts index 6feac4cc..4d52540a 100644 --- a/packages/ditto/routes/customEmojisRoute.test.ts +++ b/packages/ditto/routes/customEmojisRoute.test.ts @@ -57,11 +57,13 @@ Deno.test('customEmojisRoute', async (t) => { url: 'https://soapbox.pub/favicon.ico', static_url: 'https://soapbox.pub/favicon.ico', visible_in_picker: true, + category: 'soapbox', }, { shortcode: 'ditto', url: 'https://ditto.pub/favicon.ico', static_url: 'https://ditto.pub/favicon.ico', visible_in_picker: true, + category: 'soapbox', }]); }); }); diff --git a/packages/ditto/routes/customEmojisRoute.ts b/packages/ditto/routes/customEmojisRoute.ts index 32c7a3d0..997c9a45 100644 --- a/packages/ditto/routes/customEmojisRoute.ts +++ b/packages/ditto/routes/customEmojisRoute.ts @@ -1,6 +1,6 @@ import { userMiddleware } from '@ditto/mastoapi/middleware'; import { DittoRoute } from '@ditto/mastoapi/router'; -import { NostrFilter } from '@nostrify/nostrify'; +import { NostrFilter, NRelay } from '@nostrify/nostrify'; const route = new DittoRoute(); @@ -13,18 +13,42 @@ interface MastodonCustomEmoji { } route.get('/', userMiddleware(), async (c) => { - const { relay, user, signal } = c.var; + const { user } = c.var; const pubkey = await user.signer.getPublicKey(); + const emojis = await getCustomEmojis(pubkey, c.var); + + return c.json([...emojis.entries()].map(([shortcode, data]): MastodonCustomEmoji => { + return { + shortcode, + url: data.url.toString(), + static_url: data.url.toString(), + visible_in_picker: true, + category: data.category, + }; + })); +}); + +interface GetCustomEmojisOpts { + relay: NRelay; + signal?: AbortSignal; +} + +async function getCustomEmojis( + pubkey: string, + opts: GetCustomEmojisOpts, +): Promise> { + const { relay, signal } = opts; + + const emojis = new Map(); const [emojiList] = await relay.query([{ kinds: [10030], authors: [pubkey] }], { signal }); if (!emojiList) { - return c.json([]); + return emojis; } const a = new Set(); - const emojis = new Map(); for (const tag of emojiList.tags) { if (tag[0] === 'emoji') { @@ -32,7 +56,7 @@ route.get('/', userMiddleware(), async (c) => { if (!emojis.has(shortcode)) { try { - emojis.set(shortcode, new URL(url)); + emojis.set(shortcode, { url: new URL(url) }); } catch { // continue } @@ -56,15 +80,17 @@ route.get('/', userMiddleware(), async (c) => { } if (!filters.length) { - return c.json([]); + return new Map(); } for (const event of await relay.query(filters, { signal })) { + const d = event.tags.find(([name]) => name === 'd')?.[1]; + for (const [t, shortcode, url] of event.tags) { if (t === 'emoji') { if (!emojis.has(shortcode)) { try { - emojis.set(shortcode, new URL(url)); + emojis.set(shortcode, { url: new URL(url), category: d }); } catch { // continue } @@ -73,14 +99,7 @@ route.get('/', userMiddleware(), async (c) => { } } - return c.json([...emojis.entries()].map(([shortcode, url]): MastodonCustomEmoji => { - return { - shortcode, - url: url.toString(), - static_url: url.toString(), - visible_in_picker: true, - }; - })); -}); + return emojis; +} export default route; From 28275b76116138e481fa87213d53b2a6401f14be Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Mar 2025 00:16:52 -0500 Subject: [PATCH 03/22] Insert "emoji" tags into statuses --- packages/ditto/controllers/api/statuses.ts | 18 +++++ packages/ditto/routes/customEmojisRoute.ts | 76 +--------------------- packages/ditto/utils/custom-emoji.ts | 74 +++++++++++++++++++++ 3 files changed, 94 insertions(+), 74 deletions(-) create mode 100644 packages/ditto/utils/custom-emoji.ts diff --git a/packages/ditto/controllers/api/statuses.ts b/packages/ditto/controllers/api/statuses.ts index 53fab57a..116ad2de 100644 --- a/packages/ditto/controllers/api/statuses.ts +++ b/packages/ditto/controllers/api/statuses.ts @@ -16,6 +16,7 @@ import { lookupPubkey } from '@/utils/lookup.ts'; import { languageSchema } from '@/schema.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { assertAuthenticated, createEvent, parseBody, updateListEvent } from '@/utils/api.ts'; +import { getCustomEmojis } from '@/utils/custom-emoji.ts'; import { getInvoice, getLnurl } from '@/utils/lnurl.ts'; import { purifyEvent } from '@/utils/purify.ts'; import { getZapSplits } from '@/utils/zap-split.ts'; @@ -190,6 +191,23 @@ const createStatusController: AppController = async (c) => { } } + const shortcodes = new Set(); + + for (const [, shortcode] of data.status?.matchAll(/(? { })); }); -interface GetCustomEmojisOpts { - relay: NRelay; - signal?: AbortSignal; -} - -async function getCustomEmojis( - pubkey: string, - opts: GetCustomEmojisOpts, -): Promise> { - const { relay, signal } = opts; - - const emojis = new Map(); - - const [emojiList] = await relay.query([{ kinds: [10030], authors: [pubkey] }], { signal }); - - if (!emojiList) { - return emojis; - } - - const a = new Set(); - - for (const tag of emojiList.tags) { - if (tag[0] === 'emoji') { - const [, shortcode, url] = tag; - - if (!emojis.has(shortcode)) { - try { - emojis.set(shortcode, { url: new URL(url) }); - } catch { - // continue - } - } - } - - if (tag[0] === 'a') { - a.add(tag[1]); - } - } - - const filters: NostrFilter[] = []; - - for (const addr of a) { - const match = addr.match(/^30030:([0-9a-f]{64}):(.+)$/); - - if (match) { - const [, pubkey, d] = match; - filters.push({ kinds: [30030], authors: [pubkey], '#d': [d] }); - } - } - - if (!filters.length) { - return new Map(); - } - - for (const event of await relay.query(filters, { signal })) { - const d = event.tags.find(([name]) => name === 'd')?.[1]; - - for (const [t, shortcode, url] of event.tags) { - if (t === 'emoji') { - if (!emojis.has(shortcode)) { - try { - emojis.set(shortcode, { url: new URL(url), category: d }); - } catch { - // continue - } - } - } - } - } - - return emojis; -} - export default route; diff --git a/packages/ditto/utils/custom-emoji.ts b/packages/ditto/utils/custom-emoji.ts new file mode 100644 index 00000000..8dbcd7a8 --- /dev/null +++ b/packages/ditto/utils/custom-emoji.ts @@ -0,0 +1,74 @@ +import type { NostrFilter, NRelay } from '@nostrify/nostrify'; + +interface GetCustomEmojisOpts { + relay: NRelay; + signal?: AbortSignal; +} + +export async function getCustomEmojis( + pubkey: string, + opts: GetCustomEmojisOpts, +): Promise> { + const { relay, signal } = opts; + + const emojis = new Map(); + + const [emojiList] = await relay.query([{ kinds: [10030], authors: [pubkey] }], { signal }); + + if (!emojiList) { + return emojis; + } + + const a = new Set(); + + for (const tag of emojiList.tags) { + if (tag[0] === 'emoji') { + const [, shortcode, url] = tag; + + if (!emojis.has(shortcode)) { + try { + emojis.set(shortcode, { url: new URL(url) }); + } catch { + // continue + } + } + } + + if (tag[0] === 'a') { + a.add(tag[1]); + } + } + + const filters: NostrFilter[] = []; + + for (const addr of a) { + const match = addr.match(/^30030:([0-9a-f]{64}):(.+)$/); + + if (match) { + const [, pubkey, d] = match; + filters.push({ kinds: [30030], authors: [pubkey], '#d': [d] }); + } + } + + if (!filters.length) { + return new Map(); + } + + for (const event of await relay.query(filters, { signal })) { + const d = event.tags.find(([name]) => name === 'd')?.[1]; + + for (const [t, shortcode, url] of event.tags) { + if (t === 'emoji') { + if (!emojis.has(shortcode)) { + try { + emojis.set(shortcode, { url: new URL(url), category: d }); + } catch { + // continue + } + } + } + } + } + + return emojis; +} From 974e07981ed34a7d7f40840d17de1577994526ec Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Mar 2025 00:19:50 -0500 Subject: [PATCH 04/22] getCustomEmojis: new Map() -> emojis --- packages/ditto/utils/custom-emoji.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ditto/utils/custom-emoji.ts b/packages/ditto/utils/custom-emoji.ts index 8dbcd7a8..062cb46a 100644 --- a/packages/ditto/utils/custom-emoji.ts +++ b/packages/ditto/utils/custom-emoji.ts @@ -51,7 +51,7 @@ export async function getCustomEmojis( } if (!filters.length) { - return new Map(); + return emojis; } for (const event of await relay.query(filters, { signal })) { From e5dd285e6bb12294a0516f0ae2dfaa65cf4191bc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Mar 2025 12:48:54 -0500 Subject: [PATCH 05/22] Allow custom_emojis endpoint to be accessed without a user --- packages/ditto/routes/customEmojisRoute.test.ts | 8 ++++++++ packages/ditto/routes/customEmojisRoute.ts | 6 +++++- packages/ditto/utils/custom-emoji.ts | 12 +++++------- packages/mastoapi/middleware/userMiddleware.test.ts | 12 ++++++++++++ packages/mastoapi/middleware/userMiddleware.ts | 6 ++++-- 5 files changed, 34 insertions(+), 10 deletions(-) diff --git a/packages/ditto/routes/customEmojisRoute.test.ts b/packages/ditto/routes/customEmojisRoute.test.ts index 4d52540a..c8022ca0 100644 --- a/packages/ditto/routes/customEmojisRoute.test.ts +++ b/packages/ditto/routes/customEmojisRoute.test.ts @@ -10,6 +10,14 @@ Deno.test('customEmojisRoute', async (t) => { await using test = new TestApp(route); const { relay } = test.var; + await t.step('unauth', async () => { + const response = await test.api.get('/'); + const body = await response.json(); + + assertEquals(response.status, 200); + assertEquals(body, []); + }); + const sk = generateSecretKey(); const user = test.user({ relay, signer: new NSecSigner(sk) }); const pubkey = await user.signer.getPublicKey(); diff --git a/packages/ditto/routes/customEmojisRoute.ts b/packages/ditto/routes/customEmojisRoute.ts index 156d2cf7..5dc50f28 100644 --- a/packages/ditto/routes/customEmojisRoute.ts +++ b/packages/ditto/routes/customEmojisRoute.ts @@ -13,9 +13,13 @@ interface MastodonCustomEmoji { category?: string; } -route.get('/', userMiddleware(), async (c) => { +route.get('/', userMiddleware({ required: false }), async (c) => { const { user } = c.var; + if (!user) { + return c.json([]); + } + const pubkey = await user.signer.getPublicKey(); const emojis = await getCustomEmojis(pubkey, c.var); diff --git a/packages/ditto/utils/custom-emoji.ts b/packages/ditto/utils/custom-emoji.ts index 062cb46a..a7a83a31 100644 --- a/packages/ditto/utils/custom-emoji.ts +++ b/packages/ditto/utils/custom-emoji.ts @@ -58,13 +58,11 @@ export async function getCustomEmojis( const d = event.tags.find(([name]) => name === 'd')?.[1]; for (const [t, shortcode, url] of event.tags) { - if (t === 'emoji') { - if (!emojis.has(shortcode)) { - try { - emojis.set(shortcode, { url: new URL(url), category: d }); - } catch { - // continue - } + if (t === 'emoji' && /^\w+$/.test(shortcode) && !emojis.has(shortcode)) { + try { + emojis.set(shortcode, { url: new URL(url), category: d }); + } catch { + // continue } } } diff --git a/packages/mastoapi/middleware/userMiddleware.test.ts b/packages/mastoapi/middleware/userMiddleware.test.ts index fdd0a09a..1edf5673 100644 --- a/packages/mastoapi/middleware/userMiddleware.test.ts +++ b/packages/mastoapi/middleware/userMiddleware.test.ts @@ -10,6 +10,18 @@ Deno.test('no user 401', async () => { assertEquals(response.status, 401); }); +Deno.test('no user required false', async () => { + await using app = new TestApp(); + + app + .use(userMiddleware({ required: false })) + .get('/', (c) => c.text('ok')); + + const response = await app.request('/'); + + assertEquals(response.status, 200); +}); + Deno.test('unsupported signer 400', async () => { await using app = new TestApp(); diff --git a/packages/mastoapi/middleware/userMiddleware.ts b/packages/mastoapi/middleware/userMiddleware.ts index 2b964362..b40f8dea 100644 --- a/packages/mastoapi/middleware/userMiddleware.ts +++ b/packages/mastoapi/middleware/userMiddleware.ts @@ -12,6 +12,7 @@ interface UserMiddlewareOpts { enc?: 'nip04' | 'nip44'; role?: string; verify?: boolean; + required?: boolean; } export function userMiddleware(): DittoMiddleware<{ user: User }>; @@ -19,13 +20,14 @@ export function userMiddleware(): DittoMiddleware<{ user: User }>; export function userMiddleware( opts: UserMiddlewareOpts & { enc: 'nip44' }, ): DittoMiddleware<{ user: User }>; +export function userMiddleware(opts: UserMiddlewareOpts & { required: false }): DittoMiddleware<{ user?: User }>; export function userMiddleware(opts: UserMiddlewareOpts): DittoMiddleware<{ user: User }>; export function userMiddleware(opts: UserMiddlewareOpts = {}): DittoMiddleware<{ user: User }> { return async (c, next) => { const { conf, user, relay } = c.var; - const { enc, role, verify } = opts; + const { enc, role, verify, required = true } = opts; - if (!user) { + if (!user && required) { throw new HTTPException(401, { message: 'Authorization required' }); } From 66dcf98a22e7858b4a96b77c48d8e48adc9842b3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Mar 2025 13:24:02 -0500 Subject: [PATCH 06/22] Allow reacting with custom emoji --- packages/ditto/controllers/api/reactions.ts | 56 ++++++++++++++------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/packages/ditto/controllers/api/reactions.ts b/packages/ditto/controllers/api/reactions.ts index 74e499d4..5f459ab0 100644 --- a/packages/ditto/controllers/api/reactions.ts +++ b/packages/ditto/controllers/api/reactions.ts @@ -4,37 +4,47 @@ import { hydrateEvents } from '@/storages/hydrate.ts'; import { createEvent } from '@/utils/api.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; +import { HTTPException } from '@hono/hono/http-exception'; + +import { getCustomEmojis } from '@/utils/custom-emoji.ts'; /** * React to a status. * https://docs.pleroma.social/backend/development/API/pleroma_api/#put-apiv1pleromastatusesidreactionsemoji */ const reactionController: AppController = async (c) => { - const { relay, user } = c.var; - const id = c.req.param('id'); - const emoji = c.req.param('emoji'); + const { relay, user, conf, signal } = c.var; + const { type, value } = parseEmojiParam(c.req.param('emoji')); - if (!/^\p{RGI_Emoji}$/v.test(emoji)) { - return c.json({ error: 'Invalid emoji' }, 400); - } - - const [event] = await relay.query([{ kinds: [1, 20], ids: [id], limit: 1 }]); + const pubkey = await user!.signer.getPublicKey(); + const [event] = await relay.query([{ ids: [c.req.param('id')] }], { signal }); if (!event) { - return c.json({ error: 'Status not found' }, 404); + return c.json({ error: 'Event not found' }, 404); } - await createEvent({ - kind: 7, - content: emoji, - created_at: Math.floor(Date.now() / 1000), - tags: [['e', id], ['p', event.pubkey]], - }, c); + const tags: string[][] = [ + ['e', event.id, conf.relay, event.pubkey], + ['p', event.pubkey, conf.relay], + ]; + if (type === 'custom') { + const emojis = await getCustomEmojis(pubkey, c.var); + const emoji = emojis.get(value); + + if (!emoji) { + return c.json({ error: 'Custom emoji not found' }, 404); + } + + tags.push(['emoji', value, emoji.url.href]); + } + + const content = type === 'custom' ? `:${value}:` : value; + + await createEvent({ kind: 7, content, tags }, c); await hydrateEvents({ ...c.var, events: [event] }); - const status = await renderStatus(relay, event, { viewerPubkey: await user!.signer.getPublicKey() }); - + const status = await renderStatus(relay, event, { viewerPubkey: pubkey }); return c.json(status); }; @@ -125,4 +135,16 @@ const reactionsController: AppController = async (c) => { return c.json(results); }; +function parseEmojiParam(input: string): { type: 'native' | 'custom'; value: string } { + if (/^\p{RGI_Emoji}$/v.test(input)) { + return { type: 'native', value: input }; + } + + if (/^\w+$/.test(input)) { + return { type: 'custom', value: input }; + } + + throw new HTTPException(400, { message: 'Invalid emoji' }); +} + export { deleteReactionController, reactionController, reactionsController }; From c9b0ffb537041317f3db8aaee02f02eb647a37da Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Mar 2025 13:26:55 -0500 Subject: [PATCH 07/22] Remove redundant userMiddleware usages in app.ts --- packages/ditto/app.ts | 32 +++++--------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index 6432668c..6e896571 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -476,45 +476,23 @@ app.get('/api/v1/ditto/statuses/:id{[0-9a-f]{64}}/zapped_by', zappedByController app.route('/api/v1/ditto/cashu', cashuApp); app.post('/api/v1/reports', userMiddleware(), reportController); -app.get('/api/v1/admin/reports', userMiddleware(), userMiddleware({ role: 'admin' }), adminReportsController); -app.get( - '/api/v1/admin/reports/:id{[0-9a-f]{64}}', - userMiddleware(), - userMiddleware({ role: 'admin' }), - adminReportController, -); +app.get('/api/v1/admin/reports', userMiddleware({ role: 'admin' }), adminReportsController); +app.get('/api/v1/admin/reports/:id{[0-9a-f]{64}}', userMiddleware({ role: 'admin' }), adminReportController); app.post( '/api/v1/admin/reports/:id{[0-9a-f]{64}}/resolve', - userMiddleware(), userMiddleware({ role: 'admin' }), adminReportResolveController, ); app.post( '/api/v1/admin/reports/:id{[0-9a-f]{64}}/reopen', - userMiddleware(), userMiddleware({ role: 'admin' }), adminReportReopenController, ); app.get('/api/v1/admin/accounts', userMiddleware({ role: 'admin' }), adminAccountsController); -app.post( - '/api/v1/admin/accounts/:id{[0-9a-f]{64}}/action', - userMiddleware(), - userMiddleware({ role: 'admin' }), - adminActionController, -); -app.post( - '/api/v1/admin/accounts/:id{[0-9a-f]{64}}/approve', - userMiddleware(), - userMiddleware({ role: 'admin' }), - adminApproveController, -); -app.post( - '/api/v1/admin/accounts/:id{[0-9a-f]{64}}/reject', - userMiddleware(), - userMiddleware({ role: 'admin' }), - adminRejectController, -); +app.post('/api/v1/admin/accounts/:id{[0-9a-f]{64}}/action', userMiddleware({ role: 'admin' }), adminActionController); +app.post('/api/v1/admin/accounts/:id{[0-9a-f]{64}}/approve', userMiddleware({ role: 'admin' }), adminApproveController); +app.post('/api/v1/admin/accounts/:id{[0-9a-f]{64}}/reject', userMiddleware({ role: 'admin' }), adminRejectController); app.put('/api/v1/pleroma/admin/users/tag', userMiddleware({ role: 'admin' }), pleromaAdminTagController); app.delete('/api/v1/pleroma/admin/users/tag', userMiddleware({ role: 'admin' }), pleromaAdminUntagController); From 2f65be7dc4e1b1263c609b0b5cabc433ae13a29b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Mar 2025 13:49:42 -0500 Subject: [PATCH 08/22] Move reaction controllers to pleromaStatusesRoute --- packages/ditto/app.ts | 7 ++--- .../pleromaStatusesRoute.ts} | 26 +++++++++++-------- 2 files changed, 17 insertions(+), 16 deletions(-) rename packages/ditto/{controllers/api/reactions.ts => routes/pleromaStatusesRoute.ts} (90%) diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index 6e896571..bd86ac51 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -87,7 +87,6 @@ import { } from '@/controllers/api/pleroma.ts'; import { preferencesController } from '@/controllers/api/preferences.ts'; import { getSubscriptionController, pushSubscribeController } from '@/controllers/api/push.ts'; -import { deleteReactionController, reactionController, reactionsController } from '@/controllers/api/reactions.ts'; import { relayController } from '@/controllers/nostr/relay.ts'; import { adminReportController, @@ -150,6 +149,7 @@ import { logiMiddleware } from '@/middleware/logiMiddleware.ts'; import customEmojisRoute from '@/routes/customEmojisRoute.ts'; import dittoNamesRoute from '@/routes/dittoNamesRoute.ts'; import pleromaAdminPermissionGroupsRoute from '@/routes/pleromaAdminPermissionGroupsRoute.ts'; +import pleromaStatusesRoute from '@/routes/pleromaStatusesRoute.ts'; import { DittoRelayStore } from '@/storages/DittoRelayStore.ts'; export interface AppEnv extends DittoEnv { @@ -435,10 +435,7 @@ app.post('/api/v1/markers', userMiddleware({ verify: true }), updateMarkersContr app.get('/api/v1/push/subscription', userMiddleware(), getSubscriptionController); app.post('/api/v1/push/subscription', userMiddleware({ verify: true }), pushSubscribeController); -app.get('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/reactions', reactionsController); -app.get('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/reactions/:emoji', reactionsController); -app.put('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), reactionController); -app.delete('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), deleteReactionController); +app.route('/api/v1/pleroma/statuses', pleromaStatusesRoute); app.get('/api/v1/pleroma/admin/config', userMiddleware({ role: 'admin' }), configController); app.post('/api/v1/pleroma/admin/config', userMiddleware({ role: 'admin' }), updateConfigController); diff --git a/packages/ditto/controllers/api/reactions.ts b/packages/ditto/routes/pleromaStatusesRoute.ts similarity index 90% rename from packages/ditto/controllers/api/reactions.ts rename to packages/ditto/routes/pleromaStatusesRoute.ts index 5f459ab0..b17c95cc 100644 --- a/packages/ditto/controllers/api/reactions.ts +++ b/packages/ditto/routes/pleromaStatusesRoute.ts @@ -1,4 +1,6 @@ -import { AppController } from '@/app.ts'; +import { userMiddleware } from '@ditto/mastoapi/middleware'; +import { DittoRoute } from '@ditto/mastoapi/router'; + import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { createEvent } from '@/utils/api.ts'; @@ -8,11 +10,13 @@ import { HTTPException } from '@hono/hono/http-exception'; import { getCustomEmojis } from '@/utils/custom-emoji.ts'; -/** +const route = new DittoRoute(); + +/* * React to a status. * https://docs.pleroma.social/backend/development/API/pleroma_api/#put-apiv1pleromastatusesidreactionsemoji */ -const reactionController: AppController = async (c) => { +route.put('/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), async (c) => { const { relay, user, conf, signal } = c.var; const { type, value } = parseEmojiParam(c.req.param('emoji')); @@ -46,13 +50,13 @@ const reactionController: AppController = async (c) => { const status = await renderStatus(relay, event, { viewerPubkey: pubkey }); return c.json(status); -}; +}); -/** +/* * Delete reactions to a status. * https://docs.pleroma.social/backend/development/API/pleroma_api/#delete-apiv1pleromastatusesidreactionsemoji */ -const deleteReactionController: AppController = async (c) => { +route.delete('/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), async (c) => { const { relay, user } = c.var; const id = c.req.param('id'); @@ -89,13 +93,13 @@ const deleteReactionController: AppController = async (c) => { const status = renderStatus(relay, event, { viewerPubkey: pubkey }); return c.json(status); -}; +}); -/** +/* * Get an object of emoji to account mappings with accounts that reacted to the post. * https://docs.pleroma.social/backend/development/API/pleroma_api/#get-apiv1pleromastatusesidreactions */ -const reactionsController: AppController = async (c) => { +route.get('/:id{[0-9a-f]{64}}/reactions', userMiddleware({ required: false }), async (c) => { const { relay, user } = c.var; const id = c.req.param('id'); @@ -133,7 +137,7 @@ const reactionsController: AppController = async (c) => { ); return c.json(results); -}; +}); function parseEmojiParam(input: string): { type: 'native' | 'custom'; value: string } { if (/^\p{RGI_Emoji}$/v.test(input)) { @@ -147,4 +151,4 @@ function parseEmojiParam(input: string): { type: 'native' | 'custom'; value: str throw new HTTPException(400, { message: 'Invalid emoji' }); } -export { deleteReactionController, reactionController, reactionsController }; +export default route; From f27dac342f08f6673a680a11a68d8889b85ffeb1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Mar 2025 13:57:06 -0500 Subject: [PATCH 09/22] Small refactors to pleromaStatusesRoute --- packages/ditto/routes/pleromaStatusesRoute.ts | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/ditto/routes/pleromaStatusesRoute.ts b/packages/ditto/routes/pleromaStatusesRoute.ts index b17c95cc..0daa497c 100644 --- a/packages/ditto/routes/pleromaStatusesRoute.ts +++ b/packages/ditto/routes/pleromaStatusesRoute.ts @@ -17,12 +17,14 @@ const route = new DittoRoute(); * https://docs.pleroma.social/backend/development/API/pleroma_api/#put-apiv1pleromastatusesidreactionsemoji */ route.put('/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), async (c) => { + const params = c.req.param(); + const { relay, user, conf, signal } = c.var; - const { type, value } = parseEmojiParam(c.req.param('emoji')); + const { type, value } = parseEmojiParam(params.emoji); - const pubkey = await user!.signer.getPublicKey(); + const pubkey = await user.signer.getPublicKey(); - const [event] = await relay.query([{ ids: [c.req.param('id')] }], { signal }); + const [event] = await relay.query([{ ids: [params.id] }], { signal }); if (!event) { return c.json({ error: 'Event not found' }, 404); } @@ -57,11 +59,10 @@ route.put('/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), async (c) => * https://docs.pleroma.social/backend/development/API/pleroma_api/#delete-apiv1pleromastatusesidreactionsemoji */ route.delete('/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), async (c) => { + const { id, emoji } = c.req.param(); const { relay, user } = c.var; - const id = c.req.param('id'); - const emoji = c.req.param('emoji'); - const pubkey = await user!.signer.getPublicKey(); + const pubkey = await user.signer.getPublicKey(); if (!/^\p{RGI_Emoji}$/v.test(emoji)) { return c.json({ error: 'Invalid emoji' }, 400); @@ -99,12 +100,11 @@ route.delete('/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), async (c) * Get an object of emoji to account mappings with accounts that reacted to the post. * https://docs.pleroma.social/backend/development/API/pleroma_api/#get-apiv1pleromastatusesidreactions */ -route.get('/:id{[0-9a-f]{64}}/reactions', userMiddleware({ required: false }), async (c) => { +route.get('/:id{[0-9a-f]{64}}/reactions/:emoji?', userMiddleware({ required: false }), async (c) => { + const { id, emoji } = c.req.param(); const { relay, user } = c.var; - const id = c.req.param('id'); const pubkey = await user?.signer.getPublicKey(); - const emoji = c.req.param('emoji') as string | undefined; if (typeof emoji === 'string' && !/^\p{RGI_Emoji}$/v.test(emoji)) { return c.json({ error: 'Invalid emoji' }, 400); @@ -139,6 +139,7 @@ route.get('/:id{[0-9a-f]{64}}/reactions', userMiddleware({ required: false }), a return c.json(results); }); +/** Determine if the input is a native or custom emoji, returning a structured object or throwing an error. */ function parseEmojiParam(input: string): { type: 'native' | 'custom'; value: string } { if (/^\p{RGI_Emoji}$/v.test(input)) { return { type: 'native', value: input }; From 1dd50e2e63452121a02bd00f7e5ca74ee0a4b234 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Mar 2025 14:38:19 -0500 Subject: [PATCH 10/22] Add tests for pleromaStatusesRoute --- .../ditto/routes/pleromaStatusesRoute.test.ts | 56 +++++++++++++++++++ packages/ditto/routes/pleromaStatusesRoute.ts | 9 +-- packages/mastoapi/test/TestApp.ts | 12 +++- 3 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 packages/ditto/routes/pleromaStatusesRoute.test.ts diff --git a/packages/ditto/routes/pleromaStatusesRoute.test.ts b/packages/ditto/routes/pleromaStatusesRoute.test.ts new file mode 100644 index 00000000..09217e88 --- /dev/null +++ b/packages/ditto/routes/pleromaStatusesRoute.test.ts @@ -0,0 +1,56 @@ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; +import { TestApp } from '@ditto/mastoapi/test'; +import { genEvent, MockRelay } from '@nostrify/nostrify/test'; +import { assertEquals } from '@std/assert'; + +import { DittoPgStore } from '@/storages/DittoPgStore.ts'; +import { DittoRelayStore } from '@/storages/DittoRelayStore.ts'; + +import route from './pleromaStatusesRoute.ts'; + +Deno.test('Emoji reactions', async (t) => { + await using test = createTestApp(); + const { relay } = test.var; + + test.user(); + + const note = genEvent({ kind: 1 }); + await relay.event(note); + + await t.step('PUT /:id/reactions/:emoji', async () => { + const response = await test.api.put(`/${note.id}/reactions/🚀`); + const json = await response.json(); + + assertEquals(response.status, 200); + assertEquals(json.pleroma.emoji_reactions, [{ name: '🚀', me: true, count: 1 }]); + }); + + await t.step('GET /:id/reactions', async () => { + const response = await test.api.get(`/${note.id}/reactions`); + const json = await response.json(); + const [{ accounts }] = json; + + assertEquals(response.status, 200); + assertEquals(json, [{ name: '🚀', me: true, count: 1, accounts }]); + }); + + await t.step('DELETE /:id/reactions/:emoji', async () => { + const response = await test.api.delete(`/${note.id}/reactions/🚀`); + const json = await response.json(); + + assertEquals(response.status, 200); + assertEquals(json.pleroma.emoji_reactions, []); + }); +}); + +// TODO: modify `TestApp` itself to avoid this boilerplate. +function createTestApp(): TestApp { + const conf = new DittoConf(Deno.env); + const db = new DittoPolyPg(conf.databaseUrl); + const pool = new MockRelay(); + const store = new DittoPgStore({ conf, db }); + const relay = new DittoRelayStore({ conf, db, pool, relay: store }); + + return new TestApp(route, { conf, db, relay }); +} diff --git a/packages/ditto/routes/pleromaStatusesRoute.ts b/packages/ditto/routes/pleromaStatusesRoute.ts index 0daa497c..d3a5d6a4 100644 --- a/packages/ditto/routes/pleromaStatusesRoute.ts +++ b/packages/ditto/routes/pleromaStatusesRoute.ts @@ -60,7 +60,7 @@ route.put('/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), async (c) => */ route.delete('/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), async (c) => { const { id, emoji } = c.req.param(); - const { relay, user } = c.var; + const { relay, user, signal } = c.var; const pubkey = await user.signer.getPublicKey(); @@ -68,9 +68,7 @@ route.delete('/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), async (c) return c.json({ error: 'Invalid emoji' }, 400); } - const [event] = await relay.query([ - { kinds: [1, 20], ids: [id], limit: 1 }, - ]); + const [event] = await relay.query([{ ids: [id] }], { signal }); if (!event) { return c.json({ error: 'Status not found' }, 404); @@ -91,8 +89,7 @@ route.delete('/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), async (c) tags, }, c); - const status = renderStatus(relay, event, { viewerPubkey: pubkey }); - + const status = await renderStatus(relay, event, { viewerPubkey: pubkey }); return c.json(status); }); diff --git a/packages/mastoapi/test/TestApp.ts b/packages/mastoapi/test/TestApp.ts index a12f48a4..14b8e802 100644 --- a/packages/mastoapi/test/TestApp.ts +++ b/packages/mastoapi/test/TestApp.ts @@ -110,9 +110,19 @@ export class TestApp extends DittoApp implements AsyncDisposable { return await this.request(path, { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), + body: body ? JSON.stringify(body) : undefined, }); }, + put: async (path: string, body?: unknown): Promise => { + return await this.request(path, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: body ? JSON.stringify(body) : undefined, + }); + }, + delete: async (path: string): Promise => { + return await this.request(path, { method: 'DELETE' }); + }, }; async [Symbol.asyncDispose](): Promise { From 755ed884d44cf571113cf33650984d6797de5a3c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Mar 2025 15:01:07 -0500 Subject: [PATCH 11/22] Pleroma API supports custom emojis with or without colons --- packages/ditto/routes/pleromaStatusesRoute.ts | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/packages/ditto/routes/pleromaStatusesRoute.ts b/packages/ditto/routes/pleromaStatusesRoute.ts index d3a5d6a4..dc7391b7 100644 --- a/packages/ditto/routes/pleromaStatusesRoute.ts +++ b/packages/ditto/routes/pleromaStatusesRoute.ts @@ -17,11 +17,10 @@ const route = new DittoRoute(); * https://docs.pleroma.social/backend/development/API/pleroma_api/#put-apiv1pleromastatusesidreactionsemoji */ route.put('/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), async (c) => { - const params = c.req.param(); - const { relay, user, conf, signal } = c.var; - const { type, value } = parseEmojiParam(params.emoji); + const params = c.req.param(); + const result = parseEmojiParam(params.emoji); const pubkey = await user.signer.getPublicKey(); const [event] = await relay.query([{ ids: [params.id] }], { signal }); @@ -34,18 +33,18 @@ route.put('/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), async (c) => ['p', event.pubkey, conf.relay], ]; - if (type === 'custom') { + if (result.type === 'custom') { const emojis = await getCustomEmojis(pubkey, c.var); - const emoji = emojis.get(value); + const emoji = emojis.get(result.shortcode); if (!emoji) { return c.json({ error: 'Custom emoji not found' }, 404); } - tags.push(['emoji', value, emoji.url.href]); + tags.push(['emoji', result.shortcode, emoji.url.href]); } - const content = type === 'custom' ? `:${value}:` : value; + const content = result.type === 'custom' ? `:${result.shortcode}:` : result.emoji; await createEvent({ kind: 7, content, tags }, c); await hydrateEvents({ ...c.var, events: [event] }); @@ -137,13 +136,16 @@ route.get('/:id{[0-9a-f]{64}}/reactions/:emoji?', userMiddleware({ required: fal }); /** Determine if the input is a native or custom emoji, returning a structured object or throwing an error. */ -function parseEmojiParam(input: string): { type: 'native' | 'custom'; value: string } { +function parseEmojiParam(input: string): { type: 'native'; emoji: string } | { type: 'custom'; shortcode: string } { if (/^\p{RGI_Emoji}$/v.test(input)) { - return { type: 'native', value: input }; + return { type: 'native', emoji: input }; } - if (/^\w+$/.test(input)) { - return { type: 'custom', value: input }; + const match = input.match(/^:?(\w+):?$/); // Pleroma API supports with or without colons. + + if (match) { + const [, shortcode] = match; + return { type: 'custom', shortcode }; } throw new HTTPException(400, { message: 'Invalid emoji' }); From c40c6e8b300c8af147aa5bfed518397fff4506cb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Mar 2025 15:29:00 -0500 Subject: [PATCH 12/22] Support custom emoji reactions in event_stats --- packages/ditto/utils/custom-emoji.test.ts | 10 ++++ packages/ditto/utils/custom-emoji.ts | 21 +++++++ packages/ditto/utils/stats.test.ts | 14 ++++- packages/ditto/utils/stats.ts | 67 ++++++++++++++++------- 4 files changed, 88 insertions(+), 24 deletions(-) create mode 100644 packages/ditto/utils/custom-emoji.test.ts diff --git a/packages/ditto/utils/custom-emoji.test.ts b/packages/ditto/utils/custom-emoji.test.ts new file mode 100644 index 00000000..e9093e4e --- /dev/null +++ b/packages/ditto/utils/custom-emoji.test.ts @@ -0,0 +1,10 @@ +import { assertEquals } from '@std/assert'; + +import { parseEmojiInput } from './custom-emoji.ts'; + +Deno.test('parseEmojiInput', () => { + assertEquals(parseEmojiInput('+'), { type: 'basic', value: '+' }); + assertEquals(parseEmojiInput('🚀'), { type: 'native', native: '🚀' }); + assertEquals(parseEmojiInput(':ditto:'), { type: 'custom', shortcode: 'ditto' }); + assertEquals(parseEmojiInput('x'), undefined); +}); diff --git a/packages/ditto/utils/custom-emoji.ts b/packages/ditto/utils/custom-emoji.ts index a7a83a31..a95bacdb 100644 --- a/packages/ditto/utils/custom-emoji.ts +++ b/packages/ditto/utils/custom-emoji.ts @@ -70,3 +70,24 @@ export async function getCustomEmojis( return emojis; } + +/** Determine if the input is a native or custom emoji, returning a structured object or throwing an error. */ +export function parseEmojiInput(input: string): + | { type: 'basic'; value: '+' | '-' } + | { type: 'native'; native: string } + | { type: 'custom'; shortcode: string } + | undefined { + if (input === '+' || input === '-') { + return { type: 'basic', value: input }; + } + + if (/^\p{RGI_Emoji}$/v.test(input)) { + return { type: 'native', native: input }; + } + + const match = input.match(/^:(\w+):$/); + if (match) { + const [, shortcode] = match; + return { type: 'custom', shortcode }; + } +} diff --git a/packages/ditto/utils/stats.test.ts b/packages/ditto/utils/stats.test.ts index 2ebcab94..0fe7b0ae 100644 --- a/packages/ditto/utils/stats.test.ts +++ b/packages/ditto/utils/stats.test.ts @@ -141,16 +141,24 @@ Deno.test('updateStats with kind 7 increments reactions count', async () => { const { kysely, relay } = test; const note = genEvent({ kind: 1 }); - await updateStats({ ...test, event: note }); await relay.event(note); await updateStats({ ...test, event: genEvent({ kind: 7, content: '+', tags: [['e', note.id]] }) }); await updateStats({ ...test, event: genEvent({ kind: 7, content: '😂', tags: [['e', note.id]] }) }); + await updateStats({ + ...test, + event: genEvent({ + kind: 7, + content: ':ditto:', + tags: [['e', note.id], ['emoji', 'ditto', 'https://ditto.pub/favicon.ico']], + }), + }); + const stats = await getEventStats(kysely, note.id); - assertEquals(stats!.reactions, JSON.stringify({ '+': 1, '😂': 1 })); - assertEquals(stats!.reactions_count, 2); + assertEquals(stats!.reactions, JSON.stringify({ '+': 1, '😂': 1, 'ditto:https://ditto.pub/favicon.ico': 1 })); + assertEquals(stats!.reactions_count, 3); }); Deno.test('updateStats with kind 5 decrements reactions count', async () => { diff --git a/packages/ditto/utils/stats.ts b/packages/ditto/utils/stats.ts index 922d5dca..576c6e01 100644 --- a/packages/ditto/utils/stats.ts +++ b/packages/ditto/utils/stats.ts @@ -7,6 +7,7 @@ import { z } from 'zod'; import { findQuoteTag, findReplyTag, getTagSet } from '@/utils/tags.ts'; import type { DittoConf } from '@ditto/conf'; +import { parseEmojiInput } from '@/utils/custom-emoji.ts'; interface UpdateStatsOpts { conf: DittoConf; @@ -154,31 +155,55 @@ async function handleEvent7(opts: UpdateStatsOpts): Promise { const { kysely, event, x = 1 } = opts; const id = event.tags.findLast(([name]) => name === 'e')?.[1]; - const emoji = event.content; + const result = parseEmojiInput(event.content); - if (id && emoji && (['+', '-'].includes(emoji) || /^\p{RGI_Emoji}$/v.test(emoji))) { - await updateEventStats(kysely, id, ({ reactions }) => { - const data: Record = JSON.parse(reactions); + if (!id || !result) return; - // Increment or decrement the emoji count. - data[emoji] = (data[emoji] ?? 0) + x; + let url: URL | undefined; - // Remove reactions with a count of 0 or less. - for (const key of Object.keys(data)) { - if (data[key] < 1) { - delete data[key]; - } - } - - // Total reactions count. - const count = Object.values(data).reduce((result, value) => result + value, 0); - - return { - reactions: JSON.stringify(data), - reactions_count: count, - }; - }); + if (result.type === 'custom') { + const tag = event.tags.find(([name, value]) => name === 'emoji' && value === result.shortcode); + try { + url = new URL(tag![2]); + } catch { + return; + } } + + let key: string; + switch (result.type) { + case 'basic': + key = result.value; + break; + case 'native': + key = result.native; + break; + case 'custom': + key = `${result.shortcode}:${url}`; + break; + } + + await updateEventStats(kysely, id, ({ reactions }) => { + const data: Record = JSON.parse(reactions); + + // Increment or decrement the emoji count. + data[key] = (data[key] ?? 0) + x; + + // Remove reactions with a count of 0 or less. + for (const key of Object.keys(data)) { + if (data[key] < 1) { + delete data[key]; + } + } + + // Total reactions count. + const count = Object.values(data).reduce((result, value) => result + value, 0); + + return { + reactions: JSON.stringify(data), + reactions_count: count, + }; + }); } /** Update stats for kind 9735 event. */ From 753413f071ba3347a98ce98a898bd74e57724e91 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Mar 2025 16:32:11 -0500 Subject: [PATCH 13/22] Support custom emoji reactions --- .../ditto/routes/pleromaStatusesRoute.test.ts | 55 ++++++- packages/ditto/routes/pleromaStatusesRoute.ts | 142 +++++++++++++----- packages/ditto/views/mastodon/statuses.ts | 29 +++- packages/mastoapi/test/TestApp.ts | 5 +- 4 files changed, 185 insertions(+), 46 deletions(-) diff --git a/packages/ditto/routes/pleromaStatusesRoute.test.ts b/packages/ditto/routes/pleromaStatusesRoute.test.ts index 09217e88..138edf38 100644 --- a/packages/ditto/routes/pleromaStatusesRoute.test.ts +++ b/packages/ditto/routes/pleromaStatusesRoute.test.ts @@ -9,16 +9,23 @@ import { DittoRelayStore } from '@/storages/DittoRelayStore.ts'; import route from './pleromaStatusesRoute.ts'; +import type { MastodonStatus } from '@ditto/mastoapi/types'; + Deno.test('Emoji reactions', async (t) => { await using test = createTestApp(); const { relay } = test.var; - test.user(); + const mario = test.createUser(); + const luigi = test.createUser(); const note = genEvent({ kind: 1 }); await relay.event(note); + await relay.event(genEvent({ kind: 10030, tags: [['emoji', 'ditto', 'https://ditto.pub/favicon.ico']] }, luigi.sk)); + await t.step('PUT /:id/reactions/:emoji', async () => { + test.user(mario); + const response = await test.api.put(`/${note.id}/reactions/🚀`); const json = await response.json(); @@ -26,19 +33,61 @@ Deno.test('Emoji reactions', async (t) => { assertEquals(json.pleroma.emoji_reactions, [{ name: '🚀', me: true, count: 1 }]); }); + await t.step('PUT /:id/reactions/:emoji (custom emoji)', async () => { + test.user(luigi); + + const response = await test.api.put(`/${note.id}/reactions/:ditto:`); + const json: MastodonStatus = await response.json(); + + assertEquals( + json.pleroma.emoji_reactions.sort((a, b) => a.name.localeCompare(b.name)), + [ + { name: '🚀', me: false, count: 1 }, + { name: 'ditto', me: true, count: 1, url: 'https://ditto.pub/favicon.ico' }, + ], + ); + }); + await t.step('GET /:id/reactions', async () => { + test.user(mario); + const response = await test.api.get(`/${note.id}/reactions`); const json = await response.json(); - const [{ accounts }] = json; + + (json as MastodonStatus['pleroma']['emoji_reactions']).sort((a, b) => a.name.localeCompare(b.name)); + + const [ + { accounts: [marioAccount] }, + { accounts: [luigiAccount] }, + ] = json; assertEquals(response.status, 200); - assertEquals(json, [{ name: '🚀', me: true, count: 1, accounts }]); + + assertEquals(json, [ + { name: '🚀', me: true, count: 1, accounts: [marioAccount] }, + { name: 'ditto', me: false, count: 1, accounts: [luigiAccount], url: 'https://ditto.pub/favicon.ico' }, + ]); }); await t.step('DELETE /:id/reactions/:emoji', async () => { + test.user(mario); + const response = await test.api.delete(`/${note.id}/reactions/🚀`); const json = await response.json(); + assertEquals(response.status, 200); + + assertEquals(json.pleroma.emoji_reactions, [ + { name: 'ditto', me: false, count: 1, url: 'https://ditto.pub/favicon.ico' }, + ]); + }); + + await t.step('DELETE /:id/reactions/:emoji (custom emoji)', async () => { + test.user(luigi); + + const response = await test.api.delete(`/${note.id}/reactions/:ditto:`); + const json = await response.json(); + assertEquals(response.status, 200); assertEquals(json.pleroma.emoji_reactions, []); }); diff --git a/packages/ditto/routes/pleromaStatusesRoute.ts b/packages/ditto/routes/pleromaStatusesRoute.ts index dc7391b7..1df4d3fd 100644 --- a/packages/ditto/routes/pleromaStatusesRoute.ts +++ b/packages/ditto/routes/pleromaStatusesRoute.ts @@ -8,7 +8,7 @@ import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; import { HTTPException } from '@hono/hono/http-exception'; -import { getCustomEmojis } from '@/utils/custom-emoji.ts'; +import { getCustomEmojis, parseEmojiInput } from '@/utils/custom-emoji.ts'; const route = new DittoRoute(); @@ -44,7 +44,19 @@ route.put('/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), async (c) => tags.push(['emoji', result.shortcode, emoji.url.href]); } - const content = result.type === 'custom' ? `:${result.shortcode}:` : result.emoji; + let content: string; + + switch (result.type) { + case 'basic': + content = result.value; + break; + case 'native': + content = result.native; + break; + case 'custom': + content = `:${result.shortcode}:`; + break; + } await createEvent({ kind: 7, content, tags }, c); await hydrateEvents({ ...c.var, events: [event] }); @@ -58,36 +70,40 @@ route.put('/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), async (c) => * https://docs.pleroma.social/backend/development/API/pleroma_api/#delete-apiv1pleromastatusesidreactionsemoji */ route.delete('/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), async (c) => { - const { id, emoji } = c.req.param(); const { relay, user, signal } = c.var; + const params = c.req.param(); const pubkey = await user.signer.getPublicKey(); - if (!/^\p{RGI_Emoji}$/v.test(emoji)) { - return c.json({ error: 'Invalid emoji' }, 400); - } - - const [event] = await relay.query([{ ids: [id] }], { signal }); + const [event] = await relay.query([{ ids: [params.id] }], { signal }); if (!event) { return c.json({ error: 'Status not found' }, 404); } const events = await relay.query([ - { kinds: [7], authors: [pubkey], '#e': [id] }, - ]); + { kinds: [7], authors: [pubkey], '#e': [params.id] }, + ], { signal }); - const tags = events - .filter((event) => event.content === emoji) - .map((event) => ['e', event.id]); + const e = new Set(); + + for (const { id, content } of events) { + if (content === params.emoji || content === `:${params.emoji}:`) { + e.add(id); + } + } + + if (!e.size) { + return c.json({ error: 'Reaction not found' }, 404); + } await createEvent({ kind: 5, - content: '', - created_at: Math.floor(Date.now() / 1000), - tags, + tags: [...e].map((id) => ['e', id]), }, c); + await hydrateEvents({ ...c.var, events: [event] }); + const status = await renderStatus(relay, event, { viewerPubkey: pubkey }); return c.json(status); }); @@ -97,30 +113,79 @@ route.delete('/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), async (c) * https://docs.pleroma.social/backend/development/API/pleroma_api/#get-apiv1pleromastatusesidreactions */ route.get('/:id{[0-9a-f]{64}}/reactions/:emoji?', userMiddleware({ required: false }), async (c) => { - const { id, emoji } = c.req.param(); const { relay, user } = c.var; + const params = c.req.param(); + const result = params.emoji ? parseEmojiParam(params.emoji) : undefined; const pubkey = await user?.signer.getPublicKey(); - if (typeof emoji === 'string' && !/^\p{RGI_Emoji}$/v.test(emoji)) { - return c.json({ error: 'Invalid emoji' }, 400); - } + const events = await relay.query([{ kinds: [7], '#e': [params.id], limit: 100 }]) + .then((events) => + events.filter((event) => { + if (!result) return true; - const events = await relay.query([{ kinds: [7], '#e': [id], limit: 100 }]) - .then((events) => events.filter(({ content }) => /^\p{RGI_Emoji}$/v.test(content))) - .then((events) => events.filter((event) => !emoji || event.content === emoji)) + switch (result.type) { + case 'basic': + return event.content === result.value; + case 'native': + return event.content === result.native; + case 'custom': + return event.content === `:${result.shortcode}:`; + } + }) + ) .then((events) => hydrateEvents({ ...c.var, events })); - /** Events grouped by emoji. */ - const byEmoji = events.reduce((acc, event) => { - const emoji = event.content; - acc[emoji] = acc[emoji] || []; - acc[emoji].push(event); + /** Events grouped by emoji key. */ + const byEmojiKey = events.reduce((acc, event) => { + const result = parseEmojiInput(event.content); + if (!result) return acc; + + let url: URL | undefined; + + if (result.type === 'custom') { + const tag = event.tags.find(([name, value]) => name === 'emoji' && value === result.shortcode); + try { + url = new URL(tag![2]); + } catch { + return acc; + } + } + + let key: string; + switch (result.type) { + case 'basic': + key = result.value; + break; + case 'native': + key = result.native; + break; + case 'custom': + key = `${result.shortcode}:${url}`; + break; + } + + acc[key] = acc[key] || []; + acc[key].push(event); + return acc; }, {} as Record); const results = await Promise.all( - Object.entries(byEmoji).map(async ([name, events]) => { + Object.entries(byEmojiKey).map(async ([key, events]) => { + let name: string = key; + let url: string | undefined; + + // Custom emojis: `:` + try { + const [shortcode, ...rest] = key.split(':'); + + url = new URL(rest.join(':')).toString(); + name = shortcode; + } catch { + // fallthrough + } + return { name, count: events.length, @@ -128,6 +193,7 @@ route.get('/:id{[0-9a-f]{64}}/reactions/:emoji?', userMiddleware({ required: fal accounts: await Promise.all( events.map((event) => event.author ? renderAccount(event.author) : accountFromPubkey(event.pubkey)), ), + url, }; }), ); @@ -136,19 +202,21 @@ route.get('/:id{[0-9a-f]{64}}/reactions/:emoji?', userMiddleware({ required: fal }); /** Determine if the input is a native or custom emoji, returning a structured object or throwing an error. */ -function parseEmojiParam(input: string): { type: 'native'; emoji: string } | { type: 'custom'; shortcode: string } { - if (/^\p{RGI_Emoji}$/v.test(input)) { - return { type: 'native', emoji: input }; +function parseEmojiParam(input: string): + | { type: 'basic'; value: '+' | '-' } + | { type: 'native'; native: string } + | { type: 'custom'; shortcode: string } { + if (/^\w+$/.test(input)) { + input = `:${input}:`; // Pleroma API supports the `emoji` param with or without colons. } - const match = input.match(/^:?(\w+):?$/); // Pleroma API supports with or without colons. + const result = parseEmojiInput(input); - if (match) { - const [, shortcode] = match; - return { type: 'custom', shortcode }; + if (!result) { + throw new HTTPException(400, { message: 'Invalid emoji' }); } - throw new HTTPException(400, { message: 'Invalid emoji' }); + return result; } export default route; diff --git a/packages/ditto/views/mastodon/statuses.ts b/packages/ditto/views/mastodon/statuses.ts index a2d4a740..ba2e8d86 100644 --- a/packages/ditto/views/mastodon/statuses.ts +++ b/packages/ditto/views/mastodon/statuses.ts @@ -91,11 +91,32 @@ async function renderStatus( const subject = event.tags.find(([name]) => name === 'subject'); /** Pleroma emoji reactions object. */ - const reactions = Object.entries(event.event_stats?.reactions ?? {}).reduce((acc, [emoji, count]) => { - if (['+', '-'].includes(emoji)) return acc; - acc.push({ name: emoji, count, me: reactionEvent?.content === emoji }); + const reactions = Object.entries(event.event_stats?.reactions ?? {}).reduce((acc, [key, count]) => { + if (['+', '-'].includes(key)) return acc; // skip basic reactions (treat as likes/dislikes in Mastodon API) + + // Custom emoji reactions: `:` + try { + const [shortcode, ...rest] = key.split(':'); + const url = new URL(rest.join(':')); + const tag = reactionEvent?.tags.find((t) => t[0] === 'emoji' && t[1] === shortcode && t[2] === url.href); + + acc.push({ + name: shortcode, + me: reactionEvent?.content === `:${shortcode}:` && !!tag, + url: url.href, + count, + }); + + return acc; + } catch { + // fallthrough + } + + // Native emojis: `🚀` + acc.push({ name: key, count, me: reactionEvent?.content === key }); + return acc; - }, [] as { name: string; count: number; me: boolean }[]); + }, [] as { name: string; count: number; me: boolean; url?: string }[]); const expiresAt = new Date(Number(event.tags.find(([name]) => name === 'expiration')?.[1]) * 1000); diff --git a/packages/mastoapi/test/TestApp.ts b/packages/mastoapi/test/TestApp.ts index 14b8e802..04e1589e 100644 --- a/packages/mastoapi/test/TestApp.ts +++ b/packages/mastoapi/test/TestApp.ts @@ -95,10 +95,11 @@ export class TestApp extends DittoApp implements AsyncDisposable { return user; } - createUser(sk?: Uint8Array): User { + createUser(sk: Uint8Array = generateSecretKey()): User & { sk: Uint8Array } { return { relay: this.opts.relay, - signer: new NSecSigner(sk ?? generateSecretKey()), + signer: new NSecSigner(sk), + sk, }; } From be922bf07f9560fef1dec89f0942087947522882 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Mar 2025 16:37:00 -0500 Subject: [PATCH 14/22] Enable pleroma_custom_emoji_reactions feature in instance endpoint --- packages/ditto/controllers/api/instance.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ditto/controllers/api/instance.ts b/packages/ditto/controllers/api/instance.ts index a887fe4d..0e07f385 100644 --- a/packages/ditto/controllers/api/instance.ts +++ b/packages/ditto/controllers/api/instance.ts @@ -9,6 +9,7 @@ const features = [ 'exposable_reactions', 'mastodon_api', 'mastodon_api_streaming', + 'pleroma_custom_emoji_reactions', 'pleroma_emoji_reactions', 'quote_posting', 'v2_suggestions', From 66979c7c11669426d871f1c39b5b28896ab4e0df Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Mar 2025 16:40:58 -0500 Subject: [PATCH 15/22] Disallow +/- emojis in Pleroma API --- packages/ditto/routes/pleromaStatusesRoute.ts | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/packages/ditto/routes/pleromaStatusesRoute.ts b/packages/ditto/routes/pleromaStatusesRoute.ts index 1df4d3fd..7aa08a68 100644 --- a/packages/ditto/routes/pleromaStatusesRoute.ts +++ b/packages/ditto/routes/pleromaStatusesRoute.ts @@ -47,9 +47,6 @@ route.put('/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), async (c) => let content: string; switch (result.type) { - case 'basic': - content = result.value; - break; case 'native': content = result.native; break; @@ -125,8 +122,6 @@ route.get('/:id{[0-9a-f]{64}}/reactions/:emoji?', userMiddleware({ required: fal if (!result) return true; switch (result.type) { - case 'basic': - return event.content === result.value; case 'native': return event.content === result.native; case 'custom': @@ -139,7 +134,10 @@ route.get('/:id{[0-9a-f]{64}}/reactions/:emoji?', userMiddleware({ required: fal /** Events grouped by emoji key. */ const byEmojiKey = events.reduce((acc, event) => { const result = parseEmojiInput(event.content); - if (!result) return acc; + + if (!result || result.type === 'basic') { + return acc; + } let url: URL | undefined; @@ -154,9 +152,6 @@ route.get('/:id{[0-9a-f]{64}}/reactions/:emoji?', userMiddleware({ required: fal let key: string; switch (result.type) { - case 'basic': - key = result.value; - break; case 'native': key = result.native; break; @@ -203,7 +198,6 @@ route.get('/:id{[0-9a-f]{64}}/reactions/:emoji?', userMiddleware({ required: fal /** Determine if the input is a native or custom emoji, returning a structured object or throwing an error. */ function parseEmojiParam(input: string): - | { type: 'basic'; value: '+' | '-' } | { type: 'native'; native: string } | { type: 'custom'; shortcode: string } { if (/^\w+$/.test(input)) { @@ -212,7 +206,7 @@ function parseEmojiParam(input: string): const result = parseEmojiInput(input); - if (!result) { + if (!result || result.type === 'basic') { throw new HTTPException(400, { message: 'Invalid emoji' }); } From 006a96d0eb877e0148c5a932cbca3a0b881e5026 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Mar 2025 16:50:26 -0500 Subject: [PATCH 16/22] Disable notify in tests --- packages/ditto/routes/pleromaStatusesRoute.test.ts | 2 +- packages/ditto/test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ditto/routes/pleromaStatusesRoute.test.ts b/packages/ditto/routes/pleromaStatusesRoute.test.ts index 138edf38..15c97341 100644 --- a/packages/ditto/routes/pleromaStatusesRoute.test.ts +++ b/packages/ditto/routes/pleromaStatusesRoute.test.ts @@ -98,7 +98,7 @@ function createTestApp(): TestApp { const conf = new DittoConf(Deno.env); const db = new DittoPolyPg(conf.databaseUrl); const pool = new MockRelay(); - const store = new DittoPgStore({ conf, db }); + const store = new DittoPgStore({ conf, db, notify: false }); const relay = new DittoRelayStore({ conf, db, pool, relay: store }); return new TestApp(route, { conf, db, relay }); diff --git a/packages/ditto/test.ts b/packages/ditto/test.ts index 31b31e37..0308c870 100644 --- a/packages/ditto/test.ts +++ b/packages/ditto/test.ts @@ -22,7 +22,7 @@ export async function createTestDB(opts?: { pure?: boolean }) { conf, timeout: conf.db.timeouts.default, pure: opts?.pure ?? false, - notify: true, + notify: false, }); return { From 36ffd4283a01f658cdd5739dfb02de9c725d7241 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Mar 2025 17:03:20 -0500 Subject: [PATCH 17/22] Fix DittoPgStore req test --- packages/ditto/storages/DittoPgStore.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ditto/storages/DittoPgStore.test.ts b/packages/ditto/storages/DittoPgStore.test.ts index d243d0e4..dedde9d6 100644 --- a/packages/ditto/storages/DittoPgStore.test.ts +++ b/packages/ditto/storages/DittoPgStore.test.ts @@ -16,7 +16,7 @@ Deno.test('req streaming', async () => { const controller = new AbortController(); const promise = (async () => { - for await (const msg of relay.req([{ since: 0 }], { signal: controller.signal })) { + for await (const msg of relay.req([{ limit: 0 }], { signal: controller.signal })) { msgs.push(msg); } })(); From 88ef8087a584968e9a893ad2937d61d483268c39 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Mar 2025 17:15:37 -0500 Subject: [PATCH 18/22] Let paginationMiddleware be configurable, add pagination to reactions handler --- packages/ditto/controllers/api/statuses.ts | 2 +- packages/ditto/controllers/api/suggestions.ts | 4 +- packages/ditto/controllers/api/trends.ts | 2 +- packages/ditto/routes/pleromaStatusesRoute.ts | 155 +++++++++--------- packages/ditto/views.ts | 2 +- .../middleware/paginationMiddleware.ts | 17 +- packages/mastoapi/pagination/schema.test.ts | 2 +- packages/mastoapi/pagination/schema.ts | 35 ++-- 8 files changed, 122 insertions(+), 97 deletions(-) diff --git a/packages/ditto/controllers/api/statuses.ts b/packages/ditto/controllers/api/statuses.ts index 116ad2de..e80441b6 100644 --- a/packages/ditto/controllers/api/statuses.ts +++ b/packages/ditto/controllers/api/statuses.ts @@ -633,7 +633,7 @@ const zappedByController: AppController = async (c) => { const { db, relay } = c.var; const id = c.req.param('id'); - const { offset, limit } = paginationSchema.parse(c.req.query()); + const { offset, limit } = paginationSchema().parse(c.req.query()); const zaps = await db.kysely.selectFrom('event_zaps') .selectAll() diff --git a/packages/ditto/controllers/api/suggestions.ts b/packages/ditto/controllers/api/suggestions.ts index cb6a8206..e288ad7b 100644 --- a/packages/ditto/controllers/api/suggestions.ts +++ b/packages/ditto/controllers/api/suggestions.ts @@ -9,7 +9,7 @@ import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; export const suggestionsV1Controller: AppController = async (c) => { const { signal } = c.var; - const { offset, limit } = paginationSchema.parse(c.req.query()); + const { offset, limit } = paginationSchema().parse(c.req.query()); const suggestions = await renderV2Suggestions(c, { offset, limit }, signal); const accounts = suggestions.map(({ account }) => account); return paginatedList(c, { offset, limit }, accounts); @@ -17,7 +17,7 @@ export const suggestionsV1Controller: AppController = async (c) => { export const suggestionsV2Controller: AppController = async (c) => { const { signal } = c.var; - const { offset, limit } = paginationSchema.parse(c.req.query()); + const { offset, limit } = paginationSchema().parse(c.req.query()); const suggestions = await renderV2Suggestions(c, { offset, limit }, signal); return paginatedList(c, { offset, limit }, suggestions); }; diff --git a/packages/ditto/controllers/api/trends.ts b/packages/ditto/controllers/api/trends.ts index 4e5810ed..ca59bd34 100644 --- a/packages/ditto/controllers/api/trends.ts +++ b/packages/ditto/controllers/api/trends.ts @@ -124,7 +124,7 @@ async function getTrendingLinks(conf: DittoConf, relay: NStore): Promise { const { conf, relay } = c.var; - const { limit, offset, until } = paginationSchema.parse(c.req.query()); + const { limit, offset, until } = paginationSchema().parse(c.req.query()); const [label] = await relay.query([{ kinds: [1985], diff --git a/packages/ditto/routes/pleromaStatusesRoute.ts b/packages/ditto/routes/pleromaStatusesRoute.ts index 7aa08a68..5d7fad38 100644 --- a/packages/ditto/routes/pleromaStatusesRoute.ts +++ b/packages/ditto/routes/pleromaStatusesRoute.ts @@ -1,4 +1,4 @@ -import { userMiddleware } from '@ditto/mastoapi/middleware'; +import { paginationMiddleware, userMiddleware } from '@ditto/mastoapi/middleware'; import { DittoRoute } from '@ditto/mastoapi/router'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; @@ -109,92 +109,97 @@ route.delete('/:id{[0-9a-f]{64}}/reactions/:emoji', userMiddleware(), async (c) * Get an object of emoji to account mappings with accounts that reacted to the post. * https://docs.pleroma.social/backend/development/API/pleroma_api/#get-apiv1pleromastatusesidreactions */ -route.get('/:id{[0-9a-f]{64}}/reactions/:emoji?', userMiddleware({ required: false }), async (c) => { - const { relay, user } = c.var; +route.get( + '/:id{[0-9a-f]{64}}/reactions/:emoji?', + paginationMiddleware({ limit: 100 }), + userMiddleware({ required: false }), + async (c) => { + const { relay, user, pagination, paginate } = c.var; - const params = c.req.param(); - const result = params.emoji ? parseEmojiParam(params.emoji) : undefined; - const pubkey = await user?.signer.getPublicKey(); + const params = c.req.param(); + const result = params.emoji ? parseEmojiParam(params.emoji) : undefined; + const pubkey = await user?.signer.getPublicKey(); - const events = await relay.query([{ kinds: [7], '#e': [params.id], limit: 100 }]) - .then((events) => - events.filter((event) => { - if (!result) return true; + const events = await relay.query([{ kinds: [7], '#e': [params.id], ...pagination }]) + .then((events) => + events.filter((event) => { + if (!result) return true; - switch (result.type) { - case 'native': - return event.content === result.native; - case 'custom': - return event.content === `:${result.shortcode}:`; - } - }) - ) - .then((events) => hydrateEvents({ ...c.var, events })); + switch (result.type) { + case 'native': + return event.content === result.native; + case 'custom': + return event.content === `:${result.shortcode}:`; + } + }) + ) + .then((events) => hydrateEvents({ ...c.var, events })); - /** Events grouped by emoji key. */ - const byEmojiKey = events.reduce((acc, event) => { - const result = parseEmojiInput(event.content); + /** Events grouped by emoji key. */ + const byEmojiKey = events.reduce((acc, event) => { + const result = parseEmojiInput(event.content); - if (!result || result.type === 'basic') { - return acc; - } - - let url: URL | undefined; - - if (result.type === 'custom') { - const tag = event.tags.find(([name, value]) => name === 'emoji' && value === result.shortcode); - try { - url = new URL(tag![2]); - } catch { + if (!result || result.type === 'basic') { return acc; } - } - let key: string; - switch (result.type) { - case 'native': - key = result.native; - break; - case 'custom': - key = `${result.shortcode}:${url}`; - break; - } + let url: URL | undefined; - acc[key] = acc[key] || []; - acc[key].push(event); - - return acc; - }, {} as Record); - - const results = await Promise.all( - Object.entries(byEmojiKey).map(async ([key, events]) => { - let name: string = key; - let url: string | undefined; - - // Custom emojis: `:` - try { - const [shortcode, ...rest] = key.split(':'); - - url = new URL(rest.join(':')).toString(); - name = shortcode; - } catch { - // fallthrough + if (result.type === 'custom') { + const tag = event.tags.find(([name, value]) => name === 'emoji' && value === result.shortcode); + try { + url = new URL(tag![2]); + } catch { + return acc; + } } - return { - name, - count: events.length, - me: pubkey && events.some((event) => event.pubkey === pubkey), - accounts: await Promise.all( - events.map((event) => event.author ? renderAccount(event.author) : accountFromPubkey(event.pubkey)), - ), - url, - }; - }), - ); + let key: string; + switch (result.type) { + case 'native': + key = result.native; + break; + case 'custom': + key = `${result.shortcode}:${url}`; + break; + } - return c.json(results); -}); + acc[key] = acc[key] || []; + acc[key].push(event); + + return acc; + }, {} as Record); + + const results = await Promise.all( + Object.entries(byEmojiKey).map(async ([key, events]) => { + let name: string = key; + let url: string | undefined; + + // Custom emojis: `:` + try { + const [shortcode, ...rest] = key.split(':'); + + url = new URL(rest.join(':')).toString(); + name = shortcode; + } catch { + // fallthrough + } + + return { + name, + count: events.length, + me: pubkey && events.some((event) => event.pubkey === pubkey), + accounts: await Promise.all( + events.map((event) => event.author ? renderAccount(event.author) : accountFromPubkey(event.pubkey)), + ), + url, + }; + }), + ); + + return paginate(events, results); + }, +); /** Determine if the input is a native or custom emoji, returning a structured object or throwing an error. */ function parseEmojiParam(input: string): diff --git a/packages/ditto/views.ts b/packages/ditto/views.ts index 79379e6c..030516db 100644 --- a/packages/ditto/views.ts +++ b/packages/ditto/views.ts @@ -41,7 +41,7 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], opts?: } async function renderAccounts(c: AppContext, pubkeys: string[]) { - const { offset, limit } = paginationSchema.parse(c.req.query()); + const { offset, limit } = paginationSchema().parse(c.req.query()); const authors = pubkeys.reverse().slice(offset, offset + limit); const { relay, signal } = c.var; diff --git a/packages/mastoapi/middleware/paginationMiddleware.ts b/packages/mastoapi/middleware/paginationMiddleware.ts index 28a7f1a1..01b222f3 100644 --- a/packages/mastoapi/middleware/paginationMiddleware.ts +++ b/packages/mastoapi/middleware/paginationMiddleware.ts @@ -1,5 +1,5 @@ import { paginated, paginatedList } from '../pagination/paginate.ts'; -import { paginationSchema } from '../pagination/schema.ts'; +import { paginationSchema, type PaginationSchemaOpts } from '../pagination/schema.ts'; import type { DittoMiddleware } from '@ditto/mastoapi/router'; import type { NostrEvent } from '@nostrify/nostrify'; @@ -19,19 +19,26 @@ type HeaderRecord = Record; type PaginateFn = (events: NostrEvent[], body: object | unknown[], headers?: HeaderRecord) => Response; type ListPaginateFn = (params: ListPagination, body: object | unknown[], headers?: HeaderRecord) => Response; +interface PaginationMiddlewareOpts extends PaginationSchemaOpts { + type?: string; +} + /** Fixes compatibility with Mastodon apps by that don't use `Link` headers. */ // @ts-ignore Types are right. export function paginationMiddleware(): DittoMiddleware<{ pagination: Pagination; paginate: PaginateFn }>; export function paginationMiddleware( - type: 'list', + opts: PaginationMiddlewareOpts & { type: 'list' }, ): DittoMiddleware<{ pagination: ListPagination; paginate: ListPaginateFn }>; export function paginationMiddleware( - type?: string, + opts?: PaginationMiddlewareOpts, +): DittoMiddleware<{ pagination: Pagination; paginate: PaginateFn }>; +export function paginationMiddleware( + opts: PaginationMiddlewareOpts = {}, ): DittoMiddleware<{ pagination?: Pagination | ListPagination; paginate: PaginateFn | ListPaginateFn }> { return async (c, next) => { const { relay } = c.var; - const pagination = paginationSchema.parse(c.req.query()); + const pagination = paginationSchema(opts).parse(c.req.query()); const { max_id: maxId, @@ -59,7 +66,7 @@ export function paginationMiddleware( } } - if (type === 'list') { + if (opts.type === 'list') { c.set('pagination', { limit: pagination.limit, offset: pagination.offset, diff --git a/packages/mastoapi/pagination/schema.test.ts b/packages/mastoapi/pagination/schema.test.ts index 94be9091..b17eca16 100644 --- a/packages/mastoapi/pagination/schema.test.ts +++ b/packages/mastoapi/pagination/schema.test.ts @@ -3,7 +3,7 @@ import { assertEquals } from '@std/assert'; import { paginationSchema } from './schema.ts'; Deno.test('paginationSchema', () => { - const pagination = paginationSchema.parse({ + const pagination = paginationSchema().parse({ limit: '10', offset: '20', max_id: '1', diff --git a/packages/mastoapi/pagination/schema.ts b/packages/mastoapi/pagination/schema.ts index 5647246d..e6b35ce4 100644 --- a/packages/mastoapi/pagination/schema.ts +++ b/packages/mastoapi/pagination/schema.ts @@ -9,15 +9,28 @@ export interface Pagination { offset: number; } +export interface PaginationSchemaOpts { + limit?: number; + max?: number; +} + /** Schema to parse pagination query params. */ -export const paginationSchema: z.ZodType = z.object({ - max_id: z.string().transform((val) => { - if (!val.includes('-')) return val; - return val.split('-')[1]; - }).optional().catch(undefined), - min_id: z.string().optional().catch(undefined), - since: z.coerce.number().nonnegative().optional().catch(undefined), - until: z.coerce.number().nonnegative().optional().catch(undefined), - limit: z.coerce.number().catch(20).transform((value) => Math.min(Math.max(value, 0), 40)), - offset: z.coerce.number().nonnegative().catch(0), -}) as z.ZodType; +export function paginationSchema(opts: PaginationSchemaOpts = {}): z.ZodType { + let { limit = 20, max = 40 } = opts; + + if (limit > max) { + max = limit; + } + + return z.object({ + max_id: z.string().transform((val) => { + if (!val.includes('-')) return val; + return val.split('-')[1]; + }).optional().catch(undefined), + min_id: z.string().optional().catch(undefined), + since: z.coerce.number().nonnegative().optional().catch(undefined), + until: z.coerce.number().nonnegative().optional().catch(undefined), + limit: z.coerce.number().catch(limit).transform((value) => Math.min(Math.max(value, 0), max)), + offset: z.coerce.number().nonnegative().catch(0), + }) as z.ZodType; +} From 09b0bf8ef419433bbed785fa3588b9cd88c88820 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Mar 2025 17:20:26 -0500 Subject: [PATCH 19/22] Test paginationSchema with a custom limit --- packages/mastoapi/pagination/schema.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/mastoapi/pagination/schema.test.ts b/packages/mastoapi/pagination/schema.test.ts index b17eca16..b315e776 100644 --- a/packages/mastoapi/pagination/schema.test.ts +++ b/packages/mastoapi/pagination/schema.test.ts @@ -21,3 +21,8 @@ Deno.test('paginationSchema', () => { until: 4, }); }); + +Deno.test('paginationSchema with custom limit', () => { + const pagination = paginationSchema({ limit: 100 }).parse({}); + assertEquals(pagination.limit, 100); +}); From f27609feb8adfdcb7fc9e1a2efacc788778b4db6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Mar 2025 17:51:07 -0500 Subject: [PATCH 20/22] Set a max subscriptions size per connection --- packages/ditto/controllers/nostr/relay.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/ditto/controllers/nostr/relay.ts b/packages/ditto/controllers/nostr/relay.ts index 7a10b29f..3690b61e 100644 --- a/packages/ditto/controllers/nostr/relay.ts +++ b/packages/ditto/controllers/nostr/relay.ts @@ -153,6 +153,11 @@ function connectStream(socket: WebSocket, ip: string | undefined, opts: ConnectS async function handleReq([_, subId, ...filters]: NostrClientREQ): Promise { if (rateLimited(limiters.req)) return; + if (controllers.size > 20) { + send(['CLOSED', subId, 'error: too many subscriptions']); + return; + } + const controller = new AbortController(); controllers.get(subId)?.abort(); controllers.set(subId, controller); From 497b02002e20b5db1bfc4d2b55abd1fc571d42dc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Mar 2025 18:02:28 -0500 Subject: [PATCH 21/22] streaming: ensure close handler is called even when socket is closed by server --- packages/ditto/controllers/api/streaming.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/ditto/controllers/api/streaming.ts b/packages/ditto/controllers/api/streaming.ts index 4eebe6b3..a95cc24b 100644 --- a/packages/ditto/controllers/api/streaming.ts +++ b/packages/ditto/controllers/api/streaming.ts @@ -179,22 +179,31 @@ const streamingController: AppController = async (c) => { limiter.set(ip, count + 1, { ttl: LIMITER_WINDOW }); if (count > LIMITER_LIMIT) { - socket.close(1008, 'Rate limit exceeded'); + closeSocket(1008, 'Rate limit exceeded'); return; } } if (typeof e.data !== 'string') { - socket.close(1003, 'Invalid message'); + closeSocket(1003, 'Invalid message'); return; } }; socket.onclose = () => { + handleClose(); + }; + + function closeSocket(code?: number, reason?: string) { + socket.close(code, reason); + handleClose(); + } + + function handleClose(): void { connections.delete(socket); streamingConnectionsGauge.set(connections.size); controller.abort(); - }; + } return response; }; From 8fc00921500778b3079126a2a58c01488fe83bf6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 19 Mar 2025 15:15:44 -0500 Subject: [PATCH 22/22] Fix memory leak in relay? --- packages/ditto/controllers/nostr/relay.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/ditto/controllers/nostr/relay.ts b/packages/ditto/controllers/nostr/relay.ts index 3690b61e..bbec9b5c 100644 --- a/packages/ditto/controllers/nostr/relay.ts +++ b/packages/ditto/controllers/nostr/relay.ts @@ -171,6 +171,10 @@ function connectStream(socket: WebSocket, ip: string | undefined, opts: ConnectS } else { const [verb, , ...rest] = msg; send([verb, subId, ...rest] as NostrRelayMsg); + + if (verb === 'CLOSED') { + break; + } } } } catch (e) { @@ -182,6 +186,7 @@ function connectStream(socket: WebSocket, ip: string | undefined, opts: ConnectS send(['CLOSED', subId, 'error: something went wrong']); } } finally { + controllers.get(subId)?.abort(); controllers.delete(subId); } }