From 8e178338b74f42e8d21372e4243cc8ab70e3dc7d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 3 May 2024 16:17:36 -0500 Subject: [PATCH 1/9] Implement Markers API --- src/app.ts | 4 +++ src/controllers/api/markers.ts | 62 ++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 src/controllers/api/markers.ts diff --git a/src/app.ts b/src/app.ts index df8919d2..d80bf37a 100644 --- a/src/app.ts +++ b/src/app.ts @@ -31,6 +31,7 @@ import { blocksController } from '@/controllers/api/blocks.ts'; import { bookmarksController } from '@/controllers/api/bookmarks.ts'; import { emptyArrayController, emptyObjectController, notImplementedController } from '@/controllers/api/fallback.ts'; import { instanceController } from '@/controllers/api/instance.ts'; +import { markersController, updateMarkersController } from '@/controllers/api/markers.ts'; import { mediaController } from '@/controllers/api/media.ts'; import { mutesController } from '@/controllers/api/mutes.ts'; import { notificationsController } from '@/controllers/api/notifications.ts'; @@ -191,6 +192,9 @@ app.get('/api/v1/bookmarks', requirePubkey, bookmarksController); app.get('/api/v1/blocks', requirePubkey, blocksController); app.get('/api/v1/mutes', requirePubkey, mutesController); +app.get('/api/v1/markers', requireProof(), markersController); +app.post('/api/v1/markers', requireProof(), updateMarkersController); + app.get('/api/v1/admin/accounts', requireRole('admin'), adminAccountsController); app.get('/api/v1/pleroma/admin/config', requireRole('admin'), configController); app.post('/api/v1/pleroma/admin/config', requireRole('admin'), updateConfigController); diff --git a/src/controllers/api/markers.ts b/src/controllers/api/markers.ts new file mode 100644 index 00000000..cd90e334 --- /dev/null +++ b/src/controllers/api/markers.ts @@ -0,0 +1,62 @@ +import { z } from 'zod'; + +import { AppController } from '@/app.ts'; +import { parseBody } from '@/utils/api.ts'; + +const kv = await Deno.openKv(); + +interface Marker { + last_read_id: string; + version: number; + updated_at: string; +} + +export const markersController: AppController = async (c) => { + const pubkey = c.get('pubkey')!; + const timelines = c.req.queries('timeline[]') ?? []; + + const results = await kv.getMany( + timelines.map((timeline) => ['markers', pubkey, timeline]), + ); + + const marker = results.reduce>((acc, { key, value }) => { + if (value) { + const timeline = key[key.length - 1] as string; + acc[timeline] = value; + } + return acc; + }, {}); + + return c.json(marker); +}; + +const markerDataSchema = z.object({ + last_read_id: z.string(), +}); + +export const updateMarkersController: AppController = async (c) => { + const pubkey = c.get('pubkey')!; + const record = z.record(z.string(), markerDataSchema).parse(await parseBody(c.req.raw)); + const timelines = Object.keys(record); + + const markers: Record = {}; + + const entries = await kv.getMany( + timelines.map((timeline) => ['markers', pubkey, timeline]), + ); + + for (const timeline of timelines) { + const last = entries.find(({ key }) => key[key.length - 1] === timeline); + + const marker: Marker = { + last_read_id: record[timeline].last_read_id, + version: last?.value ? last.value.version + 1 : 1, + updated_at: new Date().toISOString(), + }; + + await kv.set(['markers', pubkey, timeline], marker); + markers[timeline] = marker; + } + + return c.json(markers); +}; From a2c5b5e61d048a22e55ae8237f78edb53a4ffc1d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 3 May 2024 16:20:07 -0500 Subject: [PATCH 2/9] Markers: only allow 'home' and 'notifications' markers --- src/controllers/api/markers.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/controllers/api/markers.ts b/src/controllers/api/markers.ts index cd90e334..ce1c4ec3 100644 --- a/src/controllers/api/markers.ts +++ b/src/controllers/api/markers.ts @@ -5,6 +5,8 @@ import { parseBody } from '@/utils/api.ts'; const kv = await Deno.openKv(); +type Timeline = 'home' | 'notifications'; + interface Marker { last_read_id: string; version: number; @@ -36,8 +38,8 @@ const markerDataSchema = z.object({ export const updateMarkersController: AppController = async (c) => { const pubkey = c.get('pubkey')!; - const record = z.record(z.string(), markerDataSchema).parse(await parseBody(c.req.raw)); - const timelines = Object.keys(record); + const record = z.record(z.enum(['home', 'notifications']), markerDataSchema).parse(await parseBody(c.req.raw)); + const timelines = Object.keys(record) as Timeline[]; const markers: Record = {}; @@ -49,7 +51,7 @@ export const updateMarkersController: AppController = async (c) => { const last = entries.find(({ key }) => key[key.length - 1] === timeline); const marker: Marker = { - last_read_id: record[timeline].last_read_id, + last_read_id: record[timeline]!.last_read_id, version: last?.value ? last.value.version + 1 : 1, updated_at: new Date().toISOString(), }; From 7efd5c1822ae23807ef578894f924040e4d24e95 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 3 May 2024 17:09:20 -0500 Subject: [PATCH 3/9] Clean up "not implemented" endpoints --- src/app.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/app.ts b/src/app.ts index d80bf37a..b382ceab 100644 --- a/src/app.ts +++ b/src/app.ts @@ -208,9 +208,7 @@ app.post('/api/v1/reports', requirePubkey, reportsController); // Not (yet) implemented. app.get('/api/v1/custom_emojis', emptyArrayController); app.get('/api/v1/filters', emptyArrayController); -app.get('/api/v1/mutes', emptyArrayController); app.get('/api/v1/domain_blocks', emptyArrayController); -app.get('/api/v1/markers', emptyObjectController); app.get('/api/v1/conversations', emptyArrayController); app.get('/api/v1/lists', emptyArrayController); From 5001567b00f7e1d39ecb0d060719c8d4920db98e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 3 May 2024 17:20:35 -0500 Subject: [PATCH 4/9] Streaming: temporarily remove UserStore (allow blocked posts through) --- src/controllers/api/streaming.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index 965855c3..8d22d5cc 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -9,7 +9,6 @@ import { bech32ToPubkey } from '@/utils.ts'; import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; -import { UserStore } from '@/storages/UserStore.ts'; const debug = Debug('ditto:streaming'); @@ -68,17 +67,14 @@ const streamingController: AppController = (c) => { const filter = await topicToFilter(stream, c.req.query(), pubkey); if (!filter) return; - const store = pubkey ? new UserStore(pubkey, Storages.admin) : Storages.admin; - try { for await (const msg of Storages.pubsub.req([filter], { signal: controller.signal })) { if (msg[0] === 'EVENT') { - const [event] = await store.query([{ ids: [msg[2].id] }]); - if (!event) continue; + const event = msg[2]; await hydrateEvents({ events: [event], - storage: store, + storage: Storages.admin, signal: AbortSignal.timeout(1000), }); From 0f3fbbcb28b6dcfbc474cc2096cf4c1224476103 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 3 May 2024 18:21:40 -0500 Subject: [PATCH 5/9] Start suggestions API --- src/app.ts | 4 +++ src/controllers/api/suggestions.ts | 50 ++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 src/controllers/api/suggestions.ts diff --git a/src/app.ts b/src/app.ts index b382ceab..ddc902fc 100644 --- a/src/app.ts +++ b/src/app.ts @@ -63,6 +63,7 @@ import { zapController, } from '@/controllers/api/statuses.ts'; import { streamingController } from '@/controllers/api/streaming.ts'; +import { suggestionsV1Controller, suggestionsV2Controller } from '@/controllers/api/suggestions.ts'; import { hashtagTimelineController, homeTimelineController, @@ -186,6 +187,9 @@ app.get('/api/pleroma/frontend_configurations', frontendConfigController); app.get('/api/v1/trends/tags', cache({ cacheName: 'web', expires: Time.minutes(15) }), trendingTagsController); app.get('/api/v1/trends', cache({ cacheName: 'web', expires: Time.minutes(15) }), trendingTagsController); +app.get('/api/v1/suggestions', suggestionsV1Controller); +app.get('/api/v2/suggestions', suggestionsV2Controller); + app.get('/api/v1/notifications', requirePubkey, notificationsController); app.get('/api/v1/favourites', requirePubkey, favouritesController); app.get('/api/v1/bookmarks', requirePubkey, bookmarksController); diff --git a/src/controllers/api/suggestions.ts b/src/controllers/api/suggestions.ts new file mode 100644 index 00000000..b85f05df --- /dev/null +++ b/src/controllers/api/suggestions.ts @@ -0,0 +1,50 @@ +import { NStore } from '@nostrify/nostrify'; + +import { AppController } from '@/app.ts'; +import { Conf } from '@/config.ts'; +import { getTagSet } from '@/tags.ts'; +import { hydrateEvents } from '@/storages/hydrate.ts'; +import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; + +export const suggestionsV1Controller: AppController = async (c) => { + const store = c.get('store'); + const signal = c.req.raw.signal; + const accounts = await renderSuggestedAccounts(store, signal); + + return c.json(accounts); +}; + +export const suggestionsV2Controller: AppController = async (c) => { + const store = c.get('store'); + const signal = c.req.raw.signal; + const accounts = await renderSuggestedAccounts(store, signal); + + const suggestions = accounts.map((account) => ({ + source: 'staff', + account, + })); + + return c.json(suggestions); +}; + +async function renderSuggestedAccounts(store: NStore, signal?: AbortSignal) { + const [follows] = await store.query( + [{ kinds: [3], authors: [Conf.pubkey], limit: 1 }], + { signal }, + ); + + const pubkeys = [...getTagSet(follows?.tags ?? [], 'p')]; + + const profiles = await store.query( + [{ kinds: [1], authors: pubkeys }], + { signal }, + ) + .then((events) => hydrateEvents({ events, storage: store, signal })); + + const accounts = await Promise.all(pubkeys.map((pubkey) => { + const profile = profiles.find((event) => event.pubkey === pubkey); + return profile ? renderAccount(profile) : accountFromPubkey(pubkey); + })); + + return accounts.filter(Boolean); +} From 4ee4266843c58ff509cf317b7b0df45d3f37b90f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 3 May 2024 18:28:07 -0500 Subject: [PATCH 6/9] instance: add 'v2_suggestions' to features --- src/controllers/api/instance.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/controllers/api/instance.ts b/src/controllers/api/instance.ts index 188d68f5..70f38e14 100644 --- a/src/controllers/api/instance.ts +++ b/src/controllers/api/instance.ts @@ -45,6 +45,7 @@ const instanceController: AppController = async (c) => { 'mastodon_api_streaming', 'exposable_reactions', 'quote_posting', + 'v2_suggestions', ], }, }, From e25372313b4c17dfea8f45c96eded8be2592d287 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 3 May 2024 18:32:35 -0500 Subject: [PATCH 7/9] suggestions: fix profile lookup, limit to 20 items for now --- src/controllers/api/suggestions.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/suggestions.ts b/src/controllers/api/suggestions.ts index b85f05df..bde09165 100644 --- a/src/controllers/api/suggestions.ts +++ b/src/controllers/api/suggestions.ts @@ -33,10 +33,11 @@ async function renderSuggestedAccounts(store: NStore, signal?: AbortSignal) { { signal }, ); - const pubkeys = [...getTagSet(follows?.tags ?? [], 'p')]; + // TODO: pagination + const pubkeys = [...getTagSet(follows?.tags ?? [], 'p')].slice(0, 20); const profiles = await store.query( - [{ kinds: [1], authors: pubkeys }], + [{ kinds: [0], authors: pubkeys, limit: pubkeys.length }], { signal }, ) .then((events) => hydrateEvents({ events, storage: store, signal })); From 0a3be0da587425f0eafc38040ce93f3ce76df144 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 3 May 2024 21:22:53 -0500 Subject: [PATCH 8/9] Notifications: fix Favourites and EmojiReacts not being displayed --- src/storages/hydrate.ts | 7 +++++++ src/views/mastodon/notifications.ts | 14 ++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 716f2518..61d82855 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -76,6 +76,13 @@ function assembleEvents( } } + if (event.kind === 7) { + const id = event.tags.find(([name]) => name === 'e')?.[1]; + if (id) { + event.reacted = b.find((e) => matchFilter({ kinds: [1], ids: [id] }, e)); + } + } + if (event.kind === 1) { const id = event.tags.find(([name]) => name === 'q')?.[1]; if (id) { diff --git a/src/views/mastodon/notifications.ts b/src/views/mastodon/notifications.ts index 266b77b0..5b618d79 100644 --- a/src/views/mastodon/notifications.ts +++ b/src/views/mastodon/notifications.ts @@ -2,6 +2,7 @@ import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { nostrDate } from '@/utils.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; +import { NostrEvent } from '@nostrify/nostrify'; interface RenderNotificationOpts { viewerPubkey: string; @@ -32,7 +33,7 @@ async function renderMention(event: DittoEvent, opts: RenderNotificationOpts) { if (!status) return; return { - id: event.id, + id: notificationId(event), type: 'mention', created_at: nostrDate(event.created_at).toISOString(), account: status.account, @@ -47,7 +48,7 @@ async function renderReblog(event: DittoEvent, opts: RenderNotificationOpts) { const account = event.author ? await renderAccount(event.author) : accountFromPubkey(event.pubkey); return { - id: event.id, + id: notificationId(event), type: 'reblog', created_at: nostrDate(event.created_at).toISOString(), account, @@ -62,7 +63,7 @@ async function renderFavourite(event: DittoEvent, opts: RenderNotificationOpts) const account = event.author ? await renderAccount(event.author) : accountFromPubkey(event.pubkey); return { - id: event.id, + id: notificationId(event), type: 'favourite', created_at: nostrDate(event.created_at).toISOString(), account, @@ -77,7 +78,7 @@ async function renderReaction(event: DittoEvent, opts: RenderNotificationOpts) { const account = event.author ? await renderAccount(event.author) : accountFromPubkey(event.pubkey); return { - id: event.id, + id: notificationId(event), type: 'pleroma:emoji_reaction', emoji: event.content, created_at: nostrDate(event.created_at).toISOString(), @@ -86,4 +87,9 @@ async function renderReaction(event: DittoEvent, opts: RenderNotificationOpts) { }; } +/** This helps notifications be sorted in the correct order. */ +function notificationId({ id, created_at }: NostrEvent): string { + return `${created_at}-${id}`; +} + export { renderNotification }; From 3770d8a0dd11833949327b290db1e3b6f7cacb17 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 4 May 2024 13:14:03 -0500 Subject: [PATCH 9/9] UnattachedMedia: return early when querying nothing --- src/db/unattached-media.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/db/unattached-media.ts b/src/db/unattached-media.ts index 415c110d..960abe8c 100644 --- a/src/db/unattached-media.ts +++ b/src/db/unattached-media.ts @@ -54,15 +54,18 @@ function deleteUnattachedMediaByUrl(url: string) { } /** Get unattached media by IDs. */ -function getUnattachedMediaByIds(ids: string[]) { +// deno-lint-ignore require-await +async function getUnattachedMediaByIds(ids: string[]) { + if (!ids.length) return []; return selectUnattachedMediaQuery() .where('id', 'in', ids) .execute(); } /** Delete rows as an event with media is being created. */ -function deleteAttachedMedia(pubkey: string, urls: string[]) { - return db.deleteFrom('unattached_media') +async function deleteAttachedMedia(pubkey: string, urls: string[]): Promise { + if (!urls.length) return; + await db.deleteFrom('unattached_media') .where('pubkey', '=', pubkey) .where('url', 'in', urls) .execute();