Merge branch 'pagination-compat' into 'main'

Fix Mastodon legacy pagination

See merge request soapbox-pub/ditto!447

src/app.ts

@@ -117,6 +117,7 @@ import { nostrController } from '@/controllers/well-known/nostr.ts';
 import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts';
 import { cspMiddleware } from '@/middleware/cspMiddleware.ts';
 import { metricsMiddleware } from '@/middleware/metricsMiddleware.ts';
+import { paginationMiddleware } from '@/middleware/paginationMiddleware.ts';
 import { rateLimitMiddleware } from '@/middleware/rateLimitMiddleware.ts';
 import { requireSigner } from '@/middleware/requireSigner.ts';
 import { signerMiddleware } from '@/middleware/signerMiddleware.ts';
@@ -131,8 +132,12 @@ interface AppEnv extends HonoEnv {
     uploader?: NUploader;
     /** NIP-98 signed event proving the pubkey is owned by the user. */
     proof?: NostrEvent;
-    /** Store */
+    /** Storage for the user, might filter out unwanted content. */
     store: NStore;
+    /** Normalized pagination params. */
+    pagination: { since?: number; until?: number; limit: number };
+    /** Normalized list pagination params. */
+    listPagination: { offset: number; limit: number };
   };
 }
 
@@ -146,7 +151,7 @@ const debug = Debug('ditto:http');
 
 app.use('*', rateLimitMiddleware(300, Time.minutes(5)));
 
-app.use('/api/*', metricsMiddleware, logger(debug));
+app.use('/api/*', metricsMiddleware, paginationMiddleware, logger(debug));
 app.use('/.well-known/*', metricsMiddleware, logger(debug));
 app.use('/users/*', metricsMiddleware, logger(debug));
 app.use('/nodeinfo/*', metricsMiddleware, logger(debug));

src/controllers/api/accounts.ts

@@ -9,7 +9,7 @@ import { booleanParamSchema, fileSchema } from '@/schema.ts';
 import { Storages } from '@/storages.ts';
 import { uploadFile } from '@/utils/upload.ts';
 import { nostrNow } from '@/utils.ts';
-import { createEvent, paginated, paginationSchema, parseBody, updateListEvent } from '@/utils/api.ts';
+import { createEvent, paginated, parseBody, updateListEvent } from '@/utils/api.ts';
 import { lookupAccount } from '@/utils/lookup.ts';
 import { renderAccounts, renderEventAccounts, renderStatuses } from '@/views.ts';
 import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts';
@@ -192,7 +192,7 @@ const accountStatusesQuerySchema = z.object({
 
 const accountStatusesController: AppController = async (c) => {
   const pubkey = c.req.param('pubkey');
-  const { since, until } = paginationSchema.parse(c.req.query());
+  const { since, until } = c.get('pagination');
   const { pinned, limit, exclude_replies, tagged } = accountStatusesQuerySchema.parse(c.req.query());
   const { signal } = c.req.raw;
 
@@ -366,7 +366,7 @@ const unfollowController: AppController = async (c) => {
 
 const followersController: AppController = (c) => {
   const pubkey = c.req.param('pubkey');
-  const params = paginationSchema.parse(c.req.query());
+  const params = c.get('pagination');
   return renderEventAccounts(c, [{ kinds: [3], '#p': [pubkey], ...params }]);
 };
 
@@ -418,7 +418,7 @@ const unmuteController: AppController = async (c) => {
 
 const favouritesController: AppController = async (c) => {
   const pubkey = await c.get('signer')?.getPublicKey()!;
-  const params = paginationSchema.parse(c.req.query());
+  const params = c.get('pagination');
   const { signal } = c.req.raw;
 
   const store = await Storages.db();

src/controllers/api/admin.ts

@@ -6,7 +6,7 @@ import { Conf } from '@/config.ts';
 import { booleanParamSchema } from '@/schema.ts';
 import { Storages } from '@/storages.ts';
 import { hydrateEvents } from '@/storages/hydrate.ts';
-import { createAdminEvent, paginated, paginationSchema, parseBody, updateEventInfo, updateUser } from '@/utils/api.ts';
+import { createAdminEvent, paginated, parseBody, updateEventInfo, updateUser } from '@/utils/api.ts';
 import { renderNameRequest } from '@/views/ditto.ts';
 import { renderAdminAccount, renderAdminAccountFromPubkey } from '@/views/mastodon/admin-accounts.ts';
 
@@ -29,7 +29,7 @@ const adminAccountQuerySchema = z.object({
 
 const adminAccountsController: AppController = async (c) => {
   const store = await Storages.db();
-  const params = paginationSchema.parse(c.req.query());
+  const params = c.get('pagination');
   const { signal } = c.req.raw;
   const {
     local,

src/controllers/api/ditto.ts

@@ -7,7 +7,7 @@ import { addTag } from '@/utils/tags.ts';
 import { AdminSigner } from '@/signers/AdminSigner.ts';
 import { booleanParamSchema } from '@/schema.ts';
 import { Conf } from '@/config.ts';
-import { createEvent, paginated, paginationSchema, parseBody } from '@/utils/api.ts';
+import { createEvent, paginated, parseBody } from '@/utils/api.ts';
 import { deleteTag } from '@/utils/tags.ts';
 import { DittoZapSplits, getZapSplits } from '@/utils/zap-split.ts';
 import { getAuthor } from '@/queries.ts';
@@ -114,7 +114,7 @@ export const nameRequestsController: AppController = async (c) => {
   const signer = c.get('signer')!;
   const pubkey = await signer.getPublicKey();
 
-  const params = paginationSchema.parse(c.req.query());
+  const params = c.get('pagination');
   const { approved, rejected } = nameRequestsSchema.parse(c.req.query());
 
   const filter: NostrFilter = {

src/controllers/api/notifications.ts

@@ -3,8 +3,9 @@ import { z } from 'zod';
 
 import { AppContext, AppController } from '@/app.ts';
 import { Conf } from '@/config.ts';
+import { DittoPagination } from '@/interfaces/DittoPagination.ts';
 import { hydrateEvents } from '@/storages/hydrate.ts';
-import { paginated, PaginationParams, paginationSchema } from '@/utils/api.ts';
+import { paginated } from '@/utils/api.ts';
 import { renderNotification } from '@/views/mastodon/notifications.ts';
 
 /** Set of known notification types across backends. */
@@ -30,7 +31,7 @@ const notificationsSchema = z.object({
 
 const notificationsController: AppController = async (c) => {
   const pubkey = await c.get('signer')?.getPublicKey()!;
-  const params = paginationSchema.parse(c.req.query());
+  const params = c.get('pagination');
 
   const types = notificationTypes
     .intersection(new Set(c.req.queries('types[]') ?? notificationTypes))
@@ -72,7 +73,7 @@ const notificationsController: AppController = async (c) => {
 async function renderNotifications(
   filters: NostrFilter[],
   types: Set<string>,
-  params: PaginationParams,
+  params: DittoPagination,
   c: AppContext,
 ) {
   const store = c.get('store');

src/controllers/api/reports.ts

@@ -3,7 +3,7 @@ import { z } from 'zod';
 
 import { type AppController } from '@/app.ts';
 import { Conf } from '@/config.ts';
-import { createEvent, paginated, paginationSchema, parseBody, updateEventInfo } from '@/utils/api.ts';
+import { createEvent, paginated, parseBody, updateEventInfo } from '@/utils/api.ts';
 import { hydrateEvents } from '@/storages/hydrate.ts';
 import { renderAdminReport } from '@/views/mastodon/reports.ts';
 import { renderReport } from '@/views/mastodon/reports.ts';
@@ -64,7 +64,7 @@ const adminReportsController: AppController = async (c) => {
   const store = c.get('store');
   const viewerPubkey = await c.get('signer')?.getPublicKey();
 
-  const params = paginationSchema.parse(c.req.query());
+  const params = c.get('pagination');
   const { resolved, account_id, target_account_id } = adminReportsSchema.parse(c.req.query());
 
   const filter: NostrFilter = {

src/controllers/api/statuses.ts

@@ -19,15 +19,7 @@ import { renderEventAccounts } from '@/views.ts';
 import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts';
 import { Storages } from '@/storages.ts';
 import { hydrateEvents, purifyEvent } from '@/storages/hydrate.ts';
-import {
-  createEvent,
-  listPaginationSchema,
-  paginated,
-  paginatedList,
-  paginationSchema,
-  parseBody,
-  updateListEvent,
-} from '@/utils/api.ts';
+import { createEvent, paginated, paginatedList, parseBody, updateListEvent } from '@/utils/api.ts';
 import { getInvoice, getLnurl } from '@/utils/lnurl.ts';
 import { getZapSplits } from '@/utils/zap-split.ts';
 
@@ -296,7 +288,7 @@ const favouriteController: AppController = async (c) => {
 
 const favouritedByController: AppController = (c) => {
   const id = c.req.param('id');
-  const params = paginationSchema.parse(c.req.query());
+  const params = c.get('pagination');
 
   return renderEventAccounts(c, [{ kinds: [7], '#e': [id], ...params }], {
     filterFn: ({ content }) => content === '+',
@@ -364,13 +356,13 @@ const unreblogStatusController: AppController = async (c) => {
 
 const rebloggedByController: AppController = (c) => {
   const id = c.req.param('id');
-  const params = paginationSchema.parse(c.req.query());
+  const params = c.get('pagination');
   return renderEventAccounts(c, [{ kinds: [6], '#e': [id], ...params }]);
 };
 
 const quotesController: AppController = async (c) => {
   const id = c.req.param('id');
-  const params = paginationSchema.parse(c.req.query());
+  const params = c.get('pagination');
   const store = await Storages.db();
 
   const [event] = await store.query([{ ids: [id], kinds: [1] }]);
@@ -571,7 +563,7 @@ const zapController: AppController = async (c) => {
 
 const zappedByController: AppController = async (c) => {
   const id = c.req.param('id');
-  const params = listPaginationSchema.parse(c.req.query());
+  const params = c.get('listPagination');
   const store = await Storages.db();
   const db = await DittoDB.getInstance();
 

src/controllers/api/suggestions.ts

@@ -4,13 +4,13 @@ import { matchFilter } from 'nostr-tools';
 import { AppContext, AppController } from '@/app.ts';
 import { Conf } from '@/config.ts';
 import { hydrateEvents } from '@/storages/hydrate.ts';
-import { listPaginationSchema, paginatedList, PaginatedListParams } from '@/utils/api.ts';
+import { paginatedList } from '@/utils/api.ts';
 import { getTagSet } from '@/utils/tags.ts';
 import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts';
 
 export const suggestionsV1Controller: AppController = async (c) => {
   const signal = c.req.raw.signal;
-  const params = listPaginationSchema.parse(c.req.query());
+  const params = c.get('listPagination');
   const suggestions = await renderV2Suggestions(c, params, signal);
   const accounts = suggestions.map(({ account }) => account);
   return paginatedList(c, params, accounts);
@@ -18,12 +18,12 @@ export const suggestionsV1Controller: AppController = async (c) => {
 
 export const suggestionsV2Controller: AppController = async (c) => {
   const signal = c.req.raw.signal;
-  const params = listPaginationSchema.parse(c.req.query());
+  const params = c.get('listPagination');
   const suggestions = await renderV2Suggestions(c, params, signal);
   return paginatedList(c, params, suggestions);
 };
 
-async function renderV2Suggestions(c: AppContext, params: PaginatedListParams, signal?: AbortSignal) {
+async function renderV2Suggestions(c: AppContext, params: { offset: number; limit: number }, signal?: AbortSignal) {
   const { offset, limit } = params;
 
   const store = c.get('store');

src/controllers/api/timelines.ts

@@ -6,12 +6,12 @@ import { Conf } from '@/config.ts';
 import { getFeedPubkeys } from '@/queries.ts';
 import { booleanParamSchema } from '@/schema.ts';
 import { hydrateEvents } from '@/storages/hydrate.ts';
-import { paginated, paginationSchema } from '@/utils/api.ts';
+import { paginated } from '@/utils/api.ts';
 import { getTagSet } from '@/utils/tags.ts';
 import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts';
 
 const homeTimelineController: AppController = async (c) => {
-  const params = paginationSchema.parse(c.req.query());
+  const params = c.get('pagination');
   const pubkey = await c.get('signer')?.getPublicKey()!;
   const authors = await getFeedPubkeys(pubkey);
   return renderStatuses(c, [{ authors, kinds: [1, 6], ...params }]);
@@ -23,7 +23,7 @@ const publicQuerySchema = z.object({
 });
 
 const publicTimelineController: AppController = (c) => {
-  const params = paginationSchema.parse(c.req.query());
+  const params = c.get('pagination');
   const { local, instance } = publicQuerySchema.parse(c.req.query());
 
   const filter: NostrFilter = { kinds: [1], ...params };
@@ -39,13 +39,13 @@ const publicTimelineController: AppController = (c) => {
 
 const hashtagTimelineController: AppController = (c) => {
   const hashtag = c.req.param('hashtag')!.toLowerCase();
-  const params = paginationSchema.parse(c.req.query());
+  const params = c.get('pagination');
   return renderStatuses(c, [{ kinds: [1], '#t': [hashtag], ...params }]);
 };
 
 const suggestedTimelineController: AppController = async (c) => {
   const store = c.get('store');
-  const params = paginationSchema.parse(c.req.query());
+  const params = c.get('pagination');
 
   const [follows] = await store.query(
     [{ kinds: [3], authors: [Conf.pubkey], limit: 1 }],

src/interfaces/DittoPagination.ts

@@ -0,0 +1,15 @@
+/** Based on Mastodon pagination. */
+export interface DittoPagination {
+  /** Lowest Nostr event `created_at` timestamp. */
+  since?: number;
+  /** Highest Nostr event `created_at` timestamp. */
+  until?: number;
+  /** @deprecated Mastodon apps are supposed to use the `Link` header. */
+  max_id?: string;
+  /** @deprecated Mastodon apps are supposed to use the `Link` header. */
+  min_id?: string;
+  /** Maximum number of results to return. Default 20, maximum 40. */
+  limit?: number;
+  /** Used by Ditto to offset tag values in Nostr list events. */
+  offset?: number;
+}

src/middleware/paginationMiddleware.ts

@@ -0,0 +1,49 @@
+import { AppMiddleware } from '@/app.ts';
+import { paginationSchema } from '@/schemas/pagination.ts';
+import { Storages } from '@/storages.ts';
+
+/** Fixes compatibility with Mastodon apps by that don't use `Link` headers. */
+export const paginationMiddleware: AppMiddleware = async (c, next) => {
+  const pagination = paginationSchema.parse(c.req.query());
+
+  const {
+    max_id: maxId,
+    min_id: minId,
+    since,
+    until,
+  } = pagination;
+
+  if ((maxId && !until) || (minId && !since)) {
+    const ids: string[] = [];
+
+    if (maxId) ids.push(maxId);
+    if (minId) ids.push(minId);
+
+    if (ids.length) {
+      const store = await Storages.db();
+
+      const events = await store.query(
+        [{ ids, limit: ids.length }],
+        { signal: c.req.raw.signal },
+      );
+
+      for (const event of events) {
+        if (!until && maxId === event.id) pagination.until = event.created_at;
+        if (!since && minId === event.id) pagination.since = event.created_at;
+      }
+    }
+  }
+
+  c.set('pagination', {
+    since: pagination.since,
+    until: pagination.until,
+    limit: pagination.limit,
+  });
+
+  c.set('listPagination', {
+    limit: pagination.limit,
+    offset: pagination.offset,
+  });
+
+  await next();
+};

src/schemas/pagination.ts

@@ -0,0 +1,11 @@
+import { z } from 'zod';
+
+/** Schema to parse pagination query params. */
+export const paginationSchema = z.object({
+  max_id: z.string().optional().catch(undefined),
+  min_id: z.string().optional().catch(undefined),
+  since: z.coerce.number().nonnegative().optional().catch(undefined),
+  until: z.coerce.number().nonnegative().optional().catch(undefined),
+  limit: z.coerce.number().catch(20).transform((value) => Math.min(Math.max(value, 0), 40)),
+  offset: z.coerce.number().nonnegative().catch(0),
+});

src/utils/api.ts

@@ -5,7 +5,6 @@ import Debug from '@soapbox/stickynotes/debug';
 import { parseFormData } from 'formdata-helper';
 import { EventTemplate } from 'nostr-tools';
 import * as TypeFest from 'type-fest';
-import { z } from 'zod';
 
 import { type AppContext } from '@/app.ts';
 import { Conf } from '@/config.ts';
@@ -176,16 +175,6 @@ async function parseBody(req: Request): Promise<unknown> {
   }
 }
 
-/** Schema to parse pagination query params. */
-const paginationSchema = z.object({
-  since: z.coerce.number().nonnegative().optional().catch(undefined),
-  until: z.coerce.number().nonnegative().optional().catch(undefined),
-  limit: z.coerce.number().catch(20).transform((value) => Math.min(Math.max(value, 0), 40)),
-});
-
-/** Mastodon API pagination query params. */
-type PaginationParams = z.infer<typeof paginationSchema>;
-
 /** Build HTTP Link header for Mastodon API pagination. */
 function buildLinkHeader(url: string, events: NostrEvent[]): string | undefined {
   if (events.length <= 1) return;
@@ -219,12 +208,6 @@ function paginated(c: AppContext, events: NostrEvent[], entities: (Entity | unde
   return c.json(results, 200, headers);
 }
 
-/** Query params for paginating a list. */
-const listPaginationSchema = z.object({
-  offset: z.coerce.number().nonnegative().catch(0),
-  limit: z.coerce.number().catch(20).transform((value) => Math.min(Math.max(value, 0), 40)),
-});
-
 /** Build HTTP Link header for paginating Nostr lists. */
 function buildListLinkHeader(url: string, params: { offset: number; limit: number }): string | undefined {
   const { origin } = Conf.url;
@@ -242,15 +225,10 @@ function buildListLinkHeader(url: string, params: { offset: number; limit: numbe
   return `<${next}>; rel="next", <${prev}>; rel="prev"`;
 }
 
-interface PaginatedListParams {
-  offset: number;
-  limit: number;
-}
-
 /** paginate a list of tags. */
 function paginatedList(
   c: AppContext,
-  params: PaginatedListParams,
+  params: { offset: number; limit: number },
   entities: unknown[],
   headers: HeaderRecord = {},
 ) {
@@ -296,13 +274,9 @@ export {
   createAdminEvent,
   createEvent,
   type EventStub,
-  listPaginationSchema,
   localRequest,
   paginated,
   paginatedList,
-  type PaginatedListParams,
-  type PaginationParams,
-  paginationSchema,
   parseBody,
   updateAdminEvent,
   updateEvent,

src/views.ts

@@ -4,7 +4,7 @@ import { AppContext } from '@/app.ts';
 import { Storages } from '@/storages.ts';
 import { renderAccount } from '@/views/mastodon/accounts.ts';
 import { renderStatus } from '@/views/mastodon/statuses.ts';
-import { listPaginationSchema, paginated, paginatedList, paginationSchema } from '@/utils/api.ts';
+import { paginated, paginatedList } from '@/utils/api.ts';
 import { hydrateEvents } from '@/storages/hydrate.ts';
 import { accountFromPubkey } from '@/views/mastodon/accounts.ts';
 
@@ -43,7 +43,7 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], opts?:
 }
 
 async function renderAccounts(c: AppContext, pubkeys: string[]) {
-  const { offset, limit } = listPaginationSchema.parse(c.req.query());
+  const { offset, limit } = c.get('listPagination');
   const authors = pubkeys.reverse().slice(offset, offset + limit);
 
   const store = await Storages.db();
@@ -73,7 +73,7 @@ async function renderStatuses(c: AppContext, ids: string[], signal = AbortSignal
   }
 
   const store = await Storages.db();
-  const { limit } = paginationSchema.parse(c.req.query());
+  const { limit } = c.get('pagination');
 
   const events = await store.query([{ kinds: [1], ids, limit }], { signal })
     .then((events) => hydrateEvents({ events, store, signal }));