Merge branch 'main' into feat-search-mime-type

deno.json

@@ -46,8 +46,9 @@
     "@negrel/webpush": "jsr:@negrel/webpush@^0.3.0",
     "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0",
     "@nostrify/db": "jsr:@nostrify/db@^0.36.1",
-    "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.36.0",
+    "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.37.0",
-    "@nostrify/policies": "jsr:@nostrify/policies@^0.35.0",
+    "@nostrify/policies": "jsr:@nostrify/policies@^0.36.1",
+    "@nostrify/types": "jsr:@nostrify/types@^0.36.0",
     "@scure/base": "npm:@scure/base@^1.1.6",
     "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs",
     "@soapbox/kysely-pglite": "jsr:@soapbox/kysely-pglite@^1.0.0",

deno.lock

@@ -26,27 +26,27 @@
     "jsr:@gleasonator/policy@0.9.1": "0.9.1",
     "jsr:@gleasonator/policy@0.9.2": "0.9.2",
     "jsr:@gleasonator/policy@0.9.3": "0.9.3",
-    "jsr:@hono/hono@^4.4.6": "4.6.2",
+    "jsr:@hono/hono@^4.4.6": "4.6.15",
     "jsr:@lambdalisue/async@^2.1.1": "2.1.1",
     "jsr:@negrel/http-ece@0.6.0": "0.6.0",
     "jsr:@negrel/webpush@0.3": "0.3.0",
     "jsr:@nostrify/db@~0.36.1": "0.36.1",
     "jsr:@nostrify/nostrify@0.31": "0.31.0",
     "jsr:@nostrify/nostrify@0.32": "0.32.0",
-    "jsr:@nostrify/nostrify@0.35": "0.35.0",
+    "jsr:@nostrify/nostrify@0.36": "0.36.2",
-    "jsr:@nostrify/nostrify@0.36": "0.36.0",
+    "jsr:@nostrify/nostrify@0.37": "0.37.0",
     "jsr:@nostrify/nostrify@~0.22.1": "0.22.5",
     "jsr:@nostrify/nostrify@~0.22.4": "0.22.4",
     "jsr:@nostrify/nostrify@~0.22.5": "0.22.5",
     "jsr:@nostrify/policies@0.33": "0.33.0",
     "jsr:@nostrify/policies@0.33.1": "0.33.1",
     "jsr:@nostrify/policies@0.34": "0.34.0",
-    "jsr:@nostrify/policies@0.35": "0.35.0",
     "jsr:@nostrify/policies@0.36": "0.36.0",
     "jsr:@nostrify/policies@~0.33.1": "0.33.1",
     "jsr:@nostrify/policies@~0.36.1": "0.36.1",
     "jsr:@nostrify/types@0.30": "0.30.1",
     "jsr:@nostrify/types@0.35": "0.35.0",
+    "jsr:@nostrify/types@0.36": "0.36.0",
     "jsr:@nostrify/types@~0.30.1": "0.30.1",
     "jsr:@soapbox/kysely-pglite@1": "1.0.0",
     "jsr:@soapbox/safe-fetch@2": "2.0.0",
@@ -60,7 +60,7 @@
     "jsr:@std/bytes@0.224.0": "0.224.0",
     "jsr:@std/bytes@^1.0.0-rc.3": "1.0.0",
     "jsr:@std/bytes@^1.0.1-rc.3": "1.0.2",
-    "jsr:@std/bytes@^1.0.2": "1.0.2",
+    "jsr:@std/bytes@^1.0.2": "1.0.4",
     "jsr:@std/bytes@^1.0.2-rc.3": "1.0.2",
     "jsr:@std/cli@0.223": "0.223.0",
     "jsr:@std/crypto@0.224": "0.224.0",
@@ -72,9 +72,9 @@
     "jsr:@std/fmt@0.213.1": "0.213.1",
     "jsr:@std/fs@0.213.1": "0.213.1",
     "jsr:@std/fs@~0.229.3": "0.229.3",
-    "jsr:@std/internal@1": "1.0.4",
+    "jsr:@std/internal@1": "1.0.5",
     "jsr:@std/io@0.223": "0.223.0",
-    "jsr:@std/io@0.224": "0.224.8",
+    "jsr:@std/io@0.224": "0.224.9",
     "jsr:@std/json@0.223": "0.223.0",
     "jsr:@std/media-types@0.224.0": "0.224.0",
     "jsr:@std/media-types@~0.224.1": "0.224.1",
@@ -139,7 +139,10 @@
       ]
     },
     "@b-fuze/deno-dom@0.1.48": {
-      "integrity": "bf5b591aef2e9e9c59adfcbb93a9ecd45bab5b7c8263625beafa5c8f1662e7da"
+      "integrity": "bf5b591aef2e9e9c59adfcbb93a9ecd45bab5b7c8263625beafa5c8f1662e7da",
+      "dependencies": [
+        "jsr:@denosaurs/plug"
+      ]
     },
     "@bradenmacdonald/s3-lite-client@0.7.6": {
       "integrity": "2b5976dca95d207dc88e23f9807e3eecbc441b0cf547dcda5784afe6668404d1",
@@ -321,6 +324,9 @@
     "@hono/hono@4.6.2": {
       "integrity": "35fcf3be4687825080b01bed7bbe2ac66f8d8b8939f0bad459661bf3b46d916f"
     },
+    "@hono/hono@4.6.15": {
+      "integrity": "935b3b12e98e4b22bcd1aa4dbe6587321e431c79829eba61f535b4ede39fd8b1"
+    },
     "@lambdalisue/async@2.1.1": {
       "integrity": "1fc9bc6f4ed50215cd2f7217842b18cea80f81c25744f88f8c5eb4be5a1c9ab4"
     },
@@ -420,8 +426,23 @@
         "npm:zod"
       ]
     },
-    "@nostrify/nostrify@0.35.0": {
+    "@nostrify/nostrify@0.36.0": {
-      "integrity": "9bfef4883838b8b4cb2e2b28a60b72de95391ca5b789bc7206a2baea054dea55",
+      "integrity": "f00dbff1f02a2c496c5e85eeeb7a84101b7dd874d87456449dc71b6d037e40fc",
+      "dependencies": [
+        "jsr:@nostrify/types@0.35",
+        "jsr:@std/crypto",
+        "jsr:@std/encoding@~0.224.1",
+        "npm:@scure/base",
+        "npm:@scure/bip32",
+        "npm:@scure/bip39",
+        "npm:lru-cache@^10.2.0",
+        "npm:nostr-tools@^2.7.0",
+        "npm:websocket-ts",
+        "npm:zod"
+      ]
+    },
+    "@nostrify/nostrify@0.36.2": {
+      "integrity": "cc4787ca170b623a2e5dfed1baa4426077daa6143af728ea7dd325d58f4d04d6",
       "dependencies": [
         "jsr:@nostrify/types@0.35",
         "jsr:@std/encoding@~0.224.1",
@@ -433,10 +454,10 @@
         "npm:zod"
       ]
     },
-    "@nostrify/nostrify@0.36.0": {
+    "@nostrify/nostrify@0.37.0": {
-      "integrity": "f00dbff1f02a2c496c5e85eeeb7a84101b7dd874d87456449dc71b6d037e40fc",
+      "integrity": "fa1439cc5e9a74986c4fb799a38a9ed7bd8663c62ae2a9363ca9b987548e27a0",
       "dependencies": [
-        "jsr:@nostrify/types@0.35",
+        "jsr:@nostrify/types@0.36",
         "jsr:@std/crypto",
         "jsr:@std/encoding@~0.224.1",
         "npm:@scure/base",
@@ -470,14 +491,6 @@
         "npm:nostr-tools@^2.7.0"
       ]
     },
-    "@nostrify/policies@0.35.0": {
-      "integrity": "b828fac9f253e460a9587c05588b7dae6a0a32c5a9c9083e449219887b9e8e20",
-      "dependencies": [
-        "jsr:@nostrify/nostrify@0.35",
-        "jsr:@nostrify/types@0.35",
-        "npm:nostr-tools@^2.7.0"
-      ]
-    },
     "@nostrify/policies@0.36.0": {
       "integrity": "ad1930de48ce03cdf34da456af1563b487581d1d86683cd416ad760ae40b1fb3",
       "dependencies": [
@@ -503,6 +516,9 @@
     "@nostrify/types@0.35.0": {
       "integrity": "b8d515563d467072694557d5626fa1600f74e83197eef45dd86a9a99c64f7fe6"
     },
+    "@nostrify/types@0.36.0": {
+      "integrity": "b3413467debcbd298d217483df4e2aae6c335a34765c90ac7811cf7c637600e7"
+    },
     "@soapbox/kysely-pglite@1.0.0": {
       "integrity": "0954b1bf3deab051c479cba966b1e6ed5a0a966aa21d1f40143ec8f5efcd475d",
       "dependencies": [
@@ -545,6 +561,9 @@
     "@std/bytes@1.0.2": {
       "integrity": "fbdee322bbd8c599a6af186a1603b3355e59a5fb1baa139f8f4c3c9a1b3e3d57"
     },
+    "@std/bytes@1.0.4": {
+      "integrity": "11a0debe522707c95c7b7ef89b478c13fb1583a7cfb9a85674cd2cc2e3a28abc"
+    },
     "@std/cli@0.223.0": {
       "integrity": "2feb7970f2028904c3edc22ea916ce9538113dfc170844f3eae03578c333c356",
       "dependencies": [
@@ -601,6 +620,9 @@
     "@std/internal@1.0.4": {
       "integrity": "62e8e4911527e5e4f307741a795c0b0a9e6958d0b3790716ae71ce085f755422"
     },
+    "@std/internal@1.0.5": {
+      "integrity": "54a546004f769c1ac9e025abd15a76b6671ddc9687e2313b67376125650dc7ba"
+    },
     "@std/io@0.223.0": {
       "integrity": "2d8c3c2ab3a515619b90da2c6ff5ea7b75a94383259ef4d02116b228393f84f1",
       "dependencies": [
@@ -650,6 +672,12 @@
         "jsr:@std/bytes@^1.0.2"
       ]
     },
+    "@std/io@0.224.9": {
+      "integrity": "4414664b6926f665102e73c969cfda06d2c4c59bd5d0c603fd4f1b1c840d6ee3",
+      "dependencies": [
+        "jsr:@std/bytes@^1.0.2"
+      ]
+    },
     "@std/json@0.223.0": {
       "integrity": "9a4a255931dd0397924c6b10bb6a72fe3e28ddd876b981ada2e3b8dd0764163f",
       "dependencies": [
@@ -2308,8 +2336,9 @@
       "jsr:@lambdalisue/async@^2.1.1",
       "jsr:@negrel/webpush@0.3",
       "jsr:@nostrify/db@~0.36.1",
-      "jsr:@nostrify/nostrify@0.36",
+      "jsr:@nostrify/nostrify@0.37",
-      "jsr:@nostrify/policies@0.35",
+      "jsr:@nostrify/policies@~0.36.1",
+      "jsr:@nostrify/types@0.36",
       "jsr:@soapbox/kysely-pglite@1",
       "jsr:@soapbox/safe-fetch@2",
       "jsr:@soapbox/stickynotes@0.4",

installation/Caddyfile

@@ -0,0 +1,35 @@
+# Cloudflare real IP configuration for rate-limiting
+# {
+#   servers {
+#     # https://www.cloudflare.com/ips/
+#     trusted_proxies static 173.245.48.0/20 103.21.244.0/22 103.22.200.0/22 103.31.4.0/22 141.101.64.0/18 108.162.192.0/18 190.93.240.0/20 188.114.96.0/20 197.234.240.0/22 198.41.128.0/17 162.158.0.0/15 104.16.0.0/13 104.24.0.0/14 172.64.0.0/13 131.0.72.0/22 2400:cb00::/32 2606:4700::/32 2803:f800::/32 2405:b500::/32 2405:8100::/32 2a06:98c0::/29 2c0f:f248::/32
+#     trusted_proxies_strict
+#   }
+# }
+
+example.com {
+  log
+  request_header X-Real-IP {client_ip}
+
+  @public path /packs/* /instance/* /images/* /favicon.ico /sw.js /sw.js.map
+
+  handle /packs/* {
+    root * /opt/ditto/public
+    header Cache-Control "public, max-age=31536000, immutable"
+    header Strict-Transport-Security "max-age=31536000"
+    file_server
+  }
+
+  handle @public {
+    root * /opt/ditto/public
+    file_server
+  }
+
+  handle /metrics {
+    respond "Access denied" 403
+  }
+
+  handle {
+    reverse_proxy :4036
+  }
+}

scripts/nostr-pull.ts

@@ -47,7 +47,7 @@ const importUsers = async (
     if (!profilesOnly) {
       matched.push(
         ...await conn.query(
-          authors.map((author) => ({ kinds: [1], authors: [author], limit: 200 })),
+          authors.map((author) => ({ kinds: [1, 20], authors: [author], limit: 200 })),
         ),
       );
     }

src/app.ts

@@ -133,6 +133,7 @@ import { DittoTranslator } from '@/interfaces/DittoTranslator.ts';
 import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts';
 import { cspMiddleware } from '@/middleware/cspMiddleware.ts';
 import { metricsMiddleware } from '@/middleware/metricsMiddleware.ts';
+import { notActivitypubMiddleware } from '@/middleware/notActivitypubMiddleware.ts';
 import { paginationMiddleware } from '@/middleware/paginationMiddleware.ts';
 import { rateLimitMiddleware } from '@/middleware/rateLimitMiddleware.ts';
 import { requireSigner } from '@/middleware/requireSigner.ts';
@@ -179,7 +180,6 @@ app.use('*', rateLimitMiddleware(300, Time.minutes(5)));
 
 app.use('/api/*', metricsMiddleware, paginationMiddleware, logger(debug));
 app.use('/.well-known/*', metricsMiddleware, logger(debug));
-app.use('/users/*', metricsMiddleware, logger(debug));
 app.use('/nodeinfo/*', metricsMiddleware, logger(debug));
 app.use('/oauth/*', metricsMiddleware, logger(debug));
 
@@ -400,18 +400,24 @@ app.use('/oauth/*', notImplementedController);
 app.get('/:acct{@.*}', frontendController);
 app.get('/:acct{@.*}/*', frontendController);
 app.get('/:bech32{^[\x21-\x7E]{1,83}1[023456789acdefghjklmnpqrstuvwxyz]{6,}$}', frontendController);
-app.get('/users/*', frontendController);
+app.get('/users/*', notActivitypubMiddleware, frontendController);
 app.get('/tags/*', frontendController);
 app.get('/statuses/*', frontendController);
 app.get('/notice/*', frontendController);
 app.get('/timeline/*', frontendController);
 
 // Known static file routes
+app.get('/sw.js', publicFiles);
 app.get('/favicon.ico', publicFiles, staticFiles);
 app.get('/images/*', publicFiles, staticFiles);
 app.get('/instance/*', publicFiles);
-app.get('/packs/*', publicFiles);
+
-app.get('/sw.js', publicFiles);
+// Packs contains immutable static files
+app.get('/packs/*', async (c, next) => {
+  c.header('Cache-Control', 'public, max-age=31536000, immutable');
+  c.header('Strict-Transport-Security', '"max-age=31536000" always');
+  await next();
+}, publicFiles);
 
 // Site index
 app.get('/', frontendController, indexController);

src/caches/pipelineEncounters.ts

@@ -0,0 +1,3 @@
+import { LRUCache } from 'lru-cache';
+
+export const pipelineEncounters = new LRUCache<string, true>({ max: 5000 });

src/config.ts

@@ -252,7 +252,7 @@ class Conf {
   }
   /** Nostr event kinds of events to listen for on the firehose. */
   static get firehoseKinds(): number[] {
-    return (Deno.env.get('FIREHOSE_KINDS') ?? '0, 1, 3, 5, 6, 7, 9735, 10002')
+    return (Deno.env.get('FIREHOSE_KINDS') ?? '0, 1, 3, 5, 6, 7, 20, 9735, 10002')
       .split(/[, ]+/g)
       .map(Number);
   }

src/controllers/api/accounts.ts

@@ -235,7 +235,7 @@ const accountStatusesController: AppController = async (c) => {
 
   const filter: NostrFilter = {
     authors: [pubkey],
-    kinds: [1, 6],
+    kinds: [1, 6, 20],
     since,
     until,
     limit,
@@ -478,7 +478,7 @@ const favouritesController: AppController = async (c) => {
     .map((event) => event.tags.find((tag) => tag[0] === 'e')?.[1])
     .filter((id): id is string => !!id);
 
-  const events1 = await store.query([{ kinds: [1], ids }], { signal })
+  const events1 = await store.query([{ kinds: [1, 20], ids }], { signal })
     .then((events) => hydrateEvents({ events, store, signal }));
 
   const viewerPubkey = await c.get('signer')?.getPublicKey();

src/controllers/api/ditto.ts

@@ -260,7 +260,7 @@ export const statusZapSplitsController: AppController = async (c) => {
   const id = c.req.param('id');
   const { signal } = c.req.raw;
 
-  const [event] = await store.query([{ kinds: [1], ids: [id], limit: 1 }], { signal });
+  const [event] = await store.query([{ kinds: [1, 20], ids: [id], limit: 1 }], { signal });
   if (!event) {
     return c.json({ error: 'Event not found' }, 404);
   }

src/controllers/api/reactions.ts

@@ -20,7 +20,7 @@ const reactionController: AppController = async (c) => {
   }
 
   const store = await Storages.db();
-  const [event] = await store.query([{ kinds: [1], ids: [id], limit: 1 }]);
+  const [event] = await store.query([{ kinds: [1, 20], ids: [id], limit: 1 }]);
 
   if (!event) {
     return c.json({ error: 'Status not found' }, 404);
@@ -56,7 +56,7 @@ const deleteReactionController: AppController = async (c) => {
   }
 
   const [event] = await store.query([
-    { kinds: [1], ids: [id], limit: 1 },
+    { kinds: [1, 20], ids: [id], limit: 1 },
   ]);
 
   if (!event) {

src/controllers/api/search.ts

@@ -166,7 +166,7 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort
   if (n.id().safeParse(q).success) {
     const filters: NostrFilter[] = [];
     if (accounts) filters.push({ kinds: [0], authors: [q] });
-    if (statuses) filters.push({ kinds: [1], ids: [q] });
+    if (statuses) filters.push({ kinds: [1, 20], ids: [q] });
     return filters;
   }
 
@@ -184,10 +184,10 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort
         if (accounts) filters.push({ kinds: [0], authors: [result.data.pubkey] });
         break;
       case 'note':
-        if (statuses) filters.push({ kinds: [1], ids: [result.data] });
+        if (statuses) filters.push({ kinds: [1, 20], ids: [result.data] });
         break;
       case 'nevent':
-        if (statuses) filters.push({ kinds: [1], ids: [result.data.id] });
+        if (statuses) filters.push({ kinds: [1, 20], ids: [result.data.id] });
         break;
     }
     return filters;

src/controllers/api/statuses.ts

@@ -151,7 +151,7 @@ const createStatusController: AppController = async (c) => {
 
   let content = await asyncReplaceAll(
     data.status ?? '',
-    /(?<![\w/])@([\w@+._]+)(?![\w/\.])/g,
+    /(?<![\w/])@([\w@+._-]+)(?![\w/\.])/g,
     async (match, username) => {
       const pubkey = await lookupPubkey(username);
       if (!pubkey) return match;
@@ -290,7 +290,7 @@ const deleteStatusController: AppController = async (c) => {
 const contextController: AppController = async (c) => {
   const id = c.req.param('id');
   const store = c.get('store');
-  const event = await getEvent(id, { kind: 1, relations: ['author', 'event_stats', 'author_stats'] });
+  const [event] = await store.query([{ kinds: [1, 20], ids: [id] }]);
   const viewerPubkey = await c.get('signer')?.getPublicKey();
 
   async function renderStatuses(events: NostrEvent[]) {
@@ -325,7 +325,8 @@ const contextController: AppController = async (c) => {
 
 const favouriteController: AppController = async (c) => {
   const id = c.req.param('id');
-  const target = await getEvent(id, { kind: 1, relations: ['author', 'event_stats', 'author_stats'] });
+  const store = await Storages.db();
+  const [target] = await store.query([{ ids: [id], kinds: [1, 20] }]);
 
   if (target) {
     await createEvent({
@@ -337,6 +338,8 @@ const favouriteController: AppController = async (c) => {
       ],
     }, c);
 
+    await hydrateEvents({ events: [target], store });
+
     const status = await renderStatus(target, { viewerPubkey: await c.get('signer')?.getPublicKey() });
 
     if (status) {
@@ -397,7 +400,7 @@ const unreblogStatusController: AppController = async (c) => {
   const pubkey = await c.get('signer')?.getPublicKey()!;
   const store = await Storages.db();
 
-  const [event] = await store.query([{ ids: [eventId], kinds: [1] }]);
+  const [event] = await store.query([{ ids: [eventId], kinds: [1, 20] }]);
   if (!event) {
     return c.json({ error: 'Record not found' }, 404);
   }
@@ -429,13 +432,13 @@ const quotesController: AppController = async (c) => {
   const params = c.get('pagination');
   const store = await Storages.db();
 
-  const [event] = await store.query([{ ids: [id], kinds: [1] }]);
+  const [event] = await store.query([{ ids: [id], kinds: [1, 20] }]);
   if (!event) {
     return c.json({ error: 'Event not found.' }, 404);
   }
 
   const quotes = await store
-    .query([{ kinds: [1], '#q': [event.id], ...params }])
+    .query([{ kinds: [1, 20], '#q': [event.id], ...params }])
     .then((events) => hydrateEvents({ events, store }));
 
   const viewerPubkey = await c.get('signer')?.getPublicKey();

src/controllers/api/streaming.ts

@@ -18,6 +18,7 @@ import { getTokenHash } from '@/utils/auth.ts';
 import { bech32ToPubkey, Time } from '@/utils.ts';
 import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts';
 import { renderNotification } from '@/views/mastodon/notifications.ts';
+import { HTTPException } from '@hono/hono/http-exception';
 
 const console = new Stickynotes('ditto:streaming');
 
@@ -214,20 +215,20 @@ async function topicToFilter(
 
   switch (topic) {
     case 'public':
-      return { kinds: [1, 6] };
+      return { kinds: [1, 6, 20] };
     case 'public:local':
-      return { kinds: [1, 6], search: `domain:${host}` };
+      return { kinds: [1, 6, 20], search: `domain:${host}` };
     case 'hashtag':
-      if (query.tag) return { kinds: [1, 6], '#t': [query.tag] };
+      if (query.tag) return { kinds: [1, 6, 20], '#t': [query.tag] };
       break;
     case 'hashtag:local':
-      if (query.tag) return { kinds: [1, 6], '#t': [query.tag], search: `domain:${host}` };
+      if (query.tag) return { kinds: [1, 6, 20], '#t': [query.tag], search: `domain:${host}` };
       break;
     case 'user':
       // HACK: this puts the user's entire contacts list into RAM,
       // and then calls `matchFilters` over it. Refreshing the page
       // is required after following a new user.
-      return pubkey ? { kinds: [1, 6], authors: [...await getFeedPubkeys(pubkey)] } : undefined;
+      return pubkey ? { kinds: [1, 6, 20], authors: [...await getFeedPubkeys(pubkey)] } : undefined;
   }
 }
 
@@ -236,13 +237,17 @@ async function getTokenPubkey(token: string): Promise<string | undefined> {
     const kysely = await Storages.kysely();
     const tokenHash = await getTokenHash(token as `token1${string}`);
 
-    const { pubkey } = await kysely
+    const row = await kysely
       .selectFrom('auth_tokens')
       .select('pubkey')
       .where('token_hash', '=', tokenHash)
-      .executeTakeFirstOrThrow();
+      .executeTakeFirst();
 
-    return pubkey;
+    if (!row) {
+      throw new HTTPException(401, { message: 'Invalid access token' });
+    }
+
+    return row.pubkey;
   } else {
     return bech32ToPubkey(token);
   }

src/controllers/api/timelines.ts

@@ -14,7 +14,7 @@ const homeTimelineController: AppController = async (c) => {
   const params = c.get('pagination');
   const pubkey = await c.get('signer')?.getPublicKey()!;
   const authors = [...await getFeedPubkeys(pubkey)];
-  return renderStatuses(c, [{ authors, kinds: [1, 6], ...params }]);
+  return renderStatuses(c, [{ authors, kinds: [1, 6, 20], ...params }]);
 };
 
 const publicQuerySchema = z.object({
@@ -33,7 +33,7 @@ const publicTimelineController: AppController = (c) => {
 
   const { local, instance, language } = result.data;
 
-  const filter: NostrFilter = { kinds: [1], ...params };
+  const filter: NostrFilter = { kinds: [1, 20], ...params };
 
   const search: `${string}:${string}`[] = [];
 
@@ -57,7 +57,7 @@ const publicTimelineController: AppController = (c) => {
 const hashtagTimelineController: AppController = (c) => {
   const hashtag = c.req.param('hashtag')!.toLowerCase();
   const params = c.get('pagination');
-  return renderStatuses(c, [{ kinds: [1], '#t': [hashtag], ...params }]);
+  return renderStatuses(c, [{ kinds: [1, 20], '#t': [hashtag], ...params }]);
 };
 
 const suggestedTimelineController: AppController = async (c) => {
@@ -70,7 +70,7 @@ const suggestedTimelineController: AppController = async (c) => {
 
   const authors = [...getTagSet(follows?.tags ?? [], 'p')];
 
-  return renderStatuses(c, [{ authors, kinds: [1], ...params }]);
+  return renderStatuses(c, [{ authors, kinds: [1, 20], ...params }]);
 };
 
 /** Render statuses for timelines. */

src/controllers/api/trends.ts

@@ -134,7 +134,7 @@ const trendingStatusesController: AppController = async (c) => {
     return c.json([]);
   }
 
-  const results = await store.query([{ kinds: [1], ids }])
+  const results = await store.query([{ kinds: [1, 20], ids }])
     .then((events) => hydrateEvents({ events, store }));
 
   // Sort events in the order they appear in the label.

src/controllers/nostr/relay.ts

@@ -138,7 +138,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) {
     relayEventsCounter.inc({ kind: event.kind.toString() });
     try {
       // This will store it (if eligible) and run other side-effects.
-      await pipeline.handleEvent(purifyEvent(event), AbortSignal.timeout(1000));
+      await pipeline.handleEvent(purifyEvent(event), { source: 'relay', signal: AbortSignal.timeout(1000) });
       send(['OK', event.id, true, '']);
     } catch (e) {
       if (e instanceof RelayError) {

src/firehose.ts

@@ -27,7 +27,7 @@ export async function startFirehose(): Promise<void> {
 
       sem.lock(async () => {
         try {
-          await pipeline.handleEvent(event, AbortSignal.timeout(5000));
+          await pipeline.handleEvent(event, { source: 'firehose', signal: AbortSignal.timeout(5000) });
         } catch (e) {
           console.warn(e);
         }

src/middleware/notActivitypubMiddleware.ts

@@ -0,0 +1,19 @@
+import { MiddlewareHandler } from '@hono/hono';
+
+const ACTIVITYPUB_TYPES = [
+  'application/activity+json',
+  'application/ld+json',
+  'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
+];
+
+/** Return 4xx errors on common (unsupported) ActivityPub routes to prevent AP traffic. */
+export const notActivitypubMiddleware: MiddlewareHandler = async (c, next) => {
+  const accept = c.req.header('accept');
+  const types = accept?.split(',')?.map((type) => type.trim()) ?? [];
+
+  if (types.every((type) => ACTIVITYPUB_TYPES.includes(type))) {
+    return c.text('ActivityPub is not supported', 406);
+  }
+
+  await next();
+};

src/notify.ts

@@ -1,24 +1,32 @@
 import { Semaphore } from '@lambdalisue/async';
+import { Stickynotes } from '@soapbox/stickynotes';
 
+import { pipelineEncounters } from '@/caches/pipelineEncounters.ts';
 import { Conf } from '@/config.ts';
 import * as pipeline from '@/pipeline.ts';
 import { Storages } from '@/storages.ts';
 
 const sem = new Semaphore(1);
+const console = new Stickynotes('ditto:notify');
 
 export async function startNotify(): Promise<void> {
   const { listen } = await Storages.database();
   const store = await Storages.db();
 
-  listen('nostr_event', (payload) => {
+  listen('nostr_event', (id) => {
+    if (pipelineEncounters.has(id)) {
+      console.debug(`Skip event ${id} because it was already in the pipeline`);
+      return;
+    }
+
     sem.lock(async () => {
       try {
-        const id = payload;
+        const signal = AbortSignal.timeout(Conf.db.timeouts.default);
-        const timeout = Conf.db.timeouts.default;
+
+        const [event] = await store.query([{ ids: [id], limit: 1 }], { signal });
 
-        const [event] = await store.query([{ ids: [id], limit: 1 }], { signal: AbortSignal.timeout(timeout) });
         if (event) {
-          await pipeline.handleEvent(event, AbortSignal.timeout(timeout));
+          await pipeline.handleEvent(event, { source: 'notify', signal });
         }
       } catch (e) {
         console.warn(e);

src/pipeline.ts

@@ -1,7 +1,6 @@
 import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify';
 import { Stickynotes } from '@soapbox/stickynotes';
 import { Kysely, sql } from 'kysely';
-import { LRUCache } from 'lru-cache';
 import { z } from 'zod';
 
 import { Conf } from '@/config.ts';
@@ -23,14 +22,25 @@ import { getTagSet } from '@/utils/tags.ts';
 import { renderWebPushNotification } from '@/views/mastodon/push.ts';
 import { policyWorker } from '@/workers/policy.ts';
 import { verifyEventWorker } from '@/workers/verify.ts';
+import { pipelineEncounters } from '@/caches/pipelineEncounters.ts';
 
 const console = new Stickynotes('ditto:pipeline');
 
+interface PipelineOpts {
+  signal: AbortSignal;
+  source: 'relay' | 'api' | 'firehose' | 'pipeline' | 'notify' | 'internal';
+}
+
 /**
  * Common pipeline function to process (and maybe store) events.
  * It is idempotent, so it can be called multiple times for the same event.
  */
-async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise<void> {
+async function handleEvent(event: DittoEvent, opts: PipelineOpts): Promise<void> {
+  // Skip events that have already been encountered.
+  if (pipelineEncounters.get(event.id)) {
+    throw new RelayError('duplicate', 'already have this event');
+  }
+  // Reject events that are too far in the future.
   if (eventAge(event) < -Time.minutes(1)) {
     throw new RelayError('invalid', 'event too far in the future');
   }
@@ -51,11 +61,12 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise<void
   if (!(await verifyEventWorker(event))) {
     throw new RelayError('invalid', 'invalid signature');
   }
-  // Skip events that have been recently encountered.
+  // Recheck encountered after async ops.
-  // We must do this after verifying the signature.
+  if (pipelineEncounters.has(event.id)) {
-  if (encounterEvent(event)) {
     throw new RelayError('duplicate', 'already have this event');
   }
+  // Set the event as encountered after verifying the signature.
+  pipelineEncounters.set(event.id, true);
 
   // Log the event.
   console.info(`NostrEvent<${event.kind}> ${event.id}`);
@@ -71,12 +82,12 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise<void
 
   // Ensure the event doesn't violate the policy.
   if (event.pubkey !== Conf.pubkey) {
-    await policyFilter(event, signal);
+    await policyFilter(event, opts.signal);
   }
 
   // Prepare the event for additional checks.
   // FIXME: This is kind of hacky. Should be reorganized to fetch only what's needed for each stage.
-  await hydrateEvent(event, signal);
+  await hydrateEvent(event, opts.signal);
 
   // Ensure that the author is not banned.
   const n = getTagSet(event.user?.tags ?? [], 'n');
@@ -93,15 +104,24 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise<void
     return;
   }
 
+  // Events received through notify are thought to already be in the database, so they only need to be streamed.
+  if (opts.source === 'notify') {
+    await Promise.all([
+      streamOut(event),
+      webPush(event),
+    ]);
+    return;
+  }
+
   const kysely = await Storages.kysely();
 
   try {
-    await storeEvent(purifyEvent(event), signal);
+    await storeEvent(purifyEvent(event), opts.signal);
   } finally {
     // This needs to run in steps, and should not block the API from responding.
     Promise.allSettled([
       handleZaps(kysely, event),
-      parseMetadata(event, signal),
+      parseMetadata(event, opts.signal),
       setLanguage(event),
       setMimeType(event),
       generateSetEvents(event),
@@ -133,17 +153,6 @@ async function policyFilter(event: NostrEvent, signal: AbortSignal): Promise<voi
   }
 }
 
-const encounters = new LRUCache<string, true>({ max: 1000 });
-
-/** Encounter the event, and return whether it has already been encountered. */
-function encounterEvent(event: NostrEvent): boolean {
-  const encountered = !!encounters.get(event.id);
-  if (!encountered) {
-    encounters.set(event.id, true);
-  }
-  return encountered;
-}
-
 /** Check whether the event has a NIP-70 `-` tag. */
 function isProtectedEvent(event: NostrEvent): boolean {
   return event.tags.some(([name]) => name === '-');
@@ -346,7 +355,7 @@ async function generateSetEvents(event: NostrEvent): Promise<void> {
       created_at: Math.floor(Date.now() / 1000),
     });
 
-    await handleEvent(rel, AbortSignal.timeout(1000));
+    await handleEvent(rel, { source: 'pipeline', signal: AbortSignal.timeout(1000) });
   }
 
   if (event.kind === 3036 && tagsAdmin) {
@@ -364,7 +373,7 @@ async function generateSetEvents(event: NostrEvent): Promise<void> {
       created_at: Math.floor(Date.now() / 1000),
     });
 
-    await handleEvent(rel, AbortSignal.timeout(1000));
+    await handleEvent(rel, { source: 'pipeline', signal: AbortSignal.timeout(1000) });
   }
 }
 

src/queries.ts

@@ -16,7 +16,7 @@ interface GetEventOpts {
   signal?: AbortSignal;
   /** Event kind. */
   kind?: number;
-  /** Relations to include on the event. */
+  /** @deprecated Relations to include on the event. */
   relations?: DittoRelation[];
 }
 

src/storages/EventsDB.ts

@@ -308,6 +308,7 @@ class EventsDB extends NPostgres {
       case 0:
         return EventsDB.buildUserSearchContent(event);
       case 1:
+      case 20:
         return nip27.replaceAll(event.content, () => '');
       case 30009:
         return EventsDB.buildTagsSearchContent(event.tags.filter(([t]) => t !== 'alt'));

src/storages/hydrate.ts

@@ -102,21 +102,21 @@ export function assembleEvents(
     if (event.kind === 1) {
       const id = findQuoteTag(event.tags)?.[1] || findQuoteInContent(event.content);
       if (id) {
-        event.quote = b.find((e) => matchFilter({ kinds: [1], ids: [id] }, e));
+        event.quote = b.find((e) => matchFilter({ kinds: [1, 20], ids: [id] }, e));
       }
     }
 
     if (event.kind === 6) {
       const id = event.tags.find(([name]) => name === 'e')?.[1];
       if (id) {
-        event.repost = b.find((e) => matchFilter({ kinds: [1], ids: [id] }, e));
+        event.repost = b.find((e) => matchFilter({ kinds: [1, 20], ids: [id] }, e));
       }
     }
 
     if (event.kind === 7) {
       const id = event.tags.findLast(([name]) => name === 'e')?.[1];
       if (id) {
-        event.reacted = b.find((e) => matchFilter({ kinds: [1], ids: [id] }, e));
+        event.reacted = b.find((e) => matchFilter({ kinds: [1, 20], ids: [id] }, e));
       }
     }
 
@@ -130,7 +130,7 @@ export function assembleEvents(
       const ids = event.tags.filter(([name]) => name === 'e').map(([_name, value]) => value);
 
       for (const id of ids) {
-        const reported = b.find((e) => matchFilter({ kinds: [1], ids: [id] }, e));
+        const reported = b.find((e) => matchFilter({ kinds: [1, 20], ids: [id] }, e));
         if (reported) {
           reportedEvents.push(reported);
         }
@@ -146,7 +146,7 @@ export function assembleEvents(
 
       const id = event.tags.find(([name]) => name === 'e')?.[1];
       if (id) {
-        event.zapped = b.find((e) => matchFilter({ kinds: [1], ids: [id] }, e));
+        event.zapped = b.find((e) => matchFilter({ kinds: [1, 20], ids: [id] }, e));
       }
 
       const zapRequestString = event?.tags?.find(([name]) => name === 'description')?.[1];
@@ -313,7 +313,7 @@ function gatherReportedNotes({ events, store, signal }: HydrateOpts): Promise<Di
   }
 
   return store.query(
-    [{ kinds: [1], ids: [...ids], limit: ids.size }],
+    [{ kinds: [1, 20], ids: [...ids], limit: ids.size }],
     { signal },
   );
 }

src/trends.ts

@@ -111,7 +111,7 @@ export async function updateTrendingTags(
       created_at: Math.floor(Date.now() / 1000),
     });
 
-    await handleEvent(label, signal);
+    await handleEvent(label, { source: 'internal', signal });
     console.info(`Trending ${l} updated.`);
   } catch (e) {
     console.error(`Error updating trending ${l}: ${e instanceof Error ? e.message : e}`);

src/utils/api.ts

@@ -161,7 +161,7 @@ async function updateNames(k: number, d: string, n: Record<string, boolean>, c:
 async function publishEvent(event: NostrEvent, c: AppContext): Promise<NostrEvent> {
   debug('EVENT', event);
   try {
-    await pipeline.handleEvent(event, c.req.raw.signal);
+    await pipeline.handleEvent(event, { source: 'api', signal: c.req.raw.signal });
     const client = await Storages.client();
     await client.event(purifyEvent(event));
   } catch (e) {
@@ -261,24 +261,6 @@ function paginatedList(
   return c.json(results, 200, headers);
 }
 
-/** JSON-LD context. */
-type LDContext = (string | Record<string, string | Record<string, string>>)[];
-
-/** Add a basic JSON-LD context to ActivityStreams object, if it doesn't already exist. */
-function maybeAddContext<T>(object: T): T & { '@context': LDContext } {
-  return {
-    '@context': ['https://www.w3.org/ns/activitystreams'],
-    ...object,
-  };
-}
-
-/** Like hono's `c.json()` except returns JSON-LD. */
-function activityJson<T, P extends string>(c: Context<any, P>, object: T) {
-  const response = c.json(maybeAddContext(object));
-  response.headers.set('content-type', 'application/activity+json; charset=UTF-8');
-  return response;
-}
-
 /** Rewrite the URL of the request object to use the local domain. */
 function localRequest(c: Context): Request {
   return Object.create(c.req.raw, {
@@ -300,7 +282,6 @@ function assertAuthenticated(c: AppContext, author: NostrEvent): void {
 }
 
 export {
-  activityJson,
   assertAuthenticated,
   createAdminEvent,
   createEvent,

src/utils/stats.ts

@@ -273,7 +273,7 @@ export async function countAuthorStats(
 ): Promise<DittoTables['author_stats']> {
   const [{ count: followers_count }, { count: notes_count }, [followList], [kind0]] = await Promise.all([
     store.count([{ kinds: [3], '#p': [pubkey] }]),
-    store.count([{ kinds: [1], authors: [pubkey] }]),
+    store.count([{ kinds: [1, 20], authors: [pubkey] }]),
     store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]),
     store.query([{ kinds: [0], authors: [pubkey], limit: 1 }]),
   ]);

src/views.ts

@@ -75,7 +75,7 @@ async function renderStatuses(c: AppContext, ids: string[], signal = AbortSignal
   const store = await Storages.db();
   const { limit } = c.get('pagination');
 
-  const events = await store.query([{ kinds: [1], ids, limit }], { signal })
+  const events = await store.query([{ kinds: [1, 20], ids, limit }], { signal })
     .then((events) => hydrateEvents({ events, store, signal }));
 
   if (!events.length) {