Merge branch 'main' into make-sizes-optional

deno.json

@@ -14,7 +14,7 @@
     "nsec": "deno run scripts/nsec.ts",
     "admin:event": "deno run -A --env-file --deny-read=.env scripts/admin-event.ts",
     "admin:role": "deno run -A --env-file --deny-read=.env scripts/admin-role.ts",
-    "setup": "deno run -A --env-file --deny-read=.env scripts/setup.ts",
+    "setup": "deno run -A --env-file scripts/setup.ts",
     "setup:kind0": "deno run -A --env-file --deny-read=.env scripts/setup-kind0.ts",
     "stats:recompute": "deno run -A --env-file --deny-read=.env scripts/stats-recompute.ts",
     "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip -o soapbox.zip && rm soapbox.zip",
@@ -50,6 +50,7 @@
     "@scure/base": "npm:@scure/base@^1.1.6",
     "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs",
     "@soapbox/kysely-pglite": "jsr:@soapbox/kysely-pglite@^1.0.0",
+    "@soapbox/safe-fetch": "jsr:@soapbox/safe-fetch@^2.0.0",
     "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0",
     "@std/assert": "jsr:@std/assert@^0.225.1",
     "@std/cli": "jsr:@std/cli@^0.223.0",
@@ -63,7 +64,6 @@
     "comlink": "npm:comlink@^4.4.1",
     "comlink-async-generator": "npm:comlink-async-generator@^0.0.1",
     "commander": "npm:commander@12.1.0",
-    "deno-safe-fetch/load": "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts",
     "deno.json": "./deno.json",
     "entities": "npm:entities@^4.5.0",
     "fast-stable-stringify": "npm:fast-stable-stringify@^1.0.0",

deno.lock

@@ -46,6 +46,7 @@
     "jsr:@nostrify/types@0.35": "0.35.0",
     "jsr:@nostrify/types@~0.30.1": "0.30.1",
     "jsr:@soapbox/kysely-pglite@1": "1.0.0",
+    "jsr:@soapbox/safe-fetch@2": "2.0.0",
     "jsr:@soapbox/stickynotes@0.4": "0.4.0",
     "jsr:@std/assert@0.223": "0.223.0",
     "jsr:@std/assert@0.224": "0.224.0",
@@ -120,6 +121,7 @@
     "npm:prom-client@^15.1.2": "15.1.2",
     "npm:sharp@~0.33.5": "0.33.5",
     "npm:tldts@^6.0.14": "6.1.18",
+    "npm:tldts@^6.1.61": "6.1.61",
     "npm:tseep@^1.2.1": "1.2.1",
     "npm:type-fest@^4.3.0": "4.18.2",
     "npm:unfurl.js@^6.4.0": "6.4.0",
@@ -483,6 +485,12 @@
         "npm:kysely@~0.27.4"
       ]
     },
+    "@soapbox/safe-fetch@2.0.0": {
+      "integrity": "f451d686501c76a0faa058fe9d2073676282a8a42c3b93c59159eb9191f11b5f",
+      "dependencies": [
+        "npm:tldts@^6.1.61"
+      ]
+    },
     "@soapbox/stickynotes@0.4.0": {
       "integrity": "60bfe61ab3d7e04bf708273b1e2d391a59534bdf29e54160e98d7afd328ca1ec"
     },
@@ -1522,6 +1530,9 @@
     "tldts-core@6.1.49": {
       "integrity": "sha512-ctRO/wzBasOCxAStJG/60Qe8/QpGmaVPsE8djdk0vioxN4uCOgKoveH71Qc2EOmVMIjVf0BjigI5p9ZDuLOygg=="
     },
+    "tldts-core@6.1.61": {
+      "integrity": "sha512-In7VffkDWUPgwa+c9picLUxvb0RltVwTkSgMNFgvlGSWveCzGBemBqTsgJCL4EDFWZ6WH0fKTsot6yNhzy3ZzQ=="
+    },
     "tldts@6.1.18": {
       "integrity": "sha512-F+6zjPFnFxZ0h6uGb8neQWwHQm8u3orZVFribsGq4eBgEVrzSkHxzWS2l6aKr19T1vXiOMFjqfff4fQt+WgJFg==",
       "dependencies": [
@@ -1534,6 +1545,12 @@
         "tldts-core@6.1.49"
       ]
     },
+    "tldts@6.1.61": {
+      "integrity": "sha512-rv8LUyez4Ygkopqn+M6OLItAOT9FF3REpPQDkdMx5ix8w4qkuE7Vo2o/vw1nxKQYmJDV8JpAMJQr1b+lTKf0FA==",
+      "dependencies": [
+        "tldts-core@6.1.61"
+      ]
+    },
     "to-regex-range@5.0.1": {
       "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
       "dependencies": [
@@ -2181,8 +2198,6 @@
     "https://deno.land/x/sentry@7.112.2/index.mjs": "04382d5c2f4e233ba389611db46f77943b2a7f6efbeaaf31193f6e586f4366ef",
     "https://esm.sh/kysely@0.17.1/dist/esm/index-nodeless.js": "9c23bfd307118e3ccd3a9f0ec1261fc3451fb5301aa34aa6f28e05156818755a",
     "https://esm.sh/v135/kysely@0.17.1/denonext/dist/esm/index-nodeless.js": "6f73bbf2d73bc7e96cdabf941c4ae8c12f58fd7b441031edec44c029aed9532b",
-    "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts": "3f74ab08cf97d4a3e6994cb79422e9b0069495e017416858121d5ff8ae04ac2a",
-    "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/mod.ts": "5f505cd265aefbcb687cde6f98c79344d3292ee1dd978e85e5ffa84a617c6682",
     "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/b4725e74ad6ca359ba0e370b55dbb8bb845a8a83/deps.ts": "b3dbecae69c30a5f161323b8c8ebd91d9af1eceb98fafab3091c7281a4b64fed",
     "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/b4725e74ad6ca359ba0e370b55dbb8bb845a8a83/mod.ts": "662438fd3909984bb8cbaf3fd44d2121e949d11301baf21d6c3f057ccf9887de",
     "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/b4725e74ad6ca359ba0e370b55dbb8bb845a8a83/src/PostgreSQLDriver.ts": "ea5a523bceeed420858b744beeb95d48976cb2b0d3f519a68b65a8229036cf6a",
@@ -2272,6 +2287,7 @@
       "jsr:@nostrify/nostrify@0.36",
       "jsr:@nostrify/policies@0.35",
       "jsr:@soapbox/kysely-pglite@1",
+      "jsr:@soapbox/safe-fetch@2",
       "jsr:@soapbox/stickynotes@0.4",
       "jsr:@std/assert@~0.225.1",
       "jsr:@std/cli@0.223",

src/schemas/activitypub.ts

@@ -1,323 +0,0 @@
-import { z } from 'zod';
-
-const apId = z.string().url();
-const recipients = z.array(z.string()).catch([]);
-const published = () => z.string().datetime().catch(new Date().toISOString());
-
-/** Validates individual items in an array, dropping any that aren't valid. */
-function filteredArray<T extends z.ZodTypeAny>(schema: T) {
-  return z.any().array()
-    .transform((arr) => (
-      arr.map((item) => {
-        const parsed = schema.safeParse(item);
-        return parsed.success ? parsed.data : undefined;
-      }).filter((item): item is z.infer<T> => Boolean(item))
-    ));
-}
-
-const imageSchema = z.object({
-  type: z.literal('Image').catch('Image'),
-  url: z.string().url(),
-});
-
-const attachmentSchema = z.object({
-  type: z.literal('Document').catch('Document'),
-  mediaType: z.string().optional().catch(undefined),
-  url: z.string().url(),
-});
-
-const mentionSchema = z.object({
-  type: z.literal('Mention'),
-  href: z.string().url(),
-  name: z.string().optional().catch(undefined),
-});
-
-const hashtagSchema = z.object({
-  type: z.literal('Hashtag'),
-  href: z.string().url(),
-  name: z.string(),
-});
-
-const emojiSchema = z.object({
-  type: z.literal('Emoji'),
-  icon: imageSchema,
-  name: z.string(),
-});
-
-const tagSchema = z.discriminatedUnion('type', [
-  mentionSchema,
-  hashtagSchema,
-  emojiSchema,
-]);
-
-const propertyValueSchema = z.object({
-  type: z.literal('PropertyValue'),
-  name: z.string(),
-  value: z.string(),
-  verified_at: z.string().nullish(),
-});
-
-/** https://codeberg.org/fediverse/fep/src/branch/main/feps/fep-fffd.md */
-const proxySchema = z.object({
-  protocol: z.string().url(),
-  proxied: z.string(),
-  authoritative: z.boolean().optional().catch(undefined),
-});
-
-const personSchema = z.object({
-  type: z.literal('Person'),
-  id: apId,
-  icon: imageSchema.optional().catch(undefined),
-  image: imageSchema.optional().catch(undefined),
-  name: z.string().catch(''),
-  preferredUsername: z.string(),
-  inbox: apId,
-  followers: apId.optional().catch(undefined),
-  following: apId.optional().catch(undefined),
-  outbox: apId.optional().catch(undefined),
-  summary: z.string().catch(''),
-  attachment: filteredArray(propertyValueSchema).catch([]),
-  tag: filteredArray(emojiSchema).catch([]),
-  endpoints: z.object({
-    sharedInbox: apId.optional(),
-  }).optional().catch({}),
-  publicKey: z.object({
-    id: apId,
-    owner: apId,
-    publicKeyPem: z.string(),
-  }).optional().catch(undefined),
-  proxyOf: z.array(proxySchema).optional().catch(undefined),
-});
-
-const applicationSchema = personSchema.merge(z.object({ type: z.literal('Application') }));
-const groupSchema = personSchema.merge(z.object({ type: z.literal('Group') }));
-const organizationSchema = personSchema.merge(z.object({ type: z.literal('Organization') }));
-const serviceSchema = personSchema.merge(z.object({ type: z.literal('Service') }));
-
-const actorSchema = z.discriminatedUnion('type', [
-  personSchema,
-  applicationSchema,
-  groupSchema,
-  organizationSchema,
-  serviceSchema,
-]);
-
-const noteSchema = z.object({
-  type: z.literal('Note'),
-  id: apId,
-  to: recipients,
-  cc: recipients,
-  content: z.string(),
-  attachment: z.array(attachmentSchema).optional().catch(undefined),
-  tag: filteredArray(tagSchema).catch([]),
-  inReplyTo: apId.optional().catch(undefined),
-  attributedTo: apId,
-  published: published(),
-  sensitive: z.boolean().optional().catch(undefined),
-  summary: z.string().nullish().catch(undefined),
-  quoteUrl: apId.optional().catch(undefined),
-  source: z.object({
-    content: z.string(),
-    mediaType: z.literal('text/markdown'),
-  }).optional().catch(undefined),
-  proxyOf: z.array(proxySchema).optional().catch(undefined),
-});
-
-const flexibleNoteSchema = noteSchema.extend({
-  quoteURL: apId.optional().catch(undefined),
-  quoteUri: apId.optional().catch(undefined),
-  _misskey_quote: apId.optional().catch(undefined),
-}).transform((note) => {
-  const { quoteUrl, quoteUri, quoteURL, _misskey_quote, ...rest } = note;
-  return {
-    quoteUrl: quoteUrl || quoteUri || quoteURL || _misskey_quote,
-    ...rest,
-  };
-});
-
-// https://github.com/colinhacks/zod/discussions/2100#discussioncomment-5109781
-const objectSchema = z.union([
-  flexibleNoteSchema,
-  personSchema,
-  applicationSchema,
-  groupSchema,
-  organizationSchema,
-  serviceSchema,
-]).pipe(
-  z.discriminatedUnion('type', [
-    noteSchema,
-    personSchema,
-    applicationSchema,
-    groupSchema,
-    organizationSchema,
-    serviceSchema,
-  ]),
-);
-
-const createNoteSchema = z.object({
-  type: z.literal('Create'),
-  id: apId,
-  to: recipients,
-  cc: recipients,
-  actor: apId,
-  object: noteSchema,
-  published: published(),
-  proxyOf: z.array(proxySchema).optional().catch(undefined),
-});
-
-const announceNoteSchema = z.object({
-  type: z.literal('Announce'),
-  id: apId,
-  to: recipients,
-  cc: recipients,
-  actor: apId,
-  object: apId.or(noteSchema),
-  published: published(),
-  proxyOf: z.array(proxySchema).optional().catch(undefined),
-});
-
-const followSchema = z.object({
-  type: z.literal('Follow'),
-  id: apId,
-  to: recipients,
-  cc: recipients,
-  actor: apId,
-  object: apId,
-  proxyOf: z.array(proxySchema).optional().catch(undefined),
-});
-
-const acceptSchema = z.object({
-  type: z.literal('Accept'),
-  id: apId,
-  actor: apId,
-  to: recipients,
-  cc: recipients,
-  object: apId.or(followSchema),
-});
-
-const likeSchema = z.object({
-  type: z.literal('Like'),
-  id: apId,
-  actor: apId,
-  object: apId,
-  to: recipients,
-  cc: recipients,
-  proxyOf: z.array(proxySchema).optional().catch(undefined),
-});
-
-const emojiReactSchema = z.object({
-  type: z.literal('EmojiReact'),
-  id: apId,
-  actor: apId,
-  object: apId,
-  content: z.string().refine((v) => /\p{Extended_Pictographic}/u.test(v)),
-  to: recipients,
-  cc: recipients,
-  proxyOf: z.array(proxySchema).optional().catch(undefined),
-});
-
-const deleteSchema = z.object({
-  type: z.literal('Delete'),
-  id: apId,
-  actor: apId,
-  object: apId,
-  to: recipients,
-  cc: recipients,
-  proxyOf: z.array(proxySchema).optional().catch(undefined),
-});
-
-const updateActorSchema = z.object({
-  type: z.literal('Update'),
-  id: apId,
-  actor: apId,
-  to: recipients,
-  cc: recipients,
-  object: actorSchema,
-  proxyOf: z.array(proxySchema).optional().catch(undefined),
-});
-
-/**
- * A custom Zap activity type we made up, based on:
- * https://github.com/nostr-protocol/nips/blob/master/57.md
- */
-const zapSchema = z.object({
-  type: z.literal('Zap'),
-  id: apId,
-  actor: apId,
-  object: apId,
-  to: recipients,
-  cc: recipients,
-  proxyOf: z.array(proxySchema).optional().catch(undefined),
-});
-
-const activitySchema = z.discriminatedUnion('type', [
-  followSchema,
-  acceptSchema,
-  createNoteSchema,
-  announceNoteSchema,
-  updateActorSchema,
-  likeSchema,
-  emojiReactSchema,
-  deleteSchema,
-  zapSchema,
-]).refine((activity) => {
-  const ids: string[] = [activity.id];
-
-  if (activity.type === 'Create') {
-    ids.push(
-      activity.object.id,
-      activity.object.attributedTo,
-    );
-  }
-
-  if (activity.type === 'Update') {
-    ids.push(activity.object.id);
-  }
-
-  const { origin: actorOrigin } = new URL(activity.actor);
-
-  // Object containment
-  return ids.every((id) => {
-    const { origin: idOrigin } = new URL(id);
-    return idOrigin === actorOrigin;
-  });
-});
-
-type Activity = z.infer<typeof activitySchema>;
-type CreateNote = z.infer<typeof createNoteSchema>;
-type Announce = z.infer<typeof announceNoteSchema>;
-type Update = z.infer<typeof updateActorSchema>;
-type Object = z.infer<typeof objectSchema>;
-type Follow = z.infer<typeof followSchema>;
-type Accept = z.infer<typeof acceptSchema>;
-type Actor = z.infer<typeof actorSchema>;
-type Note = z.infer<typeof noteSchema>;
-type Mention = z.infer<typeof mentionSchema>;
-type Hashtag = z.infer<typeof hashtagSchema>;
-type Emoji = z.infer<typeof emojiSchema>;
-type Like = z.infer<typeof likeSchema>;
-type EmojiReact = z.infer<typeof emojiReactSchema>;
-type Delete = z.infer<typeof deleteSchema>;
-type Zap = z.infer<typeof zapSchema>;
-type Proxy = z.infer<typeof proxySchema>;
-
-export { acceptSchema, activitySchema, actorSchema, emojiSchema, followSchema, imageSchema, noteSchema, objectSchema };
-export type {
-  Accept,
-  Activity,
-  Actor,
-  Announce,
-  CreateNote,
-  Delete,
-  Emoji,
-  EmojiReact,
-  Follow,
-  Hashtag,
-  Like,
-  Mention,
-  Note,
-  Object,
-  Proxy,
-  Update,
-  Zap,
-};

src/server.ts

@@ -1,5 +1,3 @@
-import 'deno-safe-fetch/load';
-
 import '@/precheck.ts';
 import '@/sentry.ts';
 import '@/nostr-wasm.ts';

src/workers/fetch.test.ts

@@ -5,9 +5,9 @@ import { fetchWorker } from '@/workers/fetch.ts';
 Deno.test({
   name: 'fetchWorker',
   async fn() {
-    const response = await fetchWorker('http://httpbin.org/get');
+    const response = await fetchWorker('https://httpbingo.org/get');
     const json = await response.json();
-    assertEquals(json.headers.Host, 'httpbin.org');
+    assertEquals(json.headers.Host, ['httpbingo.org']);
   },
   sanitizeResources: false,
 });
@@ -19,7 +19,7 @@ Deno.test({
     const signal = controller.signal;
 
     setTimeout(() => controller.abort(), 100);
-    assertRejects(() => fetchWorker('http://httpbin.org/delay/10', { signal }));
+    assertRejects(() => fetchWorker('https://httpbingo.org/delay/10', { signal }));
 
     await new Promise<void>((resolve) => {
       signal.addEventListener('abort', () => resolve(), { once: true });

src/workers/fetch.worker.ts

@@ -1,12 +1,13 @@
 /// <reference lib="webworker" />
 
-import Debug from '@soapbox/stickynotes/debug';
+import { safeFetch } from '@soapbox/safe-fetch';
+import { Stickynotes } from '@soapbox/stickynotes';
 import * as Comlink from 'comlink';
 
 import '@/workers/handlers/abortsignal.ts';
 import '@/sentry.ts';
 
-const debug = Debug('ditto:fetch.worker');
+const console = new Stickynotes('ditto:fetch.worker');
 
 export const FetchWorker = {
   async fetch(
@@ -14,8 +15,8 @@ export const FetchWorker = {
     init: Omit<RequestInit, 'signal'>,
     signal: AbortSignal | null | undefined,
   ): Promise<[BodyInit, ResponseInit]> {
-    debug(init.method, url);
+    console.debug(init.method, url);
-    const response = await fetch(url, { ...init, signal });
+    const response = await safeFetch(url, { ...init, signal });
     return [
       await response.arrayBuffer(),
       {

src/workers/policy.worker.ts

@@ -1,4 +1,4 @@
-import 'deno-safe-fetch/load';
+import '@soapbox/safe-fetch/load';
 import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify';
 import { ReadOnlyPolicy } from '@nostrify/policies';
 import * as Comlink from 'comlink';