From 56a2e622e8c372098a28dc3e75a39a4d41fec1a7 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Nov 2024 10:37:05 -0600 Subject: [PATCH 1/4] Remove `deny-read` from setup script --- deno.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deno.json b/deno.json index 38e44ff4..1425b4c4 100644 --- a/deno.json +++ b/deno.json @@ -14,7 +14,7 @@ "nsec": "deno run scripts/nsec.ts", "admin:event": "deno run -A --env-file --deny-read=.env scripts/admin-event.ts", "admin:role": "deno run -A --env-file --deny-read=.env scripts/admin-role.ts", - "setup": "deno run -A --env-file --deny-read=.env scripts/setup.ts", + "setup": "deno run -A --env-file scripts/setup.ts", "setup:kind0": "deno run -A --env-file --deny-read=.env scripts/setup-kind0.ts", "stats:recompute": "deno run -A --env-file --deny-read=.env scripts/stats-recompute.ts", "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip -o soapbox.zip && rm soapbox.zip", From 0bba4393bab753dda9ee097243fbbcf97ff9c5a2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 11 Nov 2024 19:54:59 -0600 Subject: [PATCH 2/4] Remove unused ActivityPub schema --- src/schemas/activitypub.ts | 323 ------------------------------------- 1 file changed, 323 deletions(-) delete mode 100644 src/schemas/activitypub.ts diff --git a/src/schemas/activitypub.ts b/src/schemas/activitypub.ts deleted file mode 100644 index f14c994f..00000000 --- a/src/schemas/activitypub.ts +++ /dev/null @@ -1,323 +0,0 @@ -import { z } from 'zod'; - -const apId = z.string().url(); -const recipients = z.array(z.string()).catch([]); -const published = () => z.string().datetime().catch(new Date().toISOString()); - -/** Validates individual items in an array, dropping any that aren't valid. */ -function filteredArray(schema: T) { - return z.any().array() - .transform((arr) => ( - arr.map((item) => { - const parsed = schema.safeParse(item); - return parsed.success ? parsed.data : undefined; - }).filter((item): item is z.infer => Boolean(item)) - )); -} - -const imageSchema = z.object({ - type: z.literal('Image').catch('Image'), - url: z.string().url(), -}); - -const attachmentSchema = z.object({ - type: z.literal('Document').catch('Document'), - mediaType: z.string().optional().catch(undefined), - url: z.string().url(), -}); - -const mentionSchema = z.object({ - type: z.literal('Mention'), - href: z.string().url(), - name: z.string().optional().catch(undefined), -}); - -const hashtagSchema = z.object({ - type: z.literal('Hashtag'), - href: z.string().url(), - name: z.string(), -}); - -const emojiSchema = z.object({ - type: z.literal('Emoji'), - icon: imageSchema, - name: z.string(), -}); - -const tagSchema = z.discriminatedUnion('type', [ - mentionSchema, - hashtagSchema, - emojiSchema, -]); - -const propertyValueSchema = z.object({ - type: z.literal('PropertyValue'), - name: z.string(), - value: z.string(), - verified_at: z.string().nullish(), -}); - -/** https://codeberg.org/fediverse/fep/src/branch/main/feps/fep-fffd.md */ -const proxySchema = z.object({ - protocol: z.string().url(), - proxied: z.string(), - authoritative: z.boolean().optional().catch(undefined), -}); - -const personSchema = z.object({ - type: z.literal('Person'), - id: apId, - icon: imageSchema.optional().catch(undefined), - image: imageSchema.optional().catch(undefined), - name: z.string().catch(''), - preferredUsername: z.string(), - inbox: apId, - followers: apId.optional().catch(undefined), - following: apId.optional().catch(undefined), - outbox: apId.optional().catch(undefined), - summary: z.string().catch(''), - attachment: filteredArray(propertyValueSchema).catch([]), - tag: filteredArray(emojiSchema).catch([]), - endpoints: z.object({ - sharedInbox: apId.optional(), - }).optional().catch({}), - publicKey: z.object({ - id: apId, - owner: apId, - publicKeyPem: z.string(), - }).optional().catch(undefined), - proxyOf: z.array(proxySchema).optional().catch(undefined), -}); - -const applicationSchema = personSchema.merge(z.object({ type: z.literal('Application') })); -const groupSchema = personSchema.merge(z.object({ type: z.literal('Group') })); -const organizationSchema = personSchema.merge(z.object({ type: z.literal('Organization') })); -const serviceSchema = personSchema.merge(z.object({ type: z.literal('Service') })); - -const actorSchema = z.discriminatedUnion('type', [ - personSchema, - applicationSchema, - groupSchema, - organizationSchema, - serviceSchema, -]); - -const noteSchema = z.object({ - type: z.literal('Note'), - id: apId, - to: recipients, - cc: recipients, - content: z.string(), - attachment: z.array(attachmentSchema).optional().catch(undefined), - tag: filteredArray(tagSchema).catch([]), - inReplyTo: apId.optional().catch(undefined), - attributedTo: apId, - published: published(), - sensitive: z.boolean().optional().catch(undefined), - summary: z.string().nullish().catch(undefined), - quoteUrl: apId.optional().catch(undefined), - source: z.object({ - content: z.string(), - mediaType: z.literal('text/markdown'), - }).optional().catch(undefined), - proxyOf: z.array(proxySchema).optional().catch(undefined), -}); - -const flexibleNoteSchema = noteSchema.extend({ - quoteURL: apId.optional().catch(undefined), - quoteUri: apId.optional().catch(undefined), - _misskey_quote: apId.optional().catch(undefined), -}).transform((note) => { - const { quoteUrl, quoteUri, quoteURL, _misskey_quote, ...rest } = note; - return { - quoteUrl: quoteUrl || quoteUri || quoteURL || _misskey_quote, - ...rest, - }; -}); - -// https://github.com/colinhacks/zod/discussions/2100#discussioncomment-5109781 -const objectSchema = z.union([ - flexibleNoteSchema, - personSchema, - applicationSchema, - groupSchema, - organizationSchema, - serviceSchema, -]).pipe( - z.discriminatedUnion('type', [ - noteSchema, - personSchema, - applicationSchema, - groupSchema, - organizationSchema, - serviceSchema, - ]), -); - -const createNoteSchema = z.object({ - type: z.literal('Create'), - id: apId, - to: recipients, - cc: recipients, - actor: apId, - object: noteSchema, - published: published(), - proxyOf: z.array(proxySchema).optional().catch(undefined), -}); - -const announceNoteSchema = z.object({ - type: z.literal('Announce'), - id: apId, - to: recipients, - cc: recipients, - actor: apId, - object: apId.or(noteSchema), - published: published(), - proxyOf: z.array(proxySchema).optional().catch(undefined), -}); - -const followSchema = z.object({ - type: z.literal('Follow'), - id: apId, - to: recipients, - cc: recipients, - actor: apId, - object: apId, - proxyOf: z.array(proxySchema).optional().catch(undefined), -}); - -const acceptSchema = z.object({ - type: z.literal('Accept'), - id: apId, - actor: apId, - to: recipients, - cc: recipients, - object: apId.or(followSchema), -}); - -const likeSchema = z.object({ - type: z.literal('Like'), - id: apId, - actor: apId, - object: apId, - to: recipients, - cc: recipients, - proxyOf: z.array(proxySchema).optional().catch(undefined), -}); - -const emojiReactSchema = z.object({ - type: z.literal('EmojiReact'), - id: apId, - actor: apId, - object: apId, - content: z.string().refine((v) => /\p{Extended_Pictographic}/u.test(v)), - to: recipients, - cc: recipients, - proxyOf: z.array(proxySchema).optional().catch(undefined), -}); - -const deleteSchema = z.object({ - type: z.literal('Delete'), - id: apId, - actor: apId, - object: apId, - to: recipients, - cc: recipients, - proxyOf: z.array(proxySchema).optional().catch(undefined), -}); - -const updateActorSchema = z.object({ - type: z.literal('Update'), - id: apId, - actor: apId, - to: recipients, - cc: recipients, - object: actorSchema, - proxyOf: z.array(proxySchema).optional().catch(undefined), -}); - -/** - * A custom Zap activity type we made up, based on: - * https://github.com/nostr-protocol/nips/blob/master/57.md - */ -const zapSchema = z.object({ - type: z.literal('Zap'), - id: apId, - actor: apId, - object: apId, - to: recipients, - cc: recipients, - proxyOf: z.array(proxySchema).optional().catch(undefined), -}); - -const activitySchema = z.discriminatedUnion('type', [ - followSchema, - acceptSchema, - createNoteSchema, - announceNoteSchema, - updateActorSchema, - likeSchema, - emojiReactSchema, - deleteSchema, - zapSchema, -]).refine((activity) => { - const ids: string[] = [activity.id]; - - if (activity.type === 'Create') { - ids.push( - activity.object.id, - activity.object.attributedTo, - ); - } - - if (activity.type === 'Update') { - ids.push(activity.object.id); - } - - const { origin: actorOrigin } = new URL(activity.actor); - - // Object containment - return ids.every((id) => { - const { origin: idOrigin } = new URL(id); - return idOrigin === actorOrigin; - }); -}); - -type Activity = z.infer; -type CreateNote = z.infer; -type Announce = z.infer; -type Update = z.infer; -type Object = z.infer; -type Follow = z.infer; -type Accept = z.infer; -type Actor = z.infer; -type Note = z.infer; -type Mention = z.infer; -type Hashtag = z.infer; -type Emoji = z.infer; -type Like = z.infer; -type EmojiReact = z.infer; -type Delete = z.infer; -type Zap = z.infer; -type Proxy = z.infer; - -export { acceptSchema, activitySchema, actorSchema, emojiSchema, followSchema, imageSchema, noteSchema, objectSchema }; -export type { - Accept, - Activity, - Actor, - Announce, - CreateNote, - Delete, - Emoji, - EmojiReact, - Follow, - Hashtag, - Like, - Mention, - Note, - Object, - Proxy, - Update, - Zap, -}; From 674781daf1a8714c97053b158e70ed202409a234 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 13 Nov 2024 15:23:42 -0600 Subject: [PATCH 3/4] Upgrade safe-fetch --- deno.json | 2 +- deno.lock | 20 ++++++++++++++++++-- src/server.ts | 2 -- src/workers/fetch.worker.ts | 9 +++++---- src/workers/policy.worker.ts | 2 +- 5 files changed, 25 insertions(+), 10 deletions(-) diff --git a/deno.json b/deno.json index 1425b4c4..d86b3f99 100644 --- a/deno.json +++ b/deno.json @@ -50,6 +50,7 @@ "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-pglite": "jsr:@soapbox/kysely-pglite@^1.0.0", + "@soapbox/safe-fetch": "jsr:@soapbox/safe-fetch@^2.0.0", "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0", "@std/assert": "jsr:@std/assert@^0.225.1", "@std/cli": "jsr:@std/cli@^0.223.0", @@ -63,7 +64,6 @@ "comlink": "npm:comlink@^4.4.1", "comlink-async-generator": "npm:comlink-async-generator@^0.0.1", "commander": "npm:commander@12.1.0", - "deno-safe-fetch/load": "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts", "deno.json": "./deno.json", "entities": "npm:entities@^4.5.0", "fast-stable-stringify": "npm:fast-stable-stringify@^1.0.0", diff --git a/deno.lock b/deno.lock index 14cc1086..7e2b01b2 100644 --- a/deno.lock +++ b/deno.lock @@ -46,6 +46,7 @@ "jsr:@nostrify/types@0.35": "0.35.0", "jsr:@nostrify/types@~0.30.1": "0.30.1", "jsr:@soapbox/kysely-pglite@1": "1.0.0", + "jsr:@soapbox/safe-fetch@2": "2.0.0", "jsr:@soapbox/stickynotes@0.4": "0.4.0", "jsr:@std/assert@0.223": "0.223.0", "jsr:@std/assert@0.224": "0.224.0", @@ -120,6 +121,7 @@ "npm:prom-client@^15.1.2": "15.1.2", "npm:sharp@~0.33.5": "0.33.5", "npm:tldts@^6.0.14": "6.1.18", + "npm:tldts@^6.1.61": "6.1.61", "npm:tseep@^1.2.1": "1.2.1", "npm:type-fest@^4.3.0": "4.18.2", "npm:unfurl.js@^6.4.0": "6.4.0", @@ -483,6 +485,12 @@ "npm:kysely@~0.27.4" ] }, + "@soapbox/safe-fetch@2.0.0": { + "integrity": "f451d686501c76a0faa058fe9d2073676282a8a42c3b93c59159eb9191f11b5f", + "dependencies": [ + "npm:tldts@^6.1.61" + ] + }, "@soapbox/stickynotes@0.4.0": { "integrity": "60bfe61ab3d7e04bf708273b1e2d391a59534bdf29e54160e98d7afd328ca1ec" }, @@ -1522,6 +1530,9 @@ "tldts-core@6.1.49": { "integrity": "sha512-ctRO/wzBasOCxAStJG/60Qe8/QpGmaVPsE8djdk0vioxN4uCOgKoveH71Qc2EOmVMIjVf0BjigI5p9ZDuLOygg==" }, + "tldts-core@6.1.61": { + "integrity": "sha512-In7VffkDWUPgwa+c9picLUxvb0RltVwTkSgMNFgvlGSWveCzGBemBqTsgJCL4EDFWZ6WH0fKTsot6yNhzy3ZzQ==" + }, "tldts@6.1.18": { "integrity": "sha512-F+6zjPFnFxZ0h6uGb8neQWwHQm8u3orZVFribsGq4eBgEVrzSkHxzWS2l6aKr19T1vXiOMFjqfff4fQt+WgJFg==", "dependencies": [ @@ -1534,6 +1545,12 @@ "tldts-core@6.1.49" ] }, + "tldts@6.1.61": { + "integrity": "sha512-rv8LUyez4Ygkopqn+M6OLItAOT9FF3REpPQDkdMx5ix8w4qkuE7Vo2o/vw1nxKQYmJDV8JpAMJQr1b+lTKf0FA==", + "dependencies": [ + "tldts-core@6.1.61" + ] + }, "to-regex-range@5.0.1": { "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", "dependencies": [ @@ -2181,8 +2198,6 @@ "https://deno.land/x/sentry@7.112.2/index.mjs": "04382d5c2f4e233ba389611db46f77943b2a7f6efbeaaf31193f6e586f4366ef", "https://esm.sh/kysely@0.17.1/dist/esm/index-nodeless.js": "9c23bfd307118e3ccd3a9f0ec1261fc3451fb5301aa34aa6f28e05156818755a", "https://esm.sh/v135/kysely@0.17.1/denonext/dist/esm/index-nodeless.js": "6f73bbf2d73bc7e96cdabf941c4ae8c12f58fd7b441031edec44c029aed9532b", - "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts": "3f74ab08cf97d4a3e6994cb79422e9b0069495e017416858121d5ff8ae04ac2a", - "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/mod.ts": "5f505cd265aefbcb687cde6f98c79344d3292ee1dd978e85e5ffa84a617c6682", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/b4725e74ad6ca359ba0e370b55dbb8bb845a8a83/deps.ts": "b3dbecae69c30a5f161323b8c8ebd91d9af1eceb98fafab3091c7281a4b64fed", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/b4725e74ad6ca359ba0e370b55dbb8bb845a8a83/mod.ts": "662438fd3909984bb8cbaf3fd44d2121e949d11301baf21d6c3f057ccf9887de", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/b4725e74ad6ca359ba0e370b55dbb8bb845a8a83/src/PostgreSQLDriver.ts": "ea5a523bceeed420858b744beeb95d48976cb2b0d3f519a68b65a8229036cf6a", @@ -2272,6 +2287,7 @@ "jsr:@nostrify/nostrify@0.36", "jsr:@nostrify/policies@0.35", "jsr:@soapbox/kysely-pglite@1", + "jsr:@soapbox/safe-fetch@2", "jsr:@soapbox/stickynotes@0.4", "jsr:@std/assert@~0.225.1", "jsr:@std/cli@0.223", diff --git a/src/server.ts b/src/server.ts index f7a33dc0..4825e99d 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,5 +1,3 @@ -import 'deno-safe-fetch/load'; - import '@/precheck.ts'; import '@/sentry.ts'; import '@/nostr-wasm.ts'; diff --git a/src/workers/fetch.worker.ts b/src/workers/fetch.worker.ts index e6f98455..87d0a6c3 100644 --- a/src/workers/fetch.worker.ts +++ b/src/workers/fetch.worker.ts @@ -1,12 +1,13 @@ /// -import Debug from '@soapbox/stickynotes/debug'; +import { safeFetch } from '@soapbox/safe-fetch'; +import { Stickynotes } from '@soapbox/stickynotes'; import * as Comlink from 'comlink'; import '@/workers/handlers/abortsignal.ts'; import '@/sentry.ts'; -const debug = Debug('ditto:fetch.worker'); +const console = new Stickynotes('ditto:fetch.worker'); export const FetchWorker = { async fetch( @@ -14,8 +15,8 @@ export const FetchWorker = { init: Omit, signal: AbortSignal | null | undefined, ): Promise<[BodyInit, ResponseInit]> { - debug(init.method, url); - const response = await fetch(url, { ...init, signal }); + console.debug(init.method, url); + const response = await safeFetch(url, { ...init, signal }); return [ await response.arrayBuffer(), { diff --git a/src/workers/policy.worker.ts b/src/workers/policy.worker.ts index c5b4129d..5e9d4d4a 100644 --- a/src/workers/policy.worker.ts +++ b/src/workers/policy.worker.ts @@ -1,4 +1,4 @@ -import 'deno-safe-fetch/load'; +import '@soapbox/safe-fetch/load'; import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify'; import { ReadOnlyPolicy } from '@nostrify/policies'; import * as Comlink from 'comlink'; From 515c70f0389954249a66a1cf143d2dcdb01da7ab Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 13 Nov 2024 15:34:41 -0600 Subject: [PATCH 4/4] Switch to httpbingo in tests so testing over https works --- src/workers/fetch.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/workers/fetch.test.ts b/src/workers/fetch.test.ts index d657d1f5..e4c698d4 100644 --- a/src/workers/fetch.test.ts +++ b/src/workers/fetch.test.ts @@ -5,9 +5,9 @@ import { fetchWorker } from '@/workers/fetch.ts'; Deno.test({ name: 'fetchWorker', async fn() { - const response = await fetchWorker('http://httpbin.org/get'); + const response = await fetchWorker('https://httpbingo.org/get'); const json = await response.json(); - assertEquals(json.headers.Host, 'httpbin.org'); + assertEquals(json.headers.Host, ['httpbingo.org']); }, sanitizeResources: false, }); @@ -19,7 +19,7 @@ Deno.test({ const signal = controller.signal; setTimeout(() => controller.abort(), 100); - assertRejects(() => fetchWorker('http://httpbin.org/delay/10', { signal })); + assertRejects(() => fetchWorker('https://httpbingo.org/delay/10', { signal })); await new Promise((resolve) => { signal.addEventListener('abort', () => resolve(), { once: true });