From 66277041143630c382fcf5d9ab585151c0c1ec58 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 28 Apr 2024 17:07:41 -0300 Subject: [PATCH 01/65] feat: save user preferences (kind 30078) & encrypt it --- src/controllers/api/accounts.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 02897993..f6440fee 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -18,6 +18,7 @@ import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { renderRelationship } from '@/views/mastodon/relationships.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; +import { APISigner } from '@/signers/APISigner.ts'; const usernameSchema = z .string().min(1).max(30) @@ -186,6 +187,7 @@ const updateCredentialsSchema = z.object({ bot: z.boolean().optional(), discoverable: z.boolean().optional(), nip05: z.string().optional(), + pleroma_settings_store: z.object({ soapbox_fe: z.object({ themeMode: z.string() }).passthrough() }).optional(), }); const updateCredentialsController: AppController = async (c) => { @@ -225,6 +227,16 @@ const updateCredentialsController: AppController = async (c) => { tags: [], }, c); + const soapbox_fe = result.data.pleroma_settings_store?.soapbox_fe; + if (soapbox_fe) { + const signer = new APISigner(c); + await createEvent({ + kind: 30078, + tags: [['d', 'pub.ditto.preferences']], + content: await signer.nip44.encrypt(pubkey, JSON.stringify(soapbox_fe)), + }, c); + } + const account = await renderAccount(event, { withSource: true }); return c.json(account); }; From bb82df14c6c4d0cf9612a6556fbcdb994517ddbb Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 28 Apr 2024 21:42:57 -0300 Subject: [PATCH 02/65] refactor: user preference in create & verify credentials --- src/controllers/api/accounts.ts | 39 ++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index f6440fee..5f0840e3 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -50,7 +50,21 @@ const verifyCredentialsController: AppController = async (c) => { const event = await getAuthor(pubkey, { relations: ['author_stats'] }); if (event) { - return c.json(await renderAccount(event, { withSource: true })); + const account = await renderAccount(event, { withSource: true }); + + const [userPreferencesEvent] = await eventsDB.query([{ + authors: [pubkey], + kinds: [30078], + '#d': ['pub.ditto.pleroma_settings_store'], + limit: 1, + }]); + if (userPreferencesEvent) { + const signer = new APISigner(c); + const userPreference = JSON.parse(await signer.nip44.decrypt(pubkey, userPreferencesEvent.content)); + (account.pleroma as any).settings_store = userPreference; + } + + return c.json(account); } else { return c.json(await accountFromPubkey(pubkey, { withSource: true })); } @@ -187,7 +201,7 @@ const updateCredentialsSchema = z.object({ bot: z.boolean().optional(), discoverable: z.boolean().optional(), nip05: z.string().optional(), - pleroma_settings_store: z.object({ soapbox_fe: z.object({ themeMode: z.string() }).passthrough() }).optional(), + pleroma_settings_store: z.object({ soapbox_fe: z.record(z.string(), z.unknown()) }).optional(), }); const updateCredentialsController: AppController = async (c) => { @@ -227,17 +241,30 @@ const updateCredentialsController: AppController = async (c) => { tags: [], }, c); - const soapbox_fe = result.data.pleroma_settings_store?.soapbox_fe; - if (soapbox_fe) { + const pleroma_frontend = result.data.pleroma_settings_store; + if (pleroma_frontend) { const signer = new APISigner(c); await createEvent({ kind: 30078, - tags: [['d', 'pub.ditto.preferences']], - content: await signer.nip44.encrypt(pubkey, JSON.stringify(soapbox_fe)), + tags: [['d', 'pub.ditto.pleroma_settings_store']], + content: await signer.nip44.encrypt(pubkey, JSON.stringify(pleroma_frontend)), }, c); } const account = await renderAccount(event, { withSource: true }); + + const [userPreferencesEvent] = await eventsDB.query([{ + authors: [pubkey], + kinds: [30078], + '#d': ['pub.ditto.pleroma_settings_store'], + limit: 1, + }]); + if (userPreferencesEvent) { + const signer = new APISigner(c); + const userPreference = JSON.parse(await signer.nip44.decrypt(pubkey, userPreferencesEvent.content)); + (account.pleroma as any).settings_store = userPreference; + } + return c.json(account); }; From e61cbecb3e840100d6a4561a795f6ccab17d5cb4 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 16 May 2024 10:29:14 -0300 Subject: [PATCH 03/65] refactor(unreblog): update error messages and query with Storages.db() --- src/controllers/api/statuses.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 98173b0b..984dfdbc 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -261,21 +261,19 @@ const reblogStatusController: AppController = async (c) => { const unreblogStatusController: AppController = async (c) => { const eventId = c.req.param('id'); const pubkey = await c.get('signer')?.getPublicKey()!; - - const event = await getEvent(eventId, { kind: 1 }); - - if (!event) { - return c.json({ error: 'Event not found.' }, 404); - } - const store = await Storages.db(); + const [event] = await store.query([{ ids: [eventId], kinds: [1] }]); + if (!event) { + return c.json({ error: 'Record not found' }, 404); + } + const [repostedEvent] = await store.query( [{ kinds: [6], authors: [pubkey], '#e': [event.id], limit: 1 }], ); if (!repostedEvent) { - return c.json({ error: 'Event not found.' }, 404); + return c.json({ error: 'Record not found' }, 404); } await createEvent({ From 6c3f0849b231400dea4b78f0ea9b2a0fbb943d85 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 16 May 2024 12:57:01 -0500 Subject: [PATCH 04/65] Upgrade Nostrify to v0.19.2, fix crash on mixed filters --- deno.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deno.json b/deno.json index 25ec2ed2..b783f646 100644 --- a/deno.json +++ b/deno.json @@ -21,7 +21,7 @@ "@db/sqlite": "jsr:@db/sqlite@^0.11.1", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.19.1", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.19.2", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0", From 00d4bf23448826fffdcacdc449270ba86a4e911e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 16 May 2024 15:42:32 -0500 Subject: [PATCH 05/65] Upgrade Nostrify to v0.20.0, enable Postgres FTS --- deno.json | 2 +- src/db/migrations/020_pgfts.ts | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 src/db/migrations/020_pgfts.ts diff --git a/deno.json b/deno.json index b783f646..e8719a46 100644 --- a/deno.json +++ b/deno.json @@ -21,7 +21,7 @@ "@db/sqlite": "jsr:@db/sqlite@^0.11.1", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.19.2", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.20.0", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0", diff --git a/src/db/migrations/020_pgfts.ts b/src/db/migrations/020_pgfts.ts new file mode 100644 index 00000000..8b3cfa0c --- /dev/null +++ b/src/db/migrations/020_pgfts.ts @@ -0,0 +1,19 @@ +import { Kysely, sql } from 'kysely'; + +import { Conf } from '@/config.ts'; + +export async function up(db: Kysely): Promise { + if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + await db.schema.createTable('nostr_pgfts') + .ifNotExists() + .addColumn('event_id', 'text', (c) => c.primaryKey().references('nostr_events.id').onDelete('cascade')) + .addColumn('search_vec', sql`tsvector`, (c) => c.notNull()) + .execute(); + } +} + +export async function down(db: Kysely): Promise { + if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + await db.schema.dropTable('nostr_pgfts').ifExists().execute(); + } +} From baa698688094195ed3533c50b0eb2450a06180ae Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 16 May 2024 15:48:22 -0500 Subject: [PATCH 06/65] EventsDB: enable fts conditionally based on DATABASE_URL --- src/storages/EventsDB.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index aac8e522..ef51a892 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -42,8 +42,17 @@ class EventsDB implements NStore { }; constructor(private kysely: Kysely) { + let fts: 'sqlite' | 'postgres' | undefined; + + if (Conf.databaseUrl.protocol === 'sqlite:') { + fts = 'sqlite'; + } + if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + fts = 'postgres'; + } + this.store = new NDatabase(kysely, { - fts5: Conf.databaseUrl.protocol === 'sqlite:', + fts, indexTags: EventsDB.indexTags, searchText: EventsDB.searchText, }); From 5aacbe7af5717b2d032064dbd18fd333e55cf8fe Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 16 May 2024 18:53:04 -0500 Subject: [PATCH 07/65] Fix media uploads due to 'awaiting' a query builder instance --- src/controllers/api/statuses.ts | 4 +++- src/db/unattached-media.ts | 15 +++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 984dfdbc..00f9a98e 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -4,6 +4,7 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; +import { DittoDB } from '@/db/DittoDB.ts'; import { getUnattachedMediaByIds } from '@/db/unattached-media.ts'; import { getAncestors, getAuthor, getDescendants, getEvent } from '@/queries.ts'; import { addTag, deleteTag } from '@/tags.ts'; @@ -56,6 +57,7 @@ const statusController: AppController = async (c) => { const createStatusController: AppController = async (c) => { const body = await parseBody(c.req.raw); const result = createStatusSchema.safeParse(body); + const kysely = await DittoDB.getInstance(); if (!result.success) { return c.json({ error: 'Bad request', schema: result.error }, 400); @@ -92,7 +94,7 @@ const createStatusController: AppController = async (c) => { const viewerPubkey = await c.get('signer')?.getPublicKey(); if (data.media_ids?.length) { - const media = await getUnattachedMediaByIds(data.media_ids) + const media = await getUnattachedMediaByIds(kysely, data.media_ids) .then((media) => media.filter(({ pubkey }) => pubkey === viewerPubkey)) .then((media) => media.map(({ url, data }) => ['media', url, data])); diff --git a/src/db/unattached-media.ts b/src/db/unattached-media.ts index 80636283..cee1e3a3 100644 --- a/src/db/unattached-media.ts +++ b/src/db/unattached-media.ts @@ -1,6 +1,8 @@ +import { Kysely } from 'kysely'; import uuid62 from 'uuid62'; import { DittoDB } from '@/db/DittoDB.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; import { type MediaData } from '@/schemas/nostr.ts'; interface UnattachedMedia { @@ -28,8 +30,7 @@ async function insertUnattachedMedia(media: Omit) { return kysely.selectFrom('unattached_media') .select([ 'unattached_media.id', @@ -41,9 +42,8 @@ async function selectUnattachedMediaQuery() { } /** Find attachments that exist but aren't attached to any events. */ -async function getUnattachedMedia(until: Date) { - const query = await selectUnattachedMediaQuery(); - return query +function getUnattachedMedia(kysely: Kysely, until: Date) { + return selectUnattachedMediaQuery(kysely) .leftJoin('nostr_tags', 'unattached_media.url', 'nostr_tags.value') .where('uploaded_at', '<', until.getTime()) .execute(); @@ -58,10 +58,9 @@ async function deleteUnattachedMediaByUrl(url: string) { } /** Get unattached media by IDs. */ -async function getUnattachedMediaByIds(ids: string[]) { +async function getUnattachedMediaByIds(kysely: Kysely, ids: string[]) { if (!ids.length) return []; - const query = await selectUnattachedMediaQuery(); - return query + return await selectUnattachedMediaQuery(kysely) .where('id', 'in', ids) .execute(); } From 2aee2e6bf6facc883b1364af8f630ec88a962cfa Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 17 May 2024 09:45:19 -0300 Subject: [PATCH 08/65] fix(renderReblog): render account from pubkey if there is no kind 0 --- src/views/mastodon/statuses.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 776f0169..aefe258b 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -123,8 +123,6 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< async function renderReblog(event: DittoEvent, opts: RenderStatusOpts) { const { viewerPubkey } = opts; - if (!event.author) return; - const repostId = event.tags.find(([name]) => name === 'e')?.[1]; if (!repostId) return; @@ -134,7 +132,7 @@ async function renderReblog(event: DittoEvent, opts: RenderStatusOpts) { return { id: event.id, - account: await renderAccount(event.author), + account: event.author ? await renderAccount(event.author) : await accountFromPubkey(event.pubkey), reblogged: true, reblog, }; From 4cc1d13d44b50dc31ad9af93f050eb19ab9d3b83 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 17 May 2024 11:25:17 -0300 Subject: [PATCH 09/65] fix: render followers & following list when no kind 0 --- src/views.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/views.ts b/src/views.ts index a7375429..863dd30c 100644 --- a/src/views.ts +++ b/src/views.ts @@ -5,6 +5,7 @@ import { renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; import { paginated, paginationSchema } from '@/utils/api.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; +import { accountFromPubkey } from '@/views/mastodon/accounts.ts'; /** Render account objects for the author of each event. */ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal = AbortSignal.timeout(1000)) { @@ -24,7 +25,13 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal .then((events) => hydrateEvents({ events, store, signal })); const accounts = await Promise.all( - authors.map((event) => renderAccount(event)), + Array.from(pubkeys).map(async (pubkey) => { + const event = authors.find((event) => event.pubkey === pubkey); + if (event) { + return await renderAccount(event); + } + return await accountFromPubkey(pubkey); + }), ); return paginated(c, events, accounts); @@ -39,7 +46,13 @@ async function renderAccounts(c: AppContext, authors: string[], signal = AbortSi .then((events) => hydrateEvents({ events, store, signal })); const accounts = await Promise.all( - events.map((event) => renderAccount(event)), + authors.map(async (pubkey) => { + const event = events.find((event) => event.pubkey === pubkey); + if (event) { + return await renderAccount(event); + } + return await accountFromPubkey(pubkey); + }), ); return paginated(c, events, accounts); From 251500fba1b82de21098776c4d98bfd0d6a40660 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 11:39:21 -0500 Subject: [PATCH 10/65] Never let stats be less than 0 --- src/storages/hydrate.ts | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 8d2d3027..9b958416 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -251,11 +251,19 @@ async function gatherAuthorStats(events: DittoEvent[]): Promise ({ + pubkey: row.pubkey, + followers_count: Math.max(0, row.followers_count), + following_count: Math.max(0, row.following_count), + notes_count: Math.max(0, row.notes_count), + })); } /** Collect event stats from the events. */ @@ -271,11 +279,19 @@ async function gatherEventStats(events: DittoEvent[]): Promise ({ + event_id: row.event_id, + reposts_count: Math.max(0, row.reposts_count), + reactions_count: Math.max(0, row.reactions_count), + replies_count: Math.max(0, row.replies_count), + })); } /** Return a normalized event without any non-standard keys. */ From a39910fa984e5f4896aa860143af07c1044e0271 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 13:12:40 -0500 Subject: [PATCH 11/65] Add a function to recalculate author stats --- scripts/stats-recompute.ts | 25 ++----------------------- src/stats.ts | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/scripts/stats-recompute.ts b/scripts/stats-recompute.ts index dcb0bc07..4037a85b 100644 --- a/scripts/stats-recompute.ts +++ b/scripts/stats-recompute.ts @@ -1,8 +1,6 @@ import { nip19 } from 'nostr-tools'; -import { DittoDB } from '@/db/DittoDB.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; -import { Storages } from '@/storages.ts'; +import { refreshAuthorStats } from '@/stats.ts'; let pubkey: string; try { @@ -17,23 +15,4 @@ try { Deno.exit(1); } -const store = await Storages.db(); -const kysely = await DittoDB.getInstance(); - -const [followList] = await store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]); - -const authorStats: DittoTables['author_stats'] = { - pubkey, - followers_count: (await store.count([{ kinds: [3], '#p': [pubkey] }])).count, - following_count: followList?.tags.filter(([name]) => name === 'p')?.length ?? 0, - notes_count: (await store.count([{ kinds: [1], authors: [pubkey] }])).count, -}; - -await kysely.insertInto('author_stats') - .values(authorStats) - .onConflict((oc) => - oc - .column('pubkey') - .doUpdateSet(authorStats) - ) - .execute(); +await refreshAuthorStats(pubkey); diff --git a/src/stats.ts b/src/stats.ts index 92040710..74242f7a 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -1,11 +1,12 @@ -import { NKinds, NostrEvent } from '@nostrify/nostrify'; +import { NKinds, NostrEvent, NStore } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; import { InsertQueryBuilder, Kysely } from 'kysely'; +import { SetRequired } from 'type-fest'; import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Storages } from '@/storages.ts'; -import { findReplyTag } from '@/tags.ts'; +import { findReplyTag, getTagSet } from '@/tags.ts'; type AuthorStat = keyof Omit; type EventStat = keyof Omit; @@ -216,4 +217,35 @@ function getFollowDiff(event: NostrEvent, prev?: NostrEvent): AuthorStatDiff[] { ]; } -export { updateStats }; +/** Refresh the author's stats in the database. */ +async function refreshAuthorStats(pubkey: string): Promise { + const store = await Storages.db(); + const stats = await countAuthorStats(store, pubkey); + + const kysely = await DittoDB.getInstance(); + await kysely.insertInto('author_stats') + .values(stats) + .onConflict((oc) => oc.column('pubkey').doUpdateSet(stats)) + .execute(); +} + +/** Calculate author stats from the database. */ +async function countAuthorStats( + store: SetRequired, + pubkey: string, +): Promise { + const [{ count: followers_count }, { count: notes_count }, [followList]] = await Promise.all([ + store.count([{ kinds: [3], '#p': [pubkey] }]), + store.count([{ kinds: [1], authors: [pubkey] }]), + store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]), + ]); + + return { + pubkey, + followers_count, + following_count: getTagSet(followList?.tags ?? [], 'p').size, + notes_count, + }; +} + +export { refreshAuthorStats, updateStats }; From 6995bd2b292810913c6c8d5877ea8f76b91e6b42 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 16:23:23 -0500 Subject: [PATCH 12/65] Upgrade Deno to the latest version --- .gitlab-ci.yml | 2 +- .tool-versions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a9dee457..8e728884 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:1.41.3 +image: denoland/deno:1.43.4 default: interruptible: true diff --git a/.tool-versions b/.tool-versions index a13fd5ff..9bbaf96d 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -deno 1.41.3 \ No newline at end of file +deno 1.43.4 \ No newline at end of file From ae9516b445332df74aa817a26e5be522e243a923 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 16:23:38 -0500 Subject: [PATCH 13/65] refreshAuthorStats: return the stats --- src/stats.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/stats.ts b/src/stats.ts index 74242f7a..08d4bb99 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -218,7 +218,7 @@ function getFollowDiff(event: NostrEvent, prev?: NostrEvent): AuthorStatDiff[] { } /** Refresh the author's stats in the database. */ -async function refreshAuthorStats(pubkey: string): Promise { +async function refreshAuthorStats(pubkey: string): Promise { const store = await Storages.db(); const stats = await countAuthorStats(store, pubkey); @@ -227,6 +227,8 @@ async function refreshAuthorStats(pubkey: string): Promise { .values(stats) .onConflict((oc) => oc.column('pubkey').doUpdateSet(stats)) .execute(); + + return stats; } /** Calculate author stats from the database. */ From 17b633019339506d2414bc27056531088544499a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 16:59:45 -0500 Subject: [PATCH 14/65] Downgrade Deno to v1.43.3 due to TypeScript issues --- .gitlab-ci.yml | 2 +- .tool-versions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8e728884..b2140dbf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:1.43.4 +image: denoland/deno:1.43.3 default: interruptible: true diff --git a/.tool-versions b/.tool-versions index 9bbaf96d..b3e19cd6 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -deno 1.43.4 \ No newline at end of file +deno 1.43.3 \ No newline at end of file From 5c2e3450a9ae2b06645f570ef2059109120a648f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 17:50:30 -0500 Subject: [PATCH 15/65] Refresh author stats: less naive way --- src/storages/hydrate.ts | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 9b958416..7b964d77 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -1,10 +1,12 @@ import { NostrEvent, NStore } from '@nostrify/nostrify'; +import { LRUCache } from 'lru-cache'; import { matchFilter } from 'nostr-tools'; import { DittoDB } from '@/db/DittoDB.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Conf } from '@/config.ts'; +import { refreshAuthorStats } from '@/stats.ts'; interface HydrateOpts { events: DittoEvent[]; @@ -55,6 +57,8 @@ async function hydrateEvents(opts: HydrateOpts): Promise { events: await gatherEventStats(cache), }; + requestMissingAuthorStats(events, stats.authors); + // Dedupe events. const results = [...new Map(cache.map((event) => [event.id, event])).values()]; @@ -266,6 +270,31 @@ async function gatherAuthorStats(events: DittoEvent[]): Promise( + events + .filter((event) => event.kind === 0) + .map((event) => event.pubkey), + ); + + const missing = pubkeys.difference( + new Set(stats.map((stat) => stat.pubkey)), + ); + + for (const pubkey of missing) { + refreshAuthorStatsDebounced(pubkey); + } +} + +const lru = new LRUCache({ max: 1000 }); + +/** Calls `refreshAuthorStats` only once per author. */ +function refreshAuthorStatsDebounced(pubkey: string): void { + if (lru.get(pubkey)) return; + lru.set(pubkey, true); + refreshAuthorStats(pubkey).catch(() => {}); +} + /** Collect event stats from the events. */ async function gatherEventStats(events: DittoEvent[]): Promise { const ids = new Set( From bf479d01625497eac9190286eaf383f381ca606b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 18:26:55 -0500 Subject: [PATCH 16/65] Move refreshAuthorStatsDebounced to stats.ts --- src/stats.ts | 12 +++++++++++- src/storages/hydrate.ts | 16 +++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/stats.ts b/src/stats.ts index 08d4bb99..43647818 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -1,6 +1,7 @@ import { NKinds, NostrEvent, NStore } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; import { InsertQueryBuilder, Kysely } from 'kysely'; +import { LRUCache } from 'lru-cache'; import { SetRequired } from 'type-fest'; import { DittoDB } from '@/db/DittoDB.ts'; @@ -250,4 +251,13 @@ async function countAuthorStats( }; } -export { refreshAuthorStats, updateStats }; +const lru = new LRUCache({ max: 1000 }); + +/** Calls `refreshAuthorStats` only once per author. */ +function refreshAuthorStatsDebounced(pubkey: string): void { + if (lru.get(pubkey)) return; + lru.set(pubkey, true); + refreshAuthorStats(pubkey).catch(() => {}); +} + +export { refreshAuthorStats, refreshAuthorStatsDebounced, updateStats }; diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 7b964d77..e5c488e3 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -1,12 +1,11 @@ import { NostrEvent, NStore } from '@nostrify/nostrify'; -import { LRUCache } from 'lru-cache'; import { matchFilter } from 'nostr-tools'; import { DittoDB } from '@/db/DittoDB.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Conf } from '@/config.ts'; -import { refreshAuthorStats } from '@/stats.ts'; +import { refreshAuthorStatsDebounced } from '@/stats.ts'; interface HydrateOpts { events: DittoEvent[]; @@ -57,7 +56,7 @@ async function hydrateEvents(opts: HydrateOpts): Promise { events: await gatherEventStats(cache), }; - requestMissingAuthorStats(events, stats.authors); + refreshMissingAuthorStats(events, stats.authors); // Dedupe events. const results = [...new Map(cache.map((event) => [event.id, event])).values()]; @@ -270,7 +269,7 @@ async function gatherAuthorStats(events: DittoEvent[]): Promise( events .filter((event) => event.kind === 0) @@ -286,15 +285,6 @@ function requestMissingAuthorStats(events: NostrEvent[], stats: DittoTables['aut } } -const lru = new LRUCache({ max: 1000 }); - -/** Calls `refreshAuthorStats` only once per author. */ -function refreshAuthorStatsDebounced(pubkey: string): void { - if (lru.get(pubkey)) return; - lru.set(pubkey, true); - refreshAuthorStats(pubkey).catch(() => {}); -} - /** Collect event stats from the events. */ async function gatherEventStats(events: DittoEvent[]): Promise { const ids = new Set( From 23a366081fa19e4e75d411641b34ec4ff97e2fb4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 18:42:45 -0500 Subject: [PATCH 17/65] stats: maybe refresh stats when updating --- src/stats.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/stats.ts b/src/stats.ts index 43647818..9f0d2573 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -39,6 +39,8 @@ async function updateStats(event: NostrEvent) { debug(JSON.stringify({ id: event.id, pubkey: event.pubkey, kind: event.kind, tags: event.tags, statDiffs })); } + pubkeyDiffs.forEach(([_, pubkey]) => refreshAuthorStatsDebounced(pubkey)); + const kysely = await DittoDB.getInstance(); if (pubkeyDiffs.length) queries.push(authorStatsQuery(kysely, pubkeyDiffs)); From f9a0055e78a582e26e88458bdc8ba1f5eaa6c798 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 19:00:56 -0500 Subject: [PATCH 18/65] stats: add a Semaphore when refreshing author stats --- deno.json | 1 + src/stats.ts | 15 +++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/deno.json b/deno.json index e8719a46..84d5351b 100644 --- a/deno.json +++ b/deno.json @@ -20,6 +20,7 @@ "@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "@db/sqlite": "jsr:@db/sqlite@^0.11.1", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", + "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.20.0", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", diff --git a/src/stats.ts b/src/stats.ts index 9f0d2573..71124961 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -1,3 +1,4 @@ +import { Semaphore } from '@lambdalisue/async'; import { NKinds, NostrEvent, NStore } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; import { InsertQueryBuilder, Kysely } from 'kysely'; @@ -253,13 +254,19 @@ async function countAuthorStats( }; } -const lru = new LRUCache({ max: 1000 }); +const authorStatsSemaphore = new Semaphore(10); +const refreshedAuthors = new LRUCache({ max: 1000 }); /** Calls `refreshAuthorStats` only once per author. */ function refreshAuthorStatsDebounced(pubkey: string): void { - if (lru.get(pubkey)) return; - lru.set(pubkey, true); - refreshAuthorStats(pubkey).catch(() => {}); + if (refreshedAuthors.get(pubkey)) { + return; + } + + refreshedAuthors.set(pubkey, true); + + authorStatsSemaphore + .lock(() => refreshAuthorStats(pubkey).catch(() => {})); } export { refreshAuthorStats, refreshAuthorStatsDebounced, updateStats }; From 3e93b42251db66e24ef25ba7bd3d2cc6aa91d0cb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 08:13:37 -0500 Subject: [PATCH 19/65] stats: add a debug call --- src/stats.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/stats.ts b/src/stats.ts index 71124961..256c570e 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -264,6 +264,7 @@ function refreshAuthorStatsDebounced(pubkey: string): void { } refreshedAuthors.set(pubkey, true); + debug('refreshing author stats:', pubkey); authorStatsSemaphore .lock(() => refreshAuthorStats(pubkey).catch(() => {})); From 6ac4c072a6caeb9216c9c818e53e70d1249d69eb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 08:20:43 -0500 Subject: [PATCH 20/65] Fix crash decoding url --- src/note.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/note.ts b/src/note.ts index 5603c539..1c5c70cd 100644 --- a/src/note.ts +++ b/src/note.ts @@ -18,7 +18,7 @@ const linkifyOpts: linkify.Opts = { return `#${tag}`; }, url: ({ content }) => { - if (nip21.test(content)) { + try { const { decoded } = nip21.parse(content); const pubkey = getDecodedPubkey(decoded); if (pubkey) { @@ -28,7 +28,7 @@ const linkifyOpts: linkify.Opts = { } else { return ''; } - } else { + } catch { return `${content}`; } }, From 4c87e723c06c3df9bc059282c790d59bd72d53b6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 10:56:34 -0500 Subject: [PATCH 21/65] Bump nostrify to v0.21.1 --- deno.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deno.json b/deno.json index 84d5351b..e40dd24c 100644 --- a/deno.json +++ b/deno.json @@ -22,7 +22,7 @@ "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.20.0", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.21.1", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0", From f97064afb4bc1a6c05e430f4d4315ca94ed4f179 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 11:35:29 -0500 Subject: [PATCH 22/65] Remove dependency on npm:mime, switch to @std/media-types --- deno.json | 2 +- src/deps.ts | 2 -- src/note.ts | 9 +++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/deno.json b/deno.json index e40dd24c..54ba4680 100644 --- a/deno.json +++ b/deno.json @@ -32,7 +32,7 @@ "@std/dotenv": "jsr:@std/dotenv@^0.224.0", "@std/encoding": "jsr:@std/encoding@^0.224.0", "@std/json": "jsr:@std/json@^0.223.0", - "@std/media-types": "jsr:@std/media-types@^0.224.0", + "@std/media-types": "jsr:@std/media-types@^0.224.1", "@std/streams": "jsr:@std/streams@^0.223.0", "comlink": "npm:comlink@^4.4.1", "deno-safe-fetch": "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts", diff --git a/src/deps.ts b/src/deps.ts index 46d8fecc..12be07f1 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -1,8 +1,6 @@ import 'deno-safe-fetch'; // @deno-types="npm:@types/lodash@4.14.194" export { default as lodash } from 'https://esm.sh/lodash@4.17.21'; -// @deno-types="npm:@types/mime@3.0.0" -export { default as mime } from 'npm:mime@^3.0.0'; // @deno-types="npm:@types/sanitize-html@2.9.0" export { default as sanitizeHtml } from 'npm:sanitize-html@^2.11.0'; export { diff --git a/src/note.ts b/src/note.ts index 1c5c70cd..7dc39f5a 100644 --- a/src/note.ts +++ b/src/note.ts @@ -1,10 +1,10 @@ +import { typeByExtension } from '@std/media-types'; import 'linkify-plugin-hashtag'; import linkifyStr from 'linkify-string'; import linkify from 'linkifyjs'; import { nip19, nip21 } from 'nostr-tools'; import { Conf } from '@/config.ts'; -import { mime } from '@/deps.ts'; import { type DittoAttachment } from '@/views/mastodon/attachments.ts'; linkify.registerCustomProtocol('nostr', true); @@ -87,12 +87,13 @@ function isLinkURL(link: Link): boolean { return link.type === 'url'; } -/** `npm:mime` treats `.com` as a file extension, so parse the full URL to get its path first. */ +/** Get the extension from the URL, then get its type. */ function getUrlMimeType(url: string): string | undefined { try { const { pathname } = new URL(url); - return mime.getType(pathname) || undefined; - } catch (_e) { + const ext = pathname.split('.').pop() ?? ''; + return typeByExtension(ext); + } catch { return undefined; } } From 5997ff0fff38bf79486818a7e0f9ceae2182e4b6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 11:52:33 -0500 Subject: [PATCH 23/65] Create utils/media.ts, move some code from note.ts there --- src/note.ts | 25 ++++++------------------- src/utils/media.test.ts | 17 +++++++++++++++++ src/utils/media.ts | 24 ++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 19 deletions(-) create mode 100644 src/utils/media.test.ts create mode 100644 src/utils/media.ts diff --git a/src/note.ts b/src/note.ts index 7dc39f5a..71dc0174 100644 --- a/src/note.ts +++ b/src/note.ts @@ -1,10 +1,10 @@ -import { typeByExtension } from '@std/media-types'; import 'linkify-plugin-hashtag'; import linkifyStr from 'linkify-string'; import linkify from 'linkifyjs'; import { nip19, nip21 } from 'nostr-tools'; import { Conf } from '@/config.ts'; +import { getUrlMediaType, isPermittedMediaType } from '@/utils/media.ts'; import { type DittoAttachment } from '@/views/mastodon/attachments.ts'; linkify.registerCustomProtocol('nostr', true); @@ -60,16 +60,14 @@ function parseNoteContent(content: string): ParsedNoteContent { function getMediaLinks(links: Link[]): DittoAttachment[] { return links.reduce((acc, link) => { - const mimeType = getUrlMimeType(link.href); - if (!mimeType) return acc; + const mediaType = getUrlMediaType(link.href); + if (!mediaType) return acc; - const [baseType, _subType] = mimeType.split('/'); - - if (['audio', 'image', 'video'].includes(baseType)) { + if (isPermittedMediaType(mediaType, ['audio', 'image', 'video'])) { acc.push({ url: link.href, data: { - mime: mimeType, + mime: mediaType, }, }); } @@ -79,7 +77,7 @@ function getMediaLinks(links: Link[]): DittoAttachment[] { } function isNonMediaLink({ href }: Link): boolean { - return /^https?:\/\//.test(href) && !getUrlMimeType(href); + return /^https?:\/\//.test(href) && !getUrlMediaType(href); } /** Ensures the Link is a URL so it can be parsed. */ @@ -87,17 +85,6 @@ function isLinkURL(link: Link): boolean { return link.type === 'url'; } -/** Get the extension from the URL, then get its type. */ -function getUrlMimeType(url: string): string | undefined { - try { - const { pathname } = new URL(url); - const ext = pathname.split('.').pop() ?? ''; - return typeByExtension(ext); - } catch { - return undefined; - } -} - /** Get pubkey from decoded bech32 entity, or undefined if not applicable. */ function getDecodedPubkey(decoded: nip19.DecodeResult): string | undefined { switch (decoded.type) { diff --git a/src/utils/media.test.ts b/src/utils/media.test.ts new file mode 100644 index 00000000..e88e97da --- /dev/null +++ b/src/utils/media.test.ts @@ -0,0 +1,17 @@ +import { assertEquals } from '@std/assert'; + +import { getUrlMediaType, isPermittedMediaType } from '@/utils/media.ts'; + +Deno.test('getUrlMediaType', () => { + assertEquals(getUrlMediaType('https://example.com/image.png'), 'image/png'); + assertEquals(getUrlMediaType('https://example.com/index.html'), 'text/html'); + assertEquals(getUrlMediaType('https://example.com/yolo'), undefined); + assertEquals(getUrlMediaType('https://example.com/'), undefined); +}); + +Deno.test('isPermittedMediaType', () => { + assertEquals(isPermittedMediaType('image/png', ['image', 'video']), true); + assertEquals(isPermittedMediaType('video/webm', ['image', 'video']), true); + assertEquals(isPermittedMediaType('audio/ogg', ['image', 'video']), false); + assertEquals(isPermittedMediaType('application/json', ['image', 'video']), false); +}); diff --git a/src/utils/media.ts b/src/utils/media.ts new file mode 100644 index 00000000..9c0ea9e3 --- /dev/null +++ b/src/utils/media.ts @@ -0,0 +1,24 @@ +import { typeByExtension } from '@std/media-types'; + +/** Get media type of the filename in the URL by its extension, if any. */ +export function getUrlMediaType(url: string): string | undefined { + try { + const { pathname } = new URL(url); + const ext = pathname.split('.').pop() ?? ''; + return typeByExtension(ext); + } catch { + return undefined; + } +} + +/** + * Check if the base type matches any of the permitted types. + * + * ```ts + * isPermittedMediaType('image/png', ['image', 'video']); // true + * ``` + */ +export function isPermittedMediaType(mediaType: string, permitted: string[]): boolean { + const [baseType, _subType] = mediaType.split('/'); + return permitted.includes(baseType); +} From 942260aa54f61a96a43492f886328df4cfa29dc4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 11:53:33 -0500 Subject: [PATCH 24/65] note.ts -> utils/note.ts --- src/{ => utils}/note.ts | 0 src/views/mastodon/statuses.ts | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename src/{ => utils}/note.ts (100%) diff --git a/src/note.ts b/src/utils/note.ts similarity index 100% rename from src/note.ts rename to src/utils/note.ts diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index aefe258b..1b0ebe85 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -4,10 +4,10 @@ import { nip19 } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { getMediaLinks, parseNoteContent } from '@/note.ts'; import { Storages } from '@/storages.ts'; import { findReplyTag } from '@/tags.ts'; import { nostrDate } from '@/utils.ts'; +import { getMediaLinks, parseNoteContent } from '@/utils/note.ts'; import { unfurlCardCached } from '@/utils/unfurl.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { DittoAttachment, renderAttachment } from '@/views/mastodon/attachments.ts'; From c8f9483795f12d8e79470c6142be88f8879f398d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 11:56:22 -0500 Subject: [PATCH 25/65] Add note.test.ts --- src/utils/note.test.ts | 28 ++++++++++++++++++++++++++++ src/utils/note.ts | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 src/utils/note.test.ts diff --git a/src/utils/note.test.ts b/src/utils/note.test.ts new file mode 100644 index 00000000..d123050f --- /dev/null +++ b/src/utils/note.test.ts @@ -0,0 +1,28 @@ +import { assertEquals } from '@std/assert'; + +import { getMediaLinks, parseNoteContent } from '@/utils/note.ts'; + +Deno.test('parseNoteContent', () => { + const { html, links, firstUrl } = parseNoteContent('Hello, world!'); + assertEquals(html, 'Hello, world!'); + assertEquals(links, []); + assertEquals(firstUrl, undefined); +}); + +Deno.test('getMediaLinks', () => { + const links = [ + { href: 'https://example.com/image.png' }, + { href: 'https://example.com/index.html' }, + { href: 'https://example.com/yolo' }, + { href: 'https://example.com/' }, + ]; + const mediaLinks = getMediaLinks(links); + assertEquals(mediaLinks, [ + { + url: 'https://example.com/image.png', + data: { + mime: 'image/png', + }, + }, + ]); +}); diff --git a/src/utils/note.ts b/src/utils/note.ts index 71dc0174..580c2072 100644 --- a/src/utils/note.ts +++ b/src/utils/note.ts @@ -58,7 +58,7 @@ function parseNoteContent(content: string): ParsedNoteContent { }; } -function getMediaLinks(links: Link[]): DittoAttachment[] { +function getMediaLinks(links: Pick[]): DittoAttachment[] { return links.reduce((acc, link) => { const mediaType = getUrlMediaType(link.href); if (!mediaType) return acc; From 7d34b9401e8b191e91801d205c85c3685a876769 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 13:22:20 -0500 Subject: [PATCH 26/65] Support imeta tags --- src/controllers/api/statuses.ts | 2 +- src/db/unattached-media.ts | 3 +-- src/schemas/nostr.ts | 25 +-------------------- src/upload.ts | 37 ++++++++++++++++++++----------- src/views/mastodon/attachments.ts | 16 ++++++++----- src/views/mastodon/statuses.ts | 12 ++++++---- 6 files changed, 46 insertions(+), 49 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 00f9a98e..8904b2b8 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -96,7 +96,7 @@ const createStatusController: AppController = async (c) => { if (data.media_ids?.length) { const media = await getUnattachedMediaByIds(kysely, data.media_ids) .then((media) => media.filter(({ pubkey }) => pubkey === viewerPubkey)) - .then((media) => media.map(({ url, data }) => ['media', url, data])); + .then((media) => media.map(({ data }) => ['imeta', ...data])); tags.push(...media); } diff --git a/src/db/unattached-media.ts b/src/db/unattached-media.ts index cee1e3a3..0628278d 100644 --- a/src/db/unattached-media.ts +++ b/src/db/unattached-media.ts @@ -3,13 +3,12 @@ import uuid62 from 'uuid62'; import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; -import { type MediaData } from '@/schemas/nostr.ts'; interface UnattachedMedia { id: string; pubkey: string; url: string; - data: MediaData; + data: string[][]; // NIP-94 tags uploaded_at: number; } diff --git a/src/schemas/nostr.ts b/src/schemas/nostr.ts index a42b9f07..d8aa29a4 100644 --- a/src/schemas/nostr.ts +++ b/src/schemas/nostr.ts @@ -9,27 +9,12 @@ const signedEventSchema = n.event() .refine((event) => event.id === getEventHash(event), 'Event ID does not match hash') .refine(verifyEvent, 'Event signature is invalid'); -/** Media data schema from `"media"` tags. */ -const mediaDataSchema = z.object({ - blurhash: z.string().optional().catch(undefined), - cid: z.string().optional().catch(undefined), - description: z.string().max(200).optional().catch(undefined), - height: z.number().int().positive().optional().catch(undefined), - mime: z.string().optional().catch(undefined), - name: z.string().optional().catch(undefined), - size: z.number().int().positive().optional().catch(undefined), - width: z.number().int().positive().optional().catch(undefined), -}); - /** Kind 0 content schema for the Ditto server admin user. */ const serverMetaSchema = n.metadata().and(z.object({ tagline: z.string().optional().catch(undefined), email: z.string().optional().catch(undefined), })); -/** Media data from `"media"` tags. */ -type MediaData = z.infer; - /** NIP-11 Relay Information Document. */ const relayInfoDocSchema = z.object({ name: z.string().transform((val) => val.slice(0, 30)).optional().catch(undefined), @@ -47,12 +32,4 @@ const emojiTagSchema = z.tuple([z.literal('emoji'), z.string(), z.string().url() /** NIP-30 custom emoji tag. */ type EmojiTag = z.infer; -export { - type EmojiTag, - emojiTagSchema, - type MediaData, - mediaDataSchema, - relayInfoDocSchema, - serverMetaSchema, - signedEventSchema, -}; +export { type EmojiTag, emojiTagSchema, relayInfoDocSchema, serverMetaSchema, signedEventSchema }; diff --git a/src/upload.ts b/src/upload.ts index 632dbabf..4f5fd14f 100644 --- a/src/upload.ts +++ b/src/upload.ts @@ -8,26 +8,37 @@ interface FileMeta { } /** Upload a file, track it in the database, and return the resulting media object. */ -async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal) { - const { name, type, size } = file; +async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Promise { + const { type, size } = file; const { pubkey, description } = meta; if (file.size > Conf.maxUploadSize) { throw new Error('File size is too large.'); } - const { url } = await uploader.upload(file, { signal }); + const { url, sha256, cid } = await uploader.upload(file, { signal }); - return insertUnattachedMedia({ - pubkey, - url, - data: { - name, - size, - description, - mime: type, - }, - }); + const data: string[][] = [ + ['url', url], + ['m', type], + ['size', size.toString()], + ]; + + if (sha256) { + data.push(['x', sha256]); + } + + if (cid) { + data.push(['cid', cid]); + } + + if (description) { + data.push(['alt', description]); + } + + await insertUnattachedMedia({ pubkey, url, data }); + + return data; } export { uploadFile }; diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 3ea989e6..18fe0319 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -6,15 +6,21 @@ type DittoAttachment = TypeFest.SetOptional name === 'm')?.[1]; + const alt = data.find(([name]) => name === 'alt')?.[1]; + const cid = data.find(([name]) => name === 'cid')?.[1]; + const blurhash = data.find(([name]) => name === 'blurhash')?.[1]; + return { - id: id ?? url ?? data.cid, - type: getAttachmentType(data.mime ?? ''), + id: id ?? url, + type: getAttachmentType(m ?? ''), url, preview_url: url, remote_url: null, - description: data.description ?? '', - blurhash: data.blurhash || null, - cid: data.cid, + description: alt ?? '', + blurhash: blurhash || null, + cid: cid, }; } diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 1b0ebe85..8428e9a1 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -1,4 +1,4 @@ -import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; +import { NostrEvent } from '@nostrify/nostrify'; import { isCWTag } from 'https://gitlab.com/soapbox-pub/mostr/-/raw/c67064aee5ade5e01597c6d23e22e53c628ef0e2/src/nostr/tags.ts'; import { nip19 } from 'nostr-tools'; @@ -12,7 +12,6 @@ import { unfurlCardCached } from '@/utils/unfurl.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { DittoAttachment, renderAttachment } from '@/views/mastodon/attachments.ts'; import { renderEmojis } from '@/views/mastodon/emojis.ts'; -import { mediaDataSchema } from '@/schemas/nostr.ts'; interface RenderStatusOpts { viewerPubkey?: string; @@ -80,8 +79,13 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< const mediaLinks = getMediaLinks(links); const mediaTags: DittoAttachment[] = event.tags - .filter((tag) => tag[0] === 'media') - .map(([_, url, json]) => ({ url, data: n.json().pipe(mediaDataSchema).parse(json) })); + .filter(([name]) => name === 'imeta') + .map(([_, ...entries]) => { + const data = entries.map((entry) => entry.split(' ')); + const url = data.find(([name]) => name === 'url')?.[1]; + return { url, data }; + }) + .filter((media): media is DittoAttachment => !!media.url); const media = [...mediaLinks, ...mediaTags]; From 611a94bdcf640efcd22615ff12d1f911fbf61cc4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 14:32:50 -0500 Subject: [PATCH 27/65] Fix uploading (almost) --- src/controllers/api/statuses.ts | 12 +++++++++--- src/db/unattached-media.ts | 9 +++++++++ src/utils/api.ts | 2 ++ src/utils/note.ts | 16 +++++++--------- src/views/mastodon/attachments.ts | 14 +++++++------- src/views/mastodon/statuses.ts | 13 ++++--------- 6 files changed, 38 insertions(+), 28 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 8904b2b8..d8186f59 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -5,7 +5,7 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; -import { getUnattachedMediaByIds } from '@/db/unattached-media.ts'; +import { getUnattachedMediaByUrls } from '@/db/unattached-media.ts'; import { getAncestors, getAuthor, getDescendants, getEvent } from '@/queries.ts'; import { addTag, deleteTag } from '@/tags.ts'; import { createEvent, paginationSchema, parseBody, updateListEvent } from '@/utils/api.ts'; @@ -94,9 +94,15 @@ const createStatusController: AppController = async (c) => { const viewerPubkey = await c.get('signer')?.getPublicKey(); if (data.media_ids?.length) { - const media = await getUnattachedMediaByIds(kysely, data.media_ids) + const media = await getUnattachedMediaByUrls(kysely, data.media_ids) .then((media) => media.filter(({ pubkey }) => pubkey === viewerPubkey)) - .then((media) => media.map(({ data }) => ['imeta', ...data])); + .then((media) => + media.map(({ data }) => { + const tags: string[][] = JSON.parse(data); + const values: string[] = tags.map((tag) => tag.join(' ')); + return ['imeta', ...values]; + }) + ); tags.push(...media); } diff --git a/src/db/unattached-media.ts b/src/db/unattached-media.ts index 0628278d..397a1f77 100644 --- a/src/db/unattached-media.ts +++ b/src/db/unattached-media.ts @@ -64,6 +64,14 @@ async function getUnattachedMediaByIds(kysely: Kysely, ids: string[ .execute(); } +/** Get unattached media by URLs. */ +async function getUnattachedMediaByUrls(kysely: Kysely, urls: string[]) { + if (!urls.length) return []; + return await selectUnattachedMediaQuery(kysely) + .where('url', 'in', urls) + .execute(); +} + /** Delete rows as an event with media is being created. */ async function deleteAttachedMedia(pubkey: string, urls: string[]): Promise { if (!urls.length) return; @@ -79,6 +87,7 @@ export { deleteUnattachedMediaByUrl, getUnattachedMedia, getUnattachedMediaByIds, + getUnattachedMediaByUrls, insertUnattachedMedia, type UnattachedMedia, }; diff --git a/src/utils/api.ts b/src/utils/api.ts index dceede7a..c54f5aad 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -29,6 +29,8 @@ async function createEvent(t: EventStub, c: AppContext): Promise { }); } + console.log(t); + const event = await signer.signEvent({ content: '', created_at: nostrNow(), diff --git a/src/utils/note.ts b/src/utils/note.ts index 580c2072..20cb83aa 100644 --- a/src/utils/note.ts +++ b/src/utils/note.ts @@ -5,7 +5,6 @@ import { nip19, nip21 } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { getUrlMediaType, isPermittedMediaType } from '@/utils/media.ts'; -import { type DittoAttachment } from '@/views/mastodon/attachments.ts'; linkify.registerCustomProtocol('nostr', true); linkify.registerCustomProtocol('wss'); @@ -58,18 +57,17 @@ function parseNoteContent(content: string): ParsedNoteContent { }; } -function getMediaLinks(links: Pick[]): DittoAttachment[] { - return links.reduce((acc, link) => { +/** Returns a matrix of tags. Each item is a list of NIP-94 tags representing a file. */ +function getMediaLinks(links: Pick[]): string[][][] { + return links.reduce((acc, link) => { const mediaType = getUrlMediaType(link.href); if (!mediaType) return acc; if (isPermittedMediaType(mediaType, ['audio', 'image', 'video'])) { - acc.push({ - url: link.href, - data: { - mime: mediaType, - }, - }); + acc.push([ + ['url', link.href], + ['m', mediaType], + ]); } return acc; diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 18fe0319..8922985f 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -4,16 +4,16 @@ import { UnattachedMedia } from '@/db/unattached-media.ts'; type DittoAttachment = TypeFest.SetOptional; -function renderAttachment(media: DittoAttachment) { - const { id, data, url } = media; +function renderAttachment(tags: string[][]) { + const url = tags.find(([name]) => name === 'url')?.[1]; - const m = data.find(([name]) => name === 'm')?.[1]; - const alt = data.find(([name]) => name === 'alt')?.[1]; - const cid = data.find(([name]) => name === 'cid')?.[1]; - const blurhash = data.find(([name]) => name === 'blurhash')?.[1]; + const m = tags.find(([name]) => name === 'm')?.[1]; + const alt = tags.find(([name]) => name === 'alt')?.[1]; + const cid = tags.find(([name]) => name === 'cid')?.[1]; + const blurhash = tags.find(([name]) => name === 'blurhash')?.[1]; return { - id: id ?? url, + id: url, type: getAttachmentType(m ?? ''), url, preview_url: url, diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 8428e9a1..889c23dd 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -10,7 +10,7 @@ import { nostrDate } from '@/utils.ts'; import { getMediaLinks, parseNoteContent } from '@/utils/note.ts'; import { unfurlCardCached } from '@/utils/unfurl.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; -import { DittoAttachment, renderAttachment } from '@/views/mastodon/attachments.ts'; +import { renderAttachment } from '@/views/mastodon/attachments.ts'; import { renderEmojis } from '@/views/mastodon/emojis.ts'; interface RenderStatusOpts { @@ -78,16 +78,11 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< const mediaLinks = getMediaLinks(links); - const mediaTags: DittoAttachment[] = event.tags + const imeta: string[][][] = event.tags .filter(([name]) => name === 'imeta') - .map(([_, ...entries]) => { - const data = entries.map((entry) => entry.split(' ')); - const url = data.find(([name]) => name === 'url')?.[1]; - return { url, data }; - }) - .filter((media): media is DittoAttachment => !!media.url); + .map(([_, ...entries]) => entries.map((entry) => entry.split(' '))); - const media = [...mediaLinks, ...mediaTags]; + const media = [...mediaLinks, ...imeta]; return { id: event.id, From e7d350a0e305a58191331b6664866419d215802f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 14:54:10 -0500 Subject: [PATCH 28/65] Fix uploading by URL --- deno.json | 1 - src/controllers/api/statuses.ts | 4 ++-- src/db/unattached-media.ts | 25 +++++-------------------- src/upload.ts | 11 ++++++++++- src/utils/api.ts | 2 -- src/views/mastodon/attachments.ts | 6 +++--- 6 files changed, 20 insertions(+), 29 deletions(-) diff --git a/deno.json b/deno.json index 54ba4680..946b4e02 100644 --- a/deno.json +++ b/deno.json @@ -54,7 +54,6 @@ "tseep": "npm:tseep@^1.2.1", "type-fest": "npm:type-fest@^4.3.0", "unfurl.js": "npm:unfurl.js@^6.4.0", - "uuid62": "npm:uuid62@^1.0.2", "zod": "npm:zod@^3.23.5", "~/fixtures/": "./fixtures/" }, diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index d8186f59..f7a603fe 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -5,7 +5,7 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; -import { getUnattachedMediaByUrls } from '@/db/unattached-media.ts'; +import { getUnattachedMediaByIds } from '@/db/unattached-media.ts'; import { getAncestors, getAuthor, getDescendants, getEvent } from '@/queries.ts'; import { addTag, deleteTag } from '@/tags.ts'; import { createEvent, paginationSchema, parseBody, updateListEvent } from '@/utils/api.ts'; @@ -94,7 +94,7 @@ const createStatusController: AppController = async (c) => { const viewerPubkey = await c.get('signer')?.getPublicKey(); if (data.media_ids?.length) { - const media = await getUnattachedMediaByUrls(kysely, data.media_ids) + const media = await getUnattachedMediaByIds(kysely, data.media_ids) .then((media) => media.filter(({ pubkey }) => pubkey === viewerPubkey)) .then((media) => media.map(({ data }) => { diff --git a/src/db/unattached-media.ts b/src/db/unattached-media.ts index 397a1f77..0ab46b61 100644 --- a/src/db/unattached-media.ts +++ b/src/db/unattached-media.ts @@ -1,5 +1,4 @@ import { Kysely } from 'kysely'; -import uuid62 from 'uuid62'; import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; @@ -8,24 +7,19 @@ interface UnattachedMedia { id: string; pubkey: string; url: string; - data: string[][]; // NIP-94 tags + /** NIP-94 tags. */ + data: string[][]; uploaded_at: number; } /** Add unattached media into the database. */ -async function insertUnattachedMedia(media: Omit) { - const result = { - id: uuid62.v4(), - uploaded_at: Date.now(), - ...media, - }; - +async function insertUnattachedMedia(media: UnattachedMedia) { const kysely = await DittoDB.getInstance(); await kysely.insertInto('unattached_media') - .values({ ...result, data: JSON.stringify(media.data) }) + .values({ ...media, data: JSON.stringify(media.data) }) .execute(); - return result; + return media; } /** Select query for unattached media. */ @@ -64,14 +58,6 @@ async function getUnattachedMediaByIds(kysely: Kysely, ids: string[ .execute(); } -/** Get unattached media by URLs. */ -async function getUnattachedMediaByUrls(kysely: Kysely, urls: string[]) { - if (!urls.length) return []; - return await selectUnattachedMediaQuery(kysely) - .where('url', 'in', urls) - .execute(); -} - /** Delete rows as an event with media is being created. */ async function deleteAttachedMedia(pubkey: string, urls: string[]): Promise { if (!urls.length) return; @@ -87,7 +73,6 @@ export { deleteUnattachedMediaByUrl, getUnattachedMedia, getUnattachedMediaByIds, - getUnattachedMediaByUrls, insertUnattachedMedia, type UnattachedMedia, }; diff --git a/src/upload.ts b/src/upload.ts index 4f5fd14f..d815bd83 100644 --- a/src/upload.ts +++ b/src/upload.ts @@ -36,7 +36,16 @@ async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Pro data.push(['alt', description]); } - await insertUnattachedMedia({ pubkey, url, data }); + const uuid = crypto.randomUUID(); + data.push(['uuid', uuid]); + + await insertUnattachedMedia({ + id: uuid, + pubkey, + url, + data, + uploaded_at: Date.now(), + }); return data; } diff --git a/src/utils/api.ts b/src/utils/api.ts index c54f5aad..dceede7a 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -29,8 +29,6 @@ async function createEvent(t: EventStub, c: AppContext): Promise { }); } - console.log(t); - const event = await signer.signEvent({ content: '', created_at: nostrNow(), diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 8922985f..1dbcda95 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -5,15 +5,15 @@ import { UnattachedMedia } from '@/db/unattached-media.ts'; type DittoAttachment = TypeFest.SetOptional; function renderAttachment(tags: string[][]) { - const url = tags.find(([name]) => name === 'url')?.[1]; - const m = tags.find(([name]) => name === 'm')?.[1]; + const url = tags.find(([name]) => name === 'url')?.[1]; const alt = tags.find(([name]) => name === 'alt')?.[1]; const cid = tags.find(([name]) => name === 'cid')?.[1]; + const uuid = tags.find(([name]) => name === 'uuid')?.[1]; const blurhash = tags.find(([name]) => name === 'blurhash')?.[1]; return { - id: url, + id: uuid, type: getAttachmentType(m ?? ''), url, preview_url: url, From 91ea4577f1549be74b8f036293ca3fa0a56d7bd5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 14:58:48 -0500 Subject: [PATCH 29/65] Filter out attachments with no url --- src/views/mastodon/attachments.ts | 4 +++- src/views/mastodon/statuses.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 1dbcda95..9f8e5c35 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -12,8 +12,10 @@ function renderAttachment(tags: string[][]) { const uuid = tags.find(([name]) => name === 'uuid')?.[1]; const blurhash = tags.find(([name]) => name === 'blurhash')?.[1]; + if (!url) return; + return { - id: uuid, + id: uuid ?? url, type: getAttachmentType(m ?? ''), url, preview_url: url, diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 889c23dd..16ba4822 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -106,7 +106,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< pinned: Boolean(pinEvent), reblog: null, application: null, - media_attachments: media.map(renderAttachment), + media_attachments: media.map(renderAttachment).filter(Boolean), mentions, tags: [], emojis: renderEmojis(event), From b1b341d3b8a947b2a07325bfa07f2060c01cadcd Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 15:29:12 -0500 Subject: [PATCH 30/65] Insert media URL into text --- src/controllers/api/statuses.ts | 27 +++++++++++++-------------- src/db/unattached-media.ts | 10 ++++++++-- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index f7a603fe..e620b930 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -91,21 +91,14 @@ const createStatusController: AppController = async (c) => { tags.push(['subject', data.spoiler_text]); } - const viewerPubkey = await c.get('signer')?.getPublicKey(); + const media = data.media_ids?.length ? await getUnattachedMediaByIds(kysely, data.media_ids) : []; - if (data.media_ids?.length) { - const media = await getUnattachedMediaByIds(kysely, data.media_ids) - .then((media) => media.filter(({ pubkey }) => pubkey === viewerPubkey)) - .then((media) => - media.map(({ data }) => { - const tags: string[][] = JSON.parse(data); - const values: string[] = tags.map((tag) => tag.join(' ')); - return ['imeta', ...values]; - }) - ); + const imeta: string[][] = media.map(({ data }) => { + const values: string[] = data.map((tag) => tag.join(' ')); + return ['imeta', ...values]; + }); - tags.push(...media); - } + tags.push(...imeta); const pubkeys = new Set(); @@ -137,9 +130,15 @@ const createStatusController: AppController = async (c) => { tags.push(['t', match[1]]); } + const mediaUrls: string[] = media + .map(({ data }) => data.find(([name]) => name === 'url')?.[1]) + .filter((url): url is string => Boolean(url)); + + const mediaCompat: string = mediaUrls.length ? ['', '', ...mediaUrls].join('\n') : ''; + const event = await createEvent({ kind: 1, - content, + content: content + mediaCompat, tags, }, c); diff --git a/src/db/unattached-media.ts b/src/db/unattached-media.ts index 0ab46b61..0e0aeea6 100644 --- a/src/db/unattached-media.ts +++ b/src/db/unattached-media.ts @@ -51,11 +51,17 @@ async function deleteUnattachedMediaByUrl(url: string) { } /** Get unattached media by IDs. */ -async function getUnattachedMediaByIds(kysely: Kysely, ids: string[]) { +async function getUnattachedMediaByIds(kysely: Kysely, ids: string[]): Promise { if (!ids.length) return []; - return await selectUnattachedMediaQuery(kysely) + + const results = await selectUnattachedMediaQuery(kysely) .where('id', 'in', ids) .execute(); + + return results.map((row) => ({ + ...row, + data: JSON.parse(row.data), + })); } /** Delete rows as an event with media is being created. */ From c8b999a1f7a808e89aab968c24ae2bf467fb4a8f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 15:36:17 -0500 Subject: [PATCH 31/65] imeta: don't get attachment ID from a tag --- src/upload.ts | 13 ++++--------- src/views/mastodon/attachments.ts | 14 +++++--------- src/views/mastodon/statuses.ts | 2 +- 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/src/upload.ts b/src/upload.ts index d815bd83..40184f09 100644 --- a/src/upload.ts +++ b/src/upload.ts @@ -1,5 +1,5 @@ import { Conf } from '@/config.ts'; -import { insertUnattachedMedia } from '@/db/unattached-media.ts'; +import { insertUnattachedMedia, UnattachedMedia } from '@/db/unattached-media.ts'; import { configUploader as uploader } from '@/uploaders/config.ts'; interface FileMeta { @@ -8,7 +8,7 @@ interface FileMeta { } /** Upload a file, track it in the database, and return the resulting media object. */ -async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Promise { +async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Promise { const { type, size } = file; const { pubkey, description } = meta; @@ -36,18 +36,13 @@ async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Pro data.push(['alt', description]); } - const uuid = crypto.randomUUID(); - data.push(['uuid', uuid]); - - await insertUnattachedMedia({ - id: uuid, + return insertUnattachedMedia({ + id: crypto.randomUUID(), pubkey, url, data, uploaded_at: Date.now(), }); - - return data; } export { uploadFile }; diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 9f8e5c35..273b460d 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -1,21 +1,17 @@ -import * as TypeFest from 'type-fest'; +/** Render Mastodon media attachment. */ +function renderAttachment(media: { id?: string; data: string[][] }) { + const { id, data: tags } = media; -import { UnattachedMedia } from '@/db/unattached-media.ts'; - -type DittoAttachment = TypeFest.SetOptional; - -function renderAttachment(tags: string[][]) { const m = tags.find(([name]) => name === 'm')?.[1]; const url = tags.find(([name]) => name === 'url')?.[1]; const alt = tags.find(([name]) => name === 'alt')?.[1]; const cid = tags.find(([name]) => name === 'cid')?.[1]; - const uuid = tags.find(([name]) => name === 'uuid')?.[1]; const blurhash = tags.find(([name]) => name === 'blurhash')?.[1]; if (!url) return; return { - id: uuid ?? url, + id: id ?? url, type: getAttachmentType(m ?? ''), url, preview_url: url, @@ -40,4 +36,4 @@ function getAttachmentType(mime: string): string { } } -export { type DittoAttachment, renderAttachment }; +export { renderAttachment }; diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 16ba4822..c674e16d 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -106,7 +106,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< pinned: Boolean(pinEvent), reblog: null, application: null, - media_attachments: media.map(renderAttachment).filter(Boolean), + media_attachments: media.map((m) => renderAttachment({ data: m })).filter(Boolean), mentions, tags: [], emojis: renderEmojis(event), From cbf0bc35940b87c84d30feb1a91d0c900ac47cd5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 15:46:28 -0500 Subject: [PATCH 32/65] Fix note test --- src/utils/note.test.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/utils/note.test.ts b/src/utils/note.test.ts index d123050f..9c8fad7e 100644 --- a/src/utils/note.test.ts +++ b/src/utils/note.test.ts @@ -17,12 +17,8 @@ Deno.test('getMediaLinks', () => { { href: 'https://example.com/' }, ]; const mediaLinks = getMediaLinks(links); - assertEquals(mediaLinks, [ - { - url: 'https://example.com/image.png', - data: { - mime: 'image/png', - }, - }, - ]); + assertEquals(mediaLinks, [[ + ['url', 'https://example.com/image.png'], + ['m', 'image/png'], + ]]); }); From c89be75e5b9f240c8eba9fa180cad7ebc6301d76 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 16:22:24 -0500 Subject: [PATCH 33/65] Add a nostr.build uploader --- fixtures/nostrbuild-gif.json | 34 ++++++++++++++++++++++++++++++++++ fixtures/nostrbuild-mp3.json | 29 +++++++++++++++++++++++++++++ src/config.ts | 4 ++++ src/schemas/nostrbuild.ts | 18 ++++++++++++++++++ src/upload.ts | 6 +++++- src/uploaders/config.ts | 3 +++ src/uploaders/nostrbuild.ts | 33 +++++++++++++++++++++++++++++++++ src/uploaders/types.ts | 2 ++ 8 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 fixtures/nostrbuild-gif.json create mode 100644 fixtures/nostrbuild-mp3.json create mode 100644 src/schemas/nostrbuild.ts create mode 100644 src/uploaders/nostrbuild.ts diff --git a/fixtures/nostrbuild-gif.json b/fixtures/nostrbuild-gif.json new file mode 100644 index 00000000..49a969af --- /dev/null +++ b/fixtures/nostrbuild-gif.json @@ -0,0 +1,34 @@ +{ + "status": "success", + "message": "Upload successful.", + "data": [ + { + "input_name": "APIv2", + "name": "e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "sha256": "0a71f1c9dd982079bc52e96403368209cbf9507c5f6956134686f56e684b6377", + "original_sha256": "e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3", + "type": "picture", + "mime": "image/gif", + "size": 1796276, + "blurhash": "LGH-S^Vwm]x]04kX-qR-R]SL5FxZ", + "dimensions": { + "width": 360, + "height": 216 + }, + "dimensionsString": "360x216", + "url": "https://image.nostr.build/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "thumbnail": "https://image.nostr.build/thumb/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "responsive": { + "240p": "https://image.nostr.build/resp/240p/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "360p": "https://image.nostr.build/resp/360p/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "480p": "https://image.nostr.build/resp/480p/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "720p": "https://image.nostr.build/resp/720p/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "1080p": "https://image.nostr.build/resp/1080p/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif" + }, + "metadata": { + "date:create": "2024-05-18T02:11:39+00:00", + "date:modify": "2024-05-18T02:11:39+00:00" + } + } + ] +} \ No newline at end of file diff --git a/fixtures/nostrbuild-mp3.json b/fixtures/nostrbuild-mp3.json new file mode 100644 index 00000000..42a60b44 --- /dev/null +++ b/fixtures/nostrbuild-mp3.json @@ -0,0 +1,29 @@ +{ + "status": "success", + "message": "Upload successful.", + "data": [ + { + "id": 0, + "input_name": "APIv2", + "name": "f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "url": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "thumbnail": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "responsive": { + "240p": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "360p": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "480p": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "720p": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "1080p": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3" + }, + "blurhash": "", + "sha256": "f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725", + "original_sha256": "f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725", + "type": "video", + "mime": "audio/mpeg", + "size": 1519616, + "metadata": [], + "dimensions": [], + "dimensionsString": "0x0" + } + ] +} \ No newline at end of file diff --git a/src/config.ts b/src/config.ts index 6fe62b9e..f3b472ea 100644 --- a/src/config.ts +++ b/src/config.ts @@ -136,6 +136,10 @@ class Conf { return Deno.env.get('IPFS_API_URL') || 'http://localhost:5001'; }, }; + /** nostr.build API endpoint when the `nostrbuild` uploader is used. */ + static get nostrbuildEndpoint(): string { + return Deno.env.get('NOSTRBUILD_ENDPOINT') || 'https://nostr.build/api/v2/upload/files'; + } /** Module to upload files with. */ static get uploader() { return Deno.env.get('DITTO_UPLOADER'); diff --git a/src/schemas/nostrbuild.ts b/src/schemas/nostrbuild.ts new file mode 100644 index 00000000..c9fd6802 --- /dev/null +++ b/src/schemas/nostrbuild.ts @@ -0,0 +1,18 @@ +import { z } from 'zod'; + +export const nostrbuildFileSchema = z.object({ + name: z.string(), + url: z.string().url(), + thumbnail: z.string(), + blurhash: z.string(), + sha256: z.string(), + mime: z.string(), + dimensions: z.object({ + width: z.number(), + height: z.number(), + }), +}); + +export const nostrbuildSchema = z.object({ + data: nostrbuildFileSchema.array().min(1), +}); diff --git a/src/upload.ts b/src/upload.ts index 40184f09..5b43f397 100644 --- a/src/upload.ts +++ b/src/upload.ts @@ -16,7 +16,7 @@ async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Pro throw new Error('File size is too large.'); } - const { url, sha256, cid } = await uploader.upload(file, { signal }); + const { url, sha256, cid, blurhash } = await uploader.upload(file, { signal }); const data: string[][] = [ ['url', url], @@ -32,6 +32,10 @@ async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Pro data.push(['cid', cid]); } + if (blurhash) { + data.push(['blurhash', blurhash]); + } + if (description) { data.push(['alt', description]); } diff --git a/src/uploaders/config.ts b/src/uploaders/config.ts index 5b4c7aff..8ce22b6a 100644 --- a/src/uploaders/config.ts +++ b/src/uploaders/config.ts @@ -2,6 +2,7 @@ import { Conf } from '@/config.ts'; import { ipfsUploader } from '@/uploaders/ipfs.ts'; import { localUploader } from '@/uploaders/local.ts'; +import { nostrbuildUploader } from '@/uploaders/nostrbuild.ts'; import { s3Uploader } from '@/uploaders/s3.ts'; import type { Uploader } from './types.ts'; @@ -25,6 +26,8 @@ function uploader() { return ipfsUploader; case 'local': return localUploader; + case 'nostrbuild': + return nostrbuildUploader; default: throw new Error('No `DITTO_UPLOADER` configured. Uploads are disabled.'); } diff --git a/src/uploaders/nostrbuild.ts b/src/uploaders/nostrbuild.ts new file mode 100644 index 00000000..d9eed242 --- /dev/null +++ b/src/uploaders/nostrbuild.ts @@ -0,0 +1,33 @@ +import { Conf } from '@/config.ts'; +import { nostrbuildSchema } from '@/schemas/nostrbuild.ts'; + +import type { Uploader } from './types.ts'; + +/** nostr.build uploader. */ +export const nostrbuildUploader: Uploader = { + async upload(file) { + const formData = new FormData(); + formData.append('fileToUpload', file); + + const response = await fetch(Conf.nostrbuildEndpoint, { + method: 'POST', + body: formData, + }); + + const json = await response.json(); + console.log(JSON.stringify(json)); + + const [data] = nostrbuildSchema.parse(json).data; + + return { + id: data.url, + sha256: data.sha256, + url: data.url, + blurhash: data.blurhash, + }; + }, + // deno-lint-ignore require-await + async delete(): Promise { + return; + }, +}; diff --git a/src/uploaders/types.ts b/src/uploaders/types.ts index c514ad1b..ac5bf05b 100644 --- a/src/uploaders/types.ts +++ b/src/uploaders/types.ts @@ -14,6 +14,8 @@ interface UploadResult { url: string; /** SHA-256 hash of the file. */ sha256?: string; + /** Blurhash of the file. */ + blurhash?: string; /** IPFS CID of the file. */ cid?: string; } From ce49c500ae2fe62f6a83607055803aa18034c714 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 16:47:47 -0500 Subject: [PATCH 34/65] renderStatus: fix duplicated attachments --- src/views/mastodon/statuses.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index c674e16d..c707ebba 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -76,13 +76,11 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< const cw = event.tags.find(isCWTag); const subject = event.tags.find((tag) => tag[0] === 'subject'); - const mediaLinks = getMediaLinks(links); - const imeta: string[][][] = event.tags .filter(([name]) => name === 'imeta') .map(([_, ...entries]) => entries.map((entry) => entry.split(' '))); - const media = [...mediaLinks, ...imeta]; + const media = imeta.length ? imeta : getMediaLinks(links); return { id: event.id, From 353111051a79a37aa3b7be1c44b4c54ffbeb9904 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 16:53:17 -0500 Subject: [PATCH 35/65] Use dimensions from nostr.build --- src/schemas/nostrbuild.ts | 3 ++- src/upload.ts | 6 +++++- src/uploaders/nostrbuild.ts | 4 ++-- src/uploaders/types.ts | 4 ++++ src/views/mastodon/attachments.ts | 14 ++++++++++++++ 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/schemas/nostrbuild.ts b/src/schemas/nostrbuild.ts index c9fd6802..db9f6074 100644 --- a/src/schemas/nostrbuild.ts +++ b/src/schemas/nostrbuild.ts @@ -6,11 +6,12 @@ export const nostrbuildFileSchema = z.object({ thumbnail: z.string(), blurhash: z.string(), sha256: z.string(), + original_sha256: z.string(), mime: z.string(), dimensions: z.object({ width: z.number(), height: z.number(), - }), + }).optional().catch(undefined), }); export const nostrbuildSchema = z.object({ diff --git a/src/upload.ts b/src/upload.ts index 5b43f397..1da5a7de 100644 --- a/src/upload.ts +++ b/src/upload.ts @@ -16,7 +16,7 @@ async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Pro throw new Error('File size is too large.'); } - const { url, sha256, cid, blurhash } = await uploader.upload(file, { signal }); + const { url, sha256, cid, blurhash, width, height } = await uploader.upload(file, { signal }); const data: string[][] = [ ['url', url], @@ -24,6 +24,10 @@ async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Pro ['size', size.toString()], ]; + if (typeof width === 'number' && typeof height === 'number') { + data.push(['dim', `${width}x${height}`]); + } + if (sha256) { data.push(['x', sha256]); } diff --git a/src/uploaders/nostrbuild.ts b/src/uploaders/nostrbuild.ts index d9eed242..d9d08650 100644 --- a/src/uploaders/nostrbuild.ts +++ b/src/uploaders/nostrbuild.ts @@ -15,8 +15,6 @@ export const nostrbuildUploader: Uploader = { }); const json = await response.json(); - console.log(JSON.stringify(json)); - const [data] = nostrbuildSchema.parse(json).data; return { @@ -24,6 +22,8 @@ export const nostrbuildUploader: Uploader = { sha256: data.sha256, url: data.url, blurhash: data.blurhash, + width: data.dimensions?.width, + height: data.dimensions?.height, }; }, // deno-lint-ignore require-await diff --git a/src/uploaders/types.ts b/src/uploaders/types.ts index ac5bf05b..e423028c 100644 --- a/src/uploaders/types.ts +++ b/src/uploaders/types.ts @@ -18,6 +18,10 @@ interface UploadResult { blurhash?: string; /** IPFS CID of the file. */ cid?: string; + /** Width of the file, if applicable. */ + width?: number; + /** Height of the file, if applicable. */ + height?: number; } export type { Uploader }; diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 273b460d..0b1b8eb1 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -6,10 +6,23 @@ function renderAttachment(media: { id?: string; data: string[][] }) { const url = tags.find(([name]) => name === 'url')?.[1]; const alt = tags.find(([name]) => name === 'alt')?.[1]; const cid = tags.find(([name]) => name === 'cid')?.[1]; + const dim = tags.find(([name]) => name === 'dim')?.[1]; const blurhash = tags.find(([name]) => name === 'blurhash')?.[1]; if (!url) return; + const [width, height] = dim?.split('x').map(Number) ?? [null, null]; + + const meta = (typeof width === 'number' && typeof height === 'number') + ? { + original: { + width, + height, + aspect: width / height, + }, + } + : undefined; + return { id: id ?? url, type: getAttachmentType(m ?? ''), @@ -18,6 +31,7 @@ function renderAttachment(media: { id?: string; data: string[][] }) { remote_url: null, description: alt ?? '', blurhash: blurhash || null, + meta, cid: cid, }; } From e5595d34be546a99d5594eb0af185cd4119f4c88 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 17:08:30 -0500 Subject: [PATCH 36/65] Strip imeta links from the end of the content --- src/utils/note.ts | 28 +++++++++++++++++++++++++++- src/views/mastodon/statuses.ts | 4 ++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/utils/note.ts b/src/utils/note.ts index 20cb83aa..03da2def 100644 --- a/src/utils/note.ts +++ b/src/utils/note.ts @@ -57,6 +57,32 @@ function parseNoteContent(content: string): ParsedNoteContent { }; } +/** Remove imeta links. */ +function stripimeta(content: string, tags: string[][]): string { + const imeta = tags.filter(([name]) => name === 'imeta'); + + if (!imeta.length) { + return content; + } + + const urls = new Set( + imeta.map(([, ...values]) => values.map((v) => v.split(' ')).find(([name]) => name === 'url')?.[1]), + ); + + const lines = content.split('\n').reverse(); + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + if (line === '' || urls.has(line)) { + lines.splice(i, 1); + } else { + break; + } + } + + return lines.reverse().join('\n'); +} + /** Returns a matrix of tags. Each item is a list of NIP-94 tags representing a file. */ function getMediaLinks(links: Pick[]): string[][][] { return links.reduce((acc, link) => { @@ -93,4 +119,4 @@ function getDecodedPubkey(decoded: nip19.DecodeResult): string | undefined { } } -export { getMediaLinks, parseNoteContent }; +export { getMediaLinks, parseNoteContent, stripimeta }; diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index c707ebba..a06aac21 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -7,7 +7,7 @@ import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { Storages } from '@/storages.ts'; import { findReplyTag } from '@/tags.ts'; import { nostrDate } from '@/utils.ts'; -import { getMediaLinks, parseNoteContent } from '@/utils/note.ts'; +import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts'; import { unfurlCardCached } from '@/utils/unfurl.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { renderAttachment } from '@/views/mastodon/attachments.ts'; @@ -46,7 +46,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< [{ kinds: [0], authors: mentionedPubkeys, limit: mentionedPubkeys.length }], ); - const { html, links, firstUrl } = parseNoteContent(event.content); + const { html, links, firstUrl } = parseNoteContent(stripimeta(event.content, event.tags)); const [mentions, card, relatedEvents] = await Promise .all([ From 6090c4a6d9a0de134015ab8a02aaf8884631dc7f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 18:23:04 -0500 Subject: [PATCH 37/65] Make Uploaders return NIP-94 tags --- src/upload.ts | 30 ++++-------------------------- src/uploaders/config.ts | 4 ++-- src/uploaders/ipfs.ts | 11 ++++++----- src/uploaders/local.ts | 11 ++++++----- src/uploaders/nostrbuild.ts | 26 ++++++++++++++------------ src/uploaders/s3.ts | 12 +++++++----- src/uploaders/types.ts | 22 ++-------------------- 7 files changed, 41 insertions(+), 75 deletions(-) diff --git a/src/upload.ts b/src/upload.ts index 1da5a7de..cd9d2bb6 100644 --- a/src/upload.ts +++ b/src/upload.ts @@ -9,46 +9,24 @@ interface FileMeta { /** Upload a file, track it in the database, and return the resulting media object. */ async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Promise { - const { type, size } = file; const { pubkey, description } = meta; if (file.size > Conf.maxUploadSize) { throw new Error('File size is too large.'); } - const { url, sha256, cid, blurhash, width, height } = await uploader.upload(file, { signal }); - - const data: string[][] = [ - ['url', url], - ['m', type], - ['size', size.toString()], - ]; - - if (typeof width === 'number' && typeof height === 'number') { - data.push(['dim', `${width}x${height}`]); - } - - if (sha256) { - data.push(['x', sha256]); - } - - if (cid) { - data.push(['cid', cid]); - } - - if (blurhash) { - data.push(['blurhash', blurhash]); - } + const tags = await uploader.upload(file, { signal }); + const url = tags[0][1]; if (description) { - data.push(['alt', description]); + tags.push(['alt', description]); } return insertUnattachedMedia({ id: crypto.randomUUID(), pubkey, url, - data, + data: tags, uploaded_at: Date.now(), }); } diff --git a/src/uploaders/config.ts b/src/uploaders/config.ts index 8ce22b6a..3f3aac74 100644 --- a/src/uploaders/config.ts +++ b/src/uploaders/config.ts @@ -12,8 +12,8 @@ const configUploader: Uploader = { upload(file, opts) { return uploader().upload(file, opts); }, - delete(id, opts) { - return uploader().delete(id, opts); + async delete(id, opts) { + return await uploader().delete?.(id, opts); }, }; diff --git a/src/uploaders/ipfs.ts b/src/uploaders/ipfs.ts index 21619b5b..b83dc2e3 100644 --- a/src/uploaders/ipfs.ts +++ b/src/uploaders/ipfs.ts @@ -32,11 +32,12 @@ const ipfsUploader: Uploader = { const { Hash: cid } = ipfsAddResponseSchema.parse(await response.json()); - return { - id: cid, - cid, - url: new URL(`/ipfs/${cid}`, Conf.mediaDomain).toString(), - }; + return [ + ['url', new URL(`/ipfs/${cid}`, Conf.mediaDomain).toString()], + ['m', file.type], + ['cid', cid], + ['size', file.size.toString()], + ]; }, async delete(cid, opts) { const url = new URL('/api/v0/pin/rm', Conf.ipfs.apiUrl); diff --git a/src/uploaders/local.ts b/src/uploaders/local.ts index a2381a3b..d5cd46ad 100644 --- a/src/uploaders/local.ts +++ b/src/uploaders/local.ts @@ -22,11 +22,12 @@ const localUploader: Uploader = { const url = new URL(mediaDomain); const path = url.pathname === '/' ? filename : join(url.pathname, filename); - return { - id: filename, - sha256, - url: new URL(path, url).toString(), - }; + return [ + ['url', new URL(path, url).toString()], + ['m', file.type], + ['x', sha256], + ['size', file.size.toString()], + ]; }, async delete(id) { await Deno.remove(join(Conf.uploadsDir, id)); diff --git a/src/uploaders/nostrbuild.ts b/src/uploaders/nostrbuild.ts index d9d08650..8bca331b 100644 --- a/src/uploaders/nostrbuild.ts +++ b/src/uploaders/nostrbuild.ts @@ -17,17 +17,19 @@ export const nostrbuildUploader: Uploader = { const json = await response.json(); const [data] = nostrbuildSchema.parse(json).data; - return { - id: data.url, - sha256: data.sha256, - url: data.url, - blurhash: data.blurhash, - width: data.dimensions?.width, - height: data.dimensions?.height, - }; - }, - // deno-lint-ignore require-await - async delete(): Promise { - return; + const tags: [['url', string], ...string[][]] = [ + ['url', data.url], + ['m', data.mime], + ['x', data.sha256], + ['ox', data.original_sha256], + ['size', file.size.toString()], + ['blurhash', data.blurhash], + ]; + + if (data.dimensions) { + tags.push(['dim', `${data.dimensions.width}x${data.dimensions.height}`]); + } + + return tags; }, }; diff --git a/src/uploaders/s3.ts b/src/uploaders/s3.ts index 267d8172..aaff8c82 100644 --- a/src/uploaders/s3.ts +++ b/src/uploaders/s3.ts @@ -25,12 +25,14 @@ const s3Uploader: Uploader = { const { pathStyle, bucket } = Conf.s3; const path = (pathStyle && bucket) ? join(bucket, filename) : filename; + const url = new URL(path, Conf.mediaDomain).toString(); - return { - id: filename, - sha256, - url: new URL(path, Conf.mediaDomain).toString(), - }; + return [ + ['url', url], + ['m', file.type], + ['x', sha256], + ['size', file.size.toString()], + ]; }, async delete(id) { await client().deleteObject(id); diff --git a/src/uploaders/types.ts b/src/uploaders/types.ts index e423028c..81b8a0a7 100644 --- a/src/uploaders/types.ts +++ b/src/uploaders/types.ts @@ -1,27 +1,9 @@ /** Modular uploader interface, to support uploading to different backends. */ interface Uploader { /** Upload the file to the backend. */ - upload(file: File, opts?: { signal?: AbortSignal }): Promise; + upload(file: File, opts?: { signal?: AbortSignal }): Promise<[['url', string], ...string[][]]>; /** Delete the file from the backend. */ - delete(cid: string, opts?: { signal?: AbortSignal }): Promise; -} - -/** Return value from the uploader after uploading a file. */ -interface UploadResult { - /** File ID specific to the uploader, so it can later be referenced or deleted. */ - id: string; - /** URL where the file can be accessed. */ - url: string; - /** SHA-256 hash of the file. */ - sha256?: string; - /** Blurhash of the file. */ - blurhash?: string; - /** IPFS CID of the file. */ - cid?: string; - /** Width of the file, if applicable. */ - width?: number; - /** Height of the file, if applicable. */ - height?: number; + delete?(cid: string, opts?: { signal?: AbortSignal }): Promise; } export type { Uploader }; From 82c03dcb56f9d9235a4d15d1ebe41a696bd92546 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 22:00:24 -0500 Subject: [PATCH 38/65] Rewrite all the uploaders --- src/app.ts | 5 ++ src/controllers/api/accounts.ts | 9 +++- src/controllers/api/media.ts | 7 ++- src/interfaces/DittoUploader.ts | 3 ++ src/middleware/uploaderMiddleware.ts | 27 ++++++++++ src/schemas/nostrbuild.ts | 19 ------- src/upload.ts | 12 +++-- src/uploaders/DenoUploader.ts | 44 ++++++++++++++++ src/uploaders/IPFSUploader.ts | 70 ++++++++++++++++++++++++++ src/uploaders/NostrBuildUploader.ts | 65 ++++++++++++++++++++++++ src/uploaders/{s3.ts => S3Uploader.ts} | 40 +++++++++------ src/uploaders/config.ts | 36 ------------- src/uploaders/ipfs.ts | 57 --------------------- src/uploaders/local.ts | 37 -------------- src/uploaders/nostrbuild.ts | 35 ------------- src/uploaders/types.ts | 9 ---- 16 files changed, 260 insertions(+), 215 deletions(-) create mode 100644 src/interfaces/DittoUploader.ts create mode 100644 src/middleware/uploaderMiddleware.ts delete mode 100644 src/schemas/nostrbuild.ts create mode 100644 src/uploaders/DenoUploader.ts create mode 100644 src/uploaders/IPFSUploader.ts create mode 100644 src/uploaders/NostrBuildUploader.ts rename src/uploaders/{s3.ts => S3Uploader.ts} (58%) delete mode 100644 src/uploaders/config.ts delete mode 100644 src/uploaders/ipfs.ts delete mode 100644 src/uploaders/local.ts delete mode 100644 src/uploaders/nostrbuild.ts delete mode 100644 src/uploaders/types.ts diff --git a/src/app.ts b/src/app.ts index 059e883c..ddc99904 100644 --- a/src/app.ts +++ b/src/app.ts @@ -81,6 +81,7 @@ import { hostMetaController } from '@/controllers/well-known/host-meta.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; import { nostrController } from '@/controllers/well-known/nostr.ts'; import { webfingerController } from '@/controllers/well-known/webfinger.ts'; +import { DittoUploader } from '@/interfaces/DittoUploader.ts'; import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts'; import { cacheMiddleware } from '@/middleware/cacheMiddleware.ts'; import { cspMiddleware } from '@/middleware/cspMiddleware.ts'; @@ -89,11 +90,14 @@ import { signerMiddleware } from '@/middleware/signerMiddleware.ts'; import { storeMiddleware } from '@/middleware/storeMiddleware.ts'; import { blockController } from '@/controllers/api/accounts.ts'; import { unblockController } from '@/controllers/api/accounts.ts'; +import { uploaderMiddleware } from '@/middleware/uploaderMiddleware.ts'; interface AppEnv extends HonoEnv { Variables: { /** Signer to get the logged-in user's pubkey, relays, and to sign events, or `undefined` if the user isn't logged in. */ signer?: NostrSigner; + /** Uploader for the user to upload files. */ + uploader?: DittoUploader; /** NIP-98 signed event proving the pubkey is owned by the user. */ proof?: NostrEvent; /** Store */ @@ -129,6 +133,7 @@ app.use( cspMiddleware(), cors({ origin: '*', exposeHeaders: ['link'] }), signerMiddleware, + uploaderMiddleware, auth98Middleware(), storeMiddleware, ); diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 5c26ba51..d67fd888 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -202,6 +202,7 @@ const updateCredentialsSchema = z.object({ const updateCredentialsController: AppController = async (c) => { const pubkey = await c.get('signer')?.getPublicKey()!; + const uploader = c.get('uploader'); const body = await parseBody(c.req.raw); const result = updateCredentialsSchema.safeParse(body); @@ -220,9 +221,13 @@ const updateCredentialsController: AppController = async (c) => { nip05, } = result.data; + if ((avatarFile || headerFile) && !uploader) { + return c.json({ error: 'No uploader configured.' }, 500); + } + const [avatar, header] = await Promise.all([ - avatarFile ? uploadFile(avatarFile, { pubkey }) : undefined, - headerFile ? uploadFile(headerFile, { pubkey }) : undefined, + (avatarFile && uploader) ? uploadFile(uploader, avatarFile, { pubkey }) : undefined, + (headerFile && uploader) ? uploadFile(uploader, headerFile, { pubkey }) : undefined, ]); meta.name = display_name ?? meta.name; diff --git a/src/controllers/api/media.ts b/src/controllers/api/media.ts index 33b79810..101b7767 100644 --- a/src/controllers/api/media.ts +++ b/src/controllers/api/media.ts @@ -14,6 +14,11 @@ const mediaBodySchema = z.object({ }); const mediaController: AppController = async (c) => { + const uploader = c.get('uploader'); + if (!uploader) { + return c.json({ error: 'No uploader configured.' }, 500); + } + const pubkey = await c.get('signer')?.getPublicKey()!; const result = mediaBodySchema.safeParse(await parseBody(c.req.raw)); const { signal } = c.req.raw; @@ -24,7 +29,7 @@ const mediaController: AppController = async (c) => { try { const { file, description } = result.data; - const media = await uploadFile(file, { pubkey, description }, signal); + const media = await uploadFile(uploader, file, { pubkey, description }, signal); return c.json(renderAttachment(media)); } catch (e) { console.error(e); diff --git a/src/interfaces/DittoUploader.ts b/src/interfaces/DittoUploader.ts new file mode 100644 index 00000000..08cbf504 --- /dev/null +++ b/src/interfaces/DittoUploader.ts @@ -0,0 +1,3 @@ +export interface DittoUploader { + upload(file: File, opts?: { signal?: AbortSignal }): Promise<[['url', string], ...string[][]]>; +} diff --git a/src/middleware/uploaderMiddleware.ts b/src/middleware/uploaderMiddleware.ts new file mode 100644 index 00000000..8279a122 --- /dev/null +++ b/src/middleware/uploaderMiddleware.ts @@ -0,0 +1,27 @@ +import { AppMiddleware } from '@/app.ts'; +import { Conf } from '@/config.ts'; +import { DenoUploader } from '@/uploaders/DenoUploader.ts'; +import { IPFSUploader } from '@/uploaders/IPFSUploader.ts'; +import { NostrBuildUploader } from '@/uploaders/NostrBuildUploader.ts'; +import { S3Uploader } from '@/uploaders/S3Uploader.ts'; +import { fetchWorker } from '@/workers/fetch.ts'; + +/** Set an uploader for the user. */ +export const uploaderMiddleware: AppMiddleware = async (c, next) => { + switch (Conf.uploader) { + case 's3': + c.set('uploader', new S3Uploader(Conf.s3)); + break; + case 'ipfs': + c.set('uploader', new IPFSUploader({ baseUrl: Conf.mediaDomain, apiUrl: Conf.ipfs.apiUrl, fetch: fetchWorker })); + break; + case 'local': + c.set('uploader', new DenoUploader({ baseUrl: Conf.mediaDomain, dir: Conf.uploadsDir })); + break; + case 'nostrbuild': + c.set('uploader', new NostrBuildUploader({ endpoint: Conf.nostrbuildEndpoint, fetch: fetchWorker })); + break; + } + + await next(); +}; diff --git a/src/schemas/nostrbuild.ts b/src/schemas/nostrbuild.ts deleted file mode 100644 index db9f6074..00000000 --- a/src/schemas/nostrbuild.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { z } from 'zod'; - -export const nostrbuildFileSchema = z.object({ - name: z.string(), - url: z.string().url(), - thumbnail: z.string(), - blurhash: z.string(), - sha256: z.string(), - original_sha256: z.string(), - mime: z.string(), - dimensions: z.object({ - width: z.number(), - height: z.number(), - }).optional().catch(undefined), -}); - -export const nostrbuildSchema = z.object({ - data: nostrbuildFileSchema.array().min(1), -}); diff --git a/src/upload.ts b/src/upload.ts index cd9d2bb6..0d1a085f 100644 --- a/src/upload.ts +++ b/src/upload.ts @@ -1,14 +1,18 @@ import { Conf } from '@/config.ts'; import { insertUnattachedMedia, UnattachedMedia } from '@/db/unattached-media.ts'; -import { configUploader as uploader } from '@/uploaders/config.ts'; - +import { DittoUploader } from '@/interfaces/DittoUploader.ts'; interface FileMeta { pubkey: string; description?: string; } /** Upload a file, track it in the database, and return the resulting media object. */ -async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Promise { +export async function uploadFile( + uploader: DittoUploader, + file: File, + meta: FileMeta, + signal?: AbortSignal, +): Promise { const { pubkey, description } = meta; if (file.size > Conf.maxUploadSize) { @@ -30,5 +34,3 @@ async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Pro uploaded_at: Date.now(), }); } - -export { uploadFile }; diff --git a/src/uploaders/DenoUploader.ts b/src/uploaders/DenoUploader.ts new file mode 100644 index 00000000..6c2e6d48 --- /dev/null +++ b/src/uploaders/DenoUploader.ts @@ -0,0 +1,44 @@ +import { join } from 'node:path'; + +import { crypto } from '@std/crypto'; +import { encodeHex } from '@std/encoding/hex'; +import { extensionsByType } from '@std/media-types'; + +import { DittoUploader } from '@/interfaces/DittoUploader.ts'; + +export interface DenoUploaderOpts { + baseUrl: string; + dir: string; +} + +/** Local Deno filesystem uploader. */ +export class DenoUploader implements DittoUploader { + constructor(private opts: DenoUploaderOpts) {} + + async upload(file: File): Promise<[['url', string], ...string[][]]> { + const { dir, baseUrl } = this.opts; + + const sha256 = encodeHex(await crypto.subtle.digest('SHA-256', file.stream())); + const ext = extensionsByType(file.type)?.[0] ?? 'bin'; + const filename = `${sha256}.${ext}`; + + await Deno.mkdir(dir, { recursive: true }); + await Deno.writeFile(join(dir, filename), file.stream()); + + const url = new URL(baseUrl); + const path = url.pathname === '/' ? filename : join(url.pathname, filename); + + return [ + ['url', new URL(path, url).toString()], + ['m', file.type], + ['x', sha256], + ['size', file.size.toString()], + ]; + } + + async delete(filename: string) { + const { dir } = this.opts; + const path = join(dir, filename); + await Deno.remove(path); + } +} diff --git a/src/uploaders/IPFSUploader.ts b/src/uploaders/IPFSUploader.ts new file mode 100644 index 00000000..ceb4e82e --- /dev/null +++ b/src/uploaders/IPFSUploader.ts @@ -0,0 +1,70 @@ +import { z } from 'zod'; + +import { DittoUploader } from '@/interfaces/DittoUploader.ts'; + +export interface IPFSUploaderOpts { + baseUrl: string; + apiUrl?: string; + fetch?: typeof fetch; +} + +/** + * IPFS uploader. It expects an IPFS node up and running. + * It will try to connect to `http://localhost:5001` by default, + * and upload the file using the REST API. + */ +export class IPFSUploader implements DittoUploader { + private baseUrl: string; + private apiUrl: string; + private fetch: typeof fetch; + + constructor(opts: IPFSUploaderOpts) { + this.baseUrl = opts.baseUrl; + this.apiUrl = opts.apiUrl ?? 'http://localhost:5001'; + this.fetch = opts.fetch ?? globalThis.fetch; + } + + async upload(file: File, opts?: { signal?: AbortSignal }): Promise<[['url', string], ...string[][]]> { + const url = new URL('/api/v0/add', this.apiUrl); + + const formData = new FormData(); + formData.append('file', file); + + const response = await this.fetch(url, { + method: 'POST', + body: formData, + signal: opts?.signal, + }); + + const { Hash: cid } = IPFSUploader.schema().parse(await response.json()); + + return [ + ['url', new URL(`/ipfs/${cid}`, this.baseUrl).toString()], + ['m', file.type], + ['cid', cid], + ['size', file.size.toString()], + ]; + } + + async delete(cid: string, opts?: { signal?: AbortSignal }): Promise { + const url = new URL('/api/v0/pin/rm', this.apiUrl); + + const query = new URLSearchParams(); + query.set('arg', cid); + url.search = query.toString(); + + await this.fetch(url, { + method: 'POST', + signal: opts?.signal, + }); + } + + /** Response schema for POST `/api/v0/add`. */ + static schema() { + return z.object({ + Name: z.string(), + Hash: z.string(), + Size: z.string(), + }); + } +} diff --git a/src/uploaders/NostrBuildUploader.ts b/src/uploaders/NostrBuildUploader.ts new file mode 100644 index 00000000..7e164481 --- /dev/null +++ b/src/uploaders/NostrBuildUploader.ts @@ -0,0 +1,65 @@ +import { z } from 'zod'; + +import { DittoUploader } from '@/interfaces/DittoUploader.ts'; + +export interface NostrBuildUploaderOpts { + endpoint?: string; + fetch?: typeof fetch; +} + +/** Upload files to nostr.build or another compatible server. */ +export class NostrBuildUploader implements DittoUploader { + constructor(private opts: NostrBuildUploaderOpts) {} + + async upload(file: File, opts?: { signal?: AbortSignal }): Promise<[['url', string], ...string[][]]> { + const { endpoint = 'https://nostr.build/api/v2/upload/files', fetch = globalThis.fetch } = this.opts; + + const formData = new FormData(); + formData.append('fileToUpload', file); + + const response = await fetch(endpoint, { + method: 'POST', + body: formData, + signal: opts?.signal, + }); + + const json = await response.json(); + const [data] = NostrBuildUploader.schema().parse(json).data; + + const tags: [['url', string], ...string[][]] = [ + ['url', data.url], + ['m', data.mime], + ['x', data.sha256], + ['ox', data.original_sha256], + ['size', data.size.toString()], + ]; + + if (data.dimensions) { + tags.push(['dim', `${data.dimensions.width}x${data.dimensions.height}`]); + } + + if (data.blurhash) { + tags.push(['blurhash', data.blurhash]); + } + + return tags; + } + + /** nostr.build API response schema. */ + private static schema() { + return z.object({ + data: z.object({ + url: z.string().url(), + blurhash: z.string().optional().catch(undefined), + sha256: z.string(), + original_sha256: z.string(), + mime: z.string(), + size: z.number(), + dimensions: z.object({ + width: z.number(), + height: z.number(), + }).optional().catch(undefined), + }).array().min(1), + }); + } +} diff --git a/src/uploaders/s3.ts b/src/uploaders/S3Uploader.ts similarity index 58% rename from src/uploaders/s3.ts rename to src/uploaders/S3Uploader.ts index aaff8c82..f210ce87 100644 --- a/src/uploaders/s3.ts +++ b/src/uploaders/S3Uploader.ts @@ -6,17 +6,34 @@ import { encodeHex } from '@std/encoding/hex'; import { extensionsByType } from '@std/media-types'; import { Conf } from '@/config.ts'; +import { DittoUploader } from '@/interfaces/DittoUploader.ts'; -import type { Uploader } from './types.ts'; +export interface S3UploaderOpts { + endPoint: string; + region: string; + accessKey?: string; + secretKey?: string; + bucket?: string; + pathStyle?: boolean; + port?: number; + sessionToken?: string; + useSSL?: boolean; +} /** S3-compatible uploader for AWS, Wasabi, DigitalOcean Spaces, and more. */ -const s3Uploader: Uploader = { - async upload(file) { +export class S3Uploader implements DittoUploader { + private client: S3Client; + + constructor(opts: S3UploaderOpts) { + this.client = new S3Client(opts); + } + + async upload(file: File): Promise<[['url', string], ...string[][]]> { const sha256 = encodeHex(await crypto.subtle.digest('SHA-256', file.stream())); const ext = extensionsByType(file.type)?.[0] ?? 'bin'; const filename = `${sha256}.${ext}`; - await client().putObject(filename, file.stream(), { + await this.client.putObject(filename, file.stream(), { metadata: { 'Content-Type': file.type, 'x-amz-acl': 'public-read', @@ -24,6 +41,7 @@ const s3Uploader: Uploader = { }); const { pathStyle, bucket } = Conf.s3; + const path = (pathStyle && bucket) ? join(bucket, filename) : filename; const url = new URL(path, Conf.mediaDomain).toString(); @@ -33,15 +51,9 @@ const s3Uploader: Uploader = { ['x', sha256], ['size', file.size.toString()], ]; - }, - async delete(id) { - await client().deleteObject(id); - }, -}; + } -/** Build S3 client from config. */ -function client() { - return new S3Client({ ...Conf.s3 }); + async delete(objectName: string) { + await this.client.deleteObject(objectName); + } } - -export { s3Uploader }; diff --git a/src/uploaders/config.ts b/src/uploaders/config.ts deleted file mode 100644 index 3f3aac74..00000000 --- a/src/uploaders/config.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { Conf } from '@/config.ts'; - -import { ipfsUploader } from '@/uploaders/ipfs.ts'; -import { localUploader } from '@/uploaders/local.ts'; -import { nostrbuildUploader } from '@/uploaders/nostrbuild.ts'; -import { s3Uploader } from '@/uploaders/s3.ts'; - -import type { Uploader } from './types.ts'; - -/** Meta-uploader determined from configuration. */ -const configUploader: Uploader = { - upload(file, opts) { - return uploader().upload(file, opts); - }, - async delete(id, opts) { - return await uploader().delete?.(id, opts); - }, -}; - -/** Get the uploader module based on configuration. */ -function uploader() { - switch (Conf.uploader) { - case 's3': - return s3Uploader; - case 'ipfs': - return ipfsUploader; - case 'local': - return localUploader; - case 'nostrbuild': - return nostrbuildUploader; - default: - throw new Error('No `DITTO_UPLOADER` configured. Uploads are disabled.'); - } -} - -export { configUploader }; diff --git a/src/uploaders/ipfs.ts b/src/uploaders/ipfs.ts deleted file mode 100644 index b83dc2e3..00000000 --- a/src/uploaders/ipfs.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { z } from 'zod'; - -import { Conf } from '@/config.ts'; -import { fetchWorker } from '@/workers/fetch.ts'; - -import type { Uploader } from './types.ts'; - -/** Response schema for POST `/api/v0/add`. */ -const ipfsAddResponseSchema = z.object({ - Name: z.string(), - Hash: z.string(), - Size: z.string(), -}); - -/** - * IPFS uploader. It expects an IPFS node up and running. - * It will try to connect to `http://localhost:5001` by default, - * and upload the file using the REST API. - */ -const ipfsUploader: Uploader = { - async upload(file, opts) { - const url = new URL('/api/v0/add', Conf.ipfs.apiUrl); - - const formData = new FormData(); - formData.append('file', file); - - const response = await fetchWorker(url, { - method: 'POST', - body: formData, - signal: opts?.signal, - }); - - const { Hash: cid } = ipfsAddResponseSchema.parse(await response.json()); - - return [ - ['url', new URL(`/ipfs/${cid}`, Conf.mediaDomain).toString()], - ['m', file.type], - ['cid', cid], - ['size', file.size.toString()], - ]; - }, - async delete(cid, opts) { - const url = new URL('/api/v0/pin/rm', Conf.ipfs.apiUrl); - - const query = new URLSearchParams(); - query.set('arg', cid); - - url.search = query.toString(); - - await fetchWorker(url, { - method: 'POST', - signal: opts?.signal, - }); - }, -}; - -export { ipfsUploader }; diff --git a/src/uploaders/local.ts b/src/uploaders/local.ts deleted file mode 100644 index d5cd46ad..00000000 --- a/src/uploaders/local.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { join } from 'node:path'; - -import { crypto } from '@std/crypto'; -import { encodeHex } from '@std/encoding/hex'; -import { extensionsByType } from '@std/media-types'; - -import { Conf } from '@/config.ts'; - -import type { Uploader } from './types.ts'; - -/** Local filesystem uploader. */ -const localUploader: Uploader = { - async upload(file) { - const sha256 = encodeHex(await crypto.subtle.digest('SHA-256', file.stream())); - const ext = extensionsByType(file.type)?.[0] ?? 'bin'; - const filename = `${sha256}.${ext}`; - - await Deno.mkdir(Conf.uploadsDir, { recursive: true }); - await Deno.writeFile(join(Conf.uploadsDir, filename), file.stream()); - - const { mediaDomain } = Conf; - const url = new URL(mediaDomain); - const path = url.pathname === '/' ? filename : join(url.pathname, filename); - - return [ - ['url', new URL(path, url).toString()], - ['m', file.type], - ['x', sha256], - ['size', file.size.toString()], - ]; - }, - async delete(id) { - await Deno.remove(join(Conf.uploadsDir, id)); - }, -}; - -export { localUploader }; diff --git a/src/uploaders/nostrbuild.ts b/src/uploaders/nostrbuild.ts deleted file mode 100644 index 8bca331b..00000000 --- a/src/uploaders/nostrbuild.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { Conf } from '@/config.ts'; -import { nostrbuildSchema } from '@/schemas/nostrbuild.ts'; - -import type { Uploader } from './types.ts'; - -/** nostr.build uploader. */ -export const nostrbuildUploader: Uploader = { - async upload(file) { - const formData = new FormData(); - formData.append('fileToUpload', file); - - const response = await fetch(Conf.nostrbuildEndpoint, { - method: 'POST', - body: formData, - }); - - const json = await response.json(); - const [data] = nostrbuildSchema.parse(json).data; - - const tags: [['url', string], ...string[][]] = [ - ['url', data.url], - ['m', data.mime], - ['x', data.sha256], - ['ox', data.original_sha256], - ['size', file.size.toString()], - ['blurhash', data.blurhash], - ]; - - if (data.dimensions) { - tags.push(['dim', `${data.dimensions.width}x${data.dimensions.height}`]); - } - - return tags; - }, -}; diff --git a/src/uploaders/types.ts b/src/uploaders/types.ts deleted file mode 100644 index 81b8a0a7..00000000 --- a/src/uploaders/types.ts +++ /dev/null @@ -1,9 +0,0 @@ -/** Modular uploader interface, to support uploading to different backends. */ -interface Uploader { - /** Upload the file to the backend. */ - upload(file: File, opts?: { signal?: AbortSignal }): Promise<[['url', string], ...string[][]]>; - /** Delete the file from the backend. */ - delete?(cid: string, opts?: { signal?: AbortSignal }): Promise; -} - -export type { Uploader }; From 6542d6a77789dd1a662696a7142497334fe9df5a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 22:04:43 -0500 Subject: [PATCH 39/65] Move uploader.ts to utils, make it kind of like api.ts --- src/controllers/api/accounts.ts | 11 +++-------- src/controllers/api/media.ts | 9 ++------- src/{ => utils}/upload.ts | 12 ++++++++++-- 3 files changed, 15 insertions(+), 17 deletions(-) rename src/{ => utils}/upload.ts (75%) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index d67fd888..4777f56f 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -8,7 +8,7 @@ import { getAuthor, getFollowedPubkeys } from '@/queries.ts'; import { booleanParamSchema, fileSchema } from '@/schema.ts'; import { Storages } from '@/storages.ts'; import { addTag, deleteTag, findReplyTag, getTagSet } from '@/tags.ts'; -import { uploadFile } from '@/upload.ts'; +import { uploadFile } from '@/utils/upload.ts'; import { nostrNow } from '@/utils.ts'; import { createEvent, paginated, paginationSchema, parseBody, updateListEvent } from '@/utils/api.ts'; import { lookupAccount } from '@/utils/lookup.ts'; @@ -202,7 +202,6 @@ const updateCredentialsSchema = z.object({ const updateCredentialsController: AppController = async (c) => { const pubkey = await c.get('signer')?.getPublicKey()!; - const uploader = c.get('uploader'); const body = await parseBody(c.req.raw); const result = updateCredentialsSchema.safeParse(body); @@ -221,13 +220,9 @@ const updateCredentialsController: AppController = async (c) => { nip05, } = result.data; - if ((avatarFile || headerFile) && !uploader) { - return c.json({ error: 'No uploader configured.' }, 500); - } - const [avatar, header] = await Promise.all([ - (avatarFile && uploader) ? uploadFile(uploader, avatarFile, { pubkey }) : undefined, - (headerFile && uploader) ? uploadFile(uploader, headerFile, { pubkey }) : undefined, + avatarFile ? uploadFile(c, avatarFile, { pubkey }) : undefined, + headerFile ? uploadFile(c, headerFile, { pubkey }) : undefined, ]); meta.name = display_name ?? meta.name; diff --git a/src/controllers/api/media.ts b/src/controllers/api/media.ts index 101b7767..71b3e782 100644 --- a/src/controllers/api/media.ts +++ b/src/controllers/api/media.ts @@ -4,7 +4,7 @@ import { AppController } from '@/app.ts'; import { fileSchema } from '@/schema.ts'; import { parseBody } from '@/utils/api.ts'; import { renderAttachment } from '@/views/mastodon/attachments.ts'; -import { uploadFile } from '@/upload.ts'; +import { uploadFile } from '@/utils/upload.ts'; const mediaBodySchema = z.object({ file: fileSchema, @@ -14,11 +14,6 @@ const mediaBodySchema = z.object({ }); const mediaController: AppController = async (c) => { - const uploader = c.get('uploader'); - if (!uploader) { - return c.json({ error: 'No uploader configured.' }, 500); - } - const pubkey = await c.get('signer')?.getPublicKey()!; const result = mediaBodySchema.safeParse(await parseBody(c.req.raw)); const { signal } = c.req.raw; @@ -29,7 +24,7 @@ const mediaController: AppController = async (c) => { try { const { file, description } = result.data; - const media = await uploadFile(uploader, file, { pubkey, description }, signal); + const media = await uploadFile(c, file, { pubkey, description }, signal); return c.json(renderAttachment(media)); } catch (e) { console.error(e); diff --git a/src/upload.ts b/src/utils/upload.ts similarity index 75% rename from src/upload.ts rename to src/utils/upload.ts index 0d1a085f..c4f2fc58 100644 --- a/src/upload.ts +++ b/src/utils/upload.ts @@ -1,6 +1,7 @@ +import { AppContext } from '@/app.ts'; import { Conf } from '@/config.ts'; import { insertUnattachedMedia, UnattachedMedia } from '@/db/unattached-media.ts'; -import { DittoUploader } from '@/interfaces/DittoUploader.ts'; +import { HTTPException } from 'hono'; interface FileMeta { pubkey: string; description?: string; @@ -8,11 +9,18 @@ interface FileMeta { /** Upload a file, track it in the database, and return the resulting media object. */ export async function uploadFile( - uploader: DittoUploader, + c: AppContext, file: File, meta: FileMeta, signal?: AbortSignal, ): Promise { + const uploader = c.get('uploader'); + if (!uploader) { + throw new HTTPException(500, { + res: c.json({ error: 'No uploader configured.' }), + }); + } + const { pubkey, description } = meta; if (file.size > Conf.maxUploadSize) { From 24659d8edb0cf1daf9c20090382c50a59994c561 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 22:11:54 -0500 Subject: [PATCH 40/65] IPFSUploader: make schema private --- src/uploaders/IPFSUploader.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uploaders/IPFSUploader.ts b/src/uploaders/IPFSUploader.ts index ceb4e82e..9141e784 100644 --- a/src/uploaders/IPFSUploader.ts +++ b/src/uploaders/IPFSUploader.ts @@ -60,7 +60,7 @@ export class IPFSUploader implements DittoUploader { } /** Response schema for POST `/api/v0/add`. */ - static schema() { + private static schema() { return z.object({ Name: z.string(), Hash: z.string(), From acef173ac465c40b60919dbcdcb842ede7e0ff39 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 22:15:33 -0500 Subject: [PATCH 41/65] Do things the boilerplatey way just for consistency --- src/uploaders/DenoUploader.ts | 19 +++++++++++-------- src/uploaders/NostrBuildUploader.ts | 12 ++++++++---- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/uploaders/DenoUploader.ts b/src/uploaders/DenoUploader.ts index 6c2e6d48..e2224ab5 100644 --- a/src/uploaders/DenoUploader.ts +++ b/src/uploaders/DenoUploader.ts @@ -13,19 +13,23 @@ export interface DenoUploaderOpts { /** Local Deno filesystem uploader. */ export class DenoUploader implements DittoUploader { - constructor(private opts: DenoUploaderOpts) {} + baseUrl: string; + dir: string; + + constructor(opts: DenoUploaderOpts) { + this.baseUrl = opts.baseUrl; + this.dir = opts.dir; + } async upload(file: File): Promise<[['url', string], ...string[][]]> { - const { dir, baseUrl } = this.opts; - const sha256 = encodeHex(await crypto.subtle.digest('SHA-256', file.stream())); const ext = extensionsByType(file.type)?.[0] ?? 'bin'; const filename = `${sha256}.${ext}`; - await Deno.mkdir(dir, { recursive: true }); - await Deno.writeFile(join(dir, filename), file.stream()); + await Deno.mkdir(this.dir, { recursive: true }); + await Deno.writeFile(join(this.dir, filename), file.stream()); - const url = new URL(baseUrl); + const url = new URL(this.baseUrl); const path = url.pathname === '/' ? filename : join(url.pathname, filename); return [ @@ -37,8 +41,7 @@ export class DenoUploader implements DittoUploader { } async delete(filename: string) { - const { dir } = this.opts; - const path = join(dir, filename); + const path = join(this.dir, filename); await Deno.remove(path); } } diff --git a/src/uploaders/NostrBuildUploader.ts b/src/uploaders/NostrBuildUploader.ts index 7e164481..ff4a4f0e 100644 --- a/src/uploaders/NostrBuildUploader.ts +++ b/src/uploaders/NostrBuildUploader.ts @@ -9,15 +9,19 @@ export interface NostrBuildUploaderOpts { /** Upload files to nostr.build or another compatible server. */ export class NostrBuildUploader implements DittoUploader { - constructor(private opts: NostrBuildUploaderOpts) {} + private endpoint: string; + private fetch: typeof fetch; + + constructor(opts: NostrBuildUploaderOpts) { + this.endpoint = opts.endpoint ?? 'https://nostr.build/api/v2/upload/files'; + this.fetch = opts.fetch ?? globalThis.fetch; + } async upload(file: File, opts?: { signal?: AbortSignal }): Promise<[['url', string], ...string[][]]> { - const { endpoint = 'https://nostr.build/api/v2/upload/files', fetch = globalThis.fetch } = this.opts; - const formData = new FormData(); formData.append('fileToUpload', file); - const response = await fetch(endpoint, { + const response = await this.fetch(this.endpoint, { method: 'POST', body: formData, signal: opts?.signal, From 5523c3fc0eeca605cdbf5691a37b7581001b0928 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 23:02:59 -0500 Subject: [PATCH 42/65] verifyCredentials: wait up to 5 seconds --- src/controllers/api/accounts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 4777f56f..f66e0aca 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -47,7 +47,7 @@ const createAccountController: AppController = async (c) => { const verifyCredentialsController: AppController = async (c) => { const pubkey = await c.get('signer')?.getPublicKey()!; - const event = await getAuthor(pubkey, { relations: ['author_stats'] }); + const event = await getAuthor(pubkey, { signal: AbortSignal.timeout(5000) }); if (event) { return c.json(await renderAccount(event, { withSource: true })); } else { From 7f5179efcac065ac902a1b84f03d610a9f9e5f63 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 19 May 2024 09:13:53 -0500 Subject: [PATCH 43/65] renderAttachment: guess mime from url --- src/views/mastodon/attachments.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 0b1b8eb1..2d658041 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -1,9 +1,12 @@ +import { getUrlMediaType } from '@/utils/media.ts'; + /** Render Mastodon media attachment. */ function renderAttachment(media: { id?: string; data: string[][] }) { const { id, data: tags } = media; - const m = tags.find(([name]) => name === 'm')?.[1]; const url = tags.find(([name]) => name === 'url')?.[1]; + + const m = tags.find(([name]) => name === 'm')?.[1] ?? getUrlMediaType(url!); const alt = tags.find(([name]) => name === 'alt')?.[1]; const cid = tags.find(([name]) => name === 'cid')?.[1]; const dim = tags.find(([name]) => name === 'dim')?.[1]; From 540bd058a2b727df2c8e6e6af4e04ad7057f8a11 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 19 May 2024 11:33:59 -0500 Subject: [PATCH 44/65] Fix NIP-27 mentions --- src/controllers/api/statuses.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index e620b930..291d970c 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -1,5 +1,6 @@ import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; import ISO6391 from 'iso-639-1'; +import { nip19 } from 'nostr-tools'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; @@ -111,7 +112,11 @@ const createStatusController: AppController = async (c) => { pubkeys.add(pubkey); } - return `nostr:${pubkey}`; + try { + return `nostr:${nip19.npubEncode(pubkey)}`; + } catch { + return match; + } }); // Explicit addressing From 9754e29603d404d20d6db3a2c435570266f507bb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 19 May 2024 11:45:42 -0500 Subject: [PATCH 45/65] accountSearchController: respect the `limit` param --- src/controllers/api/accounts.ts | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index f66e0aca..39161ef1 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -86,25 +86,35 @@ const accountLookupController: AppController = async (c) => { } }; -const accountSearchController: AppController = async (c) => { - const q = c.req.query('q'); +const accountSearchQuerySchema = z.object({ + q: z.string().transform(decodeURIComponent), + resolve: booleanParamSchema.optional().transform(Boolean), + following: z.boolean().default(false), + limit: z.coerce.number().catch(20).transform((value) => Math.min(Math.max(value, 0), 40)), +}); - if (!q) { - return c.json({ error: 'Missing `q` query parameter.' }, 422); +const accountSearchController: AppController = async (c) => { + const result = accountSearchQuerySchema.safeParse(c.req.query()); + const { signal } = c.req.raw; + + if (!result.success) { + return c.json({ error: 'Bad request', schema: result.error }, 422); } + const { q, limit } = result.data; + const query = decodeURIComponent(q); const store = await Storages.search(); const [event, events] = await Promise.all([ lookupAccount(query), - store.query([{ kinds: [0], search: query, limit: 20 }], { signal: c.req.raw.signal }), + store.query([{ kinds: [0], search: query, limit }], { signal }), ]); const results = await hydrateEvents({ events: event ? [event, ...events] : events, store, - signal: c.req.raw.signal, + signal, }); if ((results.length < 1) && query.match(/npub1\w+/)) { From 7c5b7c5d835e23cd5fc2d2354f2d9a970e6e2d3b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 19 May 2024 15:38:42 -0500 Subject: [PATCH 46/65] Upgrade Nostrify to v0.22.0 --- deno.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deno.json b/deno.json index 946b4e02..5567cd24 100644 --- a/deno.json +++ b/deno.json @@ -22,7 +22,7 @@ "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.21.1", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.22.0", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0", From f0b247130f46875275c814da6386952d08038071 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 19 May 2024 15:42:45 -0500 Subject: [PATCH 47/65] Add support for Blossom uploader --- src/config.ts | 4 ++++ src/middleware/uploaderMiddleware.ts | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/src/config.ts b/src/config.ts index f3b472ea..cc149983 100644 --- a/src/config.ts +++ b/src/config.ts @@ -140,6 +140,10 @@ class Conf { static get nostrbuildEndpoint(): string { return Deno.env.get('NOSTRBUILD_ENDPOINT') || 'https://nostr.build/api/v2/upload/files'; } + /** Default Blossom servers to use when the `blossom` uploader is set. */ + static get blossomServers(): string[] { + return Deno.env.get('BLOSSOM_SERVERS')?.split(',') || ['https://blossom.primal.net/']; + } /** Module to upload files with. */ static get uploader() { return Deno.env.get('DITTO_UPLOADER'); diff --git a/src/middleware/uploaderMiddleware.ts b/src/middleware/uploaderMiddleware.ts index 8279a122..b0ee570a 100644 --- a/src/middleware/uploaderMiddleware.ts +++ b/src/middleware/uploaderMiddleware.ts @@ -1,3 +1,5 @@ +import { BlossomUploader } from '@nostrify/nostrify/uploaders'; + import { AppMiddleware } from '@/app.ts'; import { Conf } from '@/config.ts'; import { DenoUploader } from '@/uploaders/DenoUploader.ts'; @@ -8,6 +10,8 @@ import { fetchWorker } from '@/workers/fetch.ts'; /** Set an uploader for the user. */ export const uploaderMiddleware: AppMiddleware = async (c, next) => { + const signer = c.get('signer'); + switch (Conf.uploader) { case 's3': c.set('uploader', new S3Uploader(Conf.s3)); @@ -21,6 +25,11 @@ export const uploaderMiddleware: AppMiddleware = async (c, next) => { case 'nostrbuild': c.set('uploader', new NostrBuildUploader({ endpoint: Conf.nostrbuildEndpoint, fetch: fetchWorker })); break; + case 'blossom': + if (signer) { + c.set('uploader', new BlossomUploader({ servers: Conf.blossomServers, signer, fetch: fetchWorker })); + } + break; } await next(); From 0541287f0e0b6a5a2c8b10a2835e66b26f88d26c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 19 May 2024 15:43:24 -0500 Subject: [PATCH 48/65] Replace our NostrBuildUploader with the one from Nostrify --- src/middleware/uploaderMiddleware.ts | 5 +- src/uploaders/NostrBuildUploader.ts | 69 ---------------------------- 2 files changed, 2 insertions(+), 72 deletions(-) delete mode 100644 src/uploaders/NostrBuildUploader.ts diff --git a/src/middleware/uploaderMiddleware.ts b/src/middleware/uploaderMiddleware.ts index b0ee570a..38e8aceb 100644 --- a/src/middleware/uploaderMiddleware.ts +++ b/src/middleware/uploaderMiddleware.ts @@ -1,10 +1,9 @@ -import { BlossomUploader } from '@nostrify/nostrify/uploaders'; +import { BlossomUploader, NostrBuildUploader } from '@nostrify/nostrify/uploaders'; import { AppMiddleware } from '@/app.ts'; import { Conf } from '@/config.ts'; import { DenoUploader } from '@/uploaders/DenoUploader.ts'; import { IPFSUploader } from '@/uploaders/IPFSUploader.ts'; -import { NostrBuildUploader } from '@/uploaders/NostrBuildUploader.ts'; import { S3Uploader } from '@/uploaders/S3Uploader.ts'; import { fetchWorker } from '@/workers/fetch.ts'; @@ -23,7 +22,7 @@ export const uploaderMiddleware: AppMiddleware = async (c, next) => { c.set('uploader', new DenoUploader({ baseUrl: Conf.mediaDomain, dir: Conf.uploadsDir })); break; case 'nostrbuild': - c.set('uploader', new NostrBuildUploader({ endpoint: Conf.nostrbuildEndpoint, fetch: fetchWorker })); + c.set('uploader', new NostrBuildUploader({ endpoint: Conf.nostrbuildEndpoint, signer, fetch: fetchWorker })); break; case 'blossom': if (signer) { diff --git a/src/uploaders/NostrBuildUploader.ts b/src/uploaders/NostrBuildUploader.ts deleted file mode 100644 index ff4a4f0e..00000000 --- a/src/uploaders/NostrBuildUploader.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { z } from 'zod'; - -import { DittoUploader } from '@/interfaces/DittoUploader.ts'; - -export interface NostrBuildUploaderOpts { - endpoint?: string; - fetch?: typeof fetch; -} - -/** Upload files to nostr.build or another compatible server. */ -export class NostrBuildUploader implements DittoUploader { - private endpoint: string; - private fetch: typeof fetch; - - constructor(opts: NostrBuildUploaderOpts) { - this.endpoint = opts.endpoint ?? 'https://nostr.build/api/v2/upload/files'; - this.fetch = opts.fetch ?? globalThis.fetch; - } - - async upload(file: File, opts?: { signal?: AbortSignal }): Promise<[['url', string], ...string[][]]> { - const formData = new FormData(); - formData.append('fileToUpload', file); - - const response = await this.fetch(this.endpoint, { - method: 'POST', - body: formData, - signal: opts?.signal, - }); - - const json = await response.json(); - const [data] = NostrBuildUploader.schema().parse(json).data; - - const tags: [['url', string], ...string[][]] = [ - ['url', data.url], - ['m', data.mime], - ['x', data.sha256], - ['ox', data.original_sha256], - ['size', data.size.toString()], - ]; - - if (data.dimensions) { - tags.push(['dim', `${data.dimensions.width}x${data.dimensions.height}`]); - } - - if (data.blurhash) { - tags.push(['blurhash', data.blurhash]); - } - - return tags; - } - - /** nostr.build API response schema. */ - private static schema() { - return z.object({ - data: z.object({ - url: z.string().url(), - blurhash: z.string().optional().catch(undefined), - sha256: z.string(), - original_sha256: z.string(), - mime: z.string(), - size: z.number(), - dimensions: z.object({ - width: z.number(), - height: z.number(), - }).optional().catch(undefined), - }).array().min(1), - }); - } -} From 6f6e87525e9b7124e52af8fc2451e57cdbd62828 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 19 May 2024 15:57:04 -0500 Subject: [PATCH 49/65] Remove DittoUploader interface in favor of NUploader --- src/app.ts | 5 ++--- src/interfaces/DittoUploader.ts | 3 --- src/uploaders/DenoUploader.ts | 5 ++--- src/uploaders/IPFSUploader.ts | 5 ++--- src/uploaders/S3Uploader.ts | 4 ++-- 5 files changed, 8 insertions(+), 14 deletions(-) delete mode 100644 src/interfaces/DittoUploader.ts diff --git a/src/app.ts b/src/app.ts index ddc99904..5300b485 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,4 +1,4 @@ -import { NostrEvent, NostrSigner, NStore } from '@nostrify/nostrify'; +import { NostrEvent, NostrSigner, NStore, NUploader } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; import { type Context, Env as HonoEnv, type Handler, Hono, Input as HonoInput, type MiddlewareHandler } from 'hono'; import { cors, logger, serveStatic } from 'hono/middleware'; @@ -81,7 +81,6 @@ import { hostMetaController } from '@/controllers/well-known/host-meta.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; import { nostrController } from '@/controllers/well-known/nostr.ts'; import { webfingerController } from '@/controllers/well-known/webfinger.ts'; -import { DittoUploader } from '@/interfaces/DittoUploader.ts'; import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts'; import { cacheMiddleware } from '@/middleware/cacheMiddleware.ts'; import { cspMiddleware } from '@/middleware/cspMiddleware.ts'; @@ -97,7 +96,7 @@ interface AppEnv extends HonoEnv { /** Signer to get the logged-in user's pubkey, relays, and to sign events, or `undefined` if the user isn't logged in. */ signer?: NostrSigner; /** Uploader for the user to upload files. */ - uploader?: DittoUploader; + uploader?: NUploader; /** NIP-98 signed event proving the pubkey is owned by the user. */ proof?: NostrEvent; /** Store */ diff --git a/src/interfaces/DittoUploader.ts b/src/interfaces/DittoUploader.ts deleted file mode 100644 index 08cbf504..00000000 --- a/src/interfaces/DittoUploader.ts +++ /dev/null @@ -1,3 +0,0 @@ -export interface DittoUploader { - upload(file: File, opts?: { signal?: AbortSignal }): Promise<[['url', string], ...string[][]]>; -} diff --git a/src/uploaders/DenoUploader.ts b/src/uploaders/DenoUploader.ts index e2224ab5..fd30d8c6 100644 --- a/src/uploaders/DenoUploader.ts +++ b/src/uploaders/DenoUploader.ts @@ -1,18 +1,17 @@ import { join } from 'node:path'; +import { NUploader } from '@nostrify/nostrify'; import { crypto } from '@std/crypto'; import { encodeHex } from '@std/encoding/hex'; import { extensionsByType } from '@std/media-types'; -import { DittoUploader } from '@/interfaces/DittoUploader.ts'; - export interface DenoUploaderOpts { baseUrl: string; dir: string; } /** Local Deno filesystem uploader. */ -export class DenoUploader implements DittoUploader { +export class DenoUploader implements NUploader { baseUrl: string; dir: string; diff --git a/src/uploaders/IPFSUploader.ts b/src/uploaders/IPFSUploader.ts index 9141e784..7bf5165b 100644 --- a/src/uploaders/IPFSUploader.ts +++ b/src/uploaders/IPFSUploader.ts @@ -1,7 +1,6 @@ +import { NUploader } from '@nostrify/nostrify'; import { z } from 'zod'; -import { DittoUploader } from '@/interfaces/DittoUploader.ts'; - export interface IPFSUploaderOpts { baseUrl: string; apiUrl?: string; @@ -13,7 +12,7 @@ export interface IPFSUploaderOpts { * It will try to connect to `http://localhost:5001` by default, * and upload the file using the REST API. */ -export class IPFSUploader implements DittoUploader { +export class IPFSUploader implements NUploader { private baseUrl: string; private apiUrl: string; private fetch: typeof fetch; diff --git a/src/uploaders/S3Uploader.ts b/src/uploaders/S3Uploader.ts index f210ce87..b74796ab 100644 --- a/src/uploaders/S3Uploader.ts +++ b/src/uploaders/S3Uploader.ts @@ -1,12 +1,12 @@ import { join } from 'node:path'; import { S3Client } from '@bradenmacdonald/s3-lite-client'; +import { NUploader } from '@nostrify/nostrify'; import { crypto } from '@std/crypto'; import { encodeHex } from '@std/encoding/hex'; import { extensionsByType } from '@std/media-types'; import { Conf } from '@/config.ts'; -import { DittoUploader } from '@/interfaces/DittoUploader.ts'; export interface S3UploaderOpts { endPoint: string; @@ -21,7 +21,7 @@ export interface S3UploaderOpts { } /** S3-compatible uploader for AWS, Wasabi, DigitalOcean Spaces, and more. */ -export class S3Uploader implements DittoUploader { +export class S3Uploader implements NUploader { private client: S3Client; constructor(opts: S3UploaderOpts) { From 7b099ee5659f278dfe49802e79c9e878ae4d69a1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 20 May 2024 11:39:31 -0500 Subject: [PATCH 50/65] EventsDB: don't index the user's bio for kind 0 events --- src/storages/EventsDB.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index ef51a892..5a3839a5 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -180,8 +180,8 @@ class EventsDB implements NStore { /** Build search content for a user. */ static buildUserSearchContent(event: NostrEvent): string { - const { name, nip05, about } = n.json().pipe(n.metadata()).catch({}).parse(event.content); - return [name, nip05, about].filter(Boolean).join('\n'); + const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(event.content); + return [name, nip05].filter(Boolean).join('\n'); } /** Build search content from tag values. */ From 98fd4babcebc2d61506be13a370d32758e5e49c4 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 20 May 2024 14:46:53 -0300 Subject: [PATCH 51/65] test(EventsDB): use eventFixture() --- src/storages/EventsDB.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 2f343791..16b429d4 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -9,10 +9,7 @@ import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { RelayError } from '@/RelayError.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; -import { genEvent } from '@/test.ts'; - -import event0 from '~/fixtures/events/event-0.json' with { type: 'json' }; -import event1 from '~/fixtures/events/event-1.json' with { type: 'json' }; +import { eventFixture, genEvent } from '@/test.ts'; /** Create in-memory database for testing. */ const createDB = async () => { @@ -28,6 +25,7 @@ const createDB = async () => { Deno.test('count filters', async () => { const { eventsDB } = await createDB(); + const event1 = await eventFixture('event-1'); assertEquals((await eventsDB.count([{ kinds: [1] }])).count, 0); await eventsDB.event(event1); @@ -37,6 +35,7 @@ Deno.test('count filters', async () => { Deno.test('insert and filter events', async () => { const { eventsDB } = await createDB(); + const event1 = await eventFixture('event-1'); await eventsDB.event(event1); assertEquals(await eventsDB.query([{ kinds: [1] }]), [event1]); @@ -52,6 +51,7 @@ Deno.test('insert and filter events', async () => { Deno.test('query events with domain search filter', async () => { const { eventsDB, kysely } = await createDB(); + const event1 = await eventFixture('event-1'); await eventsDB.event(event1); assertEquals(await eventsDB.query([{}]), [event1]); @@ -180,7 +180,7 @@ Deno.test('throws a RelayError when inserting an event deleted by a user', async Deno.test('inserting replaceable events', async () => { const { eventsDB } = await createDB(); - const event = event0; + const event = await eventFixture('event-0'); await eventsDB.event(event); const olderEvent = { ...event, id: '123', created_at: event.created_at - 1 }; From 6861dc1d57c8c9822a1b86a1e251643b6b8e8293 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 20 May 2024 12:49:01 -0500 Subject: [PATCH 52/65] Fix crash parsing Lightning URL Fixes https://gitlab.com/soapbox-pub/ditto/-/issues/139 --- src/utils/lnurl.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/utils/lnurl.ts b/src/utils/lnurl.ts index ea5ce8a6..af344f22 100644 --- a/src/utils/lnurl.ts +++ b/src/utils/lnurl.ts @@ -28,8 +28,12 @@ function getLnurl({ lud06, lud16 }: { lud06?: string; lud16?: string }, limit?: if (lud16) { const [name, host] = lud16.split('@'); if (name && host) { - const url = new URL(`/.well-known/lnurlp/${name}`, `https://${host}`); - return LNURL.encode(url, limit); + try { + const url = new URL(`/.well-known/lnurlp/${name}`, `https://${host}`); + return LNURL.encode(url, limit); + } catch { + return; + } } } } From 7d786d731e05cf48d26fdec56d7c27a6c9bfee86 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 21 May 2024 10:09:06 -0300 Subject: [PATCH 53/65] feat: add script that installs latest soapbox --- deno.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deno.json b/deno.json index 5567cd24..1d9ad6de 100644 --- a/deno.json +++ b/deno.json @@ -11,7 +11,8 @@ "nsec": "deno run scripts/nsec.ts", "admin:event": "deno run -A scripts/admin-event.ts", "admin:role": "deno run -A scripts/admin-role.ts", - "stats:recompute": "deno run -A scripts/stats-recompute.ts" + "stats:recompute": "deno run -A scripts/stats-recompute.ts", + "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip soapbox.zip && rm soapbox.zip" }, "unstable": ["ffi", "kv", "worker-options"], "exclude": ["./public"], From 83e51ad67eb313b730d30ad3af8a6c355a555df1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 12:21:35 -0500 Subject: [PATCH 54/65] lodash -> entities --- deno.json | 1 + src/controllers/api/oauth.ts | 6 +++--- src/deps.ts | 2 -- src/views/mastodon/accounts.ts | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/deno.json b/deno.json index 1d9ad6de..1ebc35e7 100644 --- a/deno.json +++ b/deno.json @@ -37,6 +37,7 @@ "@std/streams": "jsr:@std/streams@^0.223.0", "comlink": "npm:comlink@^4.4.1", "deno-safe-fetch": "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts", + "entities": "npm:entities@^4.5.0", "fast-stable-stringify": "npm:fast-stable-stringify@^1.0.0", "formdata-helper": "npm:formdata-helper@^0.3.0", "hono": "https://deno.land/x/hono@v3.10.1/mod.ts", diff --git a/src/controllers/api/oauth.ts b/src/controllers/api/oauth.ts index a755a4d5..c1407f43 100644 --- a/src/controllers/api/oauth.ts +++ b/src/controllers/api/oauth.ts @@ -1,9 +1,9 @@ import { encodeBase64 } from '@std/encoding/base64'; +import { escape } from 'entities'; import { nip19 } from 'nostr-tools'; import { z } from 'zod'; import { AppController } from '@/app.ts'; -import { lodash } from '@/deps.ts'; import { nostrNow } from '@/utils.ts'; import { parseBody } from '@/utils/api.ts'; import { getClientConnectUri } from '@/utils/connect.ts'; @@ -100,11 +100,11 @@ const oauthController: AppController = async (c) => {
- +

- Nostr Connect + Nostr Connect `); diff --git a/src/deps.ts b/src/deps.ts index 12be07f1..dc9d912a 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -1,6 +1,4 @@ import 'deno-safe-fetch'; -// @deno-types="npm:@types/lodash@4.14.194" -export { default as lodash } from 'https://esm.sh/lodash@4.17.21'; // @deno-types="npm:@types/sanitize-html@2.9.0" export { default as sanitizeHtml } from 'npm:sanitize-html@^2.11.0'; export { diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index b337167e..69cf1db8 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -1,8 +1,8 @@ import { NSchema as n } from '@nostrify/nostrify'; +import { escape } from 'entities'; import { nip19, UnsignedEvent } from 'nostr-tools'; import { Conf } from '@/config.ts'; -import { lodash } from '@/deps.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { getLnurl } from '@/utils/lnurl.ts'; import { nip05Cache } from '@/utils/nip05.ts'; @@ -53,7 +53,7 @@ async function renderAccount( header_static: banner, last_status_at: null, locked: false, - note: lodash.escape(about), + note: about ? escape(about) : '', roles: [], source: withSource ? { From d4a029b35b26e513715c9c0dedd8d297f7bbd528 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 12:44:54 -0500 Subject: [PATCH 55/65] sanitize-html -> isomorphic-dompurify --- deno.json | 1 + src/deps.ts | 2 -- src/utils/unfurl.ts | 10 ++++------ 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/deno.json b/deno.json index 1ebc35e7..316c65b5 100644 --- a/deno.json +++ b/deno.json @@ -43,6 +43,7 @@ "hono": "https://deno.land/x/hono@v3.10.1/mod.ts", "hono/middleware": "https://deno.land/x/hono@v3.10.1/middleware.ts", "iso-639-1": "npm:iso-639-1@2.1.15", + "isomorphic-dompurify": "npm:isomorphic-dompurify@^2.11.0", "kysely": "npm:kysely@^0.27.3", "kysely_deno_postgres": "https://deno.land/x/kysely_deno_postgres@v0.4.0/mod.ts", "linkify-plugin-hashtag": "npm:linkify-plugin-hashtag@^4.1.1", diff --git a/src/deps.ts b/src/deps.ts index dc9d912a..ae960349 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -1,6 +1,4 @@ import 'deno-safe-fetch'; -// @deno-types="npm:@types/sanitize-html@2.9.0" -export { default as sanitizeHtml } from 'npm:sanitize-html@^2.11.0'; export { type ParsedSignature, pemToPublicKey, diff --git a/src/utils/unfurl.ts b/src/utils/unfurl.ts index b028be50..e5ae428d 100644 --- a/src/utils/unfurl.ts +++ b/src/utils/unfurl.ts @@ -1,8 +1,8 @@ import TTLCache from '@isaacs/ttlcache'; import Debug from '@soapbox/stickynotes/debug'; +import DOMPurify from 'isomorphic-dompurify'; import { unfurl } from 'unfurl.js'; -import { sanitizeHtml } from '@/deps.ts'; import { Time } from '@/utils/time.ts'; import { fetchWorker } from '@/workers/fetch.ts'; @@ -44,11 +44,9 @@ async function unfurlCard(url: string, signal: AbortSignal): Promise Date: Tue, 21 May 2024 12:51:19 -0500 Subject: [PATCH 56/65] Remove ActivityPub actor endpoint, remove deps.ts --- deno.json | 3 +- src/app.ts | 3 -- src/controllers/activitypub/actor.ts | 28 ---------------- src/deps.ts | 12 ------- src/server.ts | 2 ++ src/utils/rsa.ts | 32 ------------------- src/views/activitypub/actor.ts | 48 ---------------------------- src/workers/policy.worker.ts | 2 +- src/workers/trends.worker.ts | 2 +- 9 files changed, 6 insertions(+), 126 deletions(-) delete mode 100644 src/controllers/activitypub/actor.ts delete mode 100644 src/deps.ts delete mode 100644 src/utils/rsa.ts delete mode 100644 src/views/activitypub/actor.ts diff --git a/deno.json b/deno.json index 316c65b5..75ea3332 100644 --- a/deno.json +++ b/deno.json @@ -36,7 +36,8 @@ "@std/media-types": "jsr:@std/media-types@^0.224.1", "@std/streams": "jsr:@std/streams@^0.223.0", "comlink": "npm:comlink@^4.4.1", - "deno-safe-fetch": "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts", + "deno-safe-fetch/load": "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts", + "deno-sqlite": "https://raw.githubusercontent.com/alexgleason/deno-sqlite/325f66d8c395e7f6f5ee78ebfa42a0eeea4a942b/mod.ts", "entities": "npm:entities@^4.5.0", "fast-stable-stringify": "npm:fast-stable-stringify@^1.0.0", "formdata-helper": "npm:formdata-helper@^0.3.0", diff --git a/src/app.ts b/src/app.ts index 5300b485..e7862ab0 100644 --- a/src/app.ts +++ b/src/app.ts @@ -7,7 +7,6 @@ import { Conf } from '@/config.ts'; import { startFirehose } from '@/firehose.ts'; import { Time } from '@/utils.ts'; -import { actorController } from '@/controllers/activitypub/actor.ts'; import { accountController, accountLookupController, @@ -142,8 +141,6 @@ app.get('/.well-known/host-meta', hostMetaController); app.get('/.well-known/nodeinfo', nodeInfoController); app.get('/.well-known/nostr.json', nostrController); -app.get('/users/:username', actorController); - app.get('/nodeinfo/:version', nodeInfoSchemaController); app.get('/api/v1/instance', cacheMiddleware({ cacheName: 'web', expires: Time.minutes(5) }), instanceController); diff --git a/src/controllers/activitypub/actor.ts b/src/controllers/activitypub/actor.ts deleted file mode 100644 index 19f5f100..00000000 --- a/src/controllers/activitypub/actor.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { getAuthor } from '@/queries.ts'; -import { activityJson } from '@/utils/api.ts'; -import { renderActor } from '@/views/activitypub/actor.ts'; -import { localNip05Lookup } from '@/utils/nip05.ts'; - -import type { AppContext, AppController } from '@/app.ts'; - -const actorController: AppController = async (c) => { - const username = c.req.param('username'); - const { signal } = c.req.raw; - - const pointer = await localNip05Lookup(c.get('store'), username); - if (!pointer) return notFound(c); - - const event = await getAuthor(pointer.pubkey, { signal }); - if (!event) return notFound(c); - - const actor = await renderActor(event, username); - if (!actor) return notFound(c); - - return activityJson(c, actor); -}; - -function notFound(c: AppContext) { - return c.json({ error: 'Not found' }, 404); -} - -export { actorController }; diff --git a/src/deps.ts b/src/deps.ts deleted file mode 100644 index ae960349..00000000 --- a/src/deps.ts +++ /dev/null @@ -1,12 +0,0 @@ -import 'deno-safe-fetch'; -export { - type ParsedSignature, - pemToPublicKey, - publicKeyToPem, - signRequest, - verifyRequest, -} from 'https://gitlab.com/soapbox-pub/fedisign/-/raw/v0.2.1/mod.ts'; -export { generateSeededRsa } from 'https://gitlab.com/soapbox-pub/seeded-rsa/-/raw/v1.0.0/mod.ts'; -export { - DB as Sqlite, -} from 'https://raw.githubusercontent.com/alexgleason/deno-sqlite/325f66d8c395e7f6f5ee78ebfa42a0eeea4a942b/mod.ts'; diff --git a/src/server.ts b/src/server.ts index 4825e99d..f7a33dc0 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,3 +1,5 @@ +import 'deno-safe-fetch/load'; + import '@/precheck.ts'; import '@/sentry.ts'; import '@/nostr-wasm.ts'; diff --git a/src/utils/rsa.ts b/src/utils/rsa.ts deleted file mode 100644 index 6942c435..00000000 --- a/src/utils/rsa.ts +++ /dev/null @@ -1,32 +0,0 @@ -import * as secp from '@noble/secp256k1'; -import { LRUCache } from 'lru-cache'; - -import { Conf } from '@/config.ts'; -import { generateSeededRsa, publicKeyToPem } from '@/deps.ts'; - -const opts = { - bits: 2048, -}; - -const rsaCache = new LRUCache>({ max: 1000 }); - -async function buildSeed(pubkey: string): Promise { - const key = await Conf.cryptoKey; - const data = new TextEncoder().encode(pubkey); - const signature = await window.crypto.subtle.sign('HMAC', key, data); - return secp.etc.bytesToHex(new Uint8Array(signature)); -} - -async function getPublicKeyPem(pubkey: string): Promise { - const cached = await rsaCache.get(pubkey); - if (cached) return cached; - - const seed = await buildSeed(pubkey); - const { publicKey } = await generateSeededRsa(seed, opts); - const promise = publicKeyToPem(publicKey); - - rsaCache.set(pubkey, promise); - return promise; -} - -export { getPublicKeyPem }; diff --git a/src/views/activitypub/actor.ts b/src/views/activitypub/actor.ts deleted file mode 100644 index cfd40ba1..00000000 --- a/src/views/activitypub/actor.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { NSchema as n } from '@nostrify/nostrify'; - -import { Conf } from '@/config.ts'; -import { getPublicKeyPem } from '@/utils/rsa.ts'; - -import type { NostrEvent } from '@nostrify/nostrify'; -import type { Actor } from '@/schemas/activitypub.ts'; - -/** Nostr metadata event to ActivityPub actor. */ -async function renderActor(event: NostrEvent, username: string): Promise { - const content = n.json().pipe(n.metadata()).catch({}).parse(event.content); - - return { - type: 'Person', - id: Conf.local(`/users/${username}`), - name: content?.name || '', - preferredUsername: username, - inbox: Conf.local(`/users/${username}/inbox`), - followers: Conf.local(`/users/${username}/followers`), - following: Conf.local(`/users/${username}/following`), - outbox: Conf.local(`/users/${username}/outbox`), - icon: content.picture - ? { - type: 'Image', - url: content.picture, - } - : undefined, - image: content.banner - ? { - type: 'Image', - url: content.banner, - } - : undefined, - summary: content.about ?? '', - attachment: [], - tag: [], - publicKey: { - id: Conf.local(`/users/${username}#main-key`), - owner: Conf.local(`/users/${username}`), - publicKeyPem: await getPublicKeyPem(event.pubkey), - }, - endpoints: { - sharedInbox: Conf.local('/inbox'), - }, - }; -} - -export { renderActor }; diff --git a/src/workers/policy.worker.ts b/src/workers/policy.worker.ts index 4e4bcae6..d1368bce 100644 --- a/src/workers/policy.worker.ts +++ b/src/workers/policy.worker.ts @@ -1,4 +1,4 @@ -import 'deno-safe-fetch'; +import 'deno-safe-fetch/load'; import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify'; import { ReadOnlyPolicy } from '@nostrify/nostrify/policies'; import * as Comlink from 'comlink'; diff --git a/src/workers/trends.worker.ts b/src/workers/trends.worker.ts index 33fd1a12..74a256b9 100644 --- a/src/workers/trends.worker.ts +++ b/src/workers/trends.worker.ts @@ -1,7 +1,7 @@ import { NSchema } from '@nostrify/nostrify'; import * as Comlink from 'comlink'; +import { DB as Sqlite } from 'deno-sqlite'; -import { Sqlite } from '@/deps.ts'; import { hashtagSchema } from '@/schema.ts'; import { generateDateRange, Time } from '@/utils/time.ts'; From 938e26e2a0eedb45606f8c972f3eb7e979eb904d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 12:54:10 -0500 Subject: [PATCH 57/65] Remove webfinger and host-meta --- src/app.ts | 4 - src/controllers/well-known/host-meta.ts | 20 ----- src/controllers/well-known/webfinger.ts | 97 ------------------------- src/schemas/webfinger.ts | 19 ----- 4 files changed, 140 deletions(-) delete mode 100644 src/controllers/well-known/host-meta.ts delete mode 100644 src/controllers/well-known/webfinger.ts delete mode 100644 src/schemas/webfinger.ts diff --git a/src/app.ts b/src/app.ts index e7862ab0..be2ce840 100644 --- a/src/app.ts +++ b/src/app.ts @@ -76,10 +76,8 @@ import { } from '@/controllers/api/timelines.ts'; import { trendingTagsController } from '@/controllers/api/trends.ts'; import { indexController } from '@/controllers/site.ts'; -import { hostMetaController } from '@/controllers/well-known/host-meta.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; import { nostrController } from '@/controllers/well-known/nostr.ts'; -import { webfingerController } from '@/controllers/well-known/webfinger.ts'; import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts'; import { cacheMiddleware } from '@/middleware/cacheMiddleware.ts'; import { cspMiddleware } from '@/middleware/cspMiddleware.ts'; @@ -136,8 +134,6 @@ app.use( storeMiddleware, ); -app.get('/.well-known/webfinger', webfingerController); -app.get('/.well-known/host-meta', hostMetaController); app.get('/.well-known/nodeinfo', nodeInfoController); app.get('/.well-known/nostr.json', nostrController); diff --git a/src/controllers/well-known/host-meta.ts b/src/controllers/well-known/host-meta.ts deleted file mode 100644 index 85da11bf..00000000 --- a/src/controllers/well-known/host-meta.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Conf } from '@/config.ts'; - -import type { AppController } from '@/app.ts'; - -/** https://datatracker.ietf.org/doc/html/rfc6415 */ -const hostMetaController: AppController = (c) => { - const template = Conf.local('/.well-known/webfinger?resource={uri}'); - - c.header('content-type', 'application/xrd+xml'); - - return c.body( - ` - - - -`, - ); -}; - -export { hostMetaController }; diff --git a/src/controllers/well-known/webfinger.ts b/src/controllers/well-known/webfinger.ts deleted file mode 100644 index c1c8b815..00000000 --- a/src/controllers/well-known/webfinger.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { nip19 } from 'nostr-tools'; -import { z } from 'zod'; - -import { Conf } from '@/config.ts'; -import { localNip05Lookup } from '@/utils/nip05.ts'; - -import type { AppContext, AppController } from '@/app.ts'; -import type { Webfinger } from '@/schemas/webfinger.ts'; - -const webfingerQuerySchema = z.object({ - resource: z.string().url(), -}); - -const webfingerController: AppController = (c) => { - const query = webfingerQuerySchema.safeParse(c.req.query()); - if (!query.success) { - return c.json({ error: 'Bad request', schema: query.error }, 400); - } - - const resource = new URL(query.data.resource); - - switch (resource.protocol) { - case 'acct:': { - return handleAcct(c, resource); - } - default: - return c.json({ error: 'Unsupported URI scheme' }, 400); - } -}; - -/** Transforms the resource URI into a `[username, domain]` tuple. */ -const acctSchema = z.custom((value) => value instanceof URL) - .transform((uri) => uri.pathname) - .pipe(z.string().email('Invalid acct')) - .transform((acct) => acct.split('@') as [username: string, host: string]) - .refine(([_username, host]) => host === Conf.url.hostname, { - message: 'Host must be local', - path: ['resource', 'acct'], - }); - -async function handleAcct(c: AppContext, resource: URL): Promise { - const result = acctSchema.safeParse(resource); - if (!result.success) { - return c.json({ error: 'Invalid acct URI', schema: result.error }, 400); - } - - const [username, host] = result.data; - const pointer = await localNip05Lookup(c.get('store'), username); - - if (!pointer) { - return c.json({ error: 'Not found' }, 404); - } - - const json = renderWebfinger({ - pubkey: pointer.pubkey, - username, - subject: `acct:${username}@${host}`, - }); - - c.header('content-type', 'application/jrd+json'); - return c.body(JSON.stringify(json)); -} - -interface RenderWebfingerOpts { - pubkey: string; - username: string; - subject: string; -} - -/** Present Nostr user on Webfinger. */ -function renderWebfinger({ pubkey, username, subject }: RenderWebfingerOpts): Webfinger { - const apId = Conf.local(`/users/${username}`); - - return { - subject, - aliases: [apId], - links: [ - { - rel: 'self', - type: 'application/activity+json', - href: apId, - }, - { - rel: 'self', - type: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', - href: apId, - }, - { - rel: 'self', - type: 'application/nostr+json', - href: `nostr:${nip19.npubEncode(pubkey)}`, - }, - ], - }; -} - -export { webfingerController }; diff --git a/src/schemas/webfinger.ts b/src/schemas/webfinger.ts deleted file mode 100644 index 8c9cf576..00000000 --- a/src/schemas/webfinger.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { z } from 'zod'; - -const linkSchema = z.object({ - rel: z.string().optional(), - type: z.string().optional(), - href: z.string().optional(), - template: z.string().optional(), -}); - -const webfingerSchema = z.object({ - subject: z.string(), - aliases: z.array(z.string()).catch([]), - links: z.array(linkSchema), -}); - -type Webfinger = z.infer; - -export { webfingerSchema }; -export type { Webfinger }; From f30aad11a5281e7599434486818f62ddc6177ac3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 13:04:23 -0500 Subject: [PATCH 58/65] Fix legacy quote posts --- src/storages/hydrate.ts | 5 +++-- src/tags.ts | 11 +++++++++-- src/views/mastodon/statuses.ts | 12 ++++++------ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index e5c488e3..f55d7414 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -6,6 +6,7 @@ import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Conf } from '@/config.ts'; import { refreshAuthorStatsDebounced } from '@/stats.ts'; +import { findQuoteTag } from '@/tags.ts'; interface HydrateOpts { events: DittoEvent[]; @@ -81,7 +82,7 @@ function assembleEvents( event.user = b.find((e) => matchFilter({ kinds: [30361], authors: [admin], '#d': [event.pubkey] }, e)); if (event.kind === 1) { - const id = event.tags.find(([name]) => name === 'q')?.[1]; + const id = findQuoteTag(event.tags)?.[1]; if (id) { event.quote = b.find((e) => matchFilter({ kinds: [1], ids: [id] }, e)); } @@ -169,7 +170,7 @@ function gatherQuotes({ events, store, signal }: HydrateOpts): Promise name === 'q')?.[1]; + const id = findQuoteTag(event.tags)?.[1]; if (id) { ids.add(id); } diff --git a/src/tags.ts b/src/tags.ts index a683393e..ecddaf4b 100644 --- a/src/tags.ts +++ b/src/tags.ts @@ -35,8 +35,15 @@ const isReplyTag = (tag: string[]) => tag[0] === 'e' && tag[3] === 'reply'; const isRootTag = (tag: string[]) => tag[0] === 'e' && tag[3] === 'root'; const isLegacyReplyTag = (tag: string[]) => tag[0] === 'e' && !tag[3]; -function findReplyTag(tags: string[][]) { +const isQuoteTag = (tag: string[]) => tag[0] === 'q'; +const isLegacyQuoteTag = (tag: string[]) => tag[0] === 'e' && tag[3] === 'mention'; + +function findReplyTag(tags: string[][]): string[] | undefined { return tags.find(isReplyTag) || tags.find(isRootTag) || tags.findLast(isLegacyReplyTag); } -export { addTag, deleteTag, findReplyTag, getTagSet, hasTag }; +function findQuoteTag(tags: string[][]): string[] | undefined { + return tags.find(isQuoteTag) || tags.find(isLegacyQuoteTag); +} + +export { addTag, deleteTag, findQuoteTag, findReplyTag, getTagSet, hasTag }; diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index a06aac21..15719778 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -1,11 +1,10 @@ import { NostrEvent } from '@nostrify/nostrify'; -import { isCWTag } from 'https://gitlab.com/soapbox-pub/mostr/-/raw/c67064aee5ade5e01597c6d23e22e53c628ef0e2/src/nostr/tags.ts'; import { nip19 } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { Storages } from '@/storages.ts'; -import { findReplyTag } from '@/tags.ts'; +import { findQuoteTag, findReplyTag } from '@/tags.ts'; import { nostrDate } from '@/utils.ts'; import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts'; import { unfurlCardCached } from '@/utils/unfurl.ts'; @@ -30,6 +29,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< : await accountFromPubkey(event.pubkey); const replyTag = findReplyTag(event.tags); + const quoteTag = findQuoteTag(event.tags); const mentionedPubkeys = [ ...new Set( @@ -73,8 +73,8 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< const content = buildInlineRecipients(mentions) + html; - const cw = event.tags.find(isCWTag); - const subject = event.tags.find((tag) => tag[0] === 'subject'); + const cw = event.tags.find(([name]) => name === 'content-warning'); + const subject = event.tags.find(([name]) => name === 'subject'); const imeta: string[][][] = event.tags .filter(([name]) => name === 'imeta') @@ -88,7 +88,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< card, content, created_at: nostrDate(event.created_at).toISOString(), - in_reply_to_id: replyTag ? replyTag[1] : null, + in_reply_to_id: replyTag?.[1] ?? null, in_reply_to_account_id: null, sensitive: !!cw, spoiler_text: (cw ? cw[1] : subject?.[1]) || '', @@ -110,7 +110,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< emojis: renderEmojis(event), poll: null, quote: !event.quote ? null : await renderStatus(event.quote, { depth: depth + 1 }), - quote_id: event.tags.find(([name]) => name === 'q')?.[1] ?? null, + quote_id: quoteTag?.[1] ?? null, uri: Conf.external(note), url: Conf.external(note), zapped: Boolean(zapEvent), From 9839b8138f36baa55240a38748049f67bc1b0c08 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 13:08:08 -0500 Subject: [PATCH 59/65] tags.ts -> utils/tags.ts --- src/controllers/api/accounts.ts | 2 +- src/controllers/api/admin.ts | 2 +- src/controllers/api/bookmarks.ts | 2 +- src/controllers/api/mutes.ts | 2 +- src/controllers/api/statuses.ts | 8 ++++---- src/controllers/api/suggestions.ts | 2 +- src/pipeline.ts | 2 +- src/policies/MuteListPolicy.ts | 2 +- src/queries.ts | 2 +- src/stats.ts | 2 +- src/storages/EventsDB.ts | 4 ++-- src/storages/UserStore.ts | 2 +- src/storages/hydrate.ts | 2 +- src/{ => utils}/tags.ts | 0 src/views/mastodon/relationships.ts | 2 +- src/views/mastodon/statuses.ts | 2 +- 16 files changed, 19 insertions(+), 19 deletions(-) rename src/{ => utils}/tags.ts (100%) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index f717be39..2d61adcd 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -7,7 +7,6 @@ import { Conf } from '@/config.ts'; import { getAuthor, getFollowedPubkeys } from '@/queries.ts'; import { booleanParamSchema, fileSchema } from '@/schema.ts'; import { Storages } from '@/storages.ts'; -import { addTag, deleteTag, findReplyTag, getTagSet } from '@/tags.ts'; import { uploadFile } from '@/utils/upload.ts'; import { nostrNow } from '@/utils.ts'; import { createEvent, paginated, paginationSchema, parseBody, updateListEvent } from '@/utils/api.ts'; @@ -18,6 +17,7 @@ import { renderRelationship } from '@/views/mastodon/relationships.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { bech32ToPubkey } from '@/utils.ts'; +import { addTag, deleteTag, findReplyTag, getTagSet } from '@/utils/tags.ts'; const usernameSchema = z .string().min(1).max(30) diff --git a/src/controllers/api/admin.ts b/src/controllers/api/admin.ts index 77571aa7..d7cd3658 100644 --- a/src/controllers/api/admin.ts +++ b/src/controllers/api/admin.ts @@ -5,8 +5,8 @@ import { Conf } from '@/config.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { booleanParamSchema } from '@/schema.ts'; import { Storages } from '@/storages.ts'; -import { addTag } from '@/tags.ts'; import { paginated, paginationSchema, parseBody, updateListAdminEvent } from '@/utils/api.ts'; +import { addTag } from '@/utils/tags.ts'; import { renderAdminAccount } from '@/views/mastodon/admin-accounts.ts'; const adminAccountQuerySchema = z.object({ diff --git a/src/controllers/api/bookmarks.ts b/src/controllers/api/bookmarks.ts index 76551827..6d80b500 100644 --- a/src/controllers/api/bookmarks.ts +++ b/src/controllers/api/bookmarks.ts @@ -1,6 +1,6 @@ import { type AppController } from '@/app.ts'; import { Storages } from '@/storages.ts'; -import { getTagSet } from '@/tags.ts'; +import { getTagSet } from '@/utils/tags.ts'; import { renderStatuses } from '@/views.ts'; /** https://docs.joinmastodon.org/methods/bookmarks/#get */ diff --git a/src/controllers/api/mutes.ts b/src/controllers/api/mutes.ts index 4afb6c40..31f54ee1 100644 --- a/src/controllers/api/mutes.ts +++ b/src/controllers/api/mutes.ts @@ -1,6 +1,6 @@ import { type AppController } from '@/app.ts'; import { Storages } from '@/storages.ts'; -import { getTagSet } from '@/tags.ts'; +import { getTagSet } from '@/utils/tags.ts'; import { renderAccounts } from '@/views.ts'; /** https://docs.joinmastodon.org/methods/mutes/#get */ diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 291d970c..e9c872f8 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -8,15 +8,15 @@ import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; import { getUnattachedMediaByIds } from '@/db/unattached-media.ts'; import { getAncestors, getAuthor, getDescendants, getEvent } from '@/queries.ts'; -import { addTag, deleteTag } from '@/tags.ts'; -import { createEvent, paginationSchema, parseBody, updateListEvent } from '@/utils/api.ts'; import { renderEventAccounts } from '@/views.ts'; import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts'; -import { getLnurl } from '@/utils/lnurl.ts'; -import { asyncReplaceAll } from '@/utils/text.ts'; import { Storages } from '@/storages.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; +import { createEvent, paginationSchema, parseBody, updateListEvent } from '@/utils/api.ts'; +import { getLnurl } from '@/utils/lnurl.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; +import { addTag, deleteTag } from '@/utils/tags.ts'; +import { asyncReplaceAll } from '@/utils/text.ts'; const createStatusSchema = z.object({ in_reply_to_id: z.string().regex(/[0-9a-f]{64}/).nullish(), diff --git a/src/controllers/api/suggestions.ts b/src/controllers/api/suggestions.ts index 6377bd4f..012244a1 100644 --- a/src/controllers/api/suggestions.ts +++ b/src/controllers/api/suggestions.ts @@ -2,8 +2,8 @@ import { NStore } from '@nostrify/nostrify'; import { AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { getTagSet } from '@/tags.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; +import { getTagSet } from '@/utils/tags.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; export const suggestionsV1Controller: AppController = async (c) => { diff --git a/src/pipeline.ts b/src/pipeline.ts index 15d495e7..bfb0577e 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -13,7 +13,6 @@ import { RelayError } from '@/RelayError.ts'; import { updateStats } from '@/stats.ts'; import { hydrateEvents, purifyEvent } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; -import { getTagSet } from '@/tags.ts'; import { eventAge, nostrDate, nostrNow, parseNip05, Time } from '@/utils.ts'; import { fetchWorker } from '@/workers/fetch.ts'; import { policyWorker } from '@/workers/policy.ts'; @@ -22,6 +21,7 @@ import { verifyEventWorker } from '@/workers/verify.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { lnurlCache } from '@/utils/lnurl.ts'; import { nip05Cache } from '@/utils/nip05.ts'; +import { getTagSet } from '@/utils/tags.ts'; import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; diff --git a/src/policies/MuteListPolicy.ts b/src/policies/MuteListPolicy.ts index cae08eba..130d10df 100644 --- a/src/policies/MuteListPolicy.ts +++ b/src/policies/MuteListPolicy.ts @@ -1,6 +1,6 @@ import { NostrEvent, NostrRelayOK, NPolicy, NStore } from '@nostrify/nostrify'; -import { getTagSet } from '@/tags.ts'; +import { getTagSet } from '@/utils/tags.ts'; export class MuteListPolicy implements NPolicy { constructor(private pubkey: string, private store: NStore) {} diff --git a/src/queries.ts b/src/queries.ts index 76fabfdd..6a197ea7 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -5,8 +5,8 @@ import { Conf } from '@/config.ts'; import { Storages } from '@/storages.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { type DittoRelation } from '@/interfaces/DittoFilter.ts'; -import { findReplyTag, getTagSet } from '@/tags.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; +import { findReplyTag, getTagSet } from '@/utils/tags.ts'; const debug = Debug('ditto:queries'); diff --git a/src/stats.ts b/src/stats.ts index 256c570e..6ffe5f7e 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -8,7 +8,7 @@ import { SetRequired } from 'type-fest'; import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Storages } from '@/storages.ts'; -import { findReplyTag, getTagSet } from '@/tags.ts'; +import { findReplyTag, getTagSet } from '@/utils/tags.ts'; type AuthorStat = keyof Omit; type EventStat = keyof Omit; diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 5a3839a5..a550f39b 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -7,11 +7,11 @@ import { Kysely } from 'kysely'; import { Conf } from '@/config.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { normalizeFilters } from '@/filter.ts'; +import { RelayError } from '@/RelayError.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; -import { getTagSet } from '@/tags.ts'; import { isNostrId, isURL } from '@/utils.ts'; import { abortError } from '@/utils/abort.ts'; -import { RelayError } from '@/RelayError.ts'; +import { getTagSet } from '@/utils/tags.ts'; /** Function to decide whether or not to index a tag. */ type TagCondition = ({ event, count, value }: { diff --git a/src/storages/UserStore.ts b/src/storages/UserStore.ts index c5657b6e..43c1771b 100644 --- a/src/storages/UserStore.ts +++ b/src/storages/UserStore.ts @@ -1,7 +1,7 @@ import { NostrEvent, NostrFilter, NStore } from '@nostrify/nostrify'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { getTagSet } from '@/tags.ts'; +import { getTagSet } from '@/utils/tags.ts'; export class UserStore implements NStore { constructor(private pubkey: string, private store: NStore) {} diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index f55d7414..68dc0bdb 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -6,7 +6,7 @@ import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Conf } from '@/config.ts'; import { refreshAuthorStatsDebounced } from '@/stats.ts'; -import { findQuoteTag } from '@/tags.ts'; +import { findQuoteTag } from '@/utils/tags.ts'; interface HydrateOpts { events: DittoEvent[]; diff --git a/src/tags.ts b/src/utils/tags.ts similarity index 100% rename from src/tags.ts rename to src/utils/tags.ts diff --git a/src/views/mastodon/relationships.ts b/src/views/mastodon/relationships.ts index 2f8ffdde..425ea563 100644 --- a/src/views/mastodon/relationships.ts +++ b/src/views/mastodon/relationships.ts @@ -1,5 +1,5 @@ import { Storages } from '@/storages.ts'; -import { hasTag } from '@/tags.ts'; +import { hasTag } from '@/utils/tags.ts'; async function renderRelationship(sourcePubkey: string, targetPubkey: string) { const db = await Storages.db(); diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 15719778..cc7cc36b 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -4,9 +4,9 @@ import { nip19 } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { Storages } from '@/storages.ts'; -import { findQuoteTag, findReplyTag } from '@/tags.ts'; import { nostrDate } from '@/utils.ts'; import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts'; +import { findQuoteTag, findReplyTag } from '@/utils/tags.ts'; import { unfurlCardCached } from '@/utils/unfurl.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { renderAttachment } from '@/views/mastodon/attachments.ts'; From 5e607f664e2009ed8d25172609af6ad54caf8b0b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 13:32:27 -0500 Subject: [PATCH 60/65] Add tags test --- src/utils/tags.test.ts | 45 ++++++++++++++++++++++++++++++++++++++++++ src/utils/tags.ts | 36 ++++++++++++++++++++++++++------- 2 files changed, 74 insertions(+), 7 deletions(-) create mode 100644 src/utils/tags.test.ts diff --git a/src/utils/tags.test.ts b/src/utils/tags.test.ts new file mode 100644 index 00000000..43e12359 --- /dev/null +++ b/src/utils/tags.test.ts @@ -0,0 +1,45 @@ +import { assertEquals } from '@std/assert'; + +import { addTag, deleteTag, findQuoteTag, findReplyTag, getTagSet, hasTag } from './tags.ts'; + +Deno.test('addTag', () => { + const tags = [['p', 'alex']]; + assertEquals(addTag(tags, ['p', 'alex']), [['p', 'alex']]); + assertEquals(addTag(tags, ['p', 'fiatjaf']), [['p', 'alex'], ['p', 'fiatjaf']]); +}); + +Deno.test('deleteTag', () => { + const tags = [['p', 'alex'], ['p', 'fiatjaf']]; + assertEquals(deleteTag(tags, ['p', 'alex']), [['p', 'fiatjaf']]); + assertEquals(deleteTag(tags, ['p', 'fiatjaf']), [['p', 'alex']]); +}); + +Deno.test('findQuoteTag', () => { + assertEquals(findQuoteTag([['q', '123']]), ['q', '123']); + assertEquals(findQuoteTag([['e', '', '', 'mention', '456']]), ['e', '', '', 'mention', '456']); + assertEquals(findQuoteTag([['e', '', '', 'mention', '456'], ['q', '123']]), ['q', '123']); + assertEquals(findQuoteTag([['q', '123'], ['e', '', '', 'mention', '456']]), ['q', '123']); +}); + +Deno.test('findReplyTag', () => { + const root = ['e', '123', '', 'root']; + const reply = ['e', '456', '', 'reply']; + + assertEquals(findReplyTag([root]), root); + assertEquals(findReplyTag([reply]), reply); + assertEquals(findReplyTag([root, reply]), reply); + assertEquals(findReplyTag([reply, root]), reply); + assertEquals(findReplyTag([['e', '321'], ['e', '789']]), ['e', '789']); + assertEquals(findReplyTag([reply, ['e', '789']]), reply); +}); + +Deno.test('getTagSet', () => { + const tags = [['p', 'alex'], ['p', 'fiatjaf'], ['p', 'alex']]; + assertEquals(getTagSet(tags, 'p'), new Set(['alex', 'fiatjaf'])); +}); + +Deno.test('hasTag', () => { + const tags = [['p', 'alex']]; + assertEquals(hasTag(tags, ['p', 'alex']), true); + assertEquals(hasTag(tags, ['p', 'fiatjaf']), false); +}); diff --git a/src/utils/tags.ts b/src/utils/tags.ts index ecddaf4b..6375e815 100644 --- a/src/utils/tags.ts +++ b/src/utils/tags.ts @@ -31,18 +31,40 @@ function addTag(tags: readonly string[][], tag: string[]): string[][] { } } -const isReplyTag = (tag: string[]) => tag[0] === 'e' && tag[3] === 'reply'; -const isRootTag = (tag: string[]) => tag[0] === 'e' && tag[3] === 'root'; -const isLegacyReplyTag = (tag: string[]) => tag[0] === 'e' && !tag[3]; +/** Tag is a NIP-10 root tag. */ +function isRootTag(tag: string[]): tag is ['e', string, string, 'root', ...string[]] { + return tag[0] === 'e' && tag[3] === 'root'; +} -const isQuoteTag = (tag: string[]) => tag[0] === 'q'; -const isLegacyQuoteTag = (tag: string[]) => tag[0] === 'e' && tag[3] === 'mention'; +/** Tag is a NIP-10 reply tag. */ +function isReplyTag(tag: string[]): tag is ['e', string, string, 'reply', ...string[]] { + return tag[0] === 'e' && tag[3] === 'reply'; +} -function findReplyTag(tags: string[][]): string[] | undefined { +/** Tag is a legacy "e" tag with a "mention" marker. */ +function isLegacyQuoteTag(tag: string[]): tag is ['e', string, string, 'mention', ...string[]] { + return tag[0] === 'e' && tag[3] === 'mention'; +} + +/** Tag is an "e" tag without a NIP-10 marker. */ +function isLegacyReplyTag(tag: string[]): tag is ['e', string, string] { + return tag[0] === 'e' && !tag[3]; +} + +/** Tag is a "q" tag. */ +function isQuoteTag(tag: string[]): tag is ['q', ...string[]] { + return tag[0] === 'q'; +} + +/** Get the "e" tag for the event being replied to, first according to the NIPs then falling back to the legacy way. */ +function findReplyTag(tags: string[][]): ['e', ...string[]] | undefined { return tags.find(isReplyTag) || tags.find(isRootTag) || tags.findLast(isLegacyReplyTag); } -function findQuoteTag(tags: string[][]): string[] | undefined { +/** Get the "q" tag, falling back to the legacy "e" tag with a "mention" marker. */ +function findQuoteTag( + tags: string[][], +): ['q', ...string[]] | ['e', string, string, 'mention', ...string[]] | undefined { return tags.find(isQuoteTag) || tags.find(isLegacyQuoteTag); } From 9873afab699b6c6ef24ce253abcf38e668bcbd7f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 13:37:24 -0500 Subject: [PATCH 61/65] Remove old tags.test.ts --- src/tags.test.ts | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 src/tags.test.ts diff --git a/src/tags.test.ts b/src/tags.test.ts deleted file mode 100644 index e49d31ab..00000000 --- a/src/tags.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { assertEquals } from '@std/assert'; - -import { addTag, deleteTag, getTagSet } from './tags.ts'; - -Deno.test('getTagSet', () => { - assertEquals(getTagSet([], 'p'), new Set()); - assertEquals(getTagSet([['p', '123']], 'p'), new Set(['123'])); - assertEquals(getTagSet([['p', '123'], ['p', '456']], 'p'), new Set(['123', '456'])); - assertEquals(getTagSet([['p', '123'], ['p', '456'], ['q', '789']], 'p'), new Set(['123', '456'])); -}); - -Deno.test('addTag', () => { - assertEquals(addTag([], ['p', '123']), [['p', '123']]); - assertEquals(addTag([['p', '123']], ['p', '123']), [['p', '123']]); - assertEquals(addTag([['p', '123'], ['p', '456']], ['p', '123']), [['p', '123'], ['p', '456']]); - assertEquals(addTag([['p', '123'], ['p', '456']], ['p', '789']), [['p', '123'], ['p', '456'], ['p', '789']]); -}); - -Deno.test('deleteTag', () => { - assertEquals(deleteTag([], ['p', '123']), []); - assertEquals(deleteTag([['p', '123']], ['p', '123']), []); - assertEquals(deleteTag([['p', '123']], ['p', '456']), [['p', '123']]); - assertEquals(deleteTag([['p', '123'], ['p', '123']], ['p', '123']), []); - assertEquals(deleteTag([['p', '123'], ['p', '456']], ['p', '456']), [['p', '123']]); -}); From bf6dc22c5a6d059e19393490e115344b5ee8829f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 14:05:48 -0500 Subject: [PATCH 62/65] Fix imeta stripping so it doesn't remove a bunch of newlines --- fixtures/events/event-imeta.json | 20 ++++++++++++++++++++ src/utils/note.test.ts | 13 ++++++++++++- src/utils/note.ts | 5 ++--- 3 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 fixtures/events/event-imeta.json diff --git a/fixtures/events/event-imeta.json b/fixtures/events/event-imeta.json new file mode 100644 index 00000000..4952ca81 --- /dev/null +++ b/fixtures/events/event-imeta.json @@ -0,0 +1,20 @@ +{ + "id": "1264cc4051db59af9a21f7fd001fdf5213424f558ea9ab16a1b014fca2250af5", + "pubkey": "6be38f8c63df7dbf84db7ec4a6e6fbbd8d19dca3b980efad18585c46f04b26f9", + "created_at": 1716306470, + "kind": 1, + "tags": [ + [ + "imeta", + "url https://image.nostr.build/258d978b91e7424cfa43b31f3cfc077d7172ae10b3b45ac956feff9e72175126.png", + "m image/png", + "x b1ceee58405ef05a41190a0946ca6b6511dff426c68013cdd165514c1ef301f9", + "ox 258d978b91e7424cfa43b31f3cfc077d7172ae10b3b45ac956feff9e72175126", + "size 114350", + "dim 1414x594", + "blurhash LDRfkC.8_4_N_3NGR*t8%gIVWBxt" + ] + ], + "content": "Today we were made aware of multiple Fediverse blog posts incorrectly attributing “vote Trump” spam on Bluesky to the Mostr.pub Bridge. \n\nThis spam is NOT coming from Mostr. From the screenshots used in these blogs, it's clear the spam is coming from an entirely different bridge called momostr.pink. This bridge is not affiliated with Mostr, and is not even a fork of Mostr. We appreciate that the authors of these posts responded quickly to us and have since corrected the blogs. \n\nMostr.pub uses stirfry policies for anti-spam filtering. This includes an anti-duplication policy that prevents spam like the recent “vote Trump” posts we’ve seen repeated over and over. \n\nIt is important to note WHY there are multiple bridges, though. \n\nWhen Mostr.pub launched, multiple major servers immediately blocked Mostr, including Mastodon.social. The moderators of Mastodon.social claimed that this was because Nostr was unregulated, and suggested to one user that if they want to bridge their account they should host their own bridge.\n\nThat is exactly what momostr.pink, the source of this spam, has done. \n\nThe obvious response to the censorship of the Mostr Bridge is to build more bridges. \n\nWhile we have opted for pro-social policies that aim to reduce spam and build better connections between decentralized platforms, other bridges built to get around censorship of the Mostr Bridge may not — as we’re already seeing.\n\nThere will inevitably be multiple bridges, and we’re working on creating solutions to the problems that arise from that. In the meantime, if the Fediverse could do itself a favor and chill with the censorship for two seconds, we might not have so many problems. \n\n\nhttps://image.nostr.build/258d978b91e7424cfa43b31f3cfc077d7172ae10b3b45ac956feff9e72175126.png", + "sig": "b950e6e2ff1dc786ef344e7dad3edf8aa315a1053ede146725bde181acf7c2c1a5fcf1e0c796552b743607d6ae161a3ff4eb3af5033ffbfd314e68213d315215" +} diff --git a/src/utils/note.test.ts b/src/utils/note.test.ts index 9c8fad7e..b351dbfd 100644 --- a/src/utils/note.test.ts +++ b/src/utils/note.test.ts @@ -1,6 +1,7 @@ import { assertEquals } from '@std/assert'; -import { getMediaLinks, parseNoteContent } from '@/utils/note.ts'; +import { eventFixture } from '@/test.ts'; +import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts'; Deno.test('parseNoteContent', () => { const { html, links, firstUrl } = parseNoteContent('Hello, world!'); @@ -22,3 +23,13 @@ Deno.test('getMediaLinks', () => { ['m', 'image/png'], ]]); }); + +Deno.test('stripimeta', async () => { + const { content, tags } = await eventFixture('event-imeta'); + + const stripped = stripimeta(content, tags); + const expected = + `Today we were made aware of multiple Fediverse blog posts incorrectly attributing “vote Trump” spam on Bluesky to the Mostr.pub Bridge. \n\nThis spam is NOT coming from Mostr. From the screenshots used in these blogs, it's clear the spam is coming from an entirely different bridge called momostr.pink. This bridge is not affiliated with Mostr, and is not even a fork of Mostr. We appreciate that the authors of these posts responded quickly to us and have since corrected the blogs. \n\nMostr.pub uses stirfry policies for anti-spam filtering. This includes an anti-duplication policy that prevents spam like the recent “vote Trump” posts we’ve seen repeated over and over. \n\nIt is important to note WHY there are multiple bridges, though. \n\nWhen Mostr.pub launched, multiple major servers immediately blocked Mostr, including Mastodon.social. The moderators of Mastodon.social claimed that this was because Nostr was unregulated, and suggested to one user that if they want to bridge their account they should host their own bridge.\n\nThat is exactly what momostr.pink, the source of this spam, has done. \n\nThe obvious response to the censorship of the Mostr Bridge is to build more bridges. \n\nWhile we have opted for pro-social policies that aim to reduce spam and build better connections between decentralized platforms, other bridges built to get around censorship of the Mostr Bridge may not — as we’re already seeing.\n\nThere will inevitably be multiple bridges, and we’re working on creating solutions to the problems that arise from that. In the meantime, if the Fediverse could do itself a favor and chill with the censorship for two seconds, we might not have so many problems. `; + + assertEquals(stripped, expected); +}); diff --git a/src/utils/note.ts b/src/utils/note.ts index 03da2def..0d1e3b20 100644 --- a/src/utils/note.ts +++ b/src/utils/note.ts @@ -71,10 +71,9 @@ function stripimeta(content: string, tags: string[][]): string { const lines = content.split('\n').reverse(); - for (let i = 0; i < lines.length; i++) { - const line = lines[i]; + for (const line of [...lines]) { if (line === '' || urls.has(line)) { - lines.splice(i, 1); + lines.splice(0, 1); } else { break; } From fc325880d2270eaf4f779087502b5cc1b2b85fc2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 14:27:49 -0500 Subject: [PATCH 63/65] Add root tags to replies --- src/controllers/api/statuses.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index e9c872f8..58526cbe 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -76,12 +76,21 @@ const createStatusController: AppController = async (c) => { const tags: string[][] = []; - if (data.quote_id) { - tags.push(['q', data.quote_id]); + if (data.in_reply_to_id) { + const ancestor = await getEvent(data.in_reply_to_id); + + if (!ancestor) { + return c.json({ error: 'Original post not found.' }, 404); + } + + const root = ancestor.tags.find((tag) => tag[0] === 'e' && tag[3] === 'root')?.[1] ?? ancestor.id; + + tags.push(['e', root, 'root']); + tags.push(['e', data.in_reply_to_id, 'reply']); } - if (data.in_reply_to_id) { - tags.push(['e', data.in_reply_to_id, 'reply']); + if (data.quote_id) { + tags.push(['q', data.quote_id]); } if (data.sensitive && data.spoiler_text) { From 9ab38203dfa921c002715baed74386aaec061ddb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 14:37:18 -0500 Subject: [PATCH 64/65] getDescendants: filter out non-replies --- src/queries.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/queries.ts b/src/queries.ts index 6a197ea7..7407077e 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -88,7 +88,11 @@ async function getAncestors(event: NostrEvent, result: NostrEvent[] = []): Promi async function getDescendants(eventId: string, signal = AbortSignal.timeout(2000)): Promise { const store = await Storages.db(); - const events = await store.query([{ kinds: [1], '#e': [eventId] }], { limit: 200, signal }); + + const events = await store + .query([{ kinds: [1], '#e': [eventId] }], { limit: 200, signal }) + .then((events) => events.filter(({ tags }) => findReplyTag(tags)?.[1] === eventId)); + return hydrateEvents({ events, store, signal }); } From 101a16bc1200f74b395934ac7053aaacd3642314 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 16:29:36 -0500 Subject: [PATCH 65/65] Index the Postgres FTS column --- src/db/migrations/021_pgfts_index.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/db/migrations/021_pgfts_index.ts diff --git a/src/db/migrations/021_pgfts_index.ts b/src/db/migrations/021_pgfts_index.ts new file mode 100644 index 00000000..d18d110b --- /dev/null +++ b/src/db/migrations/021_pgfts_index.ts @@ -0,0 +1,21 @@ +import { Kysely } from 'kysely'; + +import { Conf } from '@/config.ts'; + +export async function up(db: Kysely): Promise { + if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + await db.schema + .createIndex('nostr_pgfts_gin_search_vec') + .ifNotExists() + .on('nostr_pgfts') + .using('gin') + .column('search_vec') + .execute(); + } +} + +export async function down(db: Kysely): Promise { + if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + await db.schema.dropIndex('nostr_pgfts_gin_search_vec').ifExists().execute(); + } +}