From 616c405f0f54ec4ed5a586a0f25c3395543d6b2a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 6 Jun 2024 14:14:59 -0500 Subject: [PATCH 01/77] Add an endpoint to request a NIP-05 name --- src/app.ts | 3 ++- src/controllers/api/ditto.ts | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/app.ts b/src/app.ts index bb1c5b6b..4378445f 100644 --- a/src/app.ts +++ b/src/app.ts @@ -30,7 +30,7 @@ import { adminAccountAction, adminAccountsController } from '@/controllers/api/a import { appCredentialsController, createAppController } from '@/controllers/api/apps.ts'; import { blocksController } from '@/controllers/api/blocks.ts'; import { bookmarksController } from '@/controllers/api/bookmarks.ts'; -import { adminRelaysController, adminSetRelaysController } from '@/controllers/api/ditto.ts'; +import { adminRelaysController, adminSetRelaysController, inviteRequestController } from '@/controllers/api/ditto.ts'; import { emptyArrayController, emptyObjectController, notImplementedController } from '@/controllers/api/fallback.ts'; import { instanceController } from '@/controllers/api/instance.ts'; import { markersController, updateMarkersController } from '@/controllers/api/markers.ts'; @@ -239,6 +239,7 @@ app.delete('/api/v1/pleroma/admin/statuses/:id', requireRole('admin'), pleromaAd app.get('/api/v1/admin/ditto/relays', requireRole('admin'), adminRelaysController); app.put('/api/v1/admin/ditto/relays', requireRole('admin'), adminSetRelaysController); +app.post('/api/v1/ditto/nip05', requireSigner, inviteRequestController); app.post('/api/v1/ditto/zap', requireSigner, zapController); app.post('/api/v1/reports', requireSigner, reportController); diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index df4f210e..cf08cd3b 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -5,6 +5,7 @@ import { AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { Storages } from '@/storages.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; +import { createEvent } from '@/utils/api.ts'; const markerSchema = z.enum(['read', 'write']); @@ -58,3 +59,25 @@ function renderRelays(event: NostrEvent): RelayEntity[] { return acc; }, [] as RelayEntity[]); } + +const inviteRequestSchema = z.object({ + nip05: z.string().email(), + reason: z.string().max(500).optional(), +}); + +export const inviteRequestController: AppController = async (c) => { + const { nip05, reason } = inviteRequestSchema.parse(await c.req.json()); + + await createEvent({ + kind: 3036, + content: reason, + tags: [ + ['r', nip05], + ['L', 'nip05.domain'], + ['l', nip05.split('@')[1], 'nip05.domain'], + ['p', Conf.pubkey], + ], + }, c); + + return new Response('', { status: 204 }); +}; From ab2e9d8dd7ef126cc0c1d099d216966746ff3a60 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 6 Jun 2024 14:19:53 -0500 Subject: [PATCH 02/77] nostr.json: determine nip05 grant based on kind 30360 events --- src/controllers/well-known/nostr.ts | 5 ++++- src/utils/nip05.ts | 11 +++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/controllers/well-known/nostr.ts b/src/controllers/well-known/nostr.ts index 06698887..b6b7af09 100644 --- a/src/controllers/well-known/nostr.ts +++ b/src/controllers/well-known/nostr.ts @@ -10,9 +10,12 @@ const nameSchema = z.string().min(1).regex(/^\w+$/); * https://github.com/nostr-protocol/nips/blob/master/05.md */ const nostrController: AppController = async (c) => { + const store = c.get('store'); + const result = nameSchema.safeParse(c.req.query('name')); const name = result.success ? result.data : undefined; - const pointer = name ? await localNip05Lookup(c.get('store'), name) : undefined; + + const pointer = name ? await localNip05Lookup(store, name) : undefined; if (!name || !pointer) { return c.json({ names: {}, relays: {} }); diff --git a/src/utils/nip05.ts b/src/utils/nip05.ts index 840ef6df..a579da6c 100644 --- a/src/utils/nip05.ts +++ b/src/utils/nip05.ts @@ -45,16 +45,15 @@ const nip05Cache = new SimpleLRU( { max: 500, ttl: Time.hours(1) }, ); -async function localNip05Lookup(store: NStore, name: string): Promise { - const [label] = await store.query([{ - kinds: [1985], +async function localNip05Lookup(store: NStore, localpart: string): Promise { + const [grant] = await store.query([{ + kinds: [30360], + '#d': [`${localpart}@${Conf.url.host}`], authors: [Conf.pubkey], - '#L': ['nip05'], - '#l': [`${name}@${Conf.url.host}`], limit: 1, }]); - const pubkey = label?.tags.find(([name]) => name === 'p')?.[1]; + const pubkey = grant?.tags.find(([name]) => name === 'p')?.[1]; if (pubkey) { return { pubkey, relays: [Conf.relay] }; From 179cafcc23fe2169cc0dcc5f19bc707aa4db20cd Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 6 Jun 2024 14:44:45 -0500 Subject: [PATCH 03/77] adminAccountsController: display users with a NIP-05 grant, allow filtering "pending" users --- src/controllers/api/admin.ts | 48 +++++++++++++++++++++++++++--------- src/interfaces/DittoEvent.ts | 1 - 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/src/controllers/api/admin.ts b/src/controllers/api/admin.ts index d7cd3658..af03db58 100644 --- a/src/controllers/api/admin.ts +++ b/src/controllers/api/admin.ts @@ -1,13 +1,14 @@ +import { NostrEvent } from '@nostrify/nostrify'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { booleanParamSchema } from '@/schema.ts'; import { Storages } from '@/storages.ts'; import { paginated, paginationSchema, parseBody, updateListAdminEvent } from '@/utils/api.ts'; import { addTag } from '@/utils/tags.ts'; -import { renderAdminAccount } from '@/views/mastodon/admin-accounts.ts'; +import { renderAdminAccount, renderAdminAccountFromPubkey } from '@/views/mastodon/admin-accounts.ts'; +import { hydrateEvents } from '@/storages/hydrate.ts'; const adminAccountQuerySchema = z.object({ local: booleanParamSchema.optional(), @@ -36,25 +37,50 @@ const adminAccountsController: AppController = async (c) => { } = adminAccountQuerySchema.parse(c.req.query()); // Not supported. - if (pending || disabled || silenced || suspended || sensitized) { + if (disabled || silenced || suspended || sensitized) { return c.json([]); } const store = await Storages.db(); - const { since, until, limit } = paginationSchema.parse(c.req.query()); + const params = paginationSchema.parse(c.req.query()); const { signal } = c.req.raw; - const events = await store.query([{ kinds: [30361], authors: [Conf.pubkey], since, until, limit }], { signal }); - const pubkeys = events.map((event) => event.tags.find(([name]) => name === 'd')?.[1]!); - const authors = await store.query([{ kinds: [0], authors: pubkeys }], { signal }); + const pubkeys = new Set(); + const events: NostrEvent[] = []; - for (const event of events) { - const d = event.tags.find(([name]) => name === 'd')?.[1]; - (event as DittoEvent).d_author = authors.find((author) => author.pubkey === d); + if (pending) { + for (const event of await store.query([{ kinds: [3036], ...params }], { signal })) { + pubkeys.add(event.pubkey); + events.push(event); + } + } else { + for (const event of await store.query([{ kinds: [30360], authors: [Conf.pubkey], ...params }], { signal })) { + const pubkey = event.tags.find(([name]) => name === 'd')?.[1]; + if (pubkey) { + pubkeys.add(pubkey); + events.push(event); + } + } } + const authors = await store.query([{ kinds: [0], authors: [...pubkeys] }], { signal }) + .then((events) => hydrateEvents({ store, events, signal })); + const accounts = await Promise.all( - events.map((event) => renderAdminAccount(event)), + [...pubkeys].map(async (pubkey) => { + const author = authors.find((event) => event.pubkey === pubkey); + const account = author ? await renderAdminAccount(author) : await renderAdminAccountFromPubkey(pubkey); + const request = events.find((event) => event.kind === 3036 && event.pubkey === pubkey); + const grant = events.find( + (event) => event.kind === 30360 && event.tags.find(([name]) => name === 'd')?.[1] === pubkey, + ); + + return { + ...account, + invite_request: request ? request.content : null, + approved: !!grant, + }; + }), ); return paginated(c, events, accounts); diff --git a/src/interfaces/DittoEvent.ts b/src/interfaces/DittoEvent.ts index 6f3e1d22..fea8e1ec 100644 --- a/src/interfaces/DittoEvent.ts +++ b/src/interfaces/DittoEvent.ts @@ -21,7 +21,6 @@ export interface DittoEvent extends NostrEvent { author_domain?: string; author_stats?: AuthorStats; event_stats?: EventStats; - d_author?: DittoEvent; user?: DittoEvent; repost?: DittoEvent; quote?: DittoEvent; From 4f87287d4595e080ca5507270ec2e8bbe129acb0 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 6 Jun 2024 15:26:35 -0500 Subject: [PATCH 04/77] Add invite_request_username property to AdminAccount --- src/controllers/api/admin.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/controllers/api/admin.ts b/src/controllers/api/admin.ts index af03db58..8fc8cf3a 100644 --- a/src/controllers/api/admin.ts +++ b/src/controllers/api/admin.ts @@ -77,7 +77,8 @@ const adminAccountsController: AppController = async (c) => { return { ...account, - invite_request: request ? request.content : null, + invite_request: request?.content ?? null, + invite_request_username: request?.tags.find(([name]) => name === 'r')?.[1] ?? null, approved: !!grant, }; }), From a30e19b6b2c06bf598e713cb5510a3577899098e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 6 Jun 2024 17:35:48 -0500 Subject: [PATCH 05/77] Fix nip05 endpoints --- src/controllers/api/admin.ts | 2 +- src/controllers/api/ditto.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/admin.ts b/src/controllers/api/admin.ts index 8fc8cf3a..8c55318a 100644 --- a/src/controllers/api/admin.ts +++ b/src/controllers/api/admin.ts @@ -49,7 +49,7 @@ const adminAccountsController: AppController = async (c) => { const events: NostrEvent[] = []; if (pending) { - for (const event of await store.query([{ kinds: [3036], ...params }], { signal })) { + for (const event of await store.query([{ kinds: [3036], '#p': [Conf.pubkey], ...params }], { signal })) { pubkeys.add(event.pubkey); events.push(event); } diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index cf08cd3b..e6f398db 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -79,5 +79,5 @@ export const inviteRequestController: AppController = async (c) => { ], }, c); - return new Response('', { status: 204 }); + return new Response(null, { status: 204 }); }; From 45dee962191265134b3cbae9d1bf086c8c5e0685 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 7 Jun 2024 10:01:08 -0500 Subject: [PATCH 06/77] Support `language` param with L tags --- src/controllers/api/statuses.ts | 5 +++++ src/views/mastodon/statuses.ts | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index b5ecf01e..ce4cbfa9 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -104,6 +104,11 @@ const createStatusController: AppController = async (c) => { tags.push(['subject', data.spoiler_text]); } + if (data.language) { + tags.push(['L', 'ISO-639-1']); + tags.push(['l', data.language, 'ISO-639-1']); + } + const media = data.media_ids?.length ? await getUnattachedMediaByIds(kysely, data.media_ids) : []; const imeta: string[][] = media.map(({ data }) => { diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 04039da9..ed14c8e3 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -100,7 +100,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< sensitive: !!cw, spoiler_text: (cw ? cw[1] : subject?.[1]) || '', visibility: 'public', - language: event.tags.find((tag) => tag[0] === 'lang')?.[1] || null, + language: event.tags.find((tag) => tag[0] === 'l' && tag[2] === 'ISO-639-1')?.[1] || null, replies_count: event.event_stats?.replies_count ?? 0, reblogs_count: event.event_stats?.reposts_count ?? 0, favourites_count: event.event_stats?.reactions['+'] ?? 0, From 2c649e0fa61c727934dc540edac477c628294963 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sat, 8 Jun 2024 00:26:36 +0530 Subject: [PATCH 07/77] support npubs in admin:role script --- scripts/admin-role.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/admin-role.ts b/scripts/admin-role.ts index 6e7bfc66..2a300a5d 100644 --- a/scripts/admin-role.ts +++ b/scripts/admin-role.ts @@ -5,11 +5,14 @@ import { Conf } from '@/config.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; import { nostrNow } from '@/utils.ts'; +import { nip19 } from 'nostr-tools'; const kysely = await DittoDB.getInstance(); const eventsDB = new EventsDB(kysely); -const [pubkey, role] = Deno.args; +const [pubkeyOrNpub, role] = Deno.args; +const pubkey = pubkeyOrNpub.startsWith('npub') ? + nip19.decode(pubkeyOrNpub).data as string : pubkeyOrNpub; if (!NSchema.id().safeParse(pubkey).success) { console.error('Invalid pubkey'); From 2380234deb7ff08a1eff8df056a95cb28cd35c8b Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sat, 8 Jun 2024 00:30:22 +0530 Subject: [PATCH 08/77] fmt --- .vscode/settings.json | 5 ++++- scripts/admin-role.ts | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 23baa765..7644e548 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,5 +2,8 @@ "deno.enable": true, "deno.lint": true, "editor.defaultFormatter": "denoland.vscode-deno", - "path-intellisense.extensionOnImport": true + "path-intellisense.extensionOnImport": true, + "[typescript]": { + "editor.defaultFormatter": "denoland.vscode-deno" + } } \ No newline at end of file diff --git a/scripts/admin-role.ts b/scripts/admin-role.ts index 2a300a5d..d4feda32 100644 --- a/scripts/admin-role.ts +++ b/scripts/admin-role.ts @@ -11,8 +11,7 @@ const kysely = await DittoDB.getInstance(); const eventsDB = new EventsDB(kysely); const [pubkeyOrNpub, role] = Deno.args; -const pubkey = pubkeyOrNpub.startsWith('npub') ? - nip19.decode(pubkeyOrNpub).data as string : pubkeyOrNpub; +const pubkey = pubkeyOrNpub.startsWith('npub') ? nip19.decode(pubkeyOrNpub).data as string : pubkeyOrNpub; if (!NSchema.id().safeParse(pubkey).success) { console.error('Invalid pubkey'); From fca7825bbfc50a32fb55a1c41185ce9913d807af Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 7 Jun 2024 22:11:17 -0500 Subject: [PATCH 09/77] EventsDB: replaceable deletions support --- deno.json | 2 +- deno.lock | 29 +++++++++++++++++++-- src/storages/EventsDB.ts | 56 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/deno.json b/deno.json index 0b14069f..b225c8e7 100644 --- a/deno.json +++ b/deno.json @@ -22,7 +22,7 @@ "@db/sqlite": "jsr:@db/sqlite@^0.11.1", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.22.5", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.23.0", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", diff --git a/deno.lock b/deno.lock index e502c760..d68a374e 100644 --- a/deno.lock +++ b/deno.lock @@ -10,6 +10,7 @@ "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", + "jsr:@nostrify/nostrify@^0.23.0": "jsr:@nostrify/nostrify@0.23.0", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0": "jsr:@soapbox/kysely-deno-sqlite@2.2.0", "jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0", "jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0", @@ -17,6 +18,7 @@ "jsr:@std/assert@^0.224.0": "jsr:@std/assert@0.224.0", "jsr:@std/assert@^0.225.1": "jsr:@std/assert@0.225.3", "jsr:@std/bytes@^0.224.0": "jsr:@std/bytes@0.224.0", + "jsr:@std/bytes@^1.0.0-rc.3": "jsr:@std/bytes@1.0.0", "jsr:@std/crypto@^0.224.0": "jsr:@std/crypto@0.224.0", "jsr:@std/dotenv@^0.224.0": "jsr:@std/dotenv@0.224.0", "jsr:@std/encoding@^0.221.0": "jsr:@std/encoding@0.221.0", @@ -25,7 +27,7 @@ "jsr:@std/fmt@^0.221.0": "jsr:@std/fmt@0.221.0", "jsr:@std/fs@^0.221.0": "jsr:@std/fs@0.221.0", "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.0", - "jsr:@std/io@^0.224": "jsr:@std/io@0.224.0", + "jsr:@std/io@^0.224": "jsr:@std/io@0.224.1", "jsr:@std/media-types@^0.224.1": "jsr:@std/media-types@0.224.1", "jsr:@std/path@0.217": "jsr:@std/path@0.217.0", "jsr:@std/path@^0.221.0": "jsr:@std/path@0.221.0", @@ -119,6 +121,20 @@ "npm:zod@^3.23.8" ] }, + "@nostrify/nostrify@0.23.0": { + "integrity": "8636c0322885707d6a7b342ef55f70debf399a1eb65b83abcce7972d69e30920", + "dependencies": [ + "jsr:@std/encoding@^0.224.1", + "npm:@scure/base@^1.1.6", + "npm:@scure/bip32@^1.4.0", + "npm:@scure/bip39@^1.3.0", + "npm:kysely@^0.27.3", + "npm:lru-cache@^10.2.0", + "npm:nostr-tools@^2.7.0", + "npm:websocket-ts@^2.1.5", + "npm:zod@^3.23.8" + ] + }, "@soapbox/kysely-deno-sqlite@2.2.0": { "integrity": "668ec94600bc4b4d7bd618dd7ca65d4ef30ee61c46ffcb379b6f45203c08517a", "dependencies": [ @@ -146,6 +162,9 @@ "@std/bytes@0.224.0": { "integrity": "a2250e1d0eb7d1c5a426f21267ab9bdeac2447fa87a3d0d1a467d3f7a6058e49" }, + "@std/bytes@1.0.0": { + "integrity": "9392e72af80adccaa1197912fa19990ed091cb98d5c9c4344b0c301b22d7c632" + }, "@std/crypto@0.224.0": { "integrity": "154ef3ff08ef535562ef1a718718c5b2c5fc3808f0f9100daad69e829bfcdf2d", "dependencies": [ @@ -181,6 +200,12 @@ "jsr:@std/bytes@^0.224.0" ] }, + "@std/io@0.224.1": { + "integrity": "73de242551a5c0965eb33e36b1fc7df4834ffbc836a1a643a410ccd11253d6be", + "dependencies": [ + "jsr:@std/bytes@^1.0.0-rc.3" + ] + }, "@std/media-types@0.224.1": { "integrity": "9e69a5daed37c5b5c6d3ce4731dc191f80e67f79bed392b0957d1d03b87f11e1" }, @@ -1318,7 +1343,7 @@ "dependencies": [ "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "jsr:@db/sqlite@^0.11.1", - "jsr:@nostrify/nostrify@^0.22.5", + "jsr:@nostrify/nostrify@^0.23.0", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 20a08957..d55be2e9 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -27,6 +27,7 @@ class EventsDB implements NStore { /** Conditions for when to index certain tags. */ static tagConditions: Record = { + 'a': ({ count }) => count < 15, 'd': ({ event, count }) => count === 0 && NKinds.parameterizedReplaceable(event.kind), 'e': ({ event, count, value }) => ((event.kind === 10003) || count < 15) && isNostrId(value), 'L': ({ event, count }) => event.kind === 1985 || count === 0, @@ -77,17 +78,62 @@ class EventsDB implements NStore { /** Check if an event has been deleted by the admin. */ private async isDeletedAdmin(event: NostrEvent): Promise { - const [deletion] = await this.query([ + const filters: NostrFilter[] = [ { kinds: [5], authors: [Conf.pubkey], '#e': [event.id], limit: 1 }, - ]); - return !!deletion; + ]; + + if (NKinds.replaceable(event.kind) || NKinds.parameterizedReplaceable(event.kind)) { + const d = event.tags.find(([tag]) => tag === 'd')?.[1] ?? ''; + + filters.push({ + kinds: [5], + authors: [Conf.pubkey], + '#a': [`${event.kind}:${event.pubkey}:${d}`], + since: event.created_at, + limit: 1, + }); + } + + const events = await this.query(filters); + return events.length > 0; } /** The DITTO_NSEC can delete any event from the database. NDatabase already handles user deletions. */ private async deleteEventsAdmin(event: NostrEvent): Promise { if (event.kind === 5 && event.pubkey === Conf.pubkey) { - const ids = getTagSet(event.tags, 'e'); - await this.remove([{ ids: [...ids] }]); + const ids = new Set(event.tags.filter(([name]) => name === 'e').map(([_name, value]) => value)); + const addrs = new Set(event.tags.filter(([name]) => name === 'a').map(([_name, value]) => value)); + + const filters: NostrFilter[] = []; + + if (ids.size) { + filters.push({ ids: [...ids] }); + } + + for (const addr of addrs) { + const [k, pubkey, d] = addr.split(':'); + const kind = Number(k); + + if (!(Number.isInteger(kind) && kind >= 0)) continue; + if (!isNostrId(pubkey)) continue; + if (d === undefined) continue; + + const filter: NostrFilter = { + kinds: [kind], + authors: [pubkey], + until: event.created_at, + }; + + if (d) { + filter['#d'] = [d]; + } + + filters.push(filter); + } + + if (filters.length) { + await this.remove(filters); + } } } From b088276c5127bb2164ac13a26be5684ecb6e3e0f Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sat, 8 Jun 2024 09:10:19 -0300 Subject: [PATCH 10/77] fix: remove unused variable --- src/storages/EventsDB.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index d55be2e9..53661783 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -11,7 +11,6 @@ import { RelayError } from '@/RelayError.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; import { isNostrId, isURL } from '@/utils.ts'; import { abortError } from '@/utils/abort.ts'; -import { getTagSet } from '@/utils/tags.ts'; /** Function to decide whether or not to index a tag. */ type TagCondition = ({ event, count, value }: { From 7d54a5c7d08b3716a4f546ed0ddd871591946eb9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 8 Jun 2024 11:13:15 -0500 Subject: [PATCH 11/77] Kind 30361 -> 30382 --- docs/events.md | 40 ---------------- fixtures/events/event-30361.json | 15 ------ scripts/admin-role.ts | 42 +++++++++++++---- src/controllers/api/admin.ts | 2 +- src/db/users.ts | 75 ------------------------------ src/middleware/auth98Middleware.ts | 23 ++++----- src/storages/EventsDB.ts | 2 - src/storages/hydrate.ts | 4 +- src/views/mastodon/accounts.ts | 8 ++-- 9 files changed, 52 insertions(+), 159 deletions(-) delete mode 100644 docs/events.md delete mode 100644 fixtures/events/event-30361.json delete mode 100644 src/db/users.ts diff --git a/docs/events.md b/docs/events.md deleted file mode 100644 index 1674239a..00000000 --- a/docs/events.md +++ /dev/null @@ -1,40 +0,0 @@ -# Ditto custom events - -Instead of using database tables, the Ditto server publishes Nostr events that describe its state. It then reads these events using Nostr filters. - -## Ditto User (kind 30361) - -The Ditto server publishes kind `30361` events to represent users. These events are parameterized replaceable events of kind `30361` where the `d` tag is a pubkey. These events are published by Ditto's internal admin keypair. - -User events have the following tags: - -- `d` - pubkey of the user. -- `role` - one of `admin` or `user`. - -Example: - -```json -{ - "id": "d6ae2f320ae163612bf28080e7c6e55b228ee39bfa04ad50baab2e51022d4d59", - "kind": 30361, - "pubkey": "4cfc6ceb07bbe2f5e75f746f3e6f0eda53973e0374cd6bdbce7a930e10437e06", - "content": "", - "created_at": 1691568245, - "tags": [ - ["d", "79c2cae114ea28a981e7559b4fe7854a473521a8d22a66bbab9fa248eb820ff6"], - ["role", "user"], - ["alt", "User's account was updated by the admins of ditto.ngrok.app"] - ], - "sig": "fc12db77b1c8f8aa86c73b617f0cd4af1e6ba244239eaf3164a292de6d39363f32d6b817ffff796ace7a103d75e1d8e6a0fb7f618819b32d81a953b4a75d7507" -} -``` - -## NIP-78 - -[NIP-78](https://github.com/nostr-protocol/nips/blob/master/78.md) defines events of kind `30078` with a globally unique `d` tag. These events are queried by the `d` tag, which allows Ditto to store custom data on relays. Ditto uses reverse DNS names like `pub.ditto.` for `d` tags. - -The sections below describe the `content` field. Some are encrypted and some are not, depending on whether the data should be public. Also, some events are user events, and some are admin events. - -### `pub.ditto.pleroma.config` - -NIP-04 encrypted JSON array of Pleroma ConfigDB objects. Pleroma admin API endpoints set this config, and Ditto reads from it. diff --git a/fixtures/events/event-30361.json b/fixtures/events/event-30361.json deleted file mode 100644 index 5844000c..00000000 --- a/fixtures/events/event-30361.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "id": "d6ae2f320ae163612bf28080e7c6e55b228ee39bfa04ad50baab2e51022d4d59", - "kind": 30361, - "pubkey": "4cfc6ceb07bbe2f5e75f746f3e6f0eda53973e0374cd6bdbce7a930e10437e06", - "content": "", - "created_at": 1691568245, - "tags": [ - ["d", "79c2cae114ea28a981e7559b4fe7854a473521a8d22a66bbab9fa248eb820ff6"], - ["name", "alex"], - ["role", "user"], - ["origin", "https://ditto.ngrok.app"], - ["alt", "@alex@ditto.ngrok.app's account was updated by the admins of ditto.ngrok.app"] - ], - "sig": "fc12db77b1c8f8aa86c73b617f0cd4af1e6ba244239eaf3164a292de6d39363f32d6b817ffff796ace7a103d75e1d8e6a0fb7f618819b32d81a953b4a75d7507" -} \ No newline at end of file diff --git a/scripts/admin-role.ts b/scripts/admin-role.ts index 6e7bfc66..305b593c 100644 --- a/scripts/admin-role.ts +++ b/scripts/admin-role.ts @@ -1,7 +1,6 @@ import { NSchema } from '@nostrify/nostrify'; import { DittoDB } from '@/db/DittoDB.ts'; -import { Conf } from '@/config.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; import { nostrNow } from '@/utils.ts'; @@ -21,14 +20,39 @@ if (!['admin', 'user'].includes(role)) { Deno.exit(1); } -const event = await new AdminSigner().signEvent({ - kind: 30361, - tags: [ - ['d', pubkey], - ['role', role], - // NIP-31: https://github.com/nostr-protocol/nips/blob/master/31.md - ['alt', `User's account was updated by the admins of ${Conf.url.host}`], - ], +const signer = new AdminSigner(); +const admin = await signer.getPublicKey(); + +const [existing] = await eventsDB.query([{ + kinds: [30382], + authors: [admin], + '#d': [pubkey], + limit: 1, +}]); + +const prevTags = (existing?.tags ?? []).filter(([name, value]) => { + if (name === 'd') { + return false; + } + if (name === 'n' && value === 'admin') { + return false; + } + return true; +}); + +const tags: string[][] = [ + ['d', pubkey], +]; + +if (role === 'admin') { + tags.push(['n', 'admin']); +} + +tags.push(...prevTags); + +const event = await signer.signEvent({ + kind: 30382, + tags, content: '', created_at: nostrNow(), }); diff --git a/src/controllers/api/admin.ts b/src/controllers/api/admin.ts index d7cd3658..ae2221fb 100644 --- a/src/controllers/api/admin.ts +++ b/src/controllers/api/admin.ts @@ -44,7 +44,7 @@ const adminAccountsController: AppController = async (c) => { const { since, until, limit } = paginationSchema.parse(c.req.query()); const { signal } = c.req.raw; - const events = await store.query([{ kinds: [30361], authors: [Conf.pubkey], since, until, limit }], { signal }); + const events = await store.query([{ kinds: [30382], authors: [Conf.pubkey], since, until, limit }], { signal }); const pubkeys = events.map((event) => event.tags.find(([name]) => name === 'd')?.[1]!); const authors = await store.query([{ kinds: [0], authors: pubkeys }], { signal }); diff --git a/src/db/users.ts b/src/db/users.ts deleted file mode 100644 index bf0cab7c..00000000 --- a/src/db/users.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { NostrFilter } from '@nostrify/nostrify'; -import Debug from '@soapbox/stickynotes/debug'; - -import { Conf } from '@/config.ts'; -import * as pipeline from '@/pipeline.ts'; -import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { Storages } from '@/storages.ts'; - -const debug = Debug('ditto:users'); - -interface User { - pubkey: string; - inserted_at: Date; - admin: boolean; -} - -function buildUserEvent(user: User) { - const { origin, host } = Conf.url; - const signer = new AdminSigner(); - - return signer.signEvent({ - kind: 30361, - tags: [ - ['d', user.pubkey], - ['role', user.admin ? 'admin' : 'user'], - ['origin', origin], - // NIP-31: https://github.com/nostr-protocol/nips/blob/master/31.md - ['alt', `User's account was updated by the admins of ${host}`], - ], - content: '', - created_at: Math.floor(user.inserted_at.getTime() / 1000), - }); -} - -/** Adds a user to the database. */ -async function insertUser(user: User) { - debug('insertUser', JSON.stringify(user)); - const event = await buildUserEvent(user); - return pipeline.handleEvent(event, AbortSignal.timeout(1000)); -} - -/** - * Finds a single user based on one or more properties. - * - * ```ts - * await findUser({ username: 'alex' }); - * ``` - */ -async function findUser(user: Partial, signal?: AbortSignal): Promise { - const filter: NostrFilter = { kinds: [30361], authors: [Conf.pubkey], limit: 1 }; - - for (const [key, value] of Object.entries(user)) { - switch (key) { - case 'pubkey': - filter['#d'] = [String(value)]; - break; - case 'admin': - filter['#role'] = [value ? 'admin' : 'user']; - break; - } - } - - const store = await Storages.db(); - const [event] = await store.query([filter], { signal }); - - if (event) { - return { - pubkey: event.tags.find(([name]) => name === 'd')?.[1]!, - inserted_at: new Date(event.created_at * 1000), - admin: event.tags.find(([name]) => name === 'role')?.[1] === 'admin', - }; - } -} - -export { buildUserEvent, findUser, insertUser, type User }; diff --git a/src/middleware/auth98Middleware.ts b/src/middleware/auth98Middleware.ts index 34d69379..05b06817 100644 --- a/src/middleware/auth98Middleware.ts +++ b/src/middleware/auth98Middleware.ts @@ -2,8 +2,8 @@ import { NostrEvent } from '@nostrify/nostrify'; import { HTTPException } from 'hono'; import { type AppContext, type AppMiddleware } from '@/app.ts'; -import { findUser, User } from '@/db/users.ts'; import { ReadOnlySigner } from '@/signers/ReadOnlySigner.ts'; +import { Storages } from '@/storages.ts'; import { localRequest } from '@/utils/api.ts'; import { buildAuthEventTemplate, @@ -11,6 +11,7 @@ import { type ParseAuthRequestOpts, validateAuthEvent, } from '@/utils/nip98.ts'; +import { Conf } from '@/config.ts'; /** * NIP-98 auth. @@ -35,7 +36,14 @@ type UserRole = 'user' | 'admin'; /** Require the user to prove their role before invoking the controller. */ function requireRole(role: UserRole, opts?: ParseAuthRequestOpts): AppMiddleware { return withProof(async (_c, proof, next) => { - const user = await findUser({ pubkey: proof.pubkey }); + const store = await Storages.db(); + + const [user] = await store.query([{ + kinds: [30382], + authors: [Conf.pubkey], + '#d': [proof.pubkey], + limit: 1, + }]); if (user && matchesRole(user, role)) { await next(); @@ -53,15 +61,8 @@ function requireProof(opts?: ParseAuthRequestOpts): AppMiddleware { } /** Check whether the user fulfills the role. */ -function matchesRole(user: User, role: UserRole): boolean { - switch (role) { - case 'user': - return true; - case 'admin': - return user.admin; - default: - return false; - } +function matchesRole(user: NostrEvent, role: UserRole): boolean { + return user.tags.some(([tag, value]) => tag === 'n' && value === role); } /** HOC to obtain proof in middleware. */ diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 53661783..8dcee6cc 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -39,8 +39,6 @@ class EventsDB implements NStore { 'q': ({ event, count, value }) => count === 0 && event.kind === 1 && isNostrId(value), 'r': ({ event, count, value }) => (event.kind === 1985 ? count < 20 : count < 3) && isURL(value), 't': ({ event, count, value }) => (event.kind === 1985 ? count < 20 : count < 5) && value.length < 50, - 'name': ({ event, count }) => event.kind === 30361 && count === 0, - 'role': ({ event, count }) => event.kind === 30361 && count === 0, }; constructor(private kysely: Kysely) { diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index d80c2f46..ded03d42 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -82,7 +82,7 @@ export function assembleEvents( for (const event of a) { event.author = b.find((e) => matchFilter({ kinds: [0], authors: [event.pubkey] }, e)); - event.user = b.find((e) => matchFilter({ kinds: [30361], authors: [admin], '#d': [event.pubkey] }, e)); + event.user = b.find((e) => matchFilter({ kinds: [30382], authors: [admin], '#d': [event.pubkey] }, e)); if (event.kind === 1) { const id = findQuoteTag(event.tags)?.[1] || findQuoteInContent(event.content); @@ -201,7 +201,7 @@ function gatherUsers({ events, store, signal }: HydrateOpts): Promise event.pubkey)); return store.query( - [{ kinds: [30361], authors: [Conf.pubkey], '#d': [...pubkeys], limit: pubkeys.size }], + [{ kinds: [30382], authors: [Conf.pubkey], '#d': [...pubkeys], limit: pubkeys.size }], { signal }, ); } diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index 918d03b9..b226915e 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -6,6 +6,7 @@ import { Conf } from '@/config.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { getLnurl } from '@/utils/lnurl.ts'; import { nip05Cache } from '@/utils/nip05.ts'; +import { getTagSet } from '@/utils/tags.ts'; import { Nip05, nostrDate, nostrNow, parseNip05 } from '@/utils.ts'; import { renderEmojis } from '@/views/mastodon/emojis.ts'; @@ -33,7 +34,7 @@ async function renderAccount( const npub = nip19.npubEncode(pubkey); const parsed05 = await parseAndVerifyNip05(nip05, pubkey); - const role = event.user?.tags.find(([name]) => name === 'role')?.[1] ?? 'user'; + const roles = getTagSet(event.tags, 'n'); return { id: pubkey, @@ -74,11 +75,10 @@ async function renderAccount( username: parsed05?.nickname || npub.substring(0, 8), ditto: { accepts_zaps: Boolean(getLnurl({ lud06, lud16 })), - is_registered: Boolean(event.user), }, pleroma: { - is_admin: role === 'admin', - is_moderator: ['admin', 'moderator'].includes(role), + is_admin: roles.has('admin'), + is_moderator: roles.has('admin') || roles.has('moderator'), is_local: parsed05?.domain === Conf.url.host, settings_store: undefined as unknown, }, From b9922f96a0fa7a69c4748b906467366ba49b6c04 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 8 Jun 2024 12:17:06 -0500 Subject: [PATCH 12/77] adminActionController: mark "n" tags on the user --- src/app.ts | 4 ++-- src/controllers/api/admin.ts | 28 +++++++++++++++++----------- src/utils/api.ts | 31 +++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/app.ts b/src/app.ts index bb1c5b6b..771b8870 100644 --- a/src/app.ts +++ b/src/app.ts @@ -26,7 +26,7 @@ import { updateCredentialsController, verifyCredentialsController, } from '@/controllers/api/accounts.ts'; -import { adminAccountAction, adminAccountsController } from '@/controllers/api/admin.ts'; +import { adminAccountsController, adminActionController } from '@/controllers/api/admin.ts'; import { appCredentialsController, createAppController } from '@/controllers/api/apps.ts'; import { blocksController } from '@/controllers/api/blocks.ts'; import { bookmarksController } from '@/controllers/api/bookmarks.ts'; @@ -251,7 +251,7 @@ app.post( adminReportResolveController, ); -app.post('/api/v1/admin/accounts/:id{[0-9a-f]{64}}/action', requireSigner, requireRole('admin'), adminAccountAction); +app.post('/api/v1/admin/accounts/:id{[0-9a-f]{64}}/action', requireSigner, requireRole('admin'), adminActionController); // Not (yet) implemented. app.get('/api/v1/custom_emojis', emptyArrayController); diff --git a/src/controllers/api/admin.ts b/src/controllers/api/admin.ts index ae2221fb..1b5794c0 100644 --- a/src/controllers/api/admin.ts +++ b/src/controllers/api/admin.ts @@ -5,8 +5,7 @@ import { Conf } from '@/config.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { booleanParamSchema } from '@/schema.ts'; import { Storages } from '@/storages.ts'; -import { paginated, paginationSchema, parseBody, updateListAdminEvent } from '@/utils/api.ts'; -import { addTag } from '@/utils/tags.ts'; +import { paginated, paginationSchema, parseBody, updateUser } from '@/utils/api.ts'; import { renderAdminAccount } from '@/views/mastodon/admin-accounts.ts'; const adminAccountQuerySchema = z.object({ @@ -64,7 +63,7 @@ const adminAccountActionSchema = z.object({ type: z.enum(['none', 'sensitive', 'disable', 'silence', 'suspend']), }); -const adminAccountAction: AppController = async (c) => { +const adminActionController: AppController = async (c) => { const body = await parseBody(c.req.raw); const result = adminAccountActionSchema.safeParse(body); const authorId = c.req.param('id'); @@ -75,17 +74,24 @@ const adminAccountAction: AppController = async (c) => { const { data } = result; - if (data.type !== 'disable') { - return c.json({ error: 'Record invalid' }, 422); + const n: Record = {}; + + if (data.type === 'sensitive') { + n.sensitive = true; + } + if (data.type === 'disable') { + n.disable = true; + } + if (data.type === 'silence') { + n.silence = true; + } + if (data.type === 'suspend') { + n.suspend = true; } - await updateListAdminEvent( - { kinds: [10000], authors: [Conf.pubkey], limit: 1 }, - (tags) => addTag(tags, ['p', authorId]), - c, - ); + await updateUser(authorId, n, c); return c.json({}, 200); }; -export { adminAccountAction, adminAccountsController }; +export { adminAccountsController, adminActionController }; diff --git a/src/utils/api.ts b/src/utils/api.ts index 3cc8b7d2..80106152 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -107,6 +107,36 @@ async function updateAdminEvent( return createAdminEvent(fn(prev), c); } +async function updateUser(pubkey: string, n: Record, c: AppContext): Promise { + const signer = new AdminSigner(); + const admin = await signer.getPublicKey(); + + return updateAdminEvent( + { kinds: [30382], authors: [admin], '#d': [pubkey], limit: 1 }, + (prev) => { + const prevNames = prev?.tags.reduce((acc, [name, value]) => { + if (name === 'n') acc[value] = true; + return acc; + }, {} as Record); + + const names = { ...prevNames, ...n }; + const nTags = Object.entries(names).filter(([, value]) => value).map(([name]) => ['n', name]); + const other = prev?.tags.filter(([name]) => !['d', 'n'].includes(name)) ?? []; + + return { + kind: 30382, + content: prev?.content, + tags: [ + ['d', pubkey], + ...nTags, + ...other, + ], + }; + }, + c, + ); +} + /** Push the event through the pipeline, rethrowing any RelayError. */ async function publishEvent(event: NostrEvent, c: AppContext): Promise { debug('EVENT', event); @@ -267,4 +297,5 @@ export { updateEvent, updateListAdminEvent, updateListEvent, + updateUser, }; From a30cdec79b00d0feb7327022d6a1034e3a2a396c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 8 Jun 2024 12:22:00 -0500 Subject: [PATCH 13/77] pipeline: ensure event doesn't already exist in DB --- src/pipeline.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/pipeline.ts b/src/pipeline.ts index 3255aa7e..c85cc0b5 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -35,6 +35,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise ${event.id}`); if (event.kind !== 24133) { @@ -84,6 +85,13 @@ function encounterEvent(event: NostrEvent): boolean { return encountered; } +/** Check if the event already exists in the database. */ +async function existsInDB(event: DittoEvent): Promise { + const store = await Storages.db(); + const events = await store.query([{ ids: [event.id], limit: 1 }]); + return events.length > 0; +} + /** Hydrate the event with the user, if applicable. */ async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { await hydrateEvents({ events: [event], store: await Storages.db(), signal }); From e5fadafc7aa9a445c08e50429d04501d82ec4d1a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 8 Jun 2024 12:58:59 -0500 Subject: [PATCH 14/77] Create AdminStore to filter out banned users --- src/pipeline.ts | 19 +++++++++--------- src/storages.ts | 8 ++++---- src/storages/AdminStore.ts | 40 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 13 deletions(-) create mode 100644 src/storages/AdminStore.ts diff --git a/src/pipeline.ts b/src/pipeline.ts index c85cc0b5..f59a1eb6 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,14 +1,11 @@ import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify'; -import { PipePolicy } from '@nostrify/nostrify/policies'; import Debug from '@soapbox/stickynotes/debug'; import { sql } from 'kysely'; import { LRUCache } from 'lru-cache'; -import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; import { deleteAttachedMedia } from '@/db/unattached-media.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; import { RelayError } from '@/RelayError.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; @@ -44,6 +41,15 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { const debug = Debug('ditto:policy'); - const policy = new PipePolicy([ - new MuteListPolicy(Conf.pubkey, await Storages.admin()), - policyWorker, - ]); - try { - const result = await policy.call(event); + const result = await policyWorker.call(event); debug(JSON.stringify(result)); RelayError.assert(result); } catch (e) { diff --git a/src/storages.ts b/src/storages.ts index f8f206d1..4aaca1c2 100644 --- a/src/storages.ts +++ b/src/storages.ts @@ -3,15 +3,15 @@ import { RelayPoolWorker } from 'nostr-relaypool'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; +import { AdminStore } from '@/storages/AdminStore.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; import { PoolStore } from '@/storages/pool-store.ts'; import { SearchStore } from '@/storages/search-store.ts'; import { InternalRelay } from '@/storages/InternalRelay.ts'; -import { UserStore } from '@/storages/UserStore.ts'; export class Storages { private static _db: Promise | undefined; - private static _admin: Promise | undefined; + private static _admin: Promise | undefined; private static _client: Promise | undefined; private static _pubsub: Promise | undefined; private static _search: Promise | undefined; @@ -28,9 +28,9 @@ export class Storages { } /** Admin user storage. */ - public static async admin(): Promise { + public static async admin(): Promise { if (!this._admin) { - this._admin = Promise.resolve(new UserStore(Conf.pubkey, await this.db())); + this._admin = Promise.resolve(new AdminStore(await this.db())); } return this._admin; } diff --git a/src/storages/AdminStore.ts b/src/storages/AdminStore.ts new file mode 100644 index 00000000..6285a146 --- /dev/null +++ b/src/storages/AdminStore.ts @@ -0,0 +1,40 @@ +import { NostrEvent, NostrFilter, NStore } from '@nostrify/nostrify'; + +import { Conf } from '@/config.ts'; +import { DittoEvent } from '@/interfaces/DittoEvent.ts'; +import { getTagSet } from '@/utils/tags.ts'; + +/** A store that prevents banned users from being displayed. */ +export class AdminStore implements NStore { + constructor(private store: NStore) {} + + async event(event: NostrEvent, opts?: { signal?: AbortSignal }): Promise { + return await this.store.event(event, opts); + } + + async query(filters: NostrFilter[], opts: { signal?: AbortSignal; limit?: number } = {}): Promise { + const events = await this.store.query(filters, opts); + + const users = await this.store.query([{ + kinds: [30382], + authors: [Conf.pubkey], + '#d': events.map((event) => event.pubkey), + limit: 1, + }]); + + return events.filter((event) => { + const user = users.find( + ({ kind, pubkey, tags }) => + kind === 30382 && pubkey === Conf.pubkey && tags.find(([name]) => name === 'd')?.[1] === event.pubkey, + ); + + const n = getTagSet(user?.tags ?? [], 'n'); + + if (n.has('disable') || n.has('suspend')) { + return false; + } + + return true; + }); + } +} From 284ae9aab7e479626a0db8abfacba99cdedf7267 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 8 Jun 2024 13:14:07 -0500 Subject: [PATCH 15/77] renderAccount: fix display of roles --- src/views/mastodon/accounts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index b226915e..5c7b7769 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -34,7 +34,7 @@ async function renderAccount( const npub = nip19.npubEncode(pubkey); const parsed05 = await parseAndVerifyNip05(nip05, pubkey); - const roles = getTagSet(event.tags, 'n'); + const roles = getTagSet(event.user?.tags ?? [], 'n'); return { id: pubkey, From d2238e80f9e12f9890a629ed1770a2b3cb804af7 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 8 Jun 2024 13:44:57 -0500 Subject: [PATCH 16/77] Support Pleroma admin tags --- src/app.ts | 5 +++ src/controllers/api/pleroma.ts | 71 +++++++++++++++++++++++++++++++++- src/utils/api.ts | 5 ++- src/views/mastodon/accounts.ts | 1 + 4 files changed, 79 insertions(+), 3 deletions(-) diff --git a/src/app.ts b/src/app.ts index 771b8870..2657a287 100644 --- a/src/app.ts +++ b/src/app.ts @@ -42,6 +42,8 @@ import { configController, frontendConfigController, pleromaAdminDeleteStatusController, + pleromaAdminTagController, + pleromaAdminUntagController, updateConfigController, } from '@/controllers/api/pleroma.ts'; import { preferencesController } from '@/controllers/api/preferences.ts'; @@ -253,6 +255,9 @@ app.post( app.post('/api/v1/admin/accounts/:id{[0-9a-f]{64}}/action', requireSigner, requireRole('admin'), adminActionController); +app.put('/api/v1/pleroma/admin/users/tag', requireRole('admin'), pleromaAdminTagController); +app.delete('/api/v1/pleroma/admin/users/tag', requireRole('admin'), pleromaAdminUntagController); + // Not (yet) implemented. app.get('/api/v1/custom_emojis', emptyArrayController); app.get('/api/v1/filters', emptyArrayController); diff --git a/src/controllers/api/pleroma.ts b/src/controllers/api/pleroma.ts index 31b4fc58..57b77cf0 100644 --- a/src/controllers/api/pleroma.ts +++ b/src/controllers/api/pleroma.ts @@ -6,7 +6,8 @@ import { Conf } from '@/config.ts'; import { configSchema, elixirTupleSchema, type PleromaConfig } from '@/schemas/pleroma-api.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Storages } from '@/storages.ts'; -import { createAdminEvent } from '@/utils/api.ts'; +import { createAdminEvent, updateAdminEvent } from '@/utils/api.ts'; +import { lookupPubkey } from '@/utils/lookup.ts'; const frontendConfigController: AppController = async (c) => { const store = await Storages.db(); @@ -87,4 +88,70 @@ async function getConfigs(store: NStore, signal: AbortSignal): Promise { + const params = pleromaAdminTagsSchema.parse(await c.req.json()); + + for (const nickname of params.nicknames) { + const pubkey = await lookupPubkey(nickname); + if (!pubkey) continue; + + await updateAdminEvent( + { kinds: [30382], authors: [Conf.pubkey], '#d': [pubkey], limit: 1 }, + (prev) => { + const tags = prev?.tags ?? [['d', pubkey]]; + + for (const tag of params.tags) { + const existing = prev?.tags.some(([name, value]) => name === 't' && value === tag); + if (!existing) { + tags.push(['t', tag]); + } + } + + return { + kind: 30382, + content: prev?.content ?? '', + tags, + }; + }, + c, + ); + } + + return new Response(null, { status: 204 }); +}; + +const pleromaAdminUntagController: AppController = async (c) => { + const params = pleromaAdminTagsSchema.parse(await c.req.json()); + + for (const nickname of params.nicknames) { + const pubkey = await lookupPubkey(nickname); + if (!pubkey) continue; + + await updateAdminEvent( + { kinds: [30382], authors: [Conf.pubkey], '#d': [pubkey], limit: 1 }, + (prev) => ({ + kind: 30382, + content: prev?.content ?? '', + tags: (prev?.tags ?? [['d', pubkey]]) + .filter(([name, value]) => !(name === 't' && params.tags.includes(value))), + }), + c, + ); + } + + return new Response(null, { status: 204 }); +}; + +export { + configController, + frontendConfigController, + pleromaAdminDeleteStatusController, + pleromaAdminTagController, + pleromaAdminUntagController, + updateConfigController, +}; diff --git a/src/utils/api.ts b/src/utils/api.ts index 80106152..fdbfbfd9 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -73,6 +73,8 @@ function updateListEvent( async function createAdminEvent(t: EventStub, c: AppContext): Promise { const signer = new AdminSigner(); + console.log(t); + const event = await signer.signEvent({ content: '', created_at: nostrNow(), @@ -125,7 +127,7 @@ async function updateUser(pubkey: string, n: Record, c: AppCont return { kind: 30382, - content: prev?.content, + content: prev?.content ?? '', tags: [ ['d', pubkey], ...nTags, @@ -294,6 +296,7 @@ export { type PaginationParams, paginationSchema, parseBody, + updateAdminEvent, updateEvent, updateListAdminEvent, updateListEvent, diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index 5c7b7769..3974c3cb 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -81,6 +81,7 @@ async function renderAccount( is_moderator: roles.has('admin') || roles.has('moderator'), is_local: parsed05?.domain === Conf.url.host, settings_store: undefined as unknown, + tags: [...getTagSet(event.user?.tags ?? [], 't')], }, nostr: { pubkey, From d2df7522c4184c94ba3fa28c01bcc74a9156ae01 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 8 Jun 2024 13:57:50 -0500 Subject: [PATCH 17/77] Add Pleroma suggest/unsuggest endpoints --- src/app.ts | 4 ++++ src/controllers/api/pleroma.ts | 38 ++++++++++++++++++++++++++++++---- src/views/mastodon/accounts.ts | 7 ++++--- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/src/app.ts b/src/app.ts index 2657a287..f9ea03a9 100644 --- a/src/app.ts +++ b/src/app.ts @@ -42,7 +42,9 @@ import { configController, frontendConfigController, pleromaAdminDeleteStatusController, + pleromaAdminSuggestController, pleromaAdminTagController, + pleromaAdminUnsuggestController, pleromaAdminUntagController, updateConfigController, } from '@/controllers/api/pleroma.ts'; @@ -257,6 +259,8 @@ app.post('/api/v1/admin/accounts/:id{[0-9a-f]{64}}/action', requireSigner, requi app.put('/api/v1/pleroma/admin/users/tag', requireRole('admin'), pleromaAdminTagController); app.delete('/api/v1/pleroma/admin/users/tag', requireRole('admin'), pleromaAdminUntagController); +app.patch('/api/v1/pleroma/admin/users/suggest', requireRole('admin'), pleromaAdminSuggestController); +app.patch('/api/v1/pleroma/admin/users/unsuggest', requireRole('admin'), pleromaAdminUnsuggestController); // Not (yet) implemented. app.get('/api/v1/custom_emojis', emptyArrayController); diff --git a/src/controllers/api/pleroma.ts b/src/controllers/api/pleroma.ts index 57b77cf0..f428ce92 100644 --- a/src/controllers/api/pleroma.ts +++ b/src/controllers/api/pleroma.ts @@ -6,7 +6,7 @@ import { Conf } from '@/config.ts'; import { configSchema, elixirTupleSchema, type PleromaConfig } from '@/schemas/pleroma-api.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Storages } from '@/storages.ts'; -import { createAdminEvent, updateAdminEvent } from '@/utils/api.ts'; +import { createAdminEvent, updateAdminEvent, updateUser } from '@/utils/api.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; const frontendConfigController: AppController = async (c) => { @@ -88,13 +88,13 @@ async function getConfigs(store: NStore, signal: AbortSignal): Promise { - const params = pleromaAdminTagsSchema.parse(await c.req.json()); + const params = pleromaAdminTagSchema.parse(await c.req.json()); for (const nickname of params.nicknames) { const pubkey = await lookupPubkey(nickname); @@ -126,7 +126,7 @@ const pleromaAdminTagController: AppController = async (c) => { }; const pleromaAdminUntagController: AppController = async (c) => { - const params = pleromaAdminTagsSchema.parse(await c.req.json()); + const params = pleromaAdminTagSchema.parse(await c.req.json()); for (const nickname of params.nicknames) { const pubkey = await lookupPubkey(nickname); @@ -147,11 +147,41 @@ const pleromaAdminUntagController: AppController = async (c) => { return new Response(null, { status: 204 }); }; +const pleromaAdminSuggestSchema = z.object({ + nicknames: z.string().array(), +}); + +const pleromaAdminSuggestController: AppController = async (c) => { + const { nicknames } = pleromaAdminSuggestSchema.parse(await c.req.json()); + + for (const nickname of nicknames) { + const pubkey = await lookupPubkey(nickname); + if (!pubkey) continue; + await updateUser(pubkey, { suggest: true }, c); + } + + return new Response(null, { status: 204 }); +}; + +const pleromaAdminUnsuggestController: AppController = async (c) => { + const { nicknames } = pleromaAdminSuggestSchema.parse(await c.req.json()); + + for (const nickname of nicknames) { + const pubkey = await lookupPubkey(nickname); + if (!pubkey) continue; + await updateUser(pubkey, { suggest: false }, c); + } + + return new Response(null, { status: 204 }); +}; + export { configController, frontendConfigController, pleromaAdminDeleteStatusController, + pleromaAdminSuggestController, pleromaAdminTagController, + pleromaAdminUnsuggestController, pleromaAdminUntagController, updateConfigController, }; diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index 3974c3cb..9f2f052e 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -34,7 +34,7 @@ async function renderAccount( const npub = nip19.npubEncode(pubkey); const parsed05 = await parseAndVerifyNip05(nip05, pubkey); - const roles = getTagSet(event.user?.tags ?? [], 'n'); + const names = getTagSet(event.user?.tags ?? [], 'n'); return { id: pubkey, @@ -77,8 +77,9 @@ async function renderAccount( accepts_zaps: Boolean(getLnurl({ lud06, lud16 })), }, pleroma: { - is_admin: roles.has('admin'), - is_moderator: roles.has('admin') || roles.has('moderator'), + is_admin: names.has('admin'), + is_moderator: names.has('admin') || names.has('moderator'), + is_suggested: names.has('suggest'), is_local: parsed05?.domain === Conf.url.host, settings_store: undefined as unknown, tags: [...getTagSet(event.user?.tags ?? [], 't')], From 9c24bac0ca7448f3bdad8931171c41c4816d5251 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 8 Jun 2024 14:07:14 -0500 Subject: [PATCH 18/77] Pull suggested profiles from kind 30382 events --- src/controllers/api/suggestions.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/controllers/api/suggestions.ts b/src/controllers/api/suggestions.ts index b56851a2..c31ffc00 100644 --- a/src/controllers/api/suggestions.ts +++ b/src/controllers/api/suggestions.ts @@ -31,7 +31,7 @@ async function renderV2Suggestions(c: AppContext, params: PaginatedListParams, s const pubkey = await signer?.getPublicKey(); const filters: NostrFilter[] = [ - { kinds: [3], authors: [Conf.pubkey], limit: 1 }, + { kinds: [30382], authors: [Conf.pubkey], '#n': ['suggest'], limit }, { kinds: [1985], '#L': ['pub.ditto.trends'], '#l': [`#p`], authors: [Conf.pubkey], limit: 1 }, ]; @@ -42,8 +42,8 @@ async function renderV2Suggestions(c: AppContext, params: PaginatedListParams, s const events = await store.query(filters, { signal }); - const [suggestedEvent, followsEvent, mutesEvent, trendingEvent] = [ - events.find((event) => matchFilter({ kinds: [3], authors: [Conf.pubkey] }, event)), + const [userEvents, followsEvent, mutesEvent, trendingEvent] = [ + events.filter((event) => matchFilter({ kinds: [30382], authors: [Conf.pubkey], '#n': ['suggest'] }, event)), pubkey ? events.find((event) => matchFilter({ kinds: [3], authors: [pubkey] }, event)) : undefined, pubkey ? events.find((event) => matchFilter({ kinds: [10000], authors: [pubkey] }, event)) : undefined, events.find((event) => @@ -51,8 +51,13 @@ async function renderV2Suggestions(c: AppContext, params: PaginatedListParams, s ), ]; - const [suggested, trending, follows, mutes] = [ - getTagSet(suggestedEvent?.tags ?? [], 'p'), + const suggested = new Set( + userEvents + .map((event) => event.tags.find(([name]) => name === 'd')?.[1]) + .filter((pubkey): pubkey is string => !!pubkey), + ); + + const [trending, follows, mutes] = [ getTagSet(trendingEvent?.tags ?? [], 'p'), getTagSet(followsEvent?.tags ?? [], 'p'), getTagSet(mutesEvent?.tags ?? [], 'p'), From ca57d1be105bb5ec61bfae14b0e2d128959a23f3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 8 Jun 2024 14:28:39 -0500 Subject: [PATCH 19/77] Remove stray console.log --- src/utils/api.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/utils/api.ts b/src/utils/api.ts index fdbfbfd9..a9390b5a 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -73,8 +73,6 @@ function updateListEvent( async function createAdminEvent(t: EventStub, c: AppContext): Promise { const signer = new AdminSigner(); - console.log(t); - const event = await signer.signEvent({ content: '', created_at: nostrNow(), From a2d865d6ccea412c27fae5c543dc8aa8ea62ad9c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 8 Jun 2024 17:54:39 -0500 Subject: [PATCH 20/77] Generate an internal event for each report and invite request --- src/pipeline.ts | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/pipeline.ts b/src/pipeline.ts index f59a1eb6..7cdaa60a 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -3,10 +3,12 @@ import Debug from '@soapbox/stickynotes/debug'; import { sql } from 'kysely'; import { LRUCache } from 'lru-cache'; +import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; import { deleteAttachedMedia } from '@/db/unattached-media.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { RelayError } from '@/RelayError.ts'; +import { AdminSigner } from '@/signers/AdminSigner.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; import { eventAge, parseNip05, Time } from '@/utils.ts'; @@ -53,6 +55,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { } } +async function generateSetEvents(event: NostrEvent): Promise { + const tagsAdmin = event.tags.some(([name, value]) => ['p', 'P'].includes(name) && value === Conf.pubkey); + + if (event.kind === 1984 && tagsAdmin) { + const signer = new AdminSigner(); + + const rel = await signer.signEvent({ + kind: 30383, + content: '', + tags: [ + ['d', event.id], + ['p', event.pubkey], + ['k', '1984'], + ['n', 'open'], + ...[...getTagSet(event.tags, 'p')].map((pubkey) => ['P', pubkey]), + ...[...getTagSet(event.tags, 'e')].map((pubkey) => ['e', pubkey]), + ], + created_at: Math.floor(Date.now() / 1000), + }); + + await handleEvent(rel, AbortSignal.timeout(1000)); + } + + if (event.kind === 3036 && tagsAdmin) { + const signer = new AdminSigner(); + + const rel = await signer.signEvent({ + kind: 30383, + content: '', + tags: [ + ['d', event.id], + ['p', event.pubkey], + ['k', '3036'], + ['n', 'open'], + ], + created_at: Math.floor(Date.now() / 1000), + }); + + await handleEvent(rel, AbortSignal.timeout(1000)); + } +} + export { handleEvent }; From a14515bbe010eee99e601c7fa9c7a6749feb85fa Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 8 Jun 2024 19:48:56 -0500 Subject: [PATCH 21/77] Rework reports with event sets --- src/controllers/api/reports.ts | 80 ++++++++++++++++++++++++++-------- src/storages/EventsDB.ts | 2 +- src/utils/api.ts | 17 ++++++-- 3 files changed, 75 insertions(+), 24 deletions(-) diff --git a/src/controllers/api/reports.ts b/src/controllers/api/reports.ts index 9cb2627a..2092b8a6 100644 --- a/src/controllers/api/reports.ts +++ b/src/controllers/api/reports.ts @@ -1,12 +1,14 @@ -import { NSchema as n } from '@nostrify/nostrify'; +import { NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { createAdminEvent, createEvent, parseBody } from '@/utils/api.ts'; +import { createEvent, paginationSchema, parseBody, updateEventInfo } from '@/utils/api.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { renderAdminReport } from '@/views/mastodon/reports.ts'; import { renderReport } from '@/views/mastodon/reports.ts'; +import { getTagSet } from '@/utils/tags.ts'; +import { booleanParamSchema } from '@/schema.ts'; const reportSchema = z.object({ account_id: n.id(), @@ -52,18 +54,60 @@ const reportController: AppController = async (c) => { return c.json(await renderReport(event)); }; +const adminReportsSchema = z.object({ + resolved: booleanParamSchema.optional(), + account_id: n.id().optional(), + target_account_id: n.id().optional(), +}); + /** https://docs.joinmastodon.org/methods/admin/reports/#get */ const adminReportsController: AppController = async (c) => { const store = c.get('store'); const viewerPubkey = await c.get('signer')?.getPublicKey(); - const reports = await store.query([{ kinds: [1984], '#P': [Conf.pubkey] }]) - .then((events) => hydrateEvents({ store, events: events, signal: c.req.raw.signal })) - .then((events) => - Promise.all( - events.map((event) => renderAdminReport(event, { viewerPubkey })), - ) - ); + const params = paginationSchema.parse(c.req.query()); + const { resolved, account_id, target_account_id } = adminReportsSchema.parse(c.req.query()); + + const filter: NostrFilter = { + kinds: [30383], + authors: [Conf.pubkey], + ...params, + }; + + if (typeof resolved === 'boolean') { + filter['#n'] = [resolved ? 'closed' : 'open']; + } + if (account_id) { + filter['#p'] = [account_id]; + } + if (target_account_id) { + filter['#P'] = [target_account_id]; + } + + const orig = await store.query([filter]); + const ids = new Set(); + + for (const event of orig) { + const d = event.tags.find(([name]) => name === 'd')?.[1]; + if (d) { + ids.add(d); + } + } + + const events = await store.query([{ kinds: [1984], ids: [...ids] }]) + .then((events) => hydrateEvents({ store, events: events, signal: c.req.raw.signal })); + + const reports = await Promise.all( + events.map((event) => { + const internal = orig.find(({ tags }) => tags.some(([name, value]) => name === 'd' && value === event.id)); + const names = getTagSet(internal?.tags ?? [], 'n'); + + return renderAdminReport(event, { + viewerPubkey, + actionTaken: names.has('closed'), + }); + }), + ); return c.json(reports); }; @@ -82,12 +126,13 @@ const adminReportController: AppController = async (c) => { }], { signal }); if (!event) { - return c.json({ error: 'This action is not allowed' }, 403); + return c.json({ error: 'Not found' }, 404); } await hydrateEvents({ events: [event], store, signal }); - return c.json(await renderAdminReport(event, { viewerPubkey: pubkey })); + const report = await renderAdminReport(event, { viewerPubkey: pubkey }); + return c.json(report); }; /** https://docs.joinmastodon.org/methods/admin/reports/#resolve */ @@ -104,18 +149,15 @@ const adminReportResolveController: AppController = async (c) => { }], { signal }); if (!event) { - return c.json({ error: 'This action is not allowed' }, 403); + return c.json({ error: 'Not found' }, 404); } + await updateEventInfo(eventId, { open: false, closed: true }, c); + await hydrateEvents({ events: [event], store, signal }); - await createAdminEvent({ - kind: 5, - tags: [['e', event.id]], - content: 'Report closed.', - }, c); - - return c.json(await renderAdminReport(event, { viewerPubkey: pubkey, actionTaken: true })); + const report = await renderAdminReport(event, { viewerPubkey: pubkey, actionTaken: true }); + return c.json(report); }; export { adminReportController, adminReportResolveController, adminReportsController, reportController }; diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 8dcee6cc..6a949548 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -29,9 +29,9 @@ class EventsDB implements NStore { 'a': ({ count }) => count < 15, 'd': ({ event, count }) => count === 0 && NKinds.parameterizedReplaceable(event.kind), 'e': ({ event, count, value }) => ((event.kind === 10003) || count < 15) && isNostrId(value), + 'k': ({ count, value }) => count === 0 && Number.isInteger(Number(value)), 'L': ({ event, count }) => event.kind === 1985 || count === 0, 'l': ({ event, count }) => event.kind === 1985 || count === 0, - 'media': ({ count, value }) => (count < 4) && isURL(value), 'n': ({ count, value }) => count < 50 && value.length < 50, 'P': ({ count, value }) => count === 0 && isNostrId(value), 'p': ({ event, count, value }) => (count < 15 || event.kind === 3) && isNostrId(value), diff --git a/src/utils/api.ts b/src/utils/api.ts index a9390b5a..1fa397b5 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -107,12 +107,20 @@ async function updateAdminEvent( return createAdminEvent(fn(prev), c); } -async function updateUser(pubkey: string, n: Record, c: AppContext): Promise { +function updateUser(pubkey: string, n: Record, c: AppContext): Promise { + return updateNames(30382, pubkey, n, c); +} + +function updateEventInfo(id: string, n: Record, c: AppContext): Promise { + return updateNames(30383, id, n, c); +} + +async function updateNames(k: number, d: string, n: Record, c: AppContext): Promise { const signer = new AdminSigner(); const admin = await signer.getPublicKey(); return updateAdminEvent( - { kinds: [30382], authors: [admin], '#d': [pubkey], limit: 1 }, + { kinds: [k], authors: [admin], '#d': [d], limit: 1 }, (prev) => { const prevNames = prev?.tags.reduce((acc, [name, value]) => { if (name === 'n') acc[value] = true; @@ -124,10 +132,10 @@ async function updateUser(pubkey: string, n: Record, c: AppCont const other = prev?.tags.filter(([name]) => !['d', 'n'].includes(name)) ?? []; return { - kind: 30382, + kind: k, content: prev?.content ?? '', tags: [ - ['d', pubkey], + ['d', d], ...nTags, ...other, ], @@ -296,6 +304,7 @@ export { parseBody, updateAdminEvent, updateEvent, + updateEventInfo, updateListAdminEvent, updateListEvent, updateUser, From bd6424acf53ef73d14205ef4bdd1356818ffb3bf Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 8 Jun 2024 22:16:34 -0500 Subject: [PATCH 22/77] Add preliminary nameRequestsController --- src/app.ts | 4 ++-- src/controllers/api/ditto.ts | 29 ++++++++++++++++++----- src/interfaces/DittoEvent.ts | 2 ++ src/pipeline.ts | 2 +- src/storages/hydrate.ts | 46 ++++++++++++++++++++++++++++-------- src/views/ditto.ts | 25 ++++++++++++++++++++ 6 files changed, 89 insertions(+), 19 deletions(-) create mode 100644 src/views/ditto.ts diff --git a/src/app.ts b/src/app.ts index 198dc7f6..19641d45 100644 --- a/src/app.ts +++ b/src/app.ts @@ -30,7 +30,7 @@ import { adminAccountsController, adminActionController } from '@/controllers/ap import { appCredentialsController, createAppController } from '@/controllers/api/apps.ts'; import { blocksController } from '@/controllers/api/blocks.ts'; import { bookmarksController } from '@/controllers/api/bookmarks.ts'; -import { adminRelaysController, adminSetRelaysController, inviteRequestController } from '@/controllers/api/ditto.ts'; +import { adminRelaysController, adminSetRelaysController, nameRequestController } from '@/controllers/api/ditto.ts'; import { emptyArrayController, emptyObjectController, notImplementedController } from '@/controllers/api/fallback.ts'; import { instanceController } from '@/controllers/api/instance.ts'; import { markersController, updateMarkersController } from '@/controllers/api/markers.ts'; @@ -243,7 +243,7 @@ app.delete('/api/v1/pleroma/admin/statuses/:id', requireRole('admin'), pleromaAd app.get('/api/v1/admin/ditto/relays', requireRole('admin'), adminRelaysController); app.put('/api/v1/admin/ditto/relays', requireRole('admin'), adminSetRelaysController); -app.post('/api/v1/ditto/nip05', requireSigner, inviteRequestController); +app.post('/api/v1/ditto/names', requireSigner, nameRequestController); app.post('/api/v1/ditto/zap', requireSigner, zapController); app.post('/api/v1/reports', requireSigner, reportController); diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index e6f398db..4cf2cd45 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -3,9 +3,11 @@ import { z } from 'zod'; import { AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { Storages } from '@/storages.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; +import { Storages } from '@/storages.ts'; +import { hydrateEvents } from '@/storages/hydrate.ts'; import { createEvent } from '@/utils/api.ts'; +import { renderNameRequest } from '@/views/ditto.ts'; const markerSchema = z.enum(['read', 'write']); @@ -60,15 +62,15 @@ function renderRelays(event: NostrEvent): RelayEntity[] { }, [] as RelayEntity[]); } -const inviteRequestSchema = z.object({ +const nameRequestSchema = z.object({ nip05: z.string().email(), reason: z.string().max(500).optional(), }); -export const inviteRequestController: AppController = async (c) => { - const { nip05, reason } = inviteRequestSchema.parse(await c.req.json()); +export const nameRequestController: AppController = async (c) => { + const { nip05, reason } = nameRequestSchema.parse(await c.req.json()); - await createEvent({ + const event = await createEvent({ kind: 3036, content: reason, tags: [ @@ -79,5 +81,20 @@ export const inviteRequestController: AppController = async (c) => { ], }, c); - return new Response(null, { status: 204 }); + await hydrateEvents({ events: [event], store: await Storages.db() }); + + const nameRequest = await renderNameRequest(event); + return c.json(nameRequest); +}; + +export const nameRequestsController: AppController = async (c) => { + const store = await Storages.db(); + const signer = c.get('signer')!; + const pubkey = await signer.getPublicKey(); + + const events = await store.query([{ kinds: [3036], authors: [pubkey], limit: 20 }]) + .then((events) => hydrateEvents({ events, store })); + + const nameRequests = await Promise.all(events.map(renderNameRequest)); + return c.json(nameRequests); }; diff --git a/src/interfaces/DittoEvent.ts b/src/interfaces/DittoEvent.ts index fea8e1ec..85e11d65 100644 --- a/src/interfaces/DittoEvent.ts +++ b/src/interfaces/DittoEvent.ts @@ -34,4 +34,6 @@ export interface DittoEvent extends NostrEvent { * https://github.com/nostr-protocol/nips/blob/master/56.md */ reported_notes?: DittoEvent[]; + /** Admin event relationship. */ + info?: DittoEvent; } diff --git a/src/pipeline.ts b/src/pipeline.ts index 7cdaa60a..20fed92d 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -212,7 +212,7 @@ async function generateSetEvents(event: NostrEvent): Promise { ['d', event.id], ['p', event.pubkey], ['k', '3036'], - ['n', 'open'], + ['n', 'pending'], ], created_at: Math.floor(Date.now() / 1000), }); diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index ded03d42..a3821aa3 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -44,6 +44,10 @@ async function hydrateEvents(opts: HydrateOpts): Promise { cache.push(event); } + for (const event of await gatherInfo({ events: cache, store, signal })) { + cache.push(event); + } + for (const event of await gatherReportedProfiles({ events: cache, store, signal })) { cache.push(event); } @@ -83,6 +87,7 @@ export function assembleEvents( for (const event of a) { event.author = b.find((e) => matchFilter({ kinds: [0], authors: [event.pubkey] }, e)); event.user = b.find((e) => matchFilter({ kinds: [30382], authors: [admin], '#d': [event.pubkey] }, e)); + event.info = b.find((e) => matchFilter({ kinds: [30383], authors: [admin], '#d': [event.id] }, e)); if (event.kind === 1) { const id = findQuoteTag(event.tags)?.[1] || findQuoteInContent(event.content); @@ -106,20 +111,21 @@ export function assembleEvents( } if (event.kind === 1984) { - const targetAccountId = event.tags.find(([name]) => name === 'p')?.[1]; - if (targetAccountId) { - event.reported_profile = b.find((e) => matchFilter({ kinds: [0], authors: [targetAccountId] }, e)); + const pubkey = event.tags.find(([name]) => name === 'p')?.[1]; + if (pubkey) { + event.reported_profile = b.find((e) => matchFilter({ kinds: [0], authors: [pubkey] }, e)); } - const reportedEvents: DittoEvent[] = []; - const status_ids = event.tags.filter(([name]) => name === 'e').map((tag) => tag[1]); - if (status_ids.length > 0) { - for (const id of status_ids) { - const reportedEvent = b.find((e) => matchFilter({ kinds: [1], ids: [id] }, e)); - if (reportedEvent) reportedEvents.push(reportedEvent); + const reportedEvents: DittoEvent[] = []; + const ids = event.tags.filter(([name]) => name === 'e').map(([_name, value]) => value); + + for (const id of ids) { + const reported = b.find((e) => matchFilter({ kinds: [1], ids: [id] }, e)); + if (reported) { + reportedEvents.push(reported); } - event.reported_notes = reportedEvents; } + event.reported_notes = reportedEvents; } event.author_stats = stats.authors.find((stats) => stats.pubkey === event.pubkey); @@ -206,6 +212,26 @@ function gatherUsers({ events, store, signal }: HydrateOpts): Promise { + const ids = new Set(); + + for (const event of events) { + if (event.kind === 3036) { + ids.add(event.id); + } + } + + if (!ids.size) { + return Promise.resolve([]); + } + + return store.query( + [{ ids: [...ids], limit: ids.size }], + { signal }, + ); +} + /** Collect reported notes from the events. */ function gatherReportedNotes({ events, store, signal }: HydrateOpts): Promise { const ids = new Set(); diff --git a/src/views/ditto.ts b/src/views/ditto.ts new file mode 100644 index 00000000..708c5223 --- /dev/null +++ b/src/views/ditto.ts @@ -0,0 +1,25 @@ +import { DittoEvent } from '@/interfaces/DittoEvent.ts'; +import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; +import { getTagSet } from '@/utils/tags.ts'; + +export async function renderNameRequest(event: DittoEvent) { + const n = getTagSet(event.info?.tags ?? [], 'n'); + + let approvalStatus = 'pending'; + + if (n.has('approved')) { + approvalStatus = 'approved'; + } + if (n.has('rejected')) { + approvalStatus = 'rejected'; + } + + return { + id: event.id, + account: event.author ? await renderAccount(event.author) : accountFromPubkey(event.pubkey), + name: event.tags.find(([name]) => name === 'r')?.[1] || '', + reason: event.content, + approval_status: approvalStatus, + created_at: new Date(event.created_at * 1000).toISOString(), + }; +} From 8a7cae98419b23e665debee161936d1126c7c8a5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Jun 2024 11:03:46 -0500 Subject: [PATCH 23/77] Refactor reports more, add reopen endpoint --- src/app.ts | 7 +++ src/controllers/api/ditto.ts | 97 +++++++++++++++++++++++++++++++--- src/controllers/api/reports.ts | 46 +++++++++++----- src/storages/hydrate.ts | 8 ++- src/views/mastodon/reports.ts | 32 +++++------ 5 files changed, 153 insertions(+), 37 deletions(-) diff --git a/src/app.ts b/src/app.ts index 19641d45..e58bd53d 100644 --- a/src/app.ts +++ b/src/app.ts @@ -53,6 +53,7 @@ import { deleteReactionController, reactionController, reactionsController } fro import { relayController } from '@/controllers/nostr/relay.ts'; import { adminReportController, + adminReportReopenController, adminReportResolveController, adminReportsController, reportController, @@ -255,6 +256,12 @@ app.post( requireRole('admin'), adminReportResolveController, ); +app.post( + '/api/v1/admin/reports/:id{[0-9a-f]{64}}/reopen', + requireSigner, + requireRole('admin'), + adminReportReopenController, +); app.post('/api/v1/admin/accounts/:id{[0-9a-f]{64}}/action', requireSigner, requireRole('admin'), adminActionController); diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index 4cf2cd45..d1f9b0df 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -1,12 +1,13 @@ -import { NostrEvent } from '@nostrify/nostrify'; +import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { z } from 'zod'; import { AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; +import { booleanParamSchema } from '@/schema.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Storages } from '@/storages.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { createEvent } from '@/utils/api.ts'; +import { createEvent, paginated, paginationSchema } from '@/utils/api.ts'; import { renderNameRequest } from '@/views/ditto.ts'; const markerSchema = z.enum(['read', 'write']); @@ -87,14 +88,98 @@ export const nameRequestController: AppController = async (c) => { return c.json(nameRequest); }; +const nameRequestsSchema = z.object({ + approved: booleanParamSchema.optional(), + rejected: booleanParamSchema.optional(), +}); + export const nameRequestsController: AppController = async (c) => { const store = await Storages.db(); const signer = c.get('signer')!; const pubkey = await signer.getPublicKey(); - const events = await store.query([{ kinds: [3036], authors: [pubkey], limit: 20 }]) - .then((events) => hydrateEvents({ events, store })); + const params = paginationSchema.parse(c.req.query()); + const { approved, rejected } = nameRequestsSchema.parse(c.req.query()); - const nameRequests = await Promise.all(events.map(renderNameRequest)); - return c.json(nameRequests); + const filter: NostrFilter = { + kinds: [30383], + authors: [Conf.pubkey], + '#k': ['3036'], + '#p': [pubkey], + ...params, + }; + + if (approved) { + filter['#n'] = ['approved']; + } + if (rejected) { + filter['#n'] = ['rejected']; + } + + const orig = await store.query([filter]); + const ids = new Set(); + + for (const event of orig) { + const d = event.tags.find(([name]) => name === 'd')?.[1]; + if (d) { + ids.add(d); + } + } + + const events = await store.query([{ kinds: [3036], ids: [...ids] }]) + .then((events) => hydrateEvents({ store, events: events, signal: c.req.raw.signal })); + + const nameRequests = await Promise.all( + events.map((event) => renderNameRequest(event)), + ); + + return paginated(c, orig, nameRequests); +}; + +const adminNameRequestsSchema = z.object({ + account_id: n.id().optional(), + approved: booleanParamSchema.optional(), + rejected: booleanParamSchema.optional(), +}); + +export const adminNameRequestsController: AppController = async (c) => { + const store = await Storages.db(); + const params = paginationSchema.parse(c.req.query()); + const { account_id, approved, rejected } = adminNameRequestsSchema.parse(c.req.query()); + + const filter: NostrFilter = { + kinds: [30383], + authors: [Conf.pubkey], + '#k': ['3036'], + ...params, + }; + + if (account_id) { + filter['#p'] = [account_id]; + } + if (approved) { + filter['#n'] = ['approved']; + } + if (rejected) { + filter['#n'] = ['rejected']; + } + + const orig = await store.query([filter]); + const ids = new Set(); + + for (const event of orig) { + const d = event.tags.find(([name]) => name === 'd')?.[1]; + if (d) { + ids.add(d); + } + } + + const events = await store.query([{ kinds: [3036], ids: [...ids] }]) + .then((events) => hydrateEvents({ store, events: events, signal: c.req.raw.signal })); + + const nameRequests = await Promise.all( + events.map((event) => renderNameRequest(event)), + ); + + return paginated(c, orig, nameRequests); }; diff --git a/src/controllers/api/reports.ts b/src/controllers/api/reports.ts index 2092b8a6..9a2750ff 100644 --- a/src/controllers/api/reports.ts +++ b/src/controllers/api/reports.ts @@ -7,7 +7,6 @@ import { createEvent, paginationSchema, parseBody, updateEventInfo } from '@/uti import { hydrateEvents } from '@/storages/hydrate.ts'; import { renderAdminReport } from '@/views/mastodon/reports.ts'; import { renderReport } from '@/views/mastodon/reports.ts'; -import { getTagSet } from '@/utils/tags.ts'; import { booleanParamSchema } from '@/schema.ts'; const reportSchema = z.object({ @@ -71,6 +70,7 @@ const adminReportsController: AppController = async (c) => { const filter: NostrFilter = { kinds: [30383], authors: [Conf.pubkey], + '#k': ['1984'], ...params, }; @@ -98,15 +98,7 @@ const adminReportsController: AppController = async (c) => { .then((events) => hydrateEvents({ store, events: events, signal: c.req.raw.signal })); const reports = await Promise.all( - events.map((event) => { - const internal = orig.find(({ tags }) => tags.some(([name, value]) => name === 'd' && value === event.id)); - const names = getTagSet(internal?.tags ?? [], 'n'); - - return renderAdminReport(event, { - viewerPubkey, - actionTaken: names.has('closed'), - }); - }), + events.map((event) => renderAdminReport(event, { viewerPubkey })), ); return c.json(reports); @@ -153,11 +145,39 @@ const adminReportResolveController: AppController = async (c) => { } await updateEventInfo(eventId, { open: false, closed: true }, c); - await hydrateEvents({ events: [event], store, signal }); - const report = await renderAdminReport(event, { viewerPubkey: pubkey, actionTaken: true }); + const report = await renderAdminReport(event, { viewerPubkey: pubkey }); return c.json(report); }; -export { adminReportController, adminReportResolveController, adminReportsController, reportController }; +const adminReportReopenController: AppController = async (c) => { + const eventId = c.req.param('id'); + const { signal } = c.req.raw; + const store = c.get('store'); + const pubkey = await c.get('signer')?.getPublicKey(); + + const [event] = await store.query([{ + kinds: [1984], + ids: [eventId], + limit: 1, + }], { signal }); + + if (!event) { + return c.json({ error: 'Not found' }, 404); + } + + await updateEventInfo(eventId, { open: true, closed: false }, c); + await hydrateEvents({ events: [event], store, signal }); + + const report = await renderAdminReport(event, { viewerPubkey: pubkey }); + return c.json(report); +}; + +export { + adminReportController, + adminReportReopenController, + adminReportResolveController, + adminReportsController, + reportController, +}; diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index a3821aa3..9ec9e8c1 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -206,6 +206,10 @@ function gatherAuthors({ events, store, signal }: HydrateOpts): Promise { const pubkeys = new Set(events.map((event) => event.pubkey)); + if (!pubkeys.size) { + return Promise.resolve([]); + } + return store.query( [{ kinds: [30382], authors: [Conf.pubkey], '#d': [...pubkeys], limit: pubkeys.size }], { signal }, @@ -217,7 +221,7 @@ function gatherInfo({ events, store, signal }: HydrateOpts): Promise(); for (const event of events) { - if (event.kind === 3036) { + if (event.kind === 1984 || event.kind === 3036) { ids.add(event.id); } } @@ -227,7 +231,7 @@ function gatherInfo({ events, store, signal }: HydrateOpts): Promise name === 'p')?.[2]; + const category = event.tags.find(([name]) => name === 'p')?.[2]; const statuses = []; - if (reportEvent.reported_notes) { - for (const status of reportEvent.reported_notes) { + if (event.reported_notes) { + for (const status of event.reported_notes) { statuses.push(await renderStatus(status, { viewerPubkey })); } } - const reportedPubkey = reportEvent.tags.find(([name]) => name === 'p')?.[1]; + const reportedPubkey = event.tags.find(([name]) => name === 'p')?.[1]; if (!reportedPubkey) { return; } + const names = getTagSet(event.info?.tags ?? [], 'n'); + return { - id: reportEvent.id, - action_taken: actionTaken, + id: event.id, + action_taken: names.has('closed'), action_taken_at: null, category, - comment: reportEvent.content, + comment: event.content, forwarded: false, - created_at: nostrDate(reportEvent.created_at).toISOString(), - account: reportEvent.author - ? await renderAdminAccount(reportEvent.author) - : await renderAdminAccountFromPubkey(reportEvent.pubkey), - target_account: reportEvent.reported_profile - ? await renderAdminAccount(reportEvent.reported_profile) + created_at: nostrDate(event.created_at).toISOString(), + account: event.author ? await renderAdminAccount(event.author) : await renderAdminAccountFromPubkey(event.pubkey), + target_account: event.reported_profile + ? await renderAdminAccount(event.reported_profile) : await renderAdminAccountFromPubkey(reportedPubkey), assigned_account: null, action_taken_by_account: null, From 58a01f90de6709ce47f67693e357192aba28a363 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Jun 2024 11:07:37 -0500 Subject: [PATCH 24/77] Paginate reports --- src/controllers/api/reports.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/reports.ts b/src/controllers/api/reports.ts index 9a2750ff..da107ed9 100644 --- a/src/controllers/api/reports.ts +++ b/src/controllers/api/reports.ts @@ -3,7 +3,7 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { createEvent, paginationSchema, parseBody, updateEventInfo } from '@/utils/api.ts'; +import { createEvent, paginated, paginationSchema, parseBody, updateEventInfo } from '@/utils/api.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { renderAdminReport } from '@/views/mastodon/reports.ts'; import { renderReport } from '@/views/mastodon/reports.ts'; @@ -101,7 +101,7 @@ const adminReportsController: AppController = async (c) => { events.map((event) => renderAdminReport(event, { viewerPubkey })), ); - return c.json(reports); + return paginated(c, orig, reports); }; /** https://docs.joinmastodon.org/methods/admin/reports/#get-one */ From 8802cbd77935a9b6874f3ba564ea8ac7c413aecd Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Jun 2024 11:24:01 -0500 Subject: [PATCH 25/77] suggest -> suggested --- src/controllers/api/pleroma.ts | 4 ++-- src/controllers/api/suggestions.ts | 4 ++-- src/views/mastodon/accounts.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/controllers/api/pleroma.ts b/src/controllers/api/pleroma.ts index f428ce92..31d8545f 100644 --- a/src/controllers/api/pleroma.ts +++ b/src/controllers/api/pleroma.ts @@ -157,7 +157,7 @@ const pleromaAdminSuggestController: AppController = async (c) => { for (const nickname of nicknames) { const pubkey = await lookupPubkey(nickname); if (!pubkey) continue; - await updateUser(pubkey, { suggest: true }, c); + await updateUser(pubkey, { suggested: true }, c); } return new Response(null, { status: 204 }); @@ -169,7 +169,7 @@ const pleromaAdminUnsuggestController: AppController = async (c) => { for (const nickname of nicknames) { const pubkey = await lookupPubkey(nickname); if (!pubkey) continue; - await updateUser(pubkey, { suggest: false }, c); + await updateUser(pubkey, { suggested: false }, c); } return new Response(null, { status: 204 }); diff --git a/src/controllers/api/suggestions.ts b/src/controllers/api/suggestions.ts index c31ffc00..7e461c45 100644 --- a/src/controllers/api/suggestions.ts +++ b/src/controllers/api/suggestions.ts @@ -31,7 +31,7 @@ async function renderV2Suggestions(c: AppContext, params: PaginatedListParams, s const pubkey = await signer?.getPublicKey(); const filters: NostrFilter[] = [ - { kinds: [30382], authors: [Conf.pubkey], '#n': ['suggest'], limit }, + { kinds: [30382], authors: [Conf.pubkey], '#n': ['suggested'], limit }, { kinds: [1985], '#L': ['pub.ditto.trends'], '#l': [`#p`], authors: [Conf.pubkey], limit: 1 }, ]; @@ -43,7 +43,7 @@ async function renderV2Suggestions(c: AppContext, params: PaginatedListParams, s const events = await store.query(filters, { signal }); const [userEvents, followsEvent, mutesEvent, trendingEvent] = [ - events.filter((event) => matchFilter({ kinds: [30382], authors: [Conf.pubkey], '#n': ['suggest'] }, event)), + events.filter((event) => matchFilter({ kinds: [30382], authors: [Conf.pubkey], '#n': ['suggested'] }, event)), pubkey ? events.find((event) => matchFilter({ kinds: [3], authors: [pubkey] }, event)) : undefined, pubkey ? events.find((event) => matchFilter({ kinds: [10000], authors: [pubkey] }, event)) : undefined, events.find((event) => diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index 9f2f052e..99f69f00 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -79,7 +79,7 @@ async function renderAccount( pleroma: { is_admin: names.has('admin'), is_moderator: names.has('admin') || names.has('moderator'), - is_suggested: names.has('suggest'), + is_suggested: names.has('suggested'), is_local: parsed05?.domain === Conf.url.host, settings_store: undefined as unknown, tags: [...getTagSet(event.user?.tags ?? [], 't')], From 594f37ea3348682edebe8c48a2f9edb7b96a5e57 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Jun 2024 11:26:57 -0500 Subject: [PATCH 26/77] Use past-tense for some n-tag values --- src/controllers/api/admin.ts | 6 +++--- src/pipeline.ts | 4 ++-- src/storages/AdminStore.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/controllers/api/admin.ts b/src/controllers/api/admin.ts index 9da15bce..a9ec619b 100644 --- a/src/controllers/api/admin.ts +++ b/src/controllers/api/admin.ts @@ -107,13 +107,13 @@ const adminActionController: AppController = async (c) => { n.sensitive = true; } if (data.type === 'disable') { - n.disable = true; + n.disabled = true; } if (data.type === 'silence') { - n.silence = true; + n.silenced = true; } if (data.type === 'suspend') { - n.suspend = true; + n.suspended = true; } await updateUser(authorId, n, c); diff --git a/src/pipeline.ts b/src/pipeline.ts index 20fed92d..9f99520b 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -45,10 +45,10 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise Date: Sun, 9 Jun 2024 11:57:10 -0500 Subject: [PATCH 27/77] Add admin name approve/reject endpoints --- src/app.ts | 16 +++++++++- src/controllers/api/ditto.ts | 57 +++++++++++++++++++++++++++++++++++- src/storages/EventsDB.ts | 2 ++ 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/src/app.ts b/src/app.ts index e58bd53d..7bf29caa 100644 --- a/src/app.ts +++ b/src/app.ts @@ -30,7 +30,15 @@ import { adminAccountsController, adminActionController } from '@/controllers/ap import { appCredentialsController, createAppController } from '@/controllers/api/apps.ts'; import { blocksController } from '@/controllers/api/blocks.ts'; import { bookmarksController } from '@/controllers/api/bookmarks.ts'; -import { adminRelaysController, adminSetRelaysController, nameRequestController } from '@/controllers/api/ditto.ts'; +import { + adminNameApproveController, + adminNameRejectController, + adminNameRequestsController, + adminRelaysController, + adminSetRelaysController, + nameRequestController, + nameRequestsController, +} from '@/controllers/api/ditto.ts'; import { emptyArrayController, emptyObjectController, notImplementedController } from '@/controllers/api/fallback.ts'; import { instanceController } from '@/controllers/api/instance.ts'; import { markersController, updateMarkersController } from '@/controllers/api/markers.ts'; @@ -245,6 +253,12 @@ app.get('/api/v1/admin/ditto/relays', requireRole('admin'), adminRelaysControlle app.put('/api/v1/admin/ditto/relays', requireRole('admin'), adminSetRelaysController); app.post('/api/v1/ditto/names', requireSigner, nameRequestController); +app.get('/api/v1/ditto/names', requireSigner, nameRequestsController); + +app.get('/api/v1/admin/ditto/names', requireRole('admin'), adminNameRequestsController); +app.post('/api/v1/admin/ditto/names/:id{[0-9a-f]{64}}/approve', requireRole('admin'), adminNameApproveController); +app.post('/api/v1/admin/ditto/names/:id{[0-9a-f]{64}}/reject', requireRole('admin'), adminNameRejectController); + app.post('/api/v1/ditto/zap', requireSigner, zapController); app.post('/api/v1/reports', requireSigner, reportController); diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index d1f9b0df..f8ac5f2a 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -7,7 +7,7 @@ import { booleanParamSchema } from '@/schema.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Storages } from '@/storages.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { createEvent, paginated, paginationSchema } from '@/utils/api.ts'; +import { createAdminEvent, createEvent, paginated, paginationSchema, updateEventInfo } from '@/utils/api.ts'; import { renderNameRequest } from '@/views/ditto.ts'; const markerSchema = z.enum(['read', 'write']); @@ -183,3 +183,58 @@ export const adminNameRequestsController: AppController = async (c) => { return paginated(c, orig, nameRequests); }; + +export const adminNameApproveController: AppController = async (c) => { + const eventId = c.req.param('id'); + const store = await Storages.db(); + + const [event] = await store.query([{ kinds: [3036], ids: [eventId] }]); + if (!event) { + return c.json({ error: 'Event not found' }, 404); + } + + const r = event.tags.find(([name]) => name === 'r')?.[1]; + if (!r) { + return c.json({ error: 'NIP-05 not found' }, 404); + } + if (!z.string().email().safeParse(r).success) { + return c.json({ error: 'Invalid NIP-05' }, 400); + } + + const [existing] = await store.query([{ kinds: [30360], authors: [Conf.pubkey], '#d': [r], limit: 1 }]); + if (existing) { + return c.json({ error: 'NIP-05 already granted to another user' }, 400); + } + + await createAdminEvent({ + kind: 30360, + tags: [ + ['d', r], + ['L', 'nip05.domain'], + ['l', r.split('@')[1], 'nip05.domain'], + ['p', event.pubkey], + ], + }, c); + + await updateEventInfo(eventId, { pending: false, approved: true, rejected: false }, c); + await hydrateEvents({ events: [event], store }); + + const nameRequest = await renderNameRequest(event); + return c.json(nameRequest); +}; + +export const adminNameRejectController: AppController = async (c) => { + const eventId = c.req.param('id'); + const store = await Storages.db(); + + const [event] = await store.query([{ kinds: [3036], ids: [eventId] }]); + if (!event) { + return c.json({ error: 'Event not found' }, 404); + } + + await updateEventInfo(eventId, { pending: false, approved: false, rejected: true }, c); + await hydrateEvents({ events: [event], store }); + + const nameRequest = await renderNameRequest(event); + return c.json(nameRequest); +}; diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 6a949548..c26ebf12 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -223,6 +223,8 @@ class EventsDB implements NStore { return event.content; case 30009: return EventsDB.buildTagsSearchContent(event.tags.filter(([t]) => t !== 'alt')); + case 30360: + return event.tags.find(([name]) => name === 'd')?.[1] || ''; default: return ''; } From 5379863d36678ae348015c2fb4304120a34cc380 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Jun 2024 12:13:50 -0500 Subject: [PATCH 28/77] Tag the nip05 request in the grant event --- src/controllers/api/ditto.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index f8ac5f2a..5c8ea9c4 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -64,20 +64,20 @@ function renderRelays(event: NostrEvent): RelayEntity[] { } const nameRequestSchema = z.object({ - nip05: z.string().email(), + name: z.string().email(), reason: z.string().max(500).optional(), }); export const nameRequestController: AppController = async (c) => { - const { nip05, reason } = nameRequestSchema.parse(await c.req.json()); + const { name, reason } = nameRequestSchema.parse(await c.req.json()); const event = await createEvent({ kind: 3036, content: reason, tags: [ - ['r', nip05], + ['r', name], ['L', 'nip05.domain'], - ['l', nip05.split('@')[1], 'nip05.domain'], + ['l', name.split('@')[1], 'nip05.domain'], ['p', Conf.pubkey], ], }, c); @@ -213,6 +213,7 @@ export const adminNameApproveController: AppController = async (c) => { ['L', 'nip05.domain'], ['l', r.split('@')[1], 'nip05.domain'], ['p', event.pubkey], + ['e', event.id], ], }, c); From 07a380fb75f24cc95ea77a834b65e36c0fc132b2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Jun 2024 13:43:40 -0500 Subject: [PATCH 29/77] Rework adminAccountsController to display pending accounts from nip05 requests --- src/controllers/api/admin.ts | 112 ++++++++++++++++----------- src/views/ditto.ts | 25 +++--- src/views/mastodon/admin-accounts.ts | 22 ++++-- 3 files changed, 95 insertions(+), 64 deletions(-) diff --git a/src/controllers/api/admin.ts b/src/controllers/api/admin.ts index a9ec619b..df5bf968 100644 --- a/src/controllers/api/admin.ts +++ b/src/controllers/api/admin.ts @@ -1,13 +1,14 @@ -import { NostrEvent } from '@nostrify/nostrify'; +import { NostrFilter } from '@nostrify/nostrify'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { booleanParamSchema } from '@/schema.ts'; import { Storages } from '@/storages.ts'; -import { paginated, paginationSchema, parseBody, updateUser } from '@/utils/api.ts'; -import { renderAdminAccount, renderAdminAccountFromPubkey } from '@/views/mastodon/admin-accounts.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; +import { paginated, paginationSchema, parseBody, updateUser } from '@/utils/api.ts'; +import { renderNameRequest } from '@/views/ditto.ts'; +import { renderAdminAccount, renderAdminAccountFromPubkey } from '@/views/mastodon/admin-accounts.ts'; const adminAccountQuerySchema = z.object({ local: booleanParamSchema.optional(), @@ -27,62 +28,83 @@ const adminAccountQuerySchema = z.object({ }); const adminAccountsController: AppController = async (c) => { + const store = await Storages.db(); + const params = paginationSchema.parse(c.req.query()); + const { signal } = c.req.raw; const { + local, pending, disabled, silenced, suspended, sensitized, + staff, } = adminAccountQuerySchema.parse(c.req.query()); - // Not supported. - if (disabled || silenced || suspended || sensitized) { - return c.json([]); - } - - const store = await Storages.db(); - const params = paginationSchema.parse(c.req.query()); - const { signal } = c.req.raw; - - const pubkeys = new Set(); - const events: NostrEvent[] = []; - if (pending) { - for (const event of await store.query([{ kinds: [3036], '#p': [Conf.pubkey], ...params }], { signal })) { - pubkeys.add(event.pubkey); - events.push(event); - } - } else { - for (const event of await store.query([{ kinds: [30360], authors: [Conf.pubkey], ...params }], { signal })) { - const pubkey = event.tags.find(([name]) => name === 'd')?.[1]; - if (pubkey) { - pubkeys.add(pubkey); - events.push(event); - } + if (disabled || silenced || suspended || sensitized) { + return c.json([]); } + + const orig = await store.query( + [{ kinds: [30383], authors: [Conf.pubkey], '#k': ['3036'], ...params }], + { signal }, + ); + + const ids = new Set( + orig + .map(({ tags }) => tags.find(([name]) => name === 'd')?.[1]) + .filter((id): id is string => !!id), + ); + + const events = await store.query([{ kinds: [3036], ids: [...ids] }]) + .then((events) => hydrateEvents({ store, events, signal })); + + const nameRequests = await Promise.all(events.map(renderNameRequest)); + return paginated(c, orig, nameRequests); } - const authors = await store.query([{ kinds: [0], authors: [...pubkeys] }], { signal }) - .then((events) => hydrateEvents({ store, events, signal })); + if (disabled || silenced || suspended || sensitized) { + const n = []; - const accounts = await Promise.all( - [...pubkeys].map(async (pubkey) => { - const author = authors.find((event) => event.pubkey === pubkey); - const account = author ? await renderAdminAccount(author) : await renderAdminAccountFromPubkey(pubkey); - const request = events.find((event) => event.kind === 3036 && event.pubkey === pubkey); - const grant = events.find( - (event) => event.kind === 30360 && event.tags.find(([name]) => name === 'd')?.[1] === pubkey, - ); + if (disabled) { + n.push('disabled'); + } + if (silenced) { + n.push('silenced'); + } + if (suspended) { + n.push('suspended'); + } + if (sensitized) { + n.push('sensitized'); + } + if (staff) { + n.push('admin'); + n.push('moderator'); + } - return { - ...account, - invite_request: request?.content ?? null, - invite_request_username: request?.tags.find(([name]) => name === 'r')?.[1] ?? null, - approved: !!grant, - }; - }), - ); + const events = await store.query([{ kinds: [30382], authors: [Conf.pubkey], '#n': n, ...params }], { signal }); + const pubkeys = new Set(events.map(({ pubkey }) => pubkey)); + const authors = await store.query([{ kinds: [0], authors: [...pubkeys] }]) + .then((events) => hydrateEvents({ store, events, signal })); + const accounts = await Promise.all( + [...pubkeys].map((pubkey) => { + const author = authors.find((e) => e.pubkey === pubkey); + return author ? renderAdminAccount(author) : renderAdminAccountFromPubkey(pubkey); + }), + ); + + return paginated(c, events, accounts); + } + + const filter: NostrFilter = { kinds: [0], ...params }; + if (local) { + filter.search = `domain:${Conf.url.host}`; + } + const events = await store.query([filter], { signal }); + const accounts = await Promise.all(events.map(renderAdminAccount)); return paginated(c, events, accounts); }; @@ -104,7 +126,7 @@ const adminActionController: AppController = async (c) => { const n: Record = {}; if (data.type === 'sensitive') { - n.sensitive = true; + n.sensitized = true; } if (data.type === 'disable') { n.disabled = true; diff --git a/src/views/ditto.ts b/src/views/ditto.ts index 708c5223..ebc07b77 100644 --- a/src/views/ditto.ts +++ b/src/views/ditto.ts @@ -1,25 +1,22 @@ import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { getTagSet } from '@/utils/tags.ts'; +import { renderAdminAccount, renderAdminAccountFromPubkey } from '@/views/mastodon/admin-accounts.ts'; +/** Renders an Admin::Account entity from a name request event. */ export async function renderNameRequest(event: DittoEvent) { const n = getTagSet(event.info?.tags ?? [], 'n'); + const [username, domain] = event.tags.find(([name]) => name === 'r')?.[1]?.split('@') ?? []; - let approvalStatus = 'pending'; - - if (n.has('approved')) { - approvalStatus = 'approved'; - } - if (n.has('rejected')) { - approvalStatus = 'rejected'; - } + const adminAccount = event.author + ? await renderAdminAccount(event.author) + : await renderAdminAccountFromPubkey(event.pubkey); return { + ...adminAccount, id: event.id, - account: event.author ? await renderAccount(event.author) : accountFromPubkey(event.pubkey), - name: event.tags.find(([name]) => name === 'r')?.[1] || '', - reason: event.content, - approval_status: approvalStatus, - created_at: new Date(event.created_at * 1000).toISOString(), + approved: n.has('approved'), + username, + domain, + invite_request: event.content, }; } diff --git a/src/views/mastodon/admin-accounts.ts b/src/views/mastodon/admin-accounts.ts index 4dc85699..34b6860a 100644 --- a/src/views/mastodon/admin-accounts.ts +++ b/src/views/mastodon/admin-accounts.ts @@ -1,9 +1,20 @@ import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; -import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; +import { DittoEvent } from '@/interfaces/DittoEvent.ts'; +import { getTagSet } from '@/utils/tags.ts'; /** Expects a kind 0 fully hydrated */ async function renderAdminAccount(event: DittoEvent) { const account = await renderAccount(event); + const names = getTagSet(event.user?.tags ?? [], 'n'); + + let role = 'user'; + + if (names.has('admin')) { + role = 'admin'; + } + if (names.has('moderator')) { + role = 'moderator'; + } return { id: account.id, @@ -15,12 +26,13 @@ async function renderAdminAccount(event: DittoEvent) { ips: [], locale: '', invite_request: null, - role: event.tags.find(([name]) => name === 'role')?.[1], + role, confirmed: true, approved: true, - disabled: false, - silenced: false, - suspended: false, + disabled: names.has('disabled'), + silenced: names.has('silenced'), + suspended: names.has('suspended'), + sensitized: names.has('sensitized'), account, }; } From 2245263011a70b8f7a0ce1e5560834d1607cc6e9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Jun 2024 14:50:37 -0500 Subject: [PATCH 30/77] Add ditto:name_grant notification --- src/app.ts | 23 +++--- src/controllers/api/admin.ts | 61 ++++++++++++++- src/controllers/api/ditto.ts | 108 +-------------------------- src/controllers/api/notifications.ts | 80 +++++++++++++++++--- src/views/mastodon/notifications.ts | 19 +++++ 5 files changed, 164 insertions(+), 127 deletions(-) diff --git a/src/app.ts b/src/app.ts index 7bf29caa..074359dc 100644 --- a/src/app.ts +++ b/src/app.ts @@ -26,14 +26,16 @@ import { updateCredentialsController, verifyCredentialsController, } from '@/controllers/api/accounts.ts'; -import { adminAccountsController, adminActionController } from '@/controllers/api/admin.ts'; +import { + adminAccountsController, + adminActionController, + adminApproveController, + adminRejectController, +} from '@/controllers/api/admin.ts'; import { appCredentialsController, createAppController } from '@/controllers/api/apps.ts'; import { blocksController } from '@/controllers/api/blocks.ts'; import { bookmarksController } from '@/controllers/api/bookmarks.ts'; import { - adminNameApproveController, - adminNameRejectController, - adminNameRequestsController, adminRelaysController, adminSetRelaysController, nameRequestController, @@ -244,7 +246,6 @@ app.get('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/reactions/:emoji', reactions app.put('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/reactions/:emoji', requireSigner, reactionController); app.delete('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/reactions/:emoji', requireSigner, deleteReactionController); -app.get('/api/v1/admin/accounts', requireRole('admin'), adminAccountsController); app.get('/api/v1/pleroma/admin/config', requireRole('admin'), configController); app.post('/api/v1/pleroma/admin/config', requireRole('admin'), updateConfigController); app.delete('/api/v1/pleroma/admin/statuses/:id', requireRole('admin'), pleromaAdminDeleteStatusController); @@ -255,10 +256,6 @@ app.put('/api/v1/admin/ditto/relays', requireRole('admin'), adminSetRelaysContro app.post('/api/v1/ditto/names', requireSigner, nameRequestController); app.get('/api/v1/ditto/names', requireSigner, nameRequestsController); -app.get('/api/v1/admin/ditto/names', requireRole('admin'), adminNameRequestsController); -app.post('/api/v1/admin/ditto/names/:id{[0-9a-f]{64}}/approve', requireRole('admin'), adminNameApproveController); -app.post('/api/v1/admin/ditto/names/:id{[0-9a-f]{64}}/reject', requireRole('admin'), adminNameRejectController); - app.post('/api/v1/ditto/zap', requireSigner, zapController); app.post('/api/v1/reports', requireSigner, reportController); @@ -277,7 +274,15 @@ app.post( adminReportReopenController, ); +app.get('/api/v1/admin/accounts', requireRole('admin'), adminAccountsController); app.post('/api/v1/admin/accounts/:id{[0-9a-f]{64}}/action', requireSigner, requireRole('admin'), adminActionController); +app.post( + '/api/v1/admin/accounts/:id{[0-9a-f]{64}}/approve', + requireSigner, + requireRole('admin'), + adminApproveController, +); +app.post('/api/v1/admin/accounts/:id{[0-9a-f]{64}}/reject', requireSigner, requireRole('admin'), adminRejectController); app.put('/api/v1/pleroma/admin/users/tag', requireRole('admin'), pleromaAdminTagController); app.delete('/api/v1/pleroma/admin/users/tag', requireRole('admin'), pleromaAdminUntagController); diff --git a/src/controllers/api/admin.ts b/src/controllers/api/admin.ts index df5bf968..90afd52f 100644 --- a/src/controllers/api/admin.ts +++ b/src/controllers/api/admin.ts @@ -6,7 +6,7 @@ import { Conf } from '@/config.ts'; import { booleanParamSchema } from '@/schema.ts'; import { Storages } from '@/storages.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { paginated, paginationSchema, parseBody, updateUser } from '@/utils/api.ts'; +import { createAdminEvent, paginated, paginationSchema, parseBody, updateEventInfo, updateUser } from '@/utils/api.ts'; import { renderNameRequest } from '@/views/ditto.ts'; import { renderAdminAccount, renderAdminAccountFromPubkey } from '@/views/mastodon/admin-accounts.ts'; @@ -47,7 +47,7 @@ const adminAccountsController: AppController = async (c) => { } const orig = await store.query( - [{ kinds: [30383], authors: [Conf.pubkey], '#k': ['3036'], ...params }], + [{ kinds: [30383], authors: [Conf.pubkey], '#k': ['3036'], '#n': ['pending'], ...params }], { signal }, ); @@ -143,4 +143,59 @@ const adminActionController: AppController = async (c) => { return c.json({}, 200); }; -export { adminAccountsController, adminActionController }; +const adminApproveController: AppController = async (c) => { + const eventId = c.req.param('id'); + const store = await Storages.db(); + + const [event] = await store.query([{ kinds: [3036], ids: [eventId] }]); + if (!event) { + return c.json({ error: 'Event not found' }, 404); + } + + const r = event.tags.find(([name]) => name === 'r')?.[1]; + if (!r) { + return c.json({ error: 'NIP-05 not found' }, 404); + } + if (!z.string().email().safeParse(r).success) { + return c.json({ error: 'Invalid NIP-05' }, 400); + } + + const [existing] = await store.query([{ kinds: [30360], authors: [Conf.pubkey], '#d': [r], limit: 1 }]); + if (existing) { + return c.json({ error: 'NIP-05 already granted to another user' }, 400); + } + + await createAdminEvent({ + kind: 30360, + tags: [ + ['d', r], + ['L', 'nip05.domain'], + ['l', r.split('@')[1], 'nip05.domain'], + ['p', event.pubkey], + ['e', event.id], + ], + }, c); + + await updateEventInfo(eventId, { pending: false, approved: true, rejected: false }, c); + await hydrateEvents({ events: [event], store }); + + const nameRequest = await renderNameRequest(event); + return c.json(nameRequest); +}; + +const adminRejectController: AppController = async (c) => { + const eventId = c.req.param('id'); + const store = await Storages.db(); + + const [event] = await store.query([{ kinds: [3036], ids: [eventId] }]); + if (!event) { + return c.json({ error: 'Event not found' }, 404); + } + + await updateEventInfo(eventId, { pending: false, approved: false, rejected: true }, c); + await hydrateEvents({ events: [event], store }); + + const nameRequest = await renderNameRequest(event); + return c.json(nameRequest); +}; +export { adminAccountsController, adminActionController, adminApproveController, adminRejectController }; diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index 5c8ea9c4..5723a9e8 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -1,4 +1,4 @@ -import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; +import { NostrEvent, NostrFilter } from '@nostrify/nostrify'; import { z } from 'zod'; import { AppController } from '@/app.ts'; @@ -7,7 +7,7 @@ import { booleanParamSchema } from '@/schema.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Storages } from '@/storages.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { createAdminEvent, createEvent, paginated, paginationSchema, updateEventInfo } from '@/utils/api.ts'; +import { createEvent, paginated, paginationSchema } from '@/utils/api.ts'; import { renderNameRequest } from '@/views/ditto.ts'; const markerSchema = z.enum(['read', 'write']); @@ -135,107 +135,3 @@ export const nameRequestsController: AppController = async (c) => { return paginated(c, orig, nameRequests); }; - -const adminNameRequestsSchema = z.object({ - account_id: n.id().optional(), - approved: booleanParamSchema.optional(), - rejected: booleanParamSchema.optional(), -}); - -export const adminNameRequestsController: AppController = async (c) => { - const store = await Storages.db(); - const params = paginationSchema.parse(c.req.query()); - const { account_id, approved, rejected } = adminNameRequestsSchema.parse(c.req.query()); - - const filter: NostrFilter = { - kinds: [30383], - authors: [Conf.pubkey], - '#k': ['3036'], - ...params, - }; - - if (account_id) { - filter['#p'] = [account_id]; - } - if (approved) { - filter['#n'] = ['approved']; - } - if (rejected) { - filter['#n'] = ['rejected']; - } - - const orig = await store.query([filter]); - const ids = new Set(); - - for (const event of orig) { - const d = event.tags.find(([name]) => name === 'd')?.[1]; - if (d) { - ids.add(d); - } - } - - const events = await store.query([{ kinds: [3036], ids: [...ids] }]) - .then((events) => hydrateEvents({ store, events: events, signal: c.req.raw.signal })); - - const nameRequests = await Promise.all( - events.map((event) => renderNameRequest(event)), - ); - - return paginated(c, orig, nameRequests); -}; - -export const adminNameApproveController: AppController = async (c) => { - const eventId = c.req.param('id'); - const store = await Storages.db(); - - const [event] = await store.query([{ kinds: [3036], ids: [eventId] }]); - if (!event) { - return c.json({ error: 'Event not found' }, 404); - } - - const r = event.tags.find(([name]) => name === 'r')?.[1]; - if (!r) { - return c.json({ error: 'NIP-05 not found' }, 404); - } - if (!z.string().email().safeParse(r).success) { - return c.json({ error: 'Invalid NIP-05' }, 400); - } - - const [existing] = await store.query([{ kinds: [30360], authors: [Conf.pubkey], '#d': [r], limit: 1 }]); - if (existing) { - return c.json({ error: 'NIP-05 already granted to another user' }, 400); - } - - await createAdminEvent({ - kind: 30360, - tags: [ - ['d', r], - ['L', 'nip05.domain'], - ['l', r.split('@')[1], 'nip05.domain'], - ['p', event.pubkey], - ['e', event.id], - ], - }, c); - - await updateEventInfo(eventId, { pending: false, approved: true, rejected: false }, c); - await hydrateEvents({ events: [event], store }); - - const nameRequest = await renderNameRequest(event); - return c.json(nameRequest); -}; - -export const adminNameRejectController: AppController = async (c) => { - const eventId = c.req.param('id'); - const store = await Storages.db(); - - const [event] = await store.query([{ kinds: [3036], ids: [eventId] }]); - if (!event) { - return c.json({ error: 'Event not found' }, 404); - } - - await updateEventInfo(eventId, { pending: false, approved: false, rejected: true }, c); - await hydrateEvents({ events: [event], store }); - - const nameRequest = await renderNameRequest(event); - return c.json(nameRequest); -}; diff --git a/src/controllers/api/notifications.ts b/src/controllers/api/notifications.ts index ba15bd02..d92ccf4a 100644 --- a/src/controllers/api/notifications.ts +++ b/src/controllers/api/notifications.ts @@ -1,24 +1,87 @@ -import { NostrFilter } from '@nostrify/nostrify'; +import { NostrFilter, NSchema as n } from '@nostrify/nostrify'; +import { z } from 'zod'; import { AppContext, AppController } from '@/app.ts'; +import { Conf } from '@/config.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { paginated, paginationSchema } from '@/utils/api.ts'; +import { paginated, PaginationParams, paginationSchema } from '@/utils/api.ts'; import { renderNotification } from '@/views/mastodon/notifications.ts'; +/** Set of known notification types across backends. */ +const notificationTypes = new Set([ + 'mention', + 'status', + 'reblog', + 'follow', + 'follow_request', + 'favourite', + 'poll', + 'update', + 'admin.sign_up', + 'admin.report', + 'severed_relationships', + 'pleroma:emoji_reaction', + 'ditto:name_grant', +]); + +const notificationsSchema = z.object({ + account_id: n.id().optional(), +}); + const notificationsController: AppController = async (c) => { const pubkey = await c.get('signer')?.getPublicKey()!; - const { since, until } = paginationSchema.parse(c.req.query()); + const params = paginationSchema.parse(c.req.query()); - return renderNotifications(c, [{ kinds: [1, 6, 7], '#p': [pubkey], since, until }]); + const types = notificationTypes + .intersection(new Set(c.req.queries('types[]') ?? notificationTypes)) + .difference(new Set(c.req.queries('exclude_types[]'))); + + const { account_id } = notificationsSchema.parse(c.req.query()); + + const kinds = new Set(); + + if (types.has('mention')) { + kinds.add(1); + } + if (types.has('reblog')) { + kinds.add(6); + } + if (types.has('favourite') || types.has('pleroma:emoji_reaction')) { + kinds.add(7); + } + + const filter: NostrFilter = { + kinds: [...kinds], + '#p': [pubkey], + ...params, + }; + + const filters: NostrFilter[] = [filter]; + + if (account_id) { + filter.authors = [account_id]; + } + + if (types.has('ditto:name_grant') && !account_id) { + filters.push({ kinds: [30360], authors: [Conf.pubkey], '#p': [pubkey], ...params }); + } + + return renderNotifications(filters, types, params, c); }; -async function renderNotifications(c: AppContext, filters: NostrFilter[]) { +async function renderNotifications( + filters: NostrFilter[], + types: Set, + params: PaginationParams, + c: AppContext, +) { const store = c.get('store'); const pubkey = await c.get('signer')?.getPublicKey()!; const { signal } = c.req.raw; + const opts = { signal, limit: params.limit }; const events = await store - .query(filters, { signal }) + .query(filters, opts) .then((events) => events.filter((event) => event.pubkey !== pubkey)) .then((events) => hydrateEvents({ events, store, signal })); @@ -26,9 +89,8 @@ async function renderNotifications(c: AppContext, filters: NostrFilter[]) { return c.json([]); } - const notifications = (await Promise - .all(events.map((event) => renderNotification(event, { viewerPubkey: pubkey })))) - .filter(Boolean); + const notifications = (await Promise.all(events.map((event) => renderNotification(event, { viewerPubkey: pubkey })))) + .filter((notification) => notification && types.has(notification.type)); if (!notifications.length) { return c.json([]); diff --git a/src/views/mastodon/notifications.ts b/src/views/mastodon/notifications.ts index 5b618d79..e11d45ac 100644 --- a/src/views/mastodon/notifications.ts +++ b/src/views/mastodon/notifications.ts @@ -26,6 +26,10 @@ function renderNotification(event: DittoEvent, opts: RenderNotificationOpts) { if (event.kind === 7) { return renderReaction(event, opts); } + + if (event.kind === 30360) { + return renderNameGrant(event); + } } async function renderMention(event: DittoEvent, opts: RenderNotificationOpts) { @@ -87,6 +91,21 @@ async function renderReaction(event: DittoEvent, opts: RenderNotificationOpts) { }; } +async function renderNameGrant(event: DittoEvent) { + const d = event.tags.find(([name]) => name === 'd')?.[1]; + const account = event.author ? await renderAccount(event.author) : await accountFromPubkey(event.pubkey); + + if (!d) return; + + return { + id: notificationId(event), + type: 'ditto:name_grant', + name: d, + created_at: nostrDate(event.created_at).toISOString(), + account, + }; +} + /** This helps notifications be sorted in the correct order. */ function notificationId({ id, created_at }: NostrEvent): string { return `${created_at}-${id}`; From 42fac52e9ef80f423852ce92163514eae6046b2b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Jun 2024 15:31:14 -0500 Subject: [PATCH 31/77] Support streaming notifications --- src/controllers/api/streaming.ts | 66 ++++++++++++++++------------- src/views/mastodon/notifications.ts | 12 +++--- 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index 427c350e..552ea3bd 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -1,4 +1,4 @@ -import { NostrFilter } from '@nostrify/nostrify'; +import { NostrEvent, NostrFilter } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; import { z } from 'zod'; @@ -11,6 +11,7 @@ import { hydrateEvents } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; import { bech32ToPubkey } from '@/utils.ts'; import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts'; +import { renderNotification } from '@/views/mastodon/notifications.ts'; const debug = Debug('ditto:streaming'); @@ -52,6 +53,11 @@ const streamingController: AppController = async (c) => { const { socket, response } = Deno.upgradeWebSocket(c.req.raw, { protocol: token, idleTimeout: 30 }); + const store = await Storages.db(); + const pubsub = await Storages.pubsub(); + + const policy = pubkey ? new MuteListPolicy(pubkey, await Storages.admin()) : undefined; + function send(name: string, payload: object) { if (socket.readyState === WebSocket.OPEN) { debug('send', name, JSON.stringify(payload)); @@ -63,52 +69,54 @@ const streamingController: AppController = async (c) => { } } - socket.onopen = async () => { - if (!stream) return; - - const filter = await topicToFilter(stream, c.req.query(), pubkey); - if (!filter) return; - + async function sub(type: string, filters: NostrFilter[], render: (event: NostrEvent) => Promise) { try { - const db = await Storages.db(); - const pubsub = await Storages.pubsub(); - - for await (const msg of pubsub.req([filter], { signal: controller.signal })) { + for await (const msg of pubsub.req(filters, { signal: controller.signal })) { if (msg[0] === 'EVENT') { const event = msg[2]; - if (pubkey) { - const policy = new MuteListPolicy(pubkey, await Storages.admin()); + if (policy) { const [, , ok] = await policy.call(event); if (!ok) { continue; } } - await hydrateEvents({ - events: [event], - store: db, - signal: AbortSignal.timeout(1000), - }); + await hydrateEvents({ events: [event], store, signal: AbortSignal.timeout(1000) }); - if (event.kind === 1) { - const status = await renderStatus(event, { viewerPubkey: pubkey }); - if (status) { - send('update', status); - } - } + const result = await render(event); - if (event.kind === 6) { - const status = await renderReblog(event, { viewerPubkey: pubkey }); - if (status) { - send('update', status); - } + if (result) { + send(type, result); } } } } catch (e) { debug('streaming error:', e); } + } + + socket.onopen = async () => { + if (!stream) return; + const topicFilter = await topicToFilter(stream, c.req.query(), pubkey); + + if (topicFilter) { + sub('update', [topicFilter], async (event) => { + if (event.kind === 1) { + return await renderStatus(event, { viewerPubkey: pubkey }); + } + if (event.kind === 6) { + return await renderReblog(event, { viewerPubkey: pubkey }); + } + }); + } + + if (['user', 'user:notification'].includes(stream) && pubkey) { + sub('notification', [{ '#p': [pubkey] }], async (event) => { + return await renderNotification(event, { viewerPubkey: pubkey }); + }); + return; + } }; socket.onclose = () => { diff --git a/src/views/mastodon/notifications.ts b/src/views/mastodon/notifications.ts index e11d45ac..8f2a8a62 100644 --- a/src/views/mastodon/notifications.ts +++ b/src/views/mastodon/notifications.ts @@ -1,8 +1,10 @@ +import { NostrEvent } from '@nostrify/nostrify'; + +import { Conf } from '@/config.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { nostrDate } from '@/utils.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; -import { NostrEvent } from '@nostrify/nostrify'; interface RenderNotificationOpts { viewerPubkey: string; @@ -27,7 +29,7 @@ function renderNotification(event: DittoEvent, opts: RenderNotificationOpts) { return renderReaction(event, opts); } - if (event.kind === 30360) { + if (event.kind === 30360 && event.pubkey === Conf.pubkey) { return renderNameGrant(event); } } @@ -49,7 +51,7 @@ async function renderReblog(event: DittoEvent, opts: RenderNotificationOpts) { if (event.repost?.kind !== 1) return; const status = await renderStatus(event.repost, opts); if (!status) return; - const account = event.author ? await renderAccount(event.author) : accountFromPubkey(event.pubkey); + const account = event.author ? await renderAccount(event.author) : await accountFromPubkey(event.pubkey); return { id: notificationId(event), @@ -64,7 +66,7 @@ async function renderFavourite(event: DittoEvent, opts: RenderNotificationOpts) if (event.reacted?.kind !== 1) return; const status = await renderStatus(event.reacted, opts); if (!status) return; - const account = event.author ? await renderAccount(event.author) : accountFromPubkey(event.pubkey); + const account = event.author ? await renderAccount(event.author) : await accountFromPubkey(event.pubkey); return { id: notificationId(event), @@ -79,7 +81,7 @@ async function renderReaction(event: DittoEvent, opts: RenderNotificationOpts) { if (event.reacted?.kind !== 1) return; const status = await renderStatus(event.reacted, opts); if (!status) return; - const account = event.author ? await renderAccount(event.author) : accountFromPubkey(event.pubkey); + const account = event.author ? await renderAccount(event.author) : await accountFromPubkey(event.pubkey); return { id: notificationId(event), From 229975a752554c95c5ed04a824aa75db7cb7ba48 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Jun 2024 18:50:45 -0500 Subject: [PATCH 32/77] adminActionController: delete user's events on suspend --- src/controllers/api/admin.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/controllers/api/admin.ts b/src/controllers/api/admin.ts index 90afd52f..f61b389a 100644 --- a/src/controllers/api/admin.ts +++ b/src/controllers/api/admin.ts @@ -114,6 +114,7 @@ const adminAccountActionSchema = z.object({ const adminActionController: AppController = async (c) => { const body = await parseBody(c.req.raw); + const store = await Storages.db(); const result = adminAccountActionSchema.safeParse(body); const authorId = c.req.param('id'); @@ -136,6 +137,7 @@ const adminActionController: AppController = async (c) => { } if (data.type === 'suspend') { n.suspended = true; + store.remove([{ authors: [authorId] }]).catch(console.warn); } await updateUser(authorId, n, c); From d1ba797c93f080545713d03f3b9e37bffc3be9d5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Jun 2024 19:22:11 -0500 Subject: [PATCH 33/77] Add revoke_name admin action --- src/controllers/api/admin.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/controllers/api/admin.ts b/src/controllers/api/admin.ts index f61b389a..bcc40ce8 100644 --- a/src/controllers/api/admin.ts +++ b/src/controllers/api/admin.ts @@ -109,7 +109,7 @@ const adminAccountsController: AppController = async (c) => { }; const adminAccountActionSchema = z.object({ - type: z.enum(['none', 'sensitive', 'disable', 'silence', 'suspend']), + type: z.enum(['none', 'sensitive', 'disable', 'silence', 'suspend', 'revoke_name']), }); const adminActionController: AppController = async (c) => { @@ -139,6 +139,10 @@ const adminActionController: AppController = async (c) => { n.suspended = true; store.remove([{ authors: [authorId] }]).catch(console.warn); } + if (data.type === 'revoke_name') { + n.revoke_name = true; + store.remove([{ kinds: [30360], authors: [Conf.pubkey], '#p': [authorId] }]).catch(console.warn); + } await updateUser(authorId, n, c); From e7ed3c839c5f9256e23886429ff24740f35b19c9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Jun 2024 19:48:33 -0500 Subject: [PATCH 34/77] AdminStore: fix users check --- src/storages/AdminStore.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/storages/AdminStore.ts b/src/storages/AdminStore.ts index 8fe75765..014dcb76 100644 --- a/src/storages/AdminStore.ts +++ b/src/storages/AdminStore.ts @@ -14,12 +14,13 @@ export class AdminStore implements NStore { async query(filters: NostrFilter[], opts: { signal?: AbortSignal; limit?: number } = {}): Promise { const events = await this.store.query(filters, opts); + const pubkeys = new Set(events.map((event) => event.pubkey)); const users = await this.store.query([{ kinds: [30382], authors: [Conf.pubkey], - '#d': events.map((event) => event.pubkey), - limit: 1, + '#d': [...pubkeys], + limit: pubkeys.size, }]); return events.filter((event) => { From 6b3e01a072a9b3194e7e076fdcea4134ac13a607 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 9 Jun 2024 21:42:31 -0500 Subject: [PATCH 35/77] Upgrade nostrify to v0.23.1 --- deno.json | 2 +- deno.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deno.json b/deno.json index b225c8e7..ecf32a34 100644 --- a/deno.json +++ b/deno.json @@ -22,7 +22,7 @@ "@db/sqlite": "jsr:@db/sqlite@^0.11.1", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.23.0", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.23.1", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", diff --git a/deno.lock b/deno.lock index d68a374e..7d767abb 100644 --- a/deno.lock +++ b/deno.lock @@ -10,7 +10,7 @@ "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", - "jsr:@nostrify/nostrify@^0.23.0": "jsr:@nostrify/nostrify@0.23.0", + "jsr:@nostrify/nostrify@^0.23.1": "jsr:@nostrify/nostrify@0.23.1", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0": "jsr:@soapbox/kysely-deno-sqlite@2.2.0", "jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0", "jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0", @@ -121,8 +121,8 @@ "npm:zod@^3.23.8" ] }, - "@nostrify/nostrify@0.23.0": { - "integrity": "8636c0322885707d6a7b342ef55f70debf399a1eb65b83abcce7972d69e30920", + "@nostrify/nostrify@0.23.1": { + "integrity": "7a242dedfe33cf38131696ad96d789d54257cfbfd5b5e63748fe5d53c057d99a", "dependencies": [ "jsr:@std/encoding@^0.224.1", "npm:@scure/base@^1.1.6", @@ -1343,7 +1343,7 @@ "dependencies": [ "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "jsr:@db/sqlite@^0.11.1", - "jsr:@nostrify/nostrify@^0.23.0", + "jsr:@nostrify/nostrify@^0.23.1", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", From 1f4de9aed0428775a8956178de2c48fb8cb139f9 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sat, 8 Jun 2024 08:57:16 -0300 Subject: [PATCH 36/77] feat: add migration for 'zaps_amount' column --- src/db/migrations/025_event_stats_add_zap_count.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 src/db/migrations/025_event_stats_add_zap_count.ts diff --git a/src/db/migrations/025_event_stats_add_zap_count.ts b/src/db/migrations/025_event_stats_add_zap_count.ts new file mode 100644 index 00000000..91479907 --- /dev/null +++ b/src/db/migrations/025_event_stats_add_zap_count.ts @@ -0,0 +1,12 @@ +import { Kysely } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .alterTable('event_stats') + .addColumn('zaps_amount', 'integer', (col) => col.notNull().defaultTo(0)) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.alterTable('event_stats').dropColumn('zaps_amount').execute(); +} From d2608256600d12a2b5f8c60a3524435b943d11e3 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sat, 8 Jun 2024 09:08:14 -0300 Subject: [PATCH 37/77] feat: add 'zaps_amount' to EventStatsRow as number --- src/db/DittoTables.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index d9e320d8..aed8c8c2 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -23,6 +23,7 @@ interface EventStatsRow { reactions_count: number; quotes_count: number; reactions: string; + zaps_amount: number; } interface EventRow { From 4b58fb9bf2ee4807640d3731fd98e40751dbf723 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 10 Jun 2024 09:33:24 -0300 Subject: [PATCH 38/77] feat(updateStats): handle kind 9735 --- src/utils/stats.ts | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/utils/stats.ts b/src/utils/stats.ts index 3cc82cfd..8f0b5501 100644 --- a/src/utils/stats.ts +++ b/src/utils/stats.ts @@ -1,7 +1,7 @@ import { NostrEvent, NStore } from '@nostrify/nostrify'; import { Kysely, UpdateObject } from 'kysely'; -import { SetRequired } from 'type-fest'; +import { SetRequired } from 'type-fest'; import { DittoTables } from '@/db/DittoTables.ts'; import { findQuoteTag, findReplyTag, getTagSet } from '@/utils/tags.ts'; import { Conf } from '@/config.ts'; @@ -27,6 +27,8 @@ export async function updateStats({ event, kysely, store, x = 1 }: UpdateStatsOp return handleEvent6(kysely, event, x); case 7: return handleEvent7(kysely, event, x); + case 9735: + return handleEvent9735(kysely, event); } } @@ -132,6 +134,28 @@ async function handleEvent7(kysely: Kysely, event: NostrEvent, x: n } } +/** Update stats for kind 9735 event. */ +async function handleEvent9735(kysely: Kysely, event: NostrEvent): Promise { + // https://github.com/nostr-protocol/nips/blob/master/57.md#appendix-f-validating-zap-receipts + const id = event.tags.find(([name]) => name === 'e')?.[1]; + if (!id) return; + + let amount = '0'; + try { + const zapRequest = JSON.parse(event.tags.find(([name]) => name === 'description')?.[1]!) as NostrEvent; + amount = zapRequest.tags.find(([name]) => name === 'amount')?.[1]!; + } catch { + return; + } + if (amount === '0' || !amount || (/^\d+$/).test(amount) === false) return; + + await updateEventStats( + kysely, + id, + ({ zaps_amount }) => ({ zaps_amount: Math.max(0, zaps_amount + Number(amount)) }), + ); +} + /** Get the pubkeys that were added and removed from a follow event. */ export function getFollowDiff( tags: string[][], @@ -219,6 +243,7 @@ export async function updateEventStats( reposts_count: 0, reactions_count: 0, quotes_count: 0, + zaps_amount: 0, reactions: '{}', }; From 18648f7be37154e065ef7d5f34f6577cf241bf1a Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 10 Jun 2024 10:00:58 -0300 Subject: [PATCH 39/77] fix(hydrate): return zaps_amount in gatherEventStats --- src/storages/hydrate.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 9ec9e8c1..3c264320 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -330,6 +330,7 @@ async function gatherEventStats(events: DittoEvent[]): Promise Date: Mon, 10 Jun 2024 10:38:56 -0300 Subject: [PATCH 40/77] fix: add zaps_amount to EventStats --- src/interfaces/DittoEvent.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/interfaces/DittoEvent.ts b/src/interfaces/DittoEvent.ts index 85e11d65..2f2aef26 100644 --- a/src/interfaces/DittoEvent.ts +++ b/src/interfaces/DittoEvent.ts @@ -13,6 +13,7 @@ export interface EventStats { reposts_count: number; quotes_count: number; reactions: Record; + zaps_amount: number; } /** Internal Event representation used by Ditto, including extra keys. */ From 0b49ee4fa691db90dfe1229f499d81d9957ab844 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 10 Jun 2024 10:39:34 -0300 Subject: [PATCH 41/77] feat(renderStatus): return zaps_amount --- src/views/mastodon/statuses.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index ed14c8e3..a0874b3a 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -104,6 +104,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< replies_count: event.event_stats?.replies_count ?? 0, reblogs_count: event.event_stats?.reposts_count ?? 0, favourites_count: event.event_stats?.reactions['+'] ?? 0, + zaps_amount: event.event_stats?.zaps_amount ?? 0, favourited: reactionEvent?.content === '+', reblogged: Boolean(repostEvent), muted: false, From 899e7672dc9ca4ddeb8549d28df57ca128181988 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 10 Jun 2024 20:04:39 -0500 Subject: [PATCH 42/77] Disable Hono strict mode --- src/app.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/app.ts b/src/app.ts index 074359dc..15d4fc5c 100644 --- a/src/app.ts +++ b/src/app.ts @@ -129,7 +129,7 @@ type AppContext = Context; type AppMiddleware = MiddlewareHandler; type AppController = Handler>; -const app = new Hono(); +const app = new Hono({ strict: false }); const debug = Debug('ditto:http'); @@ -141,14 +141,12 @@ if (Conf.cronEnabled) { } app.use('/api/*', logger(debug)); -app.use('/relay/*', logger(debug)); app.use('/.well-known/*', logger(debug)); app.use('/users/*', logger(debug)); app.use('/nodeinfo/*', logger(debug)); app.use('/oauth/*', logger(debug)); app.get('/api/v1/streaming', streamingController); -app.get('/api/v1/streaming/', streamingController); app.get('/relay', relayController); app.use( @@ -297,6 +295,9 @@ app.get('/api/v1/conversations', emptyArrayController); app.get('/api/v1/lists', emptyArrayController); app.use('/api/*', notImplementedController); +app.use('/.well-known/*', notImplementedController); +app.use('/nodeinfo/*', notImplementedController); +app.use('/oauth/*', notImplementedController); const publicFiles = serveStatic({ root: './public/' }); const staticFiles = serveStatic({ root: './static/' }); From ddae4f408a9f4bf36bd1db94348162ce1106ebcf Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 11 Jun 2024 13:40:09 -0500 Subject: [PATCH 43/77] Support OAuth OOB --- src/controllers/api/oauth.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/controllers/api/oauth.ts b/src/controllers/api/oauth.ts index 5837a5ef..18e107bf 100644 --- a/src/controllers/api/oauth.ts +++ b/src/controllers/api/oauth.ts @@ -209,6 +209,10 @@ const oauthAuthorizeController: AppController = async (c) => { relays: bunker.searchParams.getAll('relay'), }); + if (redirectUri === 'urn:ietf:wg:oauth:2.0:oob') { + return c.text(token); + } + const url = addCodeToRedirectUri(redirectUri, token); return c.redirect(url); From b8546ae447e91f04f413dd1521abc0c942d90c99 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 11 Jun 2024 18:15:40 -0500 Subject: [PATCH 44/77] Include kind 3 in trending pubkeys again --- src/cron.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cron.ts b/src/cron.ts index aa0a8d8c..166e42ee 100644 --- a/src/cron.ts +++ b/src/cron.ts @@ -59,7 +59,7 @@ export function cron() { Deno.cron( 'update trending pubkeys', '0 * * * *', - () => updateTrendingTags('#p', 'p', [1, 6, 7, 9735], 40, Conf.relay), + () => updateTrendingTags('#p', 'p', [1, 3, 6, 7, 9735], 40, Conf.relay), ); Deno.cron( 'update trending zapped events', From 1151f0c28b1aa477381bd1b33ed0928cbe4fc3e1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 11 Jun 2024 18:56:11 -0500 Subject: [PATCH 45/77] EventsDB: strip nip27 mentions from search index --- src/storages/EventsDB.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index c26ebf12..98c32d54 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -3,6 +3,7 @@ import { NDatabase, NIP50, NKinds, NostrEvent, NostrFilter, NSchema as n, NStore } from '@nostrify/nostrify'; import { Stickynotes } from '@soapbox/stickynotes'; import { Kysely } from 'kysely'; +import { nip27 } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { DittoTables } from '@/db/DittoTables.ts'; @@ -220,7 +221,7 @@ class EventsDB implements NStore { case 0: return EventsDB.buildUserSearchContent(event); case 1: - return event.content; + return nip27.replaceAll(event.content, () => ''); case 30009: return EventsDB.buildTagsSearchContent(event.tags.filter(([t]) => t !== 'alt')); case 30360: From c6da216b4e789fe94e2fb502882341a7e1f2bad1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 11 Jun 2024 19:08:24 -0500 Subject: [PATCH 46/77] trends: calculate since the last label date instead of current date --- src/controllers/api/trends.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts index 349d86d5..b98fc1bf 100644 --- a/src/controllers/api/trends.ts +++ b/src/controllers/api/trends.ts @@ -164,9 +164,9 @@ export async function getTrendingTags(store: NStore, tagName: string): Promise name === tagName); - const now = new Date(); - const lastWeek = new Date(now.getTime() - Time.days(7)); - const dates = generateDateRange(lastWeek, now).reverse(); + const labelDate = new Date(label.created_at * 1000); + const lastWeek = new Date(labelDate.getTime() - Time.days(7)); + const dates = generateDateRange(lastWeek, labelDate).reverse(); return Promise.all(tags.map(async ([_, value]) => { const filters = dates.map((date) => ({ From 7dbd40a88b16ff80343eadd201657bdfc4271209 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 12 Jun 2024 12:50:22 -0500 Subject: [PATCH 47/77] Fix issues with adminAccountsController --- src/controllers/api/admin.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/admin.ts b/src/controllers/api/admin.ts index bcc40ce8..003d0cfc 100644 --- a/src/controllers/api/admin.ts +++ b/src/controllers/api/admin.ts @@ -85,7 +85,13 @@ const adminAccountsController: AppController = async (c) => { } const events = await store.query([{ kinds: [30382], authors: [Conf.pubkey], '#n': n, ...params }], { signal }); - const pubkeys = new Set(events.map(({ pubkey }) => pubkey)); + + const pubkeys = new Set( + events + .map(({ tags }) => tags.find(([name]) => name === 'd')?.[1]) + .filter((pubkey): pubkey is string => !!pubkey), + ); + const authors = await store.query([{ kinds: [0], authors: [...pubkeys] }]) .then((events) => hydrateEvents({ store, events, signal })); @@ -100,10 +106,14 @@ const adminAccountsController: AppController = async (c) => { } const filter: NostrFilter = { kinds: [0], ...params }; + if (local) { filter.search = `domain:${Conf.url.host}`; } - const events = await store.query([filter], { signal }); + + const events = await store.query([filter], { signal }) + .then((events) => hydrateEvents({ store, events, signal })); + const accounts = await Promise.all(events.map(renderAdminAccount)); return paginated(c, events, accounts); }; From edddc5384c26dd379ca7d4303ea48a59a998049d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 12 Jun 2024 18:04:04 -0500 Subject: [PATCH 48/77] Support OAuth "state" param --- src/controllers/api/oauth.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/controllers/api/oauth.ts b/src/controllers/api/oauth.ts index 18e107bf..01f80bf1 100644 --- a/src/controllers/api/oauth.ts +++ b/src/controllers/api/oauth.ts @@ -122,6 +122,7 @@ const oauthController: AppController = (c) => { return c.text('Missing `redirect_uri` query param.', 422); } + const state = c.req.query('state'); const redirectUri = maybeDecodeUri(encodedUri); return c.html(` @@ -162,6 +163,7 @@ const oauthController: AppController = (c) => {
+

Sign in with a Nostr bunker app. Please configure the app to use this relay: ${Conf.relay}

@@ -187,6 +189,7 @@ function maybeDecodeUri(uri: string): string { const oauthAuthorizeSchema = z.object({ bunker_uri: z.string().url().refine((v) => v.startsWith('bunker://')), redirect_uri: z.string().url(), + state: z.string().optional(), }); /** Controller the OAuth form is POSTed to. */ @@ -199,7 +202,7 @@ const oauthAuthorizeController: AppController = async (c) => { } // Parsed FormData values. - const { bunker_uri, redirect_uri: redirectUri } = result.data; + const { bunker_uri, redirect_uri: redirectUri, state } = result.data; const bunker = new URL(bunker_uri); @@ -213,17 +216,22 @@ const oauthAuthorizeController: AppController = async (c) => { return c.text(token); } - const url = addCodeToRedirectUri(redirectUri, token); + const url = addCodeToRedirectUri(redirectUri, token, state); return c.redirect(url); }; /** Append the given `code` as a query param to the `redirect_uri`. */ -function addCodeToRedirectUri(redirectUri: string, code: string): string { +function addCodeToRedirectUri(redirectUri: string, code: string, state?: string): string { const url = new URL(redirectUri); const q = new URLSearchParams(); q.set('code', code); + + if (state) { + q.set('state', state); + } + url.search = q.toString(); return url.toString(); From 4e0a2100411b74a41cd6280e0ce5edb36e5f2902 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 12 Jun 2024 19:19:48 -0500 Subject: [PATCH 49/77] Add a setup script to generate the .env file --- deno.json | 3 ++ deno.lock | 18 +++++++++++ scripts/setup.ts | 82 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 scripts/setup.ts diff --git a/deno.json b/deno.json index ecf32a34..a2b8733c 100644 --- a/deno.json +++ b/deno.json @@ -11,6 +11,7 @@ "nsec": "deno run scripts/nsec.ts", "admin:event": "deno run -A scripts/admin-event.ts", "admin:role": "deno run -A scripts/admin-role.ts", + "setup": "deno run -A scripts/setup.ts", "stats:recompute": "deno run -A scripts/stats-recompute.ts", "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip soapbox.zip && rm soapbox.zip" }, @@ -32,6 +33,7 @@ "@std/crypto": "jsr:@std/crypto@^0.224.0", "@std/dotenv": "jsr:@std/dotenv@^0.224.0", "@std/encoding": "jsr:@std/encoding@^0.224.0", + "@std/fs": "jsr:@std/fs@^0.229.3", "@std/json": "jsr:@std/json@^0.223.0", "@std/media-types": "jsr:@std/media-types@^0.224.1", "@std/streams": "jsr:@std/streams@^0.223.0", @@ -53,6 +55,7 @@ "nostr-relaypool": "npm:nostr-relaypool2@0.6.34", "nostr-tools": "npm:nostr-tools@2.5.1", "nostr-wasm": "npm:nostr-wasm@^0.1.0", + "question-deno": "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/mod.ts", "tldts": "npm:tldts@^6.0.14", "tseep": "npm:tseep@^1.2.1", "type-fest": "npm:type-fest@^4.3.0", diff --git a/deno.lock b/deno.lock index 7d767abb..928bf2c9 100644 --- a/deno.lock +++ b/deno.lock @@ -26,6 +26,7 @@ "jsr:@std/encoding@^0.224.1": "jsr:@std/encoding@0.224.3", "jsr:@std/fmt@^0.221.0": "jsr:@std/fmt@0.221.0", "jsr:@std/fs@^0.221.0": "jsr:@std/fs@0.221.0", + "jsr:@std/fs@^0.229.3": "jsr:@std/fs@0.229.3", "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.0", "jsr:@std/io@^0.224": "jsr:@std/io@0.224.1", "jsr:@std/media-types@^0.224.1": "jsr:@std/media-types@0.224.1", @@ -191,6 +192,9 @@ "jsr:@std/path@^0.221.0" ] }, + "@std/fs@0.229.3": { + "integrity": "783bca21f24da92e04c3893c9e79653227ab016c48e96b3078377ebd5222e6eb" + }, "@std/internal@1.0.0": { "integrity": "ac6a6dfebf838582c4b4f61a6907374e27e05bedb6ce276e0f1608fe84e7cd9a" }, @@ -1215,6 +1219,7 @@ "https://deno.land/std@0.214.0/path/windows/separator.ts": "e51c5522140eff4f8402617c5c68a201fdfa3a1a8b28dc23587cff931b665e43", "https://deno.land/std@0.214.0/path/windows/to_file_url.ts": "1cd63fd35ec8d1370feaa4752eccc4cc05ea5362a878be8dc7db733650995484", "https://deno.land/std@0.214.0/path/windows/to_namespaced_path.ts": "4ffa4fb6fae321448d5fe810b3ca741d84df4d7897e61ee29be961a6aac89a4c", + "https://deno.land/std@0.80.0/encoding/utf8.ts": "1b7e77db9a12363c67872f8a208886ca1329f160c1ca9133b13d2ed399688b99", "https://deno.land/x/hono@v3.10.1/adapter/deno/serve-static.ts": "ba10cf6aaf39da942b0d49c3b9877ddba69d41d414c6551d890beb1085f58eea", "https://deno.land/x/hono@v3.10.1/client/client.ts": "ff340f58041203879972dd368b011ed130c66914f789826610869a90603406bf", "https://deno.land/x/hono@v3.10.1/client/index.ts": "3ff4cf246f3543f827a85a2c84d66a025ac350ee927613629bda47e854bfb7ba", @@ -1279,6 +1284,8 @@ "https://deno.land/x/hono@v3.10.1/utils/url.ts": "5fc3307ef3cb2e6f34ec2a03e3d7f2126c6a9f5f0eab677222df3f0e40bd7567", "https://deno.land/x/hono@v3.10.1/validator/index.ts": "6c986e8b91dcf857ecc8164a506ae8eea8665792a4ff7215471df669c632ae7c", "https://deno.land/x/hono@v3.10.1/validator/validator.ts": "afa5e52495e0996fbba61996736fab5c486590d72d376f809e9f9ff4e0c463e9", + "https://deno.land/x/keypress@0.0.7/dep.ts": "feeb0056d332c126343249b79fe86cb0bf3abd03ea4c270cd39575c38d37a911", + "https://deno.land/x/keypress@0.0.7/mod.ts": "1130570c2397118a3a301b1137400a8e55486716cc3557b3bd5e9947b6b9c035", "https://deno.land/x/kysely_deno_postgres@v0.4.0/deps.ts": "7970f66a52a9fa0cef607cb7ef0171212af2ccb83e73ecfa7629aabc28a38793", "https://deno.land/x/kysely_deno_postgres@v0.4.0/mod.ts": "662438fd3909984bb8cbaf3fd44d2121e949d11301baf21d6c3f057ccf9887de", "https://deno.land/x/kysely_deno_postgres@v0.4.0/src/PostgreSQLDriver.ts": "590c2fa248cff38e6e0f623050983039b5fde61e9c7131593d2922fb1f0eb921", @@ -1337,6 +1344,16 @@ "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/main/mod.ts": "662438fd3909984bb8cbaf3fd44d2121e949d11301baf21d6c3f057ccf9887de", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/main/src/PostgreSQLDriver.ts": "590c2fa248cff38e6e0f623050983039b5fde61e9c7131593d2922fb1f0eb921", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/main/src/PostgreSQLDriverDatabaseConnection.ts": "2158de426860bfd4f8e73afff0289bd40a11e273c8d883d4fd6474db01a9c2a7", + "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/KeyCombo.ts": "a370b2dca76faa416d00e45479c8ce344971b5b86b44b4d0b213245c4bd2f8a3", + "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/checkbox.ts": "e337ee7396aaefe6cc8c6349a445542fe7f0760311773369c9012b3fa278d21e", + "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/config.ts": "a94a022c757f63ee7c410e29b97d3bfab1811889fb4483f56395cf376a911d1b", + "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/confirm.ts": "dde395f351e14ebff9dd882c49c1376f0c648a5978d17dfad997f9958df01d1c", + "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/input.ts": "5b7a3e194e6a5b74bb26d5ef1363d78619769772ad01244fd6f95b9c66cc6f0d", + "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/list.ts": "3b96403b043b5b5bc417d8d07b34828961c1f9d2bdbc9f24e6ef40fb9c95438e", + "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/mod.ts": "cb10c598652cf7edf600af17f73bcadcdedf6900d9f5b5647e89ba2ea378b7d5", + "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/password.ts": "3c578bd21e4fd283431aa940357f40fed2e26d3de12ad129a696d7fe38ae744d", + "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/text-util.ts": "37c0437d2030c0b6255f10afec7ccfcb6b195e9a0a011bb7956595142c3d7383", + "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/util.ts": "a8285450db7b56a3e507f478aaad68927ecb1ee545449cb869ccc4aace13fada", "https://unpkg.com/nostr-relaypool2@0.6.34/lib/nostr-relaypool.worker.js": "a336e5c58b1e6946ae8943eb4fef21b810dc2a5a233438cff92b883673e29c96" }, "workspace": { @@ -1351,6 +1368,7 @@ "jsr:@std/crypto@^0.224.0", "jsr:@std/dotenv@^0.224.0", "jsr:@std/encoding@^0.224.0", + "jsr:@std/fs@^0.229.3", "jsr:@std/json@^0.223.0", "jsr:@std/media-types@^0.224.1", "jsr:@std/streams@^0.223.0", diff --git a/scripts/setup.ts b/scripts/setup.ts new file mode 100644 index 00000000..255d8aa0 --- /dev/null +++ b/scripts/setup.ts @@ -0,0 +1,82 @@ +import { exists } from '@std/fs/exists'; +import { generateSecretKey, nip19 } from 'nostr-tools'; +import question from 'question-deno'; + +const vars: Record = {}; + +if (await exists('./.env')) { + const overwrite = await question('confirm', 'Overwrite existing .env file? (this is a destructive action)', false); + if (!overwrite) { + console.log('Aborted'); + Deno.exit(0); + } +} + +console.log('Generating secret key...'); +const sk = generateSecretKey(); +vars.DITTO_NSEC = nip19.nsecEncode(sk); + +const domain = await question('input', 'What is the domain of your instance? (eg ditto.pub)'); +vars.LOCAL_DOMAIN = `https://${domain}`; + +const database = await question('list', 'Which database do you want to use?', ['postgres', 'sqlite']); +if (database === 'sqlite') { + const path = await question('input', 'Path to SQLite database', 'data/db.sqlite3'); + vars.DATABASE_URL = `sqlite://${path}`; +} +if (database === 'postgres') { + const host = await question('input', 'Postgres host', 'localhost'); + const port = await question('input', 'Postgres port', '5432'); + const user = await question('input', 'Postgres user', 'ditto'); + const password = await question('input', 'Postgres password', 'ditto'); + const database = await question('input', 'Postgres database', 'ditto'); + vars.DATABASE_URL = `postgres://${user}:${password}@${host}:${port}/${database}`; +} + +vars.DITTO_UPLOADER = await question('list', 'How do you want to upload files?', [ + 'nostrbuild', + 'blossom', + 's3', + 'ipfs', + 'local', +]); + +if (vars.DITTO_UPLOADER === 'nostrbuild') { + vars.NOSTRBUILD_ENDPOINT = await question('input', 'nostr.build endpoint', 'https://nostr.build/api/v2/upload/files'); +} +if (vars.DITTO_UPLOADER === 'blossom') { + vars.BLOSSOM_SERVERS = await question('input', 'Blossom servers (comma separated)', 'https://blossom.primal.net/'); +} +if (vars.DITTO_UPLOADER === 's3') { + vars.S3_ACCESS_KEY = await question('input', 'S3 access key'); + vars.S3_SECRET_KEY = await question('input', 'S3 secret key'); + vars.S3_ENDPOINT = await question('input', 'S3 endpoint'); + vars.S3_BUCKET = await question('input', 'S3 bucket'); + vars.S3_REGION = await question('input', 'S3 region'); + vars.S3_PATH_STYLE = String(await question('confirm', 'Use path style?', false)); + const mediaDomain = await question('input', 'Media domain', `media.${domain}`); + vars.MEDIA_DOMAIN = `https://${mediaDomain}`; +} +if (vars.DITTO_UPLOADER === 'ipfs') { + vars.IPFS_API_URL = await question('input', 'IPFS API URL', 'http://localhost:5001'); + const mediaDomain = await question('input', 'Media domain', `media.${domain}`); + vars.MEDIA_DOMAIN = `https://${mediaDomain}`; +} +if (vars.DITTO_UPLOADER === 'local') { + vars.UPLOADS_DIR = await question('input', 'Local uploads directory', 'data/uploads'); + const mediaDomain = await question('input', 'Media domain', `media.${domain}`); + vars.MEDIA_DOMAIN = `https://${mediaDomain}`; +} + +console.log('Writing to .env file...'); + +const result = Object.entries(vars).reduce((acc, [key, value]) => { + if (value) { + return `${acc}${key}="${value}"\n`; + } + return acc; +}, ''); + +await Deno.writeTextFile('./.env', result); + +console.log('Done'); From 4285763c9dac0c35197091b070f0403938c009ad Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 12 Jun 2024 19:47:26 -0500 Subject: [PATCH 50/77] Add instance v2 controller --- src/app.ts | 12 +++-- src/controllers/api/instance.ts | 90 +++++++++++++++++++++++++++++++-- 2 files changed, 95 insertions(+), 7 deletions(-) diff --git a/src/app.ts b/src/app.ts index 15d4fc5c..5ab31d84 100644 --- a/src/app.ts +++ b/src/app.ts @@ -6,7 +6,6 @@ import { cors, logger, serveStatic } from 'hono/middleware'; import { Conf } from '@/config.ts'; import { cron } from '@/cron.ts'; import { startFirehose } from '@/firehose.ts'; -import { Time } from '@/utils.ts'; import { accountController, @@ -42,7 +41,11 @@ import { nameRequestsController, } from '@/controllers/api/ditto.ts'; import { emptyArrayController, emptyObjectController, notImplementedController } from '@/controllers/api/fallback.ts'; -import { instanceController } from '@/controllers/api/instance.ts'; +import { + instanceDescriptionController, + instanceV1Controller, + instanceV2Controller, +} from '@/controllers/api/instance.ts'; import { markersController, updateMarkersController } from '@/controllers/api/markers.ts'; import { mediaController } from '@/controllers/api/media.ts'; import { mutesController } from '@/controllers/api/mutes.ts'; @@ -103,7 +106,6 @@ import { indexController } from '@/controllers/site.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; import { nostrController } from '@/controllers/well-known/nostr.ts'; import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts'; -import { cacheMiddleware } from '@/middleware/cacheMiddleware.ts'; import { cspMiddleware } from '@/middleware/cspMiddleware.ts'; import { requireSigner } from '@/middleware/requireSigner.ts'; import { signerMiddleware } from '@/middleware/signerMiddleware.ts'; @@ -164,7 +166,9 @@ app.get('/.well-known/nostr.json', nostrController); app.get('/nodeinfo/:version', nodeInfoSchemaController); -app.get('/api/v1/instance', cacheMiddleware({ cacheName: 'web', expires: Time.minutes(5) }), instanceController); +app.get('/api/v1/instance', instanceV1Controller); +app.get('/api/v2/instance', instanceV2Controller); +app.get('/api/v1/instance/extended_description', instanceDescriptionController); app.get('/api/v1/apps/verify_credentials', appCredentialsController); app.post('/api/v1/apps', createAppController); diff --git a/src/controllers/api/instance.ts b/src/controllers/api/instance.ts index 6287325c..4c3e9812 100644 --- a/src/controllers/api/instance.ts +++ b/src/controllers/api/instance.ts @@ -3,7 +3,9 @@ import { Conf } from '@/config.ts'; import { Storages } from '@/storages.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; -const instanceController: AppController = async (c) => { +const version = '0.0.0 (compatible; Ditto 0.0.1)'; + +const instanceV1Controller: AppController = async (c) => { const { host, protocol } = Conf.url; const meta = await getInstanceMetadata(await Storages.db(), c.req.raw.signal); @@ -54,7 +56,7 @@ const instanceController: AppController = async (c) => { urls: { streaming_api: `${wsProtocol}//${host}`, }, - version: '0.0.0 (compatible; Ditto 0.0.1)', + version, email: meta.email, nostr: { pubkey: Conf.pubkey, @@ -67,4 +69,86 @@ const instanceController: AppController = async (c) => { }); }; -export { instanceController }; +const instanceV2Controller: AppController = async (c) => { + const { host, protocol } = Conf.url; + const meta = await getInstanceMetadata(await Storages.db(), c.req.raw.signal); + + /** Protocol to use for WebSocket URLs, depending on the protocol of the `LOCAL_DOMAIN`. */ + const wsProtocol = protocol === 'http:' ? 'ws:' : 'wss:'; + + return c.json({ + domain: host, + title: meta.name, + version, + source_url: 'https://gitlab.com/soapbox-pub/ditto', + description: meta.about, + usage: { + users: { + active_month: 0, + }, + }, + thumbnail: { + url: meta.picture, + blurhash: '', + versions: { + '@1x': meta.picture, + '@2x': meta.picture, + }, + }, + languages: [ + 'en', + ], + configuration: { + urls: { + streaming: `${wsProtocol}//${host}`, + }, + vapid: { + public_key: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + }, + accounts: { + max_featured_tags: 10, + max_pinned_statuses: 5, + }, + statuses: { + max_characters: Conf.postCharLimit, + max_media_attachments: 4, + characters_reserved_per_url: 23, + }, + media_attachments: { + supported_mime_types: [], + image_size_limit: 16777216, + image_matrix_limit: 33177600, + video_size_limit: 103809024, + video_frame_rate_limit: 120, + video_matrix_limit: 8294400, + }, + polls: { + max_options: 4, + max_characters_per_option: 50, + min_expiration: 300, + max_expiration: 2629746, + }, + translation: { + enabled: false, + }, + }, + registrations: { + enabled: false, + approval_required: false, + message: null, + url: null, + }, + rules: [], + }); +}; + +const instanceDescriptionController: AppController = async (c) => { + const meta = await getInstanceMetadata(await Storages.db(), c.req.raw.signal); + + return c.json({ + content: meta.about, + updated_at: new Date((meta.event?.created_at ?? 0) * 1000).toISOString(), + }); +}; + +export { instanceDescriptionController, instanceV1Controller, instanceV2Controller }; From e96cb3ed40138982f0470f78b953da8f727a4bfb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 12 Jun 2024 21:10:56 -0500 Subject: [PATCH 51/77] Update ditto.conf, remove ipfs.service --- installation/ditto.conf | 72 ++++++++++++--------------------------- installation/ipfs.service | 13 ------- 2 files changed, 22 insertions(+), 63 deletions(-) delete mode 100644 installation/ipfs.service diff --git a/installation/ditto.conf b/installation/ditto.conf index afdf65c0..773573de 100644 --- a/installation/ditto.conf +++ b/installation/ditto.conf @@ -1,4 +1,4 @@ -# Nginx configuration for Ditto with IPFS. +# Nginx configuration for Ditto. # # Edit this file to change occurences of "example.com" to your own domain. @@ -6,10 +6,6 @@ upstream ditto { server 127.0.0.1:4036; } -upstream ipfs_gateway { - server 127.0.0.1:8080; -} - server { listen 80; listen [::]:80; @@ -18,21 +14,8 @@ server { } server { - # Uncomment these lines once you acquire a certificate: - # listen 443 ssl http2; - # listen [::]:443 ssl http2; server_name example.com; - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers HIGH:!MEDIUM:!LOW:!aNULL:!NULL:!SHA; - ssl_prefer_server_ciphers on; - ssl_session_cache shared:SSL:10m; - ssl_session_tickets off; - - # Uncomment these lines once you acquire a certificate: - # ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem; - # ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem; - keepalive_timeout 70; sendfile on; client_max_body_size 100m; @@ -44,53 +27,42 @@ server { proxy_set_header Host $http_host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + root /opt/ditto/public; + + location @spa { + try_files /index.html /dev/null; + } + + location @frontend { + try_files $uri @ditto-static; + } + + location @ditto-static { + root /opt/ditto/static; + try_files $uri @spa; + } + location /packs { add_header Cache-Control "public, max-age=31536000, immutable"; add_header Strict-Transport-Security "max-age=31536000" always; root /opt/ditto/public; } - location ~ ^/(instance|sw.js$|sw.js.map$) { + location ~ ^/(instance|sw\.js$|sw\.js\.map$) { root /opt/ditto/public; + try_files $uri =404; } - location /images { + location = /favicon.ico { root /opt/ditto/static; + try_files $uri =404; } - location / { + location ~ ^/(api|relay|oauth|manifest.json|nodeinfo|.well-known/(nodeinfo|nostr.json)) { proxy_pass http://ditto; } -} - -server { - # Uncomment these lines once you acquire a certificate: - # listen 443 ssl http2; - # listen [::]:443 ssl http2; - server_name media.example.com; - - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers HIGH:!MEDIUM:!LOW:!aNULL:!NULL:!SHA; - ssl_prefer_server_ciphers on; - ssl_session_cache shared:SSL:10m; - ssl_session_tickets off; - - # Uncomment these lines once you acquire a certificate: - # ssl_certificate /etc/letsencrypt/live/media.example.com/fullchain.pem; - # ssl_certificate_key /etc/letsencrypt/live/media.example.com/privkey.pem; - - keepalive_timeout 70; - sendfile on; - client_max_body_size 1m; - ignore_invalid_headers off; - - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host $http_host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; location / { - proxy_pass http://ipfs_gateway; + try_files /dev/null @frontend; } } diff --git a/installation/ipfs.service b/installation/ipfs.service deleted file mode 100644 index e345097e..00000000 --- a/installation/ipfs.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=IPFS Daemon -Wants=network-online.target -After=network-online.target - -[Service] -Type=simple -User=ditto -ExecStart=/usr/local/bin/ipfs daemon -Restart=on-failure - -[Install] -WantedBy=multi-user.target \ No newline at end of file From f1bce3d701c5c60c090fc4ea24a0e64b87334595 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 12 Jun 2024 21:12:58 -0500 Subject: [PATCH 52/77] Add a CHANGELOG --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..04774f28 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +- Initial release + +[unreleased]: https://gitlab.com/soapbox-pub/ditto/-/compare/main...HEAD From b43aed2301bc535de378824eb9a903c5c0124d45 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 13 Jun 2024 00:48:31 -0300 Subject: [PATCH 53/77] feat: create zappedByController --- src/controllers/api/statuses.ts | 39 +++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index ce4cbfa9..4945e3b0 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -20,6 +20,7 @@ import { lookupPubkey } from '@/utils/lookup.ts'; import { addTag, deleteTag } from '@/utils/tags.ts'; import { asyncReplaceAll } from '@/utils/text.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; +import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; const createStatusSchema = z.object({ in_reply_to_id: n.id().nullish(), @@ -541,6 +542,43 @@ const zapController: AppController = async (c) => { } }; +const zappedByController: AppController = async (c) => { + const id = c.req.param('id'); + const store = await Storages.db(); + + const events: DittoEvent[] = (await store.query([{ kinds: [9735], '#e': [id], limit: 100 }])).map((event) => { + const zapRequest = event.tags.find(([name]) => name === 'description')?.[1]; + if (!zapRequest) return; + try { + return JSON.parse(zapRequest); + } catch { + return; + } + }).filter(Boolean); + + await hydrateEvents({ events, store }); + + const results = (await Promise.all( + events.map(async (event) => { + const amount = event.tags.find(([name]) => name === 'amount')?.[1]; + const onlyDigits = /^\d+$/; + if (!amount || !onlyDigits.test(amount)) return; + + const comment = event?.content ?? ''; + + const account = event?.author ? await renderAccount(event.author) : await accountFromPubkey(event.pubkey); + + return { + zap_comment: comment, + zap_amount: Number(amount), + account, + }; + }), + )).filter(Boolean); + + return c.json(results); +}; + export { bookmarkController, contextController, @@ -557,4 +595,5 @@ export { unpinController, unreblogStatusController, zapController, + zappedByController, }; From 7474c1b28836dc8715abc9227594b91df9afd1e7 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 13 Jun 2024 00:49:21 -0300 Subject: [PATCH 54/77] feat: add /api/v1/ditto/statuses/:id{[0-9a-f]{64}}/zapped_by endpoint --- src/app.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/app.ts b/src/app.ts index 5ab31d84..276f9ee3 100644 --- a/src/app.ts +++ b/src/app.ts @@ -88,6 +88,7 @@ import { unpinController, unreblogStatusController, zapController, + zappedByController, } from '@/controllers/api/statuses.ts'; import { streamingController } from '@/controllers/api/streaming.ts'; import { suggestionsV1Controller, suggestionsV2Controller } from '@/controllers/api/suggestions.ts'; @@ -259,6 +260,7 @@ app.post('/api/v1/ditto/names', requireSigner, nameRequestController); app.get('/api/v1/ditto/names', requireSigner, nameRequestsController); app.post('/api/v1/ditto/zap', requireSigner, zapController); +app.get('/api/v1/ditto/statuses/:id{[0-9a-f]{64}}/zapped_by', zappedByController); app.post('/api/v1/reports', requireSigner, reportController); app.get('/api/v1/admin/reports', requireSigner, requireRole('admin'), adminReportsController); From 22dbddb5d3bab6e0ea79beaa7eca4d1c4cf0c1df Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 13 Jun 2024 14:21:01 -0300 Subject: [PATCH 55/77] refactor: zap amount parsed with zod, change zapped_by fields name --- src/controllers/api/statuses.ts | 11 ++++------- src/utils/stats.ts | 12 +++++++----- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 4945e3b0..7b9e82cd 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -545,6 +545,7 @@ const zapController: AppController = async (c) => { const zappedByController: AppController = async (c) => { const id = c.req.param('id'); const store = await Storages.db(); + const amountSchema = z.coerce.number().int().nonnegative().catch(0); const events: DittoEvent[] = (await store.query([{ kinds: [9735], '#e': [id], limit: 100 }])).map((event) => { const zapRequest = event.tags.find(([name]) => name === 'description')?.[1]; @@ -560,17 +561,13 @@ const zappedByController: AppController = async (c) => { const results = (await Promise.all( events.map(async (event) => { - const amount = event.tags.find(([name]) => name === 'amount')?.[1]; - const onlyDigits = /^\d+$/; - if (!amount || !onlyDigits.test(amount)) return; - + const amount = amountSchema.parse(event.tags.find(([name]) => name === 'amount')?.[1]); const comment = event?.content ?? ''; - const account = event?.author ? await renderAccount(event.author) : await accountFromPubkey(event.pubkey); return { - zap_comment: comment, - zap_amount: Number(amount), + comment, + amount, account, }; }), diff --git a/src/utils/stats.ts b/src/utils/stats.ts index 8f0b5501..e6001d11 100644 --- a/src/utils/stats.ts +++ b/src/utils/stats.ts @@ -1,7 +1,8 @@ import { NostrEvent, NStore } from '@nostrify/nostrify'; import { Kysely, UpdateObject } from 'kysely'; - import { SetRequired } from 'type-fest'; +import { z } from 'zod'; + import { DittoTables } from '@/db/DittoTables.ts'; import { findQuoteTag, findReplyTag, getTagSet } from '@/utils/tags.ts'; import { Conf } from '@/config.ts'; @@ -140,19 +141,20 @@ async function handleEvent9735(kysely: Kysely, event: NostrEvent): const id = event.tags.find(([name]) => name === 'e')?.[1]; if (!id) return; - let amount = '0'; + const amountSchema = z.coerce.number().int().nonnegative().catch(0); + let amount = 0; try { const zapRequest = JSON.parse(event.tags.find(([name]) => name === 'description')?.[1]!) as NostrEvent; - amount = zapRequest.tags.find(([name]) => name === 'amount')?.[1]!; + amount = amountSchema.parse(zapRequest.tags.find(([name]) => name === 'amount')?.[1]); + if (amount <= 0) return; } catch { return; } - if (amount === '0' || !amount || (/^\d+$/).test(amount) === false) return; await updateEventStats( kysely, id, - ({ zaps_amount }) => ({ zaps_amount: Math.max(0, zaps_amount + Number(amount)) }), + ({ zaps_amount }) => ({ zaps_amount: Math.max(0, zaps_amount + amount) }), ); } From f60f9fd38ffb919d1788258152b2507d35d4a765 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 13 Jun 2024 13:48:32 -0500 Subject: [PATCH 56/77] Upgrade Deno to v1.44.2 --- .gitlab-ci.yml | 2 +- .tool-versions | 2 +- Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58a34532..20650d2e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:1.43.3 +image: denoland/deno:1.44.2 default: interruptible: true diff --git a/.tool-versions b/.tool-versions index b3e19cd6..e85b11d4 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -deno 1.43.3 \ No newline at end of file +deno 1.44.2 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f8df8159..f2dde5ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM denoland/deno:1.43.3 +FROM denoland/deno:1.44.2 EXPOSE 4036 WORKDIR /app RUN mkdir -p data && chown -R deno data From 880b09e016d3c507a91883454b9c3aaf77239b9c Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 13 Jun 2024 15:51:13 -0300 Subject: [PATCH 57/77] refactor: parse zap request with NSchema --- src/utils/stats.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/stats.ts b/src/utils/stats.ts index e6001d11..ccba0a5b 100644 --- a/src/utils/stats.ts +++ b/src/utils/stats.ts @@ -1,4 +1,4 @@ -import { NostrEvent, NStore } from '@nostrify/nostrify'; +import { NostrEvent, NSchema as n, NStore } from '@nostrify/nostrify'; import { Kysely, UpdateObject } from 'kysely'; import { SetRequired } from 'type-fest'; import { z } from 'zod'; @@ -144,7 +144,7 @@ async function handleEvent9735(kysely: Kysely, event: NostrEvent): const amountSchema = z.coerce.number().int().nonnegative().catch(0); let amount = 0; try { - const zapRequest = JSON.parse(event.tags.find(([name]) => name === 'description')?.[1]!) as NostrEvent; + const zapRequest = n.json().pipe(n.event()).parse(event.tags.find(([name]) => name === 'description')?.[1]); amount = amountSchema.parse(zapRequest.tags.find(([name]) => name === 'amount')?.[1]); if (amount <= 0) return; } catch { From 100a5056eb9401ccee3f4fe0d89988ab97fd1d72 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 13 Jun 2024 16:54:03 -0500 Subject: [PATCH 58/77] nameRequestsController: display only owned 3036 events --- src/controllers/api/ditto.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index 5723a9e8..832f487e 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -126,7 +126,11 @@ export const nameRequestsController: AppController = async (c) => { } } - const events = await store.query([{ kinds: [3036], ids: [...ids] }]) + if (ids.size === 0) { + return c.json([]); + } + + const events = await store.query([{ kinds: [3036], ids: [...ids], authors: [pubkey] }]) .then((events) => hydrateEvents({ store, events: events, signal: c.req.raw.signal })); const nameRequests = await Promise.all( From 44fe0c5e1d5446f3b97215cc4a90950d593a9c6d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 13 Jun 2024 18:43:04 -0500 Subject: [PATCH 59/77] Bump Nostrify to v0.23.2 --- deno.json | 2 +- deno.lock | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/deno.json b/deno.json index a2b8733c..1a7f5e8b 100644 --- a/deno.json +++ b/deno.json @@ -23,7 +23,7 @@ "@db/sqlite": "jsr:@db/sqlite@^0.11.1", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.23.1", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.23.2", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", diff --git a/deno.lock b/deno.lock index 928bf2c9..4a79b8f5 100644 --- a/deno.lock +++ b/deno.lock @@ -10,7 +10,7 @@ "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", - "jsr:@nostrify/nostrify@^0.23.1": "jsr:@nostrify/nostrify@0.23.1", + "jsr:@nostrify/nostrify@^0.23.2": "jsr:@nostrify/nostrify@0.23.2", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0": "jsr:@soapbox/kysely-deno-sqlite@2.2.0", "jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0", "jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0", @@ -122,11 +122,10 @@ "npm:zod@^3.23.8" ] }, - "@nostrify/nostrify@0.23.1": { - "integrity": "7a242dedfe33cf38131696ad96d789d54257cfbfd5b5e63748fe5d53c057d99a", + "@nostrify/nostrify@0.23.2": { + "integrity": "c880fd91b5fe69a6239f98cae62297ffccc2a78d160af4d376dd05899352daf0", "dependencies": [ "jsr:@std/encoding@^0.224.1", - "npm:@scure/base@^1.1.6", "npm:@scure/bip32@^1.4.0", "npm:@scure/bip39@^1.3.0", "npm:kysely@^0.27.3", @@ -1360,7 +1359,7 @@ "dependencies": [ "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "jsr:@db/sqlite@^0.11.1", - "jsr:@nostrify/nostrify@^0.23.1", + "jsr:@nostrify/nostrify@^0.23.2", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", From 5658c5db081ffb0cf4c36cc70b5a37e57e77b130 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 13 Jun 2024 23:47:12 +0000 Subject: [PATCH 60/77] nameRequestsController: `ids.size === 0` => `!ids.size` --- src/controllers/api/ditto.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index 832f487e..c9c7896b 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -126,7 +126,7 @@ export const nameRequestsController: AppController = async (c) => { } } - if (ids.size === 0) { + if (!ids.size) { return c.json([]); } From d06cafd0ddc060b214db1cfe64d83fd2589e99ee Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 13 Jun 2024 19:11:59 -0500 Subject: [PATCH 61/77] nameRequestController: prevent submitting the same name twice --- src/controllers/api/ditto.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index c9c7896b..841eb861 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -69,8 +69,17 @@ const nameRequestSchema = z.object({ }); export const nameRequestController: AppController = async (c) => { + const store = await Storages.db(); + const signer = c.get('signer')!; + const pubkey = await signer.getPublicKey(); + const { name, reason } = nameRequestSchema.parse(await c.req.json()); + const [existing] = await store.query([{ kinds: [3036], authors: [pubkey], '#r': [name], limit: 1 }]); + if (existing) { + return c.json({ error: 'Name request already exists' }, 400); + } + const event = await createEvent({ kind: 3036, content: reason, From 7186f49316061c66b27ae119b3dee7fe1e488da1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 13 Jun 2024 19:17:53 -0500 Subject: [PATCH 62/77] Remove URL restriction from r tag --- src/storages/EventsDB.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 98c32d54..8955d6d6 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -38,7 +38,7 @@ class EventsDB implements NStore { 'p': ({ event, count, value }) => (count < 15 || event.kind === 3) && isNostrId(value), 'proxy': ({ count, value }) => count === 0 && isURL(value), 'q': ({ event, count, value }) => count === 0 && event.kind === 1 && isNostrId(value), - 'r': ({ event, count, value }) => (event.kind === 1985 ? count < 20 : count < 3) && isURL(value), + 'r': ({ event, count }) => (event.kind === 1985 ? count < 20 : count < 3), 't': ({ event, count, value }) => (event.kind === 1985 ? count < 20 : count < 5) && value.length < 50, }; From 91fe7acbd22e47b9813e3f4cbf5ece0e177b0431 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 13 Jun 2024 21:36:07 -0500 Subject: [PATCH 63/77] Index nostr_tags.name --- src/db/migrations/026_tags_name_index.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/db/migrations/026_tags_name_index.ts diff --git a/src/db/migrations/026_tags_name_index.ts b/src/db/migrations/026_tags_name_index.ts new file mode 100644 index 00000000..a15587fb --- /dev/null +++ b/src/db/migrations/026_tags_name_index.ts @@ -0,0 +1,14 @@ +import { Kysely } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .createIndex('idx_tags_name') + .on('nostr_tags') + .column('name') + .ifNotExists() + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropIndex('idx_tags_name').ifExists().execute(); +} From 9c640ca86b05969b20713515fe9f540e835fa998 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 14 Jun 2024 08:31:12 -0500 Subject: [PATCH 64/77] Change verson to v1.0.0 --- CHANGELOG.md | 5 ++++- src/controllers/api/instance.ts | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04774f28..990c4f42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.0.0] - 2024-06-14 + - Initial release -[unreleased]: https://gitlab.com/soapbox-pub/ditto/-/compare/main...HEAD +[unreleased]: https://gitlab.com/soapbox-pub/ditto/-/compare/v1.0.0...HEAD +[1.0.0]: https://gitlab.com/soapbox-pub/ditto/-/tags/v1.0.0 diff --git a/src/controllers/api/instance.ts b/src/controllers/api/instance.ts index 4c3e9812..bda055dc 100644 --- a/src/controllers/api/instance.ts +++ b/src/controllers/api/instance.ts @@ -3,7 +3,7 @@ import { Conf } from '@/config.ts'; import { Storages } from '@/storages.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; -const version = '0.0.0 (compatible; Ditto 0.0.1)'; +const version = '3.0.0 (compatible; Ditto 1.0.0)'; const instanceV1Controller: AppController = async (c) => { const { host, protocol } = Conf.url; From 3b6ebc651b727623b0104c3c44099df20f54def5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 14 Jun 2024 08:34:10 -0500 Subject: [PATCH 65/77] README: check all the boxes --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6551f27d..06ca3d63 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,9 @@ For more info see: https://docs.soapbox.pub/ditto/ - [x] Reposts - [x] Notifications - [x] Profiles -- [ ] Search +- [x] Search - [x] Moderation -- [ ] Zaps +- [x] Zaps - [x] Customizable - [x] Open source - [x] Self-hosted From e3f86e1b54adc83d257742afe052149999777470 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 14 Jun 2024 08:34:54 -0500 Subject: [PATCH 66/77] README: remove WIP comment Although it is, we don't need to warn people about that anymore --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 06ca3d63..96c19fa7 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,6 @@ For more info see: https://docs.soapbox.pub/ditto/ -⚠️ This software is a work in progress. - ## Features - [x] Built-in Nostr relay From cc14f2a54993a4d794ce9c1083cff127fcfb81bf Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Fri, 14 Jun 2024 21:02:40 +0530 Subject: [PATCH 67/77] fix accidental vscode changes --- .vscode/settings.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 7644e548..23baa765 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,8 +2,5 @@ "deno.enable": true, "deno.lint": true, "editor.defaultFormatter": "denoland.vscode-deno", - "path-intellisense.extensionOnImport": true, - "[typescript]": { - "editor.defaultFormatter": "denoland.vscode-deno" - } + "path-intellisense.extensionOnImport": true } \ No newline at end of file From 72ca7af7f04bcf5c2fa3b9a8c2cfeeb47562b657 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Jun 2024 11:40:03 -0500 Subject: [PATCH 68/77] admin-role script: minor tweaks --- scripts/admin-role.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/admin-role.ts b/scripts/admin-role.ts index 6bd7fb03..503b20b0 100644 --- a/scripts/admin-role.ts +++ b/scripts/admin-role.ts @@ -1,16 +1,16 @@ import { NSchema } from '@nostrify/nostrify'; +import { nip19 } from 'nostr-tools'; import { DittoDB } from '@/db/DittoDB.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; import { nostrNow } from '@/utils.ts'; -import { nip19 } from 'nostr-tools'; const kysely = await DittoDB.getInstance(); const eventsDB = new EventsDB(kysely); const [pubkeyOrNpub, role] = Deno.args; -const pubkey = pubkeyOrNpub.startsWith('npub') ? nip19.decode(pubkeyOrNpub).data as string : pubkeyOrNpub; +const pubkey = pubkeyOrNpub.startsWith('npub1') ? nip19.decode(pubkeyOrNpub as `npub1${string}`).data : pubkeyOrNpub; if (!NSchema.id().safeParse(pubkey).success) { console.error('Invalid pubkey'); From 8d8e46eae80e099067228fa4ba2328fc23433a96 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Jun 2024 11:51:55 -0500 Subject: [PATCH 69/77] Add a script to update trends --- deno.json | 3 +- scripts/trends.ts | 44 +++++++++++ src/cron.ts | 88 ++------------------- src/trends.ts | 127 ++++++++++++++++++++++++++++++ src/trends/trending-tag-values.ts | 47 ----------- 5 files changed, 181 insertions(+), 128 deletions(-) create mode 100644 scripts/trends.ts create mode 100644 src/trends.ts delete mode 100644 src/trends/trending-tag-values.ts diff --git a/deno.json b/deno.json index 1a7f5e8b..18c88749 100644 --- a/deno.json +++ b/deno.json @@ -13,7 +13,8 @@ "admin:role": "deno run -A scripts/admin-role.ts", "setup": "deno run -A scripts/setup.ts", "stats:recompute": "deno run -A scripts/stats-recompute.ts", - "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip soapbox.zip && rm soapbox.zip" + "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip soapbox.zip && rm soapbox.zip", + "trends": "deno run -A scripts/trends.ts" }, "unstable": ["cron", "ffi", "kv", "worker-options"], "exclude": ["./public"], diff --git a/scripts/trends.ts b/scripts/trends.ts new file mode 100644 index 00000000..6600f7e2 --- /dev/null +++ b/scripts/trends.ts @@ -0,0 +1,44 @@ +import { z } from 'zod'; + +import { + updateTrendingEvents, + updateTrendingHashtags, + updateTrendingLinks, + updateTrendingPubkeys, + updateTrendingZappedEvents, +} from '@/trends.ts'; + +const trendSchema = z.enum(['pubkeys', 'zapped_events', 'events', 'hashtags', 'links']); +const trends = trendSchema.array().parse(Deno.args); + +if (!trends.length) { + trends.push('pubkeys', 'zapped_events', 'events', 'hashtags', 'links'); +} + +for (const trend of trends) { + switch (trend) { + case 'pubkeys': + console.log('Updating trending pubkeys...'); + await updateTrendingPubkeys(); + break; + case 'zapped_events': + console.log('Updating trending zapped events...'); + await updateTrendingZappedEvents(); + break; + case 'events': + console.log('Updating trending events...'); + await updateTrendingEvents(); + break; + case 'hashtags': + console.log('Updating trending hashtags...'); + await updateTrendingHashtags(); + break; + case 'links': + console.log('Updating trending links...'); + await updateTrendingLinks(); + break; + } +} + +console.log('Trends updated.'); +Deno.exit(0); diff --git a/src/cron.ts b/src/cron.ts index 166e42ee..6994561e 100644 --- a/src/cron.ts +++ b/src/cron.ts @@ -1,84 +1,12 @@ -import { Stickynotes } from '@soapbox/stickynotes'; - -import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; -import { handleEvent } from '@/pipeline.ts'; -import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { getTrendingTagValues } from '@/trends/trending-tag-values.ts'; -import { Time } from '@/utils/time.ts'; - -const console = new Stickynotes('ditto:trends'); - -async function updateTrendingTags( - l: string, - tagName: string, - kinds: number[], - limit: number, - extra = '', - aliases?: string[], -) { - console.info(`Updating trending ${l}...`); - const kysely = await DittoDB.getInstance(); - const signal = AbortSignal.timeout(1000); - - const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); - const now = Math.floor(Date.now() / 1000); - - const tagNames = aliases ? [tagName, ...aliases] : [tagName]; - - const trends = await getTrendingTagValues(kysely, tagNames, { - kinds, - since: yesterday, - until: now, - limit, - }); - - if (!trends.length) { - return; - } - - const signer = new AdminSigner(); - - const label = await signer.signEvent({ - kind: 1985, - content: '', - tags: [ - ['L', 'pub.ditto.trends'], - ['l', l, 'pub.ditto.trends'], - ...trends.map(({ value, authors, uses }) => [tagName, value, extra, authors.toString(), uses.toString()]), - ], - created_at: Math.floor(Date.now() / 1000), - }); - - await handleEvent(label, signal); - console.info(`Trending ${l} updated.`); -} +import { updateTrendingLinks } from '@/trends.ts'; +import { updateTrendingHashtags } from '@/trends.ts'; +import { updateTrendingEvents, updateTrendingPubkeys, updateTrendingZappedEvents } from '@/trends.ts'; /** Start cron jobs for the application. */ export function cron() { - Deno.cron( - 'update trending pubkeys', - '0 * * * *', - () => updateTrendingTags('#p', 'p', [1, 3, 6, 7, 9735], 40, Conf.relay), - ); - Deno.cron( - 'update trending zapped events', - '7 * * * *', - () => updateTrendingTags('zapped', 'e', [9735], 40, Conf.relay, ['q']), - ); - Deno.cron( - 'update trending events', - '15 * * * *', - () => updateTrendingTags('#e', 'e', [1, 6, 7, 9735], 40, Conf.relay, ['q']), - ); - Deno.cron( - 'update trending hashtags', - '30 * * * *', - () => updateTrendingTags('#t', 't', [1], 20), - ); - Deno.cron( - 'update trending links', - '45 * * * *', - () => updateTrendingTags('#r', 'r', [1], 20), - ); + Deno.cron('update trending pubkeys', '0 * * * *', updateTrendingPubkeys); + Deno.cron('update trending zapped events', '7 * * * *', updateTrendingZappedEvents); + Deno.cron('update trending events', '15 * * * *', updateTrendingEvents); + Deno.cron('update trending hashtags', '30 * * * *', updateTrendingHashtags); + Deno.cron('update trending links', '45 * * * *', updateTrendingLinks); } diff --git a/src/trends.ts b/src/trends.ts new file mode 100644 index 00000000..c346a492 --- /dev/null +++ b/src/trends.ts @@ -0,0 +1,127 @@ +import { NostrFilter } from '@nostrify/nostrify'; +import { Stickynotes } from '@soapbox/stickynotes'; +import { Kysely } from 'kysely'; + +import { Conf } from '@/config.ts'; +import { DittoDB } from '@/db/DittoDB.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; +import { handleEvent } from '@/pipeline.ts'; +import { AdminSigner } from '@/signers/AdminSigner.ts'; +import { Time } from '@/utils/time.ts'; + +const console = new Stickynotes('ditto:trends'); + +/** Get trending tag values for a given tag in the given time frame. */ +export async function getTrendingTagValues( + /** Kysely instance to execute queries on. */ + kysely: Kysely, + /** Tag name to filter by, eg `t` or `r`. */ + tagNames: string[], + /** Filter of eligible events. */ + filter: NostrFilter, +): Promise<{ value: string; authors: number; uses: number }[]> { + let query = kysely + .selectFrom('nostr_tags') + .innerJoin('nostr_events', 'nostr_events.id', 'nostr_tags.event_id') + .select(({ fn }) => [ + 'nostr_tags.value', + fn.agg('count', ['nostr_events.pubkey']).distinct().as('authors'), + fn.countAll().as('uses'), + ]) + .where('nostr_tags.name', 'in', tagNames) + .groupBy('nostr_tags.value') + .orderBy((c) => c.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc'); + + if (filter.kinds) { + query = query.where('nostr_events.kind', 'in', filter.kinds); + } + if (typeof filter.since === 'number') { + query = query.where('nostr_events.created_at', '>=', filter.since); + } + if (typeof filter.until === 'number') { + query = query.where('nostr_events.created_at', '<=', filter.until); + } + if (typeof filter.limit === 'number') { + query = query.limit(filter.limit); + } + + const rows = await query.execute(); + + return rows.map((row) => ({ + value: row.value, + authors: Number(row.authors), + uses: Number(row.uses), + })); +} + +/** Get trending tags and publish an event with them. */ +export async function updateTrendingTags( + l: string, + tagName: string, + kinds: number[], + limit: number, + extra = '', + aliases?: string[], +) { + console.info(`Updating trending ${l}...`); + const kysely = await DittoDB.getInstance(); + const signal = AbortSignal.timeout(1000); + + const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); + const now = Math.floor(Date.now() / 1000); + + const tagNames = aliases ? [tagName, ...aliases] : [tagName]; + + const trends = await getTrendingTagValues(kysely, tagNames, { + kinds, + since: yesterday, + until: now, + limit, + }); + + if (!trends.length) { + console.info(`No trending ${l} found. Skipping.`); + return; + } + + const signer = new AdminSigner(); + + const label = await signer.signEvent({ + kind: 1985, + content: '', + tags: [ + ['L', 'pub.ditto.trends'], + ['l', l, 'pub.ditto.trends'], + ...trends.map(({ value, authors, uses }) => [tagName, value, extra, authors.toString(), uses.toString()]), + ], + created_at: Math.floor(Date.now() / 1000), + }); + + await handleEvent(label, signal); + console.info(`Trending ${l} updated.`); +} + +/** Update trending pubkeys. */ +export function updateTrendingPubkeys(): Promise { + return updateTrendingTags('#p', 'p', [1, 3, 6, 7, 9735], 40, Conf.relay); +} + +/** Update trending zapped events. */ +export function updateTrendingZappedEvents(): Promise { + return updateTrendingTags('zapped', 'e', [9735], 40, Conf.relay, ['q']); +} + +/** Update trending events. */ +export function updateTrendingEvents(): Promise { + return updateTrendingTags('#e', 'e', [1, 6, 7, 9735], 40, Conf.relay, ['q']); +} + +/** Update trending hashtags. */ +export function updateTrendingHashtags(): Promise { + return updateTrendingTags('#t', 't', [1], 20); +} + +/** Update trending links. */ +export function updateTrendingLinks(): Promise { + return updateTrendingTags('#r', 'r', [1], 20); +} diff --git a/src/trends/trending-tag-values.ts b/src/trends/trending-tag-values.ts deleted file mode 100644 index 17ec53d2..00000000 --- a/src/trends/trending-tag-values.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { NostrFilter } from '@nostrify/nostrify'; -import { Kysely } from 'kysely'; - -import { DittoTables } from '@/db/DittoTables.ts'; - -/** Get trending tag values for a given tag in the given time frame. */ -export async function getTrendingTagValues( - /** Kysely instance to execute queries on. */ - kysely: Kysely, - /** Tag name to filter by, eg `t` or `r`. */ - tagNames: string[], - /** Filter of eligible events. */ - filter: NostrFilter, -): Promise<{ value: string; authors: number; uses: number }[]> { - let query = kysely - .selectFrom('nostr_tags') - .innerJoin('nostr_events', 'nostr_events.id', 'nostr_tags.event_id') - .select(({ fn }) => [ - 'nostr_tags.value', - fn.agg('count', ['nostr_events.pubkey']).distinct().as('authors'), - fn.countAll().as('uses'), - ]) - .where('nostr_tags.name', 'in', tagNames) - .groupBy('nostr_tags.value') - .orderBy((c) => c.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc'); - - if (filter.kinds) { - query = query.where('nostr_events.kind', 'in', filter.kinds); - } - if (typeof filter.since === 'number') { - query = query.where('nostr_events.created_at', '>=', filter.since); - } - if (typeof filter.until === 'number') { - query = query.where('nostr_events.created_at', '<=', filter.until); - } - if (typeof filter.limit === 'number') { - query = query.limit(filter.limit); - } - - const rows = await query.execute(); - - return rows.map((row) => ({ - value: row.value, - authors: Number(row.authors), - uses: Number(row.uses), - })); -} From a3b7acd1c102bb7bb7d392172323530f4d705289 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Jun 2024 17:18:55 -0500 Subject: [PATCH 70/77] Cache trends API results at a different interval than trends calculations --- src/controllers/api/trends.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts index b98fc1bf..9226d2db 100644 --- a/src/controllers/api/trends.ts +++ b/src/controllers/api/trends.ts @@ -11,7 +11,7 @@ import { renderStatus } from '@/views/mastodon/statuses.ts'; let trendingHashtagsCache = getTrendingHashtags(); -Deno.cron('update trending hashtags cache', { minute: { every: 15 } }, async () => { +Deno.cron('update trending hashtags cache', '35 * * * *', async () => { const trends = await getTrendingHashtags(); trendingHashtagsCache = Promise.resolve(trends); }); @@ -50,7 +50,7 @@ async function getTrendingHashtags() { let trendingLinksCache = getTrendingLinks(); -Deno.cron('update trending links cache', { minute: { every: 15 } }, async () => { +Deno.cron('update trending links cache', '50 * * * *', async () => { const trends = await getTrendingLinks(); trendingLinksCache = Promise.resolve(trends); }); From 1afd8a739ab1c689798e135d992ec604b60c521b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Jun 2024 17:20:39 -0500 Subject: [PATCH 71/77] Update deno.lock --- deno.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/deno.lock b/deno.lock index 4a79b8f5..faca92e0 100644 --- a/deno.lock +++ b/deno.lock @@ -126,6 +126,7 @@ "integrity": "c880fd91b5fe69a6239f98cae62297ffccc2a78d160af4d376dd05899352daf0", "dependencies": [ "jsr:@std/encoding@^0.224.1", + "npm:@scure/base@^1.1.6", "npm:@scure/bip32@^1.4.0", "npm:@scure/bip39@^1.3.0", "npm:kysely@^0.27.3", From 6ce8aae0d178ef5723853ccb974545db3ad2d3b4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Jun 2024 18:14:38 -0500 Subject: [PATCH 72/77] trends: rewrite getTrendingTags to process the history in serial, split across several queries --- src/controllers/api/trends.ts | 64 +++++++++++++++++------------------ 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts index 9226d2db..76c84f58 100644 --- a/src/controllers/api/trends.ts +++ b/src/controllers/api/trends.ts @@ -1,4 +1,4 @@ -import { NostrEvent, NStore } from '@nostrify/nostrify'; +import { NostrEvent, NostrFilter, NStore } from '@nostrify/nostrify'; import { z } from 'zod'; import { AppController } from '@/app.ts'; @@ -148,58 +148,56 @@ interface TrendingTag { } export async function getTrendingTags(store: NStore, tagName: string): Promise { - const filter = { + const [label] = await store.query([{ kinds: [1985], '#L': ['pub.ditto.trends'], '#l': [`#${tagName}`], authors: [Conf.pubkey], limit: 1, - }; + }]); - const [label] = await store.query([filter]); + if (!label) return []; - if (!label) { - return []; - } + const date = new Date(label.created_at * 1000); + const lastWeek = new Date(date.getTime() - Time.days(7)); + const dates = generateDateRange(lastWeek, date).reverse(); - const tags = label.tags.filter(([name]) => name === tagName); + const results: TrendingTag[] = []; - const labelDate = new Date(label.created_at * 1000); - const lastWeek = new Date(labelDate.getTime() - Time.days(7)); - const dates = generateDateRange(lastWeek, labelDate).reverse(); + for (const [name, value] of label.tags) { + if (name !== tagName) continue; - return Promise.all(tags.map(async ([_, value]) => { - const filters = dates.map((date) => ({ - ...filter, - [`#${tagName}`]: [value], - since: Math.floor(date.getTime() / 1000), - until: Math.floor((date.getTime() + Time.days(1)) / 1000), - })); + const history: TrendingTag['history'] = []; - const labels = await store.query(filters); + for (const date of dates) { + const [label] = await store.query([{ + kinds: [1985], + '#L': ['pub.ditto.trends'], + '#l': [`#${tagName}`], + [`#${tagName}`]: [value], + authors: [Conf.pubkey], + since: Math.floor(date.getTime() / 1000), + until: Math.floor((date.getTime() + Time.days(1)) / 1000), + limit: 1, + } as NostrFilter]); - const history = dates.map((date) => { - const label = labels.find((label) => { - const since = Math.floor(date.getTime() / 1000); - const until = Math.floor((date.getTime() + Time.days(1)) / 1000); - return label.created_at >= since && label.created_at < until; - }); + const [, , , accounts, uses] = label?.tags.find(([n, v]) => n === tagName && v === value) ?? []; - const [, , , accounts, uses] = label?.tags.find((tag) => tag[0] === tagName && tag[1] === value) ?? []; - - return { + history.push({ day: Math.floor(date.getTime() / 1000), authors: Number(accounts || 0), uses: Number(uses || 0), - }; - }); + }); + } - return { + results.push({ name: tagName, value, history, - }; - })); + }); + } + + return results; } export { trendingLinksController, trendingStatusesController, trendingTagsController }; From e63ee9b5a34e8d151eb89c6c3310de519f02bdb6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Jun 2024 19:36:34 -0500 Subject: [PATCH 73/77] Add interfaces for Mastodon entity types, hide deactivated accounts --- src/entities/MastodonAccount.ts | 58 +++++++++++++++++++++++++++++++++ src/entities/MastodonMention.ts | 6 ++++ src/entities/MastodonStatus.ts | 42 ++++++++++++++++++++++++ src/entities/PreviewCard.ts | 16 +++++++++ src/utils/unfurl.ts | 18 +--------- src/views/mastodon/accounts.ts | 14 ++++++-- src/views/mastodon/statuses.ts | 16 +++++---- 7 files changed, 143 insertions(+), 27 deletions(-) create mode 100644 src/entities/MastodonAccount.ts create mode 100644 src/entities/MastodonMention.ts create mode 100644 src/entities/MastodonStatus.ts create mode 100644 src/entities/PreviewCard.ts diff --git a/src/entities/MastodonAccount.ts b/src/entities/MastodonAccount.ts new file mode 100644 index 00000000..27cad244 --- /dev/null +++ b/src/entities/MastodonAccount.ts @@ -0,0 +1,58 @@ +/** Mastodon account entity, including supported extensions from Pleroma, etc. */ +export interface MastodonAccount { + id: string; + acct: string; + avatar: string; + avatar_static: string; + bot: boolean; + created_at: string; + discoverable: boolean; + display_name: string; + emojis: { + shortcode: string; + static_url: string; + url: string; + }[]; + fields: unknown[]; + follow_requests_count: number; + followers_count: number; + following_count: number; + fqn: string; + header: string; + header_static: string; + last_status_at: string | null; + locked: boolean; + note: string; + roles: unknown[]; + source?: { + fields: unknown[]; + language: string; + note: string; + privacy: string; + sensitive: boolean; + follow_requests_count: number; + nostr: { + nip05?: string; + }; + }; + statuses_count: number; + url: string; + username: string; + ditto: { + accepts_zaps: boolean; + }; + pleroma: { + deactivated: boolean; + is_admin: boolean; + is_moderator: boolean; + is_suggested: boolean; + is_local: boolean; + settings_store: unknown; + tags: string[]; + }; + nostr: { + pubkey: string; + lud16?: string; + }; + website?: string; +} diff --git a/src/entities/MastodonMention.ts b/src/entities/MastodonMention.ts new file mode 100644 index 00000000..e6140024 --- /dev/null +++ b/src/entities/MastodonMention.ts @@ -0,0 +1,6 @@ +export interface MastodonMention { + acct: string; + id: string; + url: string; + username: string; +} diff --git a/src/entities/MastodonStatus.ts b/src/entities/MastodonStatus.ts new file mode 100644 index 00000000..1fcbcacb --- /dev/null +++ b/src/entities/MastodonStatus.ts @@ -0,0 +1,42 @@ +import { MastodonAccount } from '@/entities/MastodonAccount.ts'; +import { PreviewCard } from '@/entities/PreviewCard.ts'; + +export interface MastodonStatus { + id: string; + account: MastodonAccount; + card: PreviewCard | null; + content: string; + created_at: string; + in_reply_to_id: string | null; + in_reply_to_account_id: string | null; + sensitive: boolean; + spoiler_text: string; + visibility: string; + language: string | null; + replies_count: number; + reblogs_count: number; + favourites_count: number; + zaps_amount: number; + favourited: boolean; + reblogged: boolean; + muted: boolean; + bookmarked: boolean; + pinned: boolean; + reblog: MastodonStatus | null; + application: unknown; + media_attachments: unknown[]; + mentions: unknown[]; + tags: unknown[]; + emojis: unknown[]; + poll: unknown; + quote?: MastodonStatus | null; + quote_id: string | null; + uri: string; + url: string; + zapped: boolean; + pleroma: { + emoji_reactions: { name: string; count: number; me: boolean }[]; + expires_at?: string; + quotes_count: number; + }; +} diff --git a/src/entities/PreviewCard.ts b/src/entities/PreviewCard.ts new file mode 100644 index 00000000..2decf926 --- /dev/null +++ b/src/entities/PreviewCard.ts @@ -0,0 +1,16 @@ +export interface PreviewCard { + url: string; + title: string; + description: string; + type: 'link' | 'photo' | 'video' | 'rich'; + author_name: string; + author_url: string; + provider_name: string; + provider_url: string; + html: string; + width: number; + height: number; + image: string | null; + embed_url: string; + blurhash: string | null; +} diff --git a/src/utils/unfurl.ts b/src/utils/unfurl.ts index e5ae428d..41ffcd3a 100644 --- a/src/utils/unfurl.ts +++ b/src/utils/unfurl.ts @@ -3,28 +3,12 @@ import Debug from '@soapbox/stickynotes/debug'; import DOMPurify from 'isomorphic-dompurify'; import { unfurl } from 'unfurl.js'; +import { PreviewCard } from '@/entities/PreviewCard.ts'; import { Time } from '@/utils/time.ts'; import { fetchWorker } from '@/workers/fetch.ts'; const debug = Debug('ditto:unfurl'); -interface PreviewCard { - url: string; - title: string; - description: string; - type: 'link' | 'photo' | 'video' | 'rich'; - author_name: string; - author_url: string; - provider_name: string; - provider_url: string; - html: string; - width: number; - height: number; - image: string | null; - embed_url: string; - blurhash: string | null; -} - async function unfurlCard(url: string, signal: AbortSignal): Promise { debug(`Unfurling ${url}...`); try { diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index 99f69f00..5de6c389 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -3,6 +3,7 @@ import { escape } from 'entities'; import { nip19, UnsignedEvent } from 'nostr-tools'; import { Conf } from '@/config.ts'; +import { MastodonAccount } from '@/entities/MastodonAccount.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { getLnurl } from '@/utils/lnurl.ts'; import { nip05Cache } from '@/utils/nip05.ts'; @@ -17,10 +18,17 @@ interface ToAccountOpts { async function renderAccount( event: Omit, opts: ToAccountOpts = {}, -) { +): Promise { const { withSource = false } = opts; const { pubkey } = event; + const names = getTagSet(event.user?.tags ?? [], 'n'); + if (names.has('disabled') || names.has('suspended')) { + const account = await accountFromPubkey(pubkey, opts); + account.pleroma.deactivated = true; + return account; + } + const { name, nip05, @@ -34,7 +42,6 @@ async function renderAccount( const npub = nip19.npubEncode(pubkey); const parsed05 = await parseAndVerifyNip05(nip05, pubkey); - const names = getTagSet(event.user?.tags ?? [], 'n'); return { id: pubkey, @@ -77,6 +84,7 @@ async function renderAccount( accepts_zaps: Boolean(getLnurl({ lud06, lud16 })), }, pleroma: { + deactivated: names.has('disabled') || names.has('suspended'), is_admin: names.has('admin'), is_moderator: names.has('admin') || names.has('moderator'), is_suggested: names.has('suggested'), @@ -92,7 +100,7 @@ async function renderAccount( }; } -function accountFromPubkey(pubkey: string, opts: ToAccountOpts = {}) { +function accountFromPubkey(pubkey: string, opts: ToAccountOpts = {}): Promise { const event: UnsignedEvent = { kind: 0, pubkey, diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index a0874b3a..4408c607 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -2,6 +2,8 @@ import { NostrEvent } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; import { Conf } from '@/config.ts'; +import { MastodonMention } from '@/entities/MastodonMention.ts'; +import { MastodonStatus } from '@/entities/MastodonStatus.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { Storages } from '@/storages.ts'; import { nostrDate } from '@/utils.ts'; @@ -17,7 +19,7 @@ interface RenderStatusOpts { depth?: number; } -async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise { +async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise { const { viewerPubkey, depth = 1 } = opts; if (depth > 2 || depth < 0) return; @@ -130,12 +132,14 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< }; } -async function renderReblog(event: DittoEvent, opts: RenderStatusOpts) { +async function renderReblog(event: DittoEvent, opts: RenderStatusOpts): Promise { const { viewerPubkey } = opts; if (!event.repost) return; const status = await renderStatus(event, {}); // omit viewerPubkey intentionally - const reblog = await renderStatus(event.repost, { viewerPubkey }); + if (!status) return; + + const reblog = await renderStatus(event.repost, { viewerPubkey }) ?? null; return { ...status, @@ -145,7 +149,7 @@ async function renderReblog(event: DittoEvent, opts: RenderStatusOpts) { }; } -async function toMention(pubkey: string, event?: NostrEvent) { +async function toMention(pubkey: string, event?: NostrEvent): Promise { const account = event ? await renderAccount(event) : undefined; if (account) { @@ -166,9 +170,7 @@ async function toMention(pubkey: string, event?: NostrEvent) { } } -type Mention = Awaited>; - -function buildInlineRecipients(mentions: Mention[]): string { +function buildInlineRecipients(mentions: MastodonMention[]): string { if (!mentions.length) return ''; const elements = mentions.reduce((acc, { url, username }) => { From c39fd2daa2b5178e0d32356ef95a903792e355a5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Jun 2024 23:41:19 -0500 Subject: [PATCH 74/77] Improve the setup script and clean up config --- scripts/setup.ts | 73 ++++++++++++++++++++++------------ src/config.ts | 68 +++++++++++++++---------------- src/views/mastodon/statuses.ts | 4 +- 3 files changed, 83 insertions(+), 62 deletions(-) diff --git a/scripts/setup.ts b/scripts/setup.ts index 255d8aa0..4223d318 100644 --- a/scripts/setup.ts +++ b/scripts/setup.ts @@ -1,22 +1,44 @@ +import nodeUrl from 'node:url'; import { exists } from '@std/fs/exists'; import { generateSecretKey, nip19 } from 'nostr-tools'; import question from 'question-deno'; -const vars: Record = {}; +import { Conf } from '@/config.ts'; + +console.log(''); +console.log('Hello! Welcome to the Ditto setup tool. We will ask you a few questions to generate a .env file for you.'); +console.log(''); +console.log('- Ditto docs: https://docs.soapbox.pub/ditto/'); if (await exists('./.env')) { - const overwrite = await question('confirm', 'Overwrite existing .env file? (this is a destructive action)', false); - if (!overwrite) { - console.log('Aborted'); - Deno.exit(0); - } + console.log('- Your existing .env file will be overwritten.'); } -console.log('Generating secret key...'); -const sk = generateSecretKey(); -vars.DITTO_NSEC = nip19.nsecEncode(sk); +console.log('- Press Ctrl+D to exit at any time.'); +console.log(''); -const domain = await question('input', 'What is the domain of your instance? (eg ditto.pub)'); +const vars: Record = {}; + +const DITTO_NSEC = Deno.env.get('DITTO_NSEC'); + +if (DITTO_NSEC) { + const choice = await question('list', 'Looks like you already have a DITTO_NSEC. Should we keep it?', [ + 'keep', + 'create new (destructive)', + ]); + if (choice === 'keep') { + vars.DITTO_NSEC = DITTO_NSEC; + } + if (choice === 'create new (destructive)') { + vars.DITTO_NSEC = nip19.nsecEncode(generateSecretKey()); + console.log(' Generated secret key\n'); + } +} else { + vars.DITTO_NSEC = nip19.nsecEncode(generateSecretKey()); + console.log(' Generated secret key\n'); +} + +const domain = await question('input', 'What is the domain of your instance? (eg ditto.pub)', Conf.url.host); vars.LOCAL_DOMAIN = `https://${domain}`; const database = await question('list', 'Which database do you want to use?', ['postgres', 'sqlite']); @@ -25,11 +47,12 @@ if (database === 'sqlite') { vars.DATABASE_URL = `sqlite://${path}`; } if (database === 'postgres') { - const host = await question('input', 'Postgres host', 'localhost'); - const port = await question('input', 'Postgres port', '5432'); - const user = await question('input', 'Postgres user', 'ditto'); - const password = await question('input', 'Postgres password', 'ditto'); - const database = await question('input', 'Postgres database', 'ditto'); + const url = nodeUrl.parse(Deno.env.get('DATABASE_URL') ?? 'postgres://ditto:ditto@localhost:5432/ditto'); + const host = await question('input', 'Postgres host', url.hostname); + const port = await question('input', 'Postgres port', url.port); + const user = await question('input', 'Postgres user', url.username); + const password = await question('input', 'Postgres password', url.password); + const database = await question('input', 'Postgres database', url.pathname.slice(1)); vars.DATABASE_URL = `postgres://${user}:${password}@${host}:${port}/${database}`; } @@ -42,28 +65,28 @@ vars.DITTO_UPLOADER = await question('list', 'How do you want to upload files?', ]); if (vars.DITTO_UPLOADER === 'nostrbuild') { - vars.NOSTRBUILD_ENDPOINT = await question('input', 'nostr.build endpoint', 'https://nostr.build/api/v2/upload/files'); + vars.NOSTRBUILD_ENDPOINT = await question('input', 'nostr.build endpoint', Conf.nostrbuildEndpoint); } if (vars.DITTO_UPLOADER === 'blossom') { - vars.BLOSSOM_SERVERS = await question('input', 'Blossom servers (comma separated)', 'https://blossom.primal.net/'); + vars.BLOSSOM_SERVERS = await question('input', 'Blossom servers (comma separated)', Conf.blossomServers.join(',')); } if (vars.DITTO_UPLOADER === 's3') { - vars.S3_ACCESS_KEY = await question('input', 'S3 access key'); - vars.S3_SECRET_KEY = await question('input', 'S3 secret key'); - vars.S3_ENDPOINT = await question('input', 'S3 endpoint'); - vars.S3_BUCKET = await question('input', 'S3 bucket'); - vars.S3_REGION = await question('input', 'S3 region'); - vars.S3_PATH_STYLE = String(await question('confirm', 'Use path style?', false)); + vars.S3_ACCESS_KEY = await question('input', 'S3 access key', Conf.s3.accessKey); + vars.S3_SECRET_KEY = await question('input', 'S3 secret key', Conf.s3.secretKey); + vars.S3_ENDPOINT = await question('input', 'S3 endpoint', Conf.s3.endPoint); + vars.S3_BUCKET = await question('input', 'S3 bucket', Conf.s3.bucket); + vars.S3_REGION = await question('input', 'S3 region', Conf.s3.region); + vars.S3_PATH_STYLE = String(await question('confirm', 'Use path style?', Conf.s3.pathStyle ?? false)); const mediaDomain = await question('input', 'Media domain', `media.${domain}`); vars.MEDIA_DOMAIN = `https://${mediaDomain}`; } if (vars.DITTO_UPLOADER === 'ipfs') { - vars.IPFS_API_URL = await question('input', 'IPFS API URL', 'http://localhost:5001'); + vars.IPFS_API_URL = await question('input', 'IPFS API URL', Conf.ipfs.apiUrl); const mediaDomain = await question('input', 'Media domain', `media.${domain}`); vars.MEDIA_DOMAIN = `https://${mediaDomain}`; } if (vars.DITTO_UPLOADER === 'local') { - vars.UPLOADS_DIR = await question('input', 'Local uploads directory', 'data/uploads'); + vars.UPLOADS_DIR = await question('input', 'Local uploads directory', Conf.uploadsDir); const mediaDomain = await question('input', 'Media domain', `media.${domain}`); vars.MEDIA_DOMAIN = `https://${mediaDomain}`; } diff --git a/src/config.ts b/src/config.ts index e1c0f103..6dc765ff 100644 --- a/src/config.ts +++ b/src/config.ts @@ -13,8 +13,9 @@ await dotenv.load({ /** Application-wide configuration. */ class Conf { + private static _pubkey: string | undefined; /** Ditto admin secret key in nip19 format. This is the way it's configured by an admin. */ - static get nsec() { + static get nsec(): `nsec1${string}` { const value = Deno.env.get('DITTO_NSEC'); if (!value) { throw new Error('Missing DITTO_NSEC'); @@ -25,13 +26,18 @@ class Conf { return value as `nsec1${string}`; } /** Ditto admin secret key in hex format. */ - static get seckey() { + static get seckey(): Uint8Array { return nip19.decode(Conf.nsec).data; } /** Ditto admin public key in hex format. */ - static pubkey = getPublicKey(Conf.seckey); + static get pubkey(): string { + if (!this._pubkey) { + this._pubkey = getPublicKey(Conf.seckey); + } + return this._pubkey; + } /** Ditto admin secret key as a Web Crypto key. */ - static get cryptoKey() { + static get cryptoKey(): Promise { return crypto.subtle.importKey( 'raw', Conf.seckey, @@ -41,7 +47,7 @@ class Conf { ); } - static get port() { + static get port(): number { return parseInt(Deno.env.get('PORT') || '4036'); } @@ -50,17 +56,13 @@ class Conf { return `${protocol === 'https:' ? 'wss:' : 'ws:'}//${host}/relay`; } /** Relay to use for NIP-50 `search` queries. */ - static get searchRelay() { + static get searchRelay(): string | undefined { return Deno.env.get('SEARCH_RELAY'); } /** Origin of the Ditto server, including the protocol and port. */ - static get localDomain() { + static get localDomain(): string { return Deno.env.get('LOCAL_DOMAIN') || `http://localhost:${Conf.port}`; } - /** URL to an external Nostr viewer. */ - static get externalDomain() { - return Deno.env.get('NOSTR_EXTERNAL') || Conf.localDomain; - } /** * Heroku-style database URL. This is used in production to connect to the * database. @@ -76,7 +78,7 @@ class Conf { } static db = { get url(): url.UrlWithStringQuery { - return url.parse(Deno.env.get('DATABASE_URL') ?? 'sqlite://data/db.sqlite3'); + return url.parse(Conf.databaseUrl); }, get dialect(): 'sqlite' | 'postgres' | undefined { switch (Conf.db.url.protocol) { @@ -90,43 +92,43 @@ class Conf { }, }; /** Character limit to enforce for posts made through Mastodon API. */ - static get postCharLimit() { + static get postCharLimit(): number { return Number(Deno.env.get('POST_CHAR_LIMIT') || 5000); } /** S3 media storage configuration. */ static s3 = { - get endPoint() { + get endPoint(): string | undefined { return Deno.env.get('S3_ENDPOINT')!; }, - get region() { + get region(): string | undefined { return Deno.env.get('S3_REGION')!; }, - get accessKey() { + get accessKey(): string | undefined { return Deno.env.get('S3_ACCESS_KEY'); }, - get secretKey() { + get secretKey(): string | undefined { return Deno.env.get('S3_SECRET_KEY'); }, - get bucket() { + get bucket(): string | undefined { return Deno.env.get('S3_BUCKET'); }, - get pathStyle() { + get pathStyle(): boolean | undefined { return optionalBooleanSchema.parse(Deno.env.get('S3_PATH_STYLE')); }, - get port() { + get port(): number | undefined { return optionalNumberSchema.parse(Deno.env.get('S3_PORT')); }, - get sessionToken() { + get sessionToken(): string | undefined { return Deno.env.get('S3_SESSION_TOKEN'); }, - get useSSL() { + get useSSL(): boolean | undefined { return optionalBooleanSchema.parse(Deno.env.get('S3_USE_SSL')); }, }; /** IPFS uploader configuration. */ static ipfs = { /** Base URL for private IPFS API calls. */ - get apiUrl() { + get apiUrl(): string { return Deno.env.get('IPFS_API_URL') || 'http://localhost:5001'; }, }; @@ -139,15 +141,15 @@ class Conf { return Deno.env.get('BLOSSOM_SERVERS')?.split(',') || ['https://blossom.primal.net/']; } /** Module to upload files with. */ - static get uploader() { + static get uploader(): string | undefined { return Deno.env.get('DITTO_UPLOADER'); } /** Location to use for local uploads. */ - static get uploadsDir() { + static get uploadsDir(): string | undefined { return Deno.env.get('UPLOADS_DIR') || 'data/uploads'; } /** Media base URL for uploads. */ - static get mediaDomain() { + static get mediaDomain(): string { const value = Deno.env.get('MEDIA_DOMAIN'); if (!value) { @@ -159,11 +161,11 @@ class Conf { return value; } /** Max upload size for files in number of bytes. Default 100MiB. */ - static get maxUploadSize() { + static get maxUploadSize(): number { return Number(Deno.env.get('MAX_UPLOAD_SIZE') || 100 * 1024 * 1024); } /** Usernames that regular users cannot sign up with. */ - static get forbiddenUsernames() { + static get forbiddenUsernames(): string[] { return Deno.env.get('FORBIDDEN_USERNAMES')?.split(',') || [ '_', 'admin', @@ -175,24 +177,20 @@ class Conf { } /** Proof-of-work configuration. */ static pow = { - get registrations() { + get registrations(): number { return Number(Deno.env.get('DITTO_POW_REGISTRATIONS') ?? 20); }, }; /** Domain of the Ditto server as a `URL` object, for easily grabbing the `hostname`, etc. */ - static get url() { + static get url(): URL { return new URL(Conf.localDomain); } /** Merges the path with the localDomain. */ static local(path: string): string { return mergePaths(Conf.localDomain, path); } - /** Get an external URL for the NIP-19 identifier. */ - static external(nip19: string): string { - return new URL(`/${nip19}`, Conf.externalDomain).toString(); - } /** URL to send Sentry errors to. */ - static get sentryDsn() { + static get sentryDsn(): string | undefined { return Deno.env.get('SENTRY_DSN'); } /** SQLite settings. */ diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 4408c607..d440b65c 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -121,8 +121,8 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< poll: null, quote: !event.quote ? null : await renderStatus(event.quote, { depth: depth + 1 }), quote_id: event.quote?.id ?? null, - uri: Conf.external(note), - url: Conf.external(note), + uri: Conf.local(`/${note}`), + url: Conf.local(`/${note}`), zapped: Boolean(zapEvent), pleroma: { emoji_reactions: reactions, From c3af8299f1b7f9b8dd5e947e5a1046bb6249ac1e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Jun 2024 23:51:50 -0500 Subject: [PATCH 75/77] Spread s3 config Fixes https://gitlab.com/soapbox-pub/ditto/-/issues/156 --- src/config.ts | 6 +++--- src/middleware/uploaderMiddleware.ts | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/config.ts b/src/config.ts index 6dc765ff..502544d8 100644 --- a/src/config.ts +++ b/src/config.ts @@ -98,10 +98,10 @@ class Conf { /** S3 media storage configuration. */ static s3 = { get endPoint(): string | undefined { - return Deno.env.get('S3_ENDPOINT')!; + return Deno.env.get('S3_ENDPOINT'); }, get region(): string | undefined { - return Deno.env.get('S3_REGION')!; + return Deno.env.get('S3_REGION'); }, get accessKey(): string | undefined { return Deno.env.get('S3_ACCESS_KEY'); @@ -145,7 +145,7 @@ class Conf { return Deno.env.get('DITTO_UPLOADER'); } /** Location to use for local uploads. */ - static get uploadsDir(): string | undefined { + static get uploadsDir(): string { return Deno.env.get('UPLOADS_DIR') || 'data/uploads'; } /** Media base URL for uploads. */ diff --git a/src/middleware/uploaderMiddleware.ts b/src/middleware/uploaderMiddleware.ts index 38e8aceb..96a47336 100644 --- a/src/middleware/uploaderMiddleware.ts +++ b/src/middleware/uploaderMiddleware.ts @@ -13,7 +13,20 @@ export const uploaderMiddleware: AppMiddleware = async (c, next) => { switch (Conf.uploader) { case 's3': - c.set('uploader', new S3Uploader(Conf.s3)); + c.set( + 'uploader', + new S3Uploader({ + accessKey: Conf.s3.accessKey, + bucket: Conf.s3.bucket, + endPoint: Conf.s3.endPoint!, + pathStyle: Conf.s3.pathStyle, + port: Conf.s3.port, + region: Conf.s3.region!, + secretKey: Conf.s3.secretKey, + sessionToken: Conf.s3.sessionToken, + useSSL: Conf.s3.useSSL, + }), + ); break; case 'ipfs': c.set('uploader', new IPFSUploader({ baseUrl: Conf.mediaDomain, apiUrl: Conf.ipfs.apiUrl, fetch: fetchWorker })); From dbacb6b63a98766a25badc7cd404a116963c139e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Jun 2024 00:14:43 -0500 Subject: [PATCH 76/77] setup: handle DATABASE_URL more transparently on re-runs --- scripts/setup.ts | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/scripts/setup.ts b/scripts/setup.ts index 4223d318..705f1e84 100644 --- a/scripts/setup.ts +++ b/scripts/setup.ts @@ -41,19 +41,24 @@ if (DITTO_NSEC) { const domain = await question('input', 'What is the domain of your instance? (eg ditto.pub)', Conf.url.host); vars.LOCAL_DOMAIN = `https://${domain}`; -const database = await question('list', 'Which database do you want to use?', ['postgres', 'sqlite']); -if (database === 'sqlite') { - const path = await question('input', 'Path to SQLite database', 'data/db.sqlite3'); - vars.DATABASE_URL = `sqlite://${path}`; -} -if (database === 'postgres') { - const url = nodeUrl.parse(Deno.env.get('DATABASE_URL') ?? 'postgres://ditto:ditto@localhost:5432/ditto'); - const host = await question('input', 'Postgres host', url.hostname); - const port = await question('input', 'Postgres port', url.port); - const user = await question('input', 'Postgres user', url.username); - const password = await question('input', 'Postgres password', url.password); - const database = await question('input', 'Postgres database', url.pathname.slice(1)); - vars.DATABASE_URL = `postgres://${user}:${password}@${host}:${port}/${database}`; +const DATABASE_URL = Deno.env.get('DATABASE_URL'); + +if (DATABASE_URL) { + vars.DATABASE_URL = await question('input', 'Database URL', DATABASE_URL); +} else { + const database = await question('list', 'Which database do you want to use?', ['postgres', 'sqlite']); + if (database === 'sqlite') { + const path = await question('input', 'Path to SQLite database', 'data/db.sqlite3'); + vars.DATABASE_URL = `sqlite://${path}`; + } + if (database === 'postgres') { + const host = await question('input', 'Postgres host', 'localhost'); + const port = await question('input', 'Postgres port', '5432'); + const user = await question('input', 'Postgres user', 'ditto'); + const password = await question('input', 'Postgres password', 'ditto'); + const database = await question('input', 'Postgres database', 'ditto'); + vars.DATABASE_URL = `postgres://${user}:${password}@${host}:${port}/${database}`; + } } vars.DITTO_UPLOADER = await question('list', 'How do you want to upload files?', [ From 198317119307b8bf4c5bcbc195a23de3e5163231 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Jun 2024 00:15:10 -0500 Subject: [PATCH 77/77] setup: removed unused node:url import --- scripts/setup.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/setup.ts b/scripts/setup.ts index 705f1e84..9a6d6f34 100644 --- a/scripts/setup.ts +++ b/scripts/setup.ts @@ -1,4 +1,3 @@ -import nodeUrl from 'node:url'; import { exists } from '@std/fs/exists'; import { generateSecretKey, nip19 } from 'nostr-tools'; import question from 'question-deno';