diff --git a/.goosehints b/.goosehints new file mode 100644 index 00000000..2d76b934 --- /dev/null +++ b/.goosehints @@ -0,0 +1,45 @@ +# Ditto + +This project is called Ditto, a self-hosted social media server written in TypeScript with Deno. It implements the [Nostr Protocol](https://raw.githubusercontent.com/nostr-protocol/nips/refs/heads/master/README.md), and parts of the [Mastodon API](https://docs.joinmastodon.org/methods/) and [Pleroma API](https://git.pleroma.social/pleroma/pleroma/-/raw/develop/docs/development/API/pleroma_api.md). + +## Project Structure + +Ditto is a monorepo with a `packages` directory. The main package is `packages/ditto`, and the main API definition is in `packages/ditto/app.ts`. + +## Deno, npm, and jsr + +Ditto uses Deno 2.x + +Dependencies are managed in `deno.json`, which are added with the `deno add` command. This command also updates the `deno.lock` file. npm packages can be added by using `deno add` and prefixing the package name with an `npm:` protocol. For example, `deno add npm:kysely` would add the `kysely` package from npm. + +[jsr](https://jsr.io/) is a modern alternative to npm. It's a completely different registry with different packages available. jsr packages can be added by using `deno add` and prefixing the package name with a `jsr:` protocol. For example, `deno add jsr:@std/assert` would add the `@std/assert` package from jsr. + +## Nostr + +Nostr is a decentralized social media protocol involving clients, relays, keys, and a unified Nostr event format. + +Specifications on Nostr are called "NIPs". NIP stands for "Nostr Implementation Possibilities". NIPs are numbered like `NIP-XX` where `XX` are two capitalized hexadecimal digits, eg `NIP-01` and `NIP-C7`. + +To learn about Nostr, use the fetch tool to read [NIP-01](https://raw.githubusercontent.com/nostr-protocol/nips/refs/heads/master/01.md). + +To read a specific NIP, construct the NIP URL following this template: `https://raw.githubusercontent.com/nostr-protocol/nips/refs/heads/master/{nip}.md` (replace `{nip}` in the URL template with the relevant NIP name, eg `07` for NIP-07, or `C7` for NIP-C7). Then use the fetch tool to read the URL. + +To read the definition of a specific kind, construct a URL following this template: `https://nostrbook.dev/kinds/{kind}.md` (replace `{kind}` in the template with the kind number, eg `https://nostrbook.dev/kinds/0.md` for kind 0). + +To discover the full list of NIPs, use the fetch tool to read the [NIPs README](https://raw.githubusercontent.com/nostr-protocol/nips/refs/heads/master/README.md). + +It's important that Ditto conforms to Nostr standards. Please read as much of the NIPs as you need to have a full understanding before adding or modifying Nostr events and filters. It is possible to add new ideas to Nostr that don't exist yet in the NIPs, but only after other options have been explored. Care must be taken when adding new Nostr ideas, to ensure they fit seamlessly within the existing Nostr ecosystem. + +## How Ditto uses Nostr and Mastodon API + +Ditto implements a full Nostr relay, available at `/relay` of the Ditto server. + +Mastodon API functionality, available at `/api/*`, is built around the Nostr relay's storage implementation. + +Ditto's goal is to enable Mastodon API clients to interact directly with Nostr. It achieves this by implementing most of Mastodon's API, and "pretending" to be a Mastodon server to client applications, while in actuality it uses Nostr as its decentralized protocol layer. + +## Testing Changes + +After making changes, please run `deno task check` to check for type errors. If there are any type errors, please try to fix them. + +Afterwards, run `deno fmt` to format the code, and then you are done. Please do not try to run the server, or run any other tests. \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index ba5b1bb4..8a761ba9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,5 +2,8 @@ "deno.enable": true, "deno.lint": true, "editor.defaultFormatter": "denoland.vscode-deno", - "path-intellisense.extensionOnImport": true + "path-intellisense.extensionOnImport": true, + "files.associations": { + ".goosehints": "markdown" + } } diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index bd86ac51..b4ecbaec 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -16,6 +16,7 @@ import { startSentry } from '@/sentry.ts'; import { DittoAPIStore } from '@/storages/DittoAPIStore.ts'; import { DittoPgStore } from '@/storages/DittoPgStore.ts'; import { DittoPool } from '@/storages/DittoPool.ts'; +import { createNip89 } from '@/utils/nip89.ts'; import { Time } from '@/utils/time.ts'; import { seedZapSplits } from '@/utils/zap-split.ts'; @@ -198,6 +199,7 @@ const pgstore = new DittoPgStore({ const pool = new DittoPool({ conf, relay: pgstore }); const relay = new DittoRelayStore({ db, conf, pool, relay: pgstore }); +await createNip89({ conf, relay }); await seedZapSplits({ conf, relay }); if (conf.firehoseEnabled) { diff --git a/packages/ditto/controllers/api/statuses.ts b/packages/ditto/controllers/api/statuses.ts index e80441b6..9d44ac56 100644 --- a/packages/ditto/controllers/api/statuses.ts +++ b/packages/ditto/controllers/api/statuses.ts @@ -17,6 +17,7 @@ import { languageSchema } from '@/schema.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { assertAuthenticated, createEvent, parseBody, updateListEvent } from '@/utils/api.ts'; import { getCustomEmojis } from '@/utils/custom-emoji.ts'; +import { getInstanceMetadata } from '@/utils/instance.ts'; import { getInvoice, getLnurl } from '@/utils/lnurl.ts'; import { purifyEvent } from '@/utils/purify.ts'; import { getZapSplits } from '@/utils/zap-split.ts'; @@ -25,6 +26,7 @@ import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts'; const createStatusSchema = z.object({ + disclose_client: z.boolean().nullish(), in_reply_to_id: n.id().nullish(), language: languageSchema.nullish(), media_ids: z.string().array().nullish(), @@ -265,6 +267,11 @@ const createStatusController: AppController = async (c) => { content += mediaUrls.join('\n'); } + if (data.disclose_client) { + const { name } = await getInstanceMetadata(c.var); + tags.push(['client', name, `31990:${await conf.signer.getPublicKey()}:ditto`, conf.relay]); + } + const event = await createEvent({ kind: 1, content, diff --git a/packages/ditto/controllers/nostr/relay-info.ts b/packages/ditto/controllers/nostr/relay-info.ts index 945e311b..f9df43d0 100644 --- a/packages/ditto/controllers/nostr/relay-info.ts +++ b/packages/ditto/controllers/nostr/relay-info.ts @@ -9,6 +9,8 @@ const relayInfoController: AppController = async (c) => { const meta = await getInstanceMetadata(c.var); c.res.headers.set('access-control-allow-origin', '*'); + c.res.headers.set('access-control-allow-headers', '*'); + c.res.headers.set('access-control-allow-methods', 'GET, POST, OPTIONS'); return c.json({ name: meta.name, @@ -16,7 +18,7 @@ const relayInfoController: AppController = async (c) => { pubkey: await conf.signer.getPublicKey(), contact: meta.email, supported_nips: [1, 5, 9, 11, 16, 45, 50, 46, 98], - software: 'Ditto', + software: 'https://gitlab.com/soapbox-pub/ditto', version: denoJson.version, limitation: { auth_required: false, diff --git a/packages/ditto/controllers/nostr/relay.ts b/packages/ditto/controllers/nostr/relay.ts index bbec9b5c..af233c86 100644 --- a/packages/ditto/controllers/nostr/relay.ts +++ b/packages/ditto/controllers/nostr/relay.ts @@ -165,6 +165,8 @@ function connectStream(socket: WebSocket, ip: string | undefined, opts: ConnectS try { for await (const msg of relay.req(filters, { limit: 100, signal, timeout: conf.db.timeouts.relay })) { + if (!controllers.has(subId)) break; + if (msg[0] === 'EVENT') { const [, , event] = msg; send(['EVENT', subId, purifyEvent(event)]); @@ -186,8 +188,8 @@ function connectStream(socket: WebSocket, ip: string | undefined, opts: ConnectS send(['CLOSED', subId, 'error: something went wrong']); } } finally { - controllers.get(subId)?.abort(); controllers.delete(subId); + controller.abort(); } } diff --git a/packages/ditto/interfaces/DittoEvent.ts b/packages/ditto/interfaces/DittoEvent.ts index cfa81c60..4f85408b 100644 --- a/packages/ditto/interfaces/DittoEvent.ts +++ b/packages/ditto/interfaces/DittoEvent.ts @@ -59,4 +59,5 @@ export interface DittoEvent extends NostrEvent { language?: LanguageCode; /** Whether or not pubkey accepts cashu. */ accepts_zaps_cashu?: boolean; + client?: DittoEvent; } diff --git a/packages/ditto/storages/DittoPgStore.ts b/packages/ditto/storages/DittoPgStore.ts index e8a969be..5152ec09 100644 --- a/packages/ditto/storages/DittoPgStore.ts +++ b/packages/ditto/storages/DittoPgStore.ts @@ -83,6 +83,7 @@ export class DittoPgStore extends NPostgres { /** Conditions for when to index certain tags. */ static tagConditions: Record = { 'a': ({ count }) => count < 15, + 'client': ({ count, value }) => count === 0 && value.length < 50, 'd': ({ event, count }) => count === 0 && NKinds.parameterizedReplaceable(event.kind), 'e': DittoPgStore.eTagCondition, 'k': ({ count }) => count < 3, @@ -324,7 +325,7 @@ export class DittoPgStore extends NPostgres { machina.push(['EOSE', subId]); }).catch((error) => { - if (error instanceof Error && error.message.includes('timeout')) { + if (error instanceof Error && (error.name === 'TimeoutError' || error.message.includes('timeout'))) { machina.push(['CLOSED', subId, 'error: the relay could not respond fast enough']); } else { machina.push(['CLOSED', subId, 'error: something went wrong']); @@ -361,7 +362,7 @@ export class DittoPgStore extends NPostgres { yield msg; } } catch (e) { - if (e instanceof Error && e.name === 'AbortError') { + if (e instanceof Error && (e.name === 'TimeoutError' || e.message.includes('timeout'))) { yield ['CLOSED', subId, 'error: the relay could not respond fast enough']; } else { yield ['CLOSED', subId, 'error: something went wrong']; @@ -521,6 +522,12 @@ export class DittoPgStore extends NPostgres { } } + const client = event.tags.find(([name]) => name === 'client')?.[2]; + + if (client && /^31990:([0-9a-f]{64}):(.+)$/.test(client)) { + ext.client = client; + } + ext.protocol = event.tags.find(([name]) => name === 'proxy')?.[2] ?? 'nostr'; return ext; diff --git a/packages/ditto/storages/hydrate.ts b/packages/ditto/storages/hydrate.ts index de6c3a82..1d135044 100644 --- a/packages/ditto/storages/hydrate.ts +++ b/packages/ditto/storages/hydrate.ts @@ -1,6 +1,6 @@ import { DittoDB, DittoTables } from '@ditto/db'; import { DittoConf } from '@ditto/conf'; -import { NStore } from '@nostrify/nostrify'; +import { type NostrFilter, NStore } from '@nostrify/nostrify'; import { Kysely } from 'kysely'; import { matchFilter } from 'nostr-tools'; import { NSchema as n } from '@nostrify/nostrify'; @@ -54,6 +54,10 @@ async function hydrateEvents(opts: HydrateOpts): Promise { cache.push(event); } + for (const event of await gatherClients({ ...opts, events: cache })) { + cache.push(event); + } + const authorStats = await gatherAuthorStats(cache, db.kysely); const eventStats = await gatherEventStats(cache, db.kysely); @@ -132,6 +136,16 @@ export function assembleEvents( event.user = b.find((e) => matchFilter({ kinds: [30382], authors: [admin], '#d': [event.pubkey] }, e)); event.info = b.find((e) => matchFilter({ kinds: [30383], authors: [admin], '#d': [event.id] }, e)); + for (const [name, _value, addr] of event.tags) { + if (name === 'client' && addr) { + const match = addr.match(/^31990:([0-9a-f]{64}):(.+)$/); + if (match) { + const [, pubkey, d] = match; + event.client = b.find((e) => matchFilter({ kinds: [31990], authors: [pubkey], '#d': [d] }, e)); + } + } + } + if (event.kind === 1) { const id = findQuoteTag(event.tags)?.[1] || findQuoteInContent(event.content); if (id) { @@ -379,6 +393,28 @@ function gatherAcceptCashu({ events, relay, signal }: HydrateOpts): Promise { + const filters: NostrFilter[] = []; + + for (const event of events) { + for (const [name, _value, addr] of event.tags) { + if (name === 'client' && addr) { + const match = addr.match(/^31990:([0-9a-f]{64}):(.+)$/); + if (match) { + const [, pubkey, d] = match; + filters.push({ kinds: [31990], authors: [pubkey], '#d': [d], limit: 1 }); + } + } + } + } + + if (!filters.length) { + return Promise.resolve([]); + } + + return relay.query(filters, { signal }); +} + /** Collect author stats from the events. */ async function gatherAuthorStats( events: DittoEvent[], diff --git a/packages/ditto/utils/instance.ts b/packages/ditto/utils/instance.ts index 52fe7358..df021478 100644 --- a/packages/ditto/utils/instance.ts +++ b/packages/ditto/utils/instance.ts @@ -44,6 +44,7 @@ export async function getInstanceMetadata(opts: GetInstanceMetadataOpts): Promis tagline: meta.tagline ?? meta.about ?? 'Nostr community server', email: meta.email ?? `postmaster@${conf.url.host}`, picture: meta.picture ?? conf.local('/images/thumbnail.png'), + website: meta.website ?? conf.localDomain, event, screenshots: meta.screenshots ?? [], }; diff --git a/packages/ditto/utils/nip89.ts b/packages/ditto/utils/nip89.ts new file mode 100644 index 00000000..b0d7ba4b --- /dev/null +++ b/packages/ditto/utils/nip89.ts @@ -0,0 +1,35 @@ +import { DittoConf } from '@ditto/conf'; + +import { getInstanceMetadata } from '@/utils/instance.ts'; + +import type { NStore } from '@nostrify/nostrify'; + +interface CreateNip89Opts { + conf: DittoConf; + relay: NStore; + signal?: AbortSignal; +} + +/** + * Creates a NIP-89 application handler event (kind 31990) + * This identifies Ditto as a client that can handle various kinds of events + */ +export async function createNip89(opts: CreateNip89Opts): Promise { + const { conf, relay, signal } = opts; + + const { event: _, ...metadata } = await getInstanceMetadata(opts); + + const event = await conf.signer.signEvent({ + kind: 31990, + tags: [ + ['d', 'ditto'], + ['k', '1'], + ['t', 'ditto'], + ['web', conf.local('/'), 'web'], + ], + content: JSON.stringify(metadata), + created_at: Math.floor(Date.now() / 1000), + }); + + await relay.event(event, { signal }); +} diff --git a/packages/ditto/views/mastodon/statuses.ts b/packages/ditto/views/mastodon/statuses.ts index 1c91173c..e8a3b0ea 100644 --- a/packages/ditto/views/mastodon/statuses.ts +++ b/packages/ditto/views/mastodon/statuses.ts @@ -1,4 +1,4 @@ -import { NostrEvent, NStore } from '@nostrify/nostrify'; +import { NostrEvent, NSchema as n, NStore } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; import { Conf } from '@/config.ts'; @@ -120,11 +120,35 @@ async function renderStatus( return acc; }, [] as { name: string; count: number; me: boolean; url?: string }[]); + let application: MastodonStatus['application'] = undefined; + + if (event.client) { + const result = n.json().pipe(n.metadata()).safeParse(event.client.content); + if (result.success) { + const name = result.data.name ?? event.tags.find(([name]) => name === 'client')?.[1]; + if (name) { + application = { + name, + website: result.data.website ?? null, + }; + } + } + } else { + const name = event.tags.find(([name]) => name === 'client')?.[1]; + if (name) { + application = { + name, + website: null, + }; + } + } + const expiresAt = new Date(Number(event.tags.find(([name]) => name === 'expiration')?.[1]) * 1000); return { id: event.id, account, + application, card: event.event_stats?.link_preview ?? null, content: compatMentions + html, created_at: nostrDate(event.created_at).toISOString(), @@ -145,7 +169,6 @@ async function renderStatus( bookmarked: Boolean(bookmarkEvent), pinned: Boolean(pinEvent), reblog: null, - application: null, media_attachments: media .map((m) => renderAttachment({ tags: m })) .filter((m): m is MastodonAttachment => Boolean(m)), diff --git a/packages/mastoapi/types/MastodonStatus.ts b/packages/mastoapi/types/MastodonStatus.ts index 5196bfce..9eecd95b 100644 --- a/packages/mastoapi/types/MastodonStatus.ts +++ b/packages/mastoapi/types/MastodonStatus.ts @@ -5,6 +5,10 @@ import type { MastodonPreviewCard } from './MastodonPreviewCard.ts'; export interface MastodonStatus { id: string; account: MastodonAccount; + application?: { + name: string; + website: string | null; + }; card: MastodonPreviewCard | null; content: string; created_at: string; @@ -25,7 +29,6 @@ export interface MastodonStatus { bookmarked: boolean; pinned: boolean; reblog: MastodonStatus | null; - application: unknown; media_attachments: MastodonAttachment[]; mentions: unknown[]; tags: unknown[];