diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b754ff1e..c15e8907 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:2.2.0 +image: denoland/deno:2.2.2 default: interruptible: true @@ -12,7 +12,7 @@ test: - deno fmt --check - deno task lint - deno task check - - deno task test --coverage=cov_profile + - deno task test --ignore=packages/transcode --coverage=cov_profile - deno coverage cov_profile coverage: /All files[^\|]*\|[^\|]*\s+([\d\.]+)/ services: diff --git a/.tool-versions b/.tool-versions index f9adf79b..9cafcaae 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -deno 2.2.0 \ No newline at end of file +deno 2.2.2 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 0b8724a0..fc50c763 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM denoland/deno:2.2.0 +FROM denoland/deno:2.2.2 ENV PORT 5000 WORKDIR /app diff --git a/deno.json b/deno.json index c8a226af..33cb119c 100644 --- a/deno.json +++ b/deno.json @@ -11,6 +11,7 @@ "./packages/nip98", "./packages/policies", "./packages/ratelimiter", + "./packages/transcode", "./packages/translators", "./packages/uploaders" ], @@ -73,6 +74,7 @@ "@soapbox/logi": "jsr:@soapbox/logi@^0.3.0", "@soapbox/safe-fetch": "jsr:@soapbox/safe-fetch@^2.0.0", "@std/assert": "jsr:@std/assert@^0.225.1", + "@std/async": "jsr:@std/async@^1.0.10", "@std/cli": "jsr:@std/cli@^0.223.0", "@std/crypto": "jsr:@std/crypto@^0.224.0", "@std/encoding": "jsr:@std/encoding@^0.224.0", diff --git a/deno.lock b/deno.lock index 1f039c17..20e8a8f8 100644 --- a/deno.lock +++ b/deno.lock @@ -58,6 +58,7 @@ "jsr:@std/assert@^1.0.10": "1.0.11", "jsr:@std/assert@~0.213.1": "0.213.1", "jsr:@std/assert@~0.225.1": "0.225.3", + "jsr:@std/async@^1.0.10": "1.0.10", "jsr:@std/bytes@0.223": "0.223.0", "jsr:@std/bytes@0.224": "0.224.0", "jsr:@std/bytes@0.224.0": "0.224.0", @@ -604,6 +605,9 @@ "jsr:@std/internal@^1.0.5" ] }, + "@std/async@1.0.10": { + "integrity": "2ff1b1c7d33d1416159989b0f69e59ec7ee8cb58510df01e454def2108b3dbec" + }, "@std/bytes@0.223.0": { "integrity": "84b75052cd8680942c397c2631318772b295019098f40aac5c36cead4cba51a8" }, @@ -2489,6 +2493,7 @@ "jsr:@soapbox/logi@0.3", "jsr:@soapbox/safe-fetch@2", "jsr:@std/assert@~0.225.1", + "jsr:@std/async@^1.0.10", "jsr:@std/cli@0.223", "jsr:@std/crypto@0.224", "jsr:@std/encoding@0.224", diff --git a/packages/captcha/assets/bg/A Large Body of Water Surrounded By Mountains.jpg b/packages/captcha/assets/bg/a-large-body-of-water-surrounded-by-mountains.jpg similarity index 100% rename from packages/captcha/assets/bg/A Large Body of Water Surrounded By Mountains.jpg rename to packages/captcha/assets/bg/a-large-body-of-water-surrounded-by-mountains.jpg diff --git a/packages/captcha/assets/bg/A Trail of Footprints In The Sand.jpg b/packages/captcha/assets/bg/a-trail-of-footprints-in-the-sand.jpg similarity index 100% rename from packages/captcha/assets/bg/A Trail of Footprints In The Sand.jpg rename to packages/captcha/assets/bg/a-trail-of-footprints-in-the-sand.jpg diff --git a/packages/captcha/assets/bg/Ashim DSilva.jpg b/packages/captcha/assets/bg/ashim-dsilva.jpg similarity index 100% rename from packages/captcha/assets/bg/Ashim DSilva.jpg rename to packages/captcha/assets/bg/ashim-dsilva.jpg diff --git a/packages/captcha/assets/bg/Canazei Granite Ridges.jpg b/packages/captcha/assets/bg/canazei-granite-ridges.jpg similarity index 100% rename from packages/captcha/assets/bg/Canazei Granite Ridges.jpg rename to packages/captcha/assets/bg/canazei-granite-ridges.jpg diff --git a/packages/captcha/assets/bg/Martin Adams.jpg b/packages/captcha/assets/bg/martin-adams.jpg similarity index 100% rename from packages/captcha/assets/bg/Martin Adams.jpg rename to packages/captcha/assets/bg/martin-adams.jpg diff --git a/packages/captcha/assets/bg/Morskie Oko.jpg b/packages/captcha/assets/bg/morskie-oko.jpg similarity index 100% rename from packages/captcha/assets/bg/Morskie Oko.jpg rename to packages/captcha/assets/bg/morskie-oko.jpg diff --git a/packages/captcha/assets/bg/Mr. Lee.jpg b/packages/captcha/assets/bg/mr-lee.jpg similarity index 100% rename from packages/captcha/assets/bg/Mr. Lee.jpg rename to packages/captcha/assets/bg/mr-lee.jpg diff --git a/packages/captcha/assets/bg/Nattu Adnan.jpg b/packages/captcha/assets/bg/nattu-adnan.jpg similarity index 100% rename from packages/captcha/assets/bg/Nattu Adnan.jpg rename to packages/captcha/assets/bg/nattu-adnan.jpg diff --git a/packages/captcha/assets/bg/Photo by SpaceX.jpg b/packages/captcha/assets/bg/photo-by-spacex.jpg similarity index 100% rename from packages/captcha/assets/bg/Photo by SpaceX.jpg rename to packages/captcha/assets/bg/photo-by-spacex.jpg diff --git a/packages/captcha/assets/bg/Photo of Valley.jpg b/packages/captcha/assets/bg/photo-of-valley.jpg similarity index 100% rename from packages/captcha/assets/bg/Photo of Valley.jpg rename to packages/captcha/assets/bg/photo-of-valley.jpg diff --git a/packages/captcha/assets/bg/Snow-Capped Mountain.jpg b/packages/captcha/assets/bg/snow-capped-mountain.jpg similarity index 100% rename from packages/captcha/assets/bg/Snow-Capped Mountain.jpg rename to packages/captcha/assets/bg/snow-capped-mountain.jpg diff --git a/packages/captcha/assets/bg/Sunset by the Pier.jpg b/packages/captcha/assets/bg/sunset-by-the-pier.jpg similarity index 100% rename from packages/captcha/assets/bg/Sunset by the Pier.jpg rename to packages/captcha/assets/bg/sunset-by-the-pier.jpg diff --git a/packages/captcha/assets/bg/Tj Holowaychuk.jpg b/packages/captcha/assets/bg/tj-holowaychuk.jpg similarity index 100% rename from packages/captcha/assets/bg/Tj Holowaychuk.jpg rename to packages/captcha/assets/bg/tj-holowaychuk.jpg diff --git a/packages/captcha/assets/bg/Viktor Forgacs.jpg b/packages/captcha/assets/bg/viktor-forgacs.jpg similarity index 100% rename from packages/captcha/assets/bg/Viktor Forgacs.jpg rename to packages/captcha/assets/bg/viktor-forgacs.jpg diff --git a/packages/captcha/deno.json b/packages/captcha/deno.json index ce71ebf8..51bdec15 100644 --- a/packages/captcha/deno.json +++ b/packages/captcha/deno.json @@ -1,6 +1,6 @@ { "name": "@ditto/captcha", - "version": "1.0.0", + "version": "0.1.0", "exports": { ".": "./mod.ts" } diff --git a/packages/conf/DittoConf.ts b/packages/conf/DittoConf.ts index f775a861..576cc7e8 100644 --- a/packages/conf/DittoConf.ts +++ b/packages/conf/DittoConf.ts @@ -279,6 +279,11 @@ export class DittoConf { return optionalBooleanSchema.parse(this.env.get('MEDIA_ANALYZE')) ?? false; } + /** Whether to transcode uploaded video files with ffmpeg. */ + get mediaTranscode(): boolean { + return optionalBooleanSchema.parse(this.env.get('MEDIA_TRANSCODE')) ?? false; + } + /** Max upload size for files in number of bytes. Default 100MiB. */ get maxUploadSize(): number { return Number(this.env.get('MAX_UPLOAD_SIZE') || 100 * 1024 * 1024); @@ -480,4 +485,14 @@ export class DittoConf { get precheck(): boolean { return optionalBooleanSchema.parse(this.env.get('DITTO_PRECHECK')) ?? true; } + + /** Path to `ffmpeg` executable. */ + get ffmpegPath(): string { + return this.env.get('FFMPEG_PATH') || 'ffmpeg'; + } + + /** Path to `ffprobe` executable. */ + get ffprobePath(): string { + return this.env.get('FFPROBE_PATH') || 'ffprobe'; + } } diff --git a/packages/conf/deno.json b/packages/conf/deno.json index 7ba0a49a..06a202de 100644 --- a/packages/conf/deno.json +++ b/packages/conf/deno.json @@ -1,6 +1,6 @@ { "name": "@ditto/conf", - "version": "1.1.0", + "version": "0.1.0", "exports": { ".": "./mod.ts" } diff --git a/packages/db/adapters/TestDB.test.ts b/packages/db/adapters/TestDB.test.ts new file mode 100644 index 00000000..f2eb67c5 --- /dev/null +++ b/packages/db/adapters/TestDB.test.ts @@ -0,0 +1,25 @@ +import { DittoConf } from '@ditto/conf'; +import { NPostgres } from '@nostrify/db'; +import { genEvent } from '@nostrify/nostrify/test'; +import { assertEquals } from '@std/assert'; + +import { DittoPolyPg } from './DittoPolyPg.ts'; +import { TestDB } from './TestDB.ts'; + +Deno.test('TestDB', async () => { + const conf = new DittoConf(Deno.env); + const orig = new DittoPolyPg(conf.databaseUrl); + + await using db = new TestDB(orig); + await db.migrate(); + await db.clear(); + + const store = new NPostgres(orig.kysely); + await store.event(genEvent()); + + assertEquals((await store.count([{}])).count, 1); + + await db.clear(); + + assertEquals((await store.count([{}])).count, 0); +}); diff --git a/packages/db/adapters/TestDB.ts b/packages/db/adapters/TestDB.ts new file mode 100644 index 00000000..49f45a5f --- /dev/null +++ b/packages/db/adapters/TestDB.ts @@ -0,0 +1,49 @@ +import { type Kysely, sql } from 'kysely'; + +import type { DittoDB } from '../DittoDB.ts'; +import type { DittoTables } from '../DittoTables.ts'; + +/** Wraps another DittoDB implementation to clear all data when disposed. */ +export class TestDB implements DittoDB { + constructor(private db: DittoDB) {} + + get kysely(): Kysely { + return this.db.kysely; + } + + get poolSize(): number { + return this.db.poolSize; + } + + get availableConnections(): number { + return this.db.availableConnections; + } + + migrate(): Promise { + return this.db.migrate(); + } + + listen(channel: string, callback: (payload: string) => void): void { + return this.db.listen(channel, callback); + } + + /** Truncate all tables. */ + async clear(): Promise { + const query = sql<{ tablename: string }>`select tablename from pg_tables where schemaname = current_schema()`; + + const { rows } = await query.execute(this.db.kysely); + + for (const { tablename } of rows) { + if (tablename.startsWith('kysely_')) { + continue; // Skip Kysely's internal tables + } else { + await sql`truncate table ${sql.ref(tablename)} cascade`.execute(this.db.kysely); + } + } + } + + async [Symbol.asyncDispose](): Promise { + await this.clear(); + await this.db[Symbol.asyncDispose](); + } +} diff --git a/packages/db/deno.json b/packages/db/deno.json index 51570d2f..a98e202a 100644 --- a/packages/db/deno.json +++ b/packages/db/deno.json @@ -1,5 +1,6 @@ { "name": "@ditto/db", + "version": "0.1.0", "exports": { ".": "./mod.ts" } diff --git a/packages/db/mod.ts b/packages/db/mod.ts index 2766e524..ae50fff7 100644 --- a/packages/db/mod.ts +++ b/packages/db/mod.ts @@ -2,6 +2,7 @@ export { DittoPglite } from './adapters/DittoPglite.ts'; export { DittoPolyPg } from './adapters/DittoPolyPg.ts'; export { DittoPostgres } from './adapters/DittoPostgres.ts'; export { DummyDB } from './adapters/DummyDB.ts'; +export { TestDB } from './adapters/TestDB.ts'; export type { DittoDB } from './DittoDB.ts'; export type { DittoTables } from './DittoTables.ts'; diff --git a/packages/ditto/DittoPush.ts b/packages/ditto/DittoPush.ts index 3a378300..c1dded38 100644 --- a/packages/ditto/DittoPush.ts +++ b/packages/ditto/DittoPush.ts @@ -14,10 +14,10 @@ export class DittoPush { private server: Promise; constructor(opts: DittoPushOpts) { - const { conf, relay } = opts; + const { conf } = opts; this.server = (async () => { - const meta = await getInstanceMetadata(relay); + const meta = await getInstanceMetadata(opts); const keys = await conf.vapidKeys; if (keys) { diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index f89448c4..931b3825 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -1,5 +1,5 @@ import { DittoConf } from '@ditto/conf'; -import { DittoDB, DittoPolyPg } from '@ditto/db'; +import { DittoPolyPg } from '@ditto/db'; import { paginationMiddleware, tokenMiddleware, userMiddleware } from '@ditto/mastoapi/middleware'; import { DittoApp, type DittoEnv } from '@ditto/mastoapi/router'; import { relayPoolRelaysSizeGauge, relayPoolSubscriptionsSizeGauge } from '@ditto/metrics'; @@ -12,6 +12,7 @@ import { NostrEvent, NostrSigner, NRelay, NUploader } from '@nostrify/nostrify'; import { cron } from '@/cron.ts'; import { startFirehose } from '@/firehose.ts'; +import { startSentry } from '@/sentry.ts'; import { DittoAPIStore } from '@/storages/DittoAPIStore.ts'; import { DittoPgStore } from '@/storages/DittoPgStore.ts'; import { DittoPool } from '@/storages/DittoPool.ts'; @@ -54,8 +55,6 @@ import { adminSetRelaysController, deleteZapSplitsController, getZapSplitsController, - nameRequestController, - nameRequestsController, statusZapSplitsController, updateInstanceController, updateZapSplitsController, @@ -148,24 +147,20 @@ import { rateLimitMiddleware } from '@/middleware/rateLimitMiddleware.ts'; import { uploaderMiddleware } from '@/middleware/uploaderMiddleware.ts'; import { translatorMiddleware } from '@/middleware/translatorMiddleware.ts'; import { logiMiddleware } from '@/middleware/logiMiddleware.ts'; +import dittoNamesRoute from '@/routes/dittoNamesRoute.ts'; +import pleromaAdminPermissionGroupsRoute from '@/routes/pleromaAdminPermissionGroupsRoute.ts'; import { DittoRelayStore } from '@/storages/DittoRelayStore.ts'; export interface AppEnv extends DittoEnv { - Variables: { - conf: DittoConf; + Variables: DittoEnv['Variables'] & { /** Uploader for the user to upload files. */ uploader?: NUploader; /** NIP-98 signed event proving the pubkey is owned by the user. */ proof?: NostrEvent; - /** Kysely instance for the database. */ - db: DittoDB; - /** Base database store. No content filtering. */ - relay: NRelay; /** Normalized pagination params. */ pagination: { since?: number; until?: number; limit: number }; /** Translation service. */ translator?: DittoTranslator; - signal: AbortSignal; user?: { /** Signer to get the logged-in user's pubkey, relays, and to sign events, or `undefined` if the user isn't logged in. */ signer: NostrSigner; @@ -182,6 +177,8 @@ type AppController

= Handler({ max: 5000 }); diff --git a/packages/ditto/controllers/api/admin.ts b/packages/ditto/controllers/api/admin.ts index c09ba6fe..34e54ac2 100644 --- a/packages/ditto/controllers/api/admin.ts +++ b/packages/ditto/controllers/api/admin.ts @@ -128,7 +128,7 @@ const adminAccountActionSchema = z.object({ }); const adminActionController: AppController = async (c) => { - const { conf, relay, signal } = c.var; + const { conf, relay, requestId, signal } = c.var; const body = await parseBody(c.req.raw); const result = adminAccountActionSchema.safeParse(body); @@ -155,7 +155,7 @@ const adminActionController: AppController = async (c) => { n.disabled = true; n.suspended = true; relay.remove!([{ authors: [authorId] }]).catch((e: unknown) => { - logi({ level: 'error', ns: 'ditto.api.admin.account.action', type: data.type, error: errorJson(e) }); + logi({ level: 'error', ns: 'ditto.api.admin.account.action', type: data.type, requestId, error: errorJson(e) }); }); } if (data.type === 'revoke_name') { @@ -180,7 +180,7 @@ const adminActionController: AppController = async (c) => { return c.json({ error: 'Name grant not found' }, 404); } } catch (e) { - logi({ level: 'error', ns: 'ditto.api.admin.account.action', type: data.type, error: errorJson(e) }); + logi({ level: 'error', ns: 'ditto.api.admin.account.action', type: data.type, requestId, error: errorJson(e) }); return c.json({ error: 'Unexpected runtime error' }, 500); } } diff --git a/packages/ditto/controllers/api/cashu.ts b/packages/ditto/controllers/api/cashu.ts index 4546dda3..1989a569 100644 --- a/packages/ditto/controllers/api/cashu.ts +++ b/packages/ditto/controllers/api/cashu.ts @@ -103,7 +103,7 @@ route.put('/wallet', userMiddleware({ enc: 'nip44' }), async (c) => { /** Gets a wallet, if it exists. */ route.get('/wallet', userMiddleware({ enc: 'nip44' }), swapNutzapsMiddleware, async (c) => { - const { conf, relay, user, signal } = c.var; + const { conf, relay, user, signal, requestId } = c.var; const pubkey = await user.signer.getPublicKey(); @@ -139,7 +139,7 @@ route.get('/wallet', userMiddleware({ enc: 'nip44' }), swapNutzapsMiddleware, as return accumulator + current.amount; }, 0); } catch (e) { - logi({ level: 'error', ns: 'ditto.api.cashu.wallet.swap', error: errorJson(e) }); + logi({ level: 'error', ns: 'ditto.api.cashu.wallet.swap', requestId, error: errorJson(e) }); } } diff --git a/packages/ditto/controllers/api/ditto.ts b/packages/ditto/controllers/api/ditto.ts index 38c72eb4..fb87c1b7 100644 --- a/packages/ditto/controllers/api/ditto.ts +++ b/packages/ditto/controllers/api/ditto.ts @@ -1,19 +1,17 @@ -import { paginated } from '@ditto/mastoapi/pagination'; -import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; +import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; import { z } from 'zod'; import { AppController } from '@/app.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { getAuthor } from '@/queries.ts'; import { addTag } from '@/utils/tags.ts'; -import { createEvent, parseBody, updateAdminEvent } from '@/utils/api.ts'; +import { parseBody, updateAdminEvent } from '@/utils/api.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; import { deleteTag } from '@/utils/tags.ts'; import { DittoZapSplits, getZapSplits } from '@/utils/zap-split.ts'; import { screenshotsSchema } from '@/schemas/nostr.ts'; -import { booleanParamSchema, percentageSchema } from '@/schema.ts'; +import { percentageSchema } from '@/schema.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; -import { renderNameRequest } from '@/views/ditto.ts'; import { accountFromPubkey } from '@/views/mastodon/accounts.ts'; import { renderAccount } from '@/views/mastodon/accounts.ts'; import { updateListAdminEvent } from '@/utils/api.ts'; @@ -81,102 +79,6 @@ function renderRelays(event: NostrEvent): RelayEntity[] { }, [] as RelayEntity[]); } -const nameRequestSchema = z.object({ - name: z.string().email(), - reason: z.string().max(500).optional(), -}); - -export const nameRequestController: AppController = async (c) => { - const { conf, relay, user } = c.var; - - const pubkey = await user!.signer.getPublicKey(); - const result = nameRequestSchema.safeParse(await c.req.json()); - - if (!result.success) { - return c.json({ error: 'Invalid username', schema: result.error }, 400); - } - - const { name, reason } = result.data; - - const [existing] = await relay.query([{ kinds: [3036], authors: [pubkey], '#r': [name.toLowerCase()], limit: 1 }]); - if (existing) { - return c.json({ error: 'Name request already exists' }, 400); - } - - const r: string[][] = [['r', name]]; - - if (name !== name.toLowerCase()) { - r.push(['r', name.toLowerCase()]); - } - - const event = await createEvent({ - kind: 3036, - content: reason, - tags: [ - ...r, - ['L', 'nip05.domain'], - ['l', name.split('@')[1], 'nip05.domain'], - ['p', await conf.signer.getPublicKey()], - ], - }, c); - - await hydrateEvents({ ...c.var, events: [event] }); - - const nameRequest = await renderNameRequest(event); - return c.json(nameRequest); -}; - -const nameRequestsSchema = z.object({ - approved: booleanParamSchema.optional(), - rejected: booleanParamSchema.optional(), -}); - -export const nameRequestsController: AppController = async (c) => { - const { conf, relay, user } = c.var; - const pubkey = await user!.signer.getPublicKey(); - - const params = c.get('pagination'); - const { approved, rejected } = nameRequestsSchema.parse(c.req.query()); - - const filter: NostrFilter = { - kinds: [30383], - authors: [await conf.signer.getPublicKey()], - '#k': ['3036'], - '#p': [pubkey], - ...params, - }; - - if (approved) { - filter['#n'] = ['approved']; - } - if (rejected) { - filter['#n'] = ['rejected']; - } - - const orig = await relay.query([filter]); - const ids = new Set(); - - for (const event of orig) { - const d = event.tags.find(([name]) => name === 'd')?.[1]; - if (d) { - ids.add(d); - } - } - - if (!ids.size) { - return c.json([]); - } - - const events = await relay.query([{ kinds: [3036], ids: [...ids], authors: [pubkey] }]) - .then((events) => hydrateEvents({ ...c.var, events })); - - const nameRequests = await Promise.all( - events.map((event) => renderNameRequest(event)), - ); - - return paginated(c, orig, nameRequests); -}; - const zapSplitSchema = z.record( n.id(), z.object({ @@ -186,7 +88,8 @@ const zapSplitSchema = z.record( ); export const updateZapSplitsController: AppController = async (c) => { - const { conf, relay } = c.var; + const { conf } = c.var; + const body = await parseBody(c.req.raw); const result = zapSplitSchema.safeParse(body); @@ -196,7 +99,7 @@ export const updateZapSplitsController: AppController = async (c) => { const adminPubkey = await conf.signer.getPublicKey(); - const dittoZapSplit = await getZapSplits(relay, adminPubkey); + const dittoZapSplit = await getZapSplits(adminPubkey, c.var); if (!dittoZapSplit) { return c.json({ error: 'Zap split not activated, restart the server.' }, 404); } @@ -223,7 +126,8 @@ export const updateZapSplitsController: AppController = async (c) => { const deleteZapSplitSchema = z.array(n.id()).min(1); export const deleteZapSplitsController: AppController = async (c) => { - const { conf, relay } = c.var; + const { conf } = c.var; + const body = await parseBody(c.req.raw); const result = deleteZapSplitSchema.safeParse(body); @@ -233,7 +137,7 @@ export const deleteZapSplitsController: AppController = async (c) => { const adminPubkey = await conf.signer.getPublicKey(); - const dittoZapSplit = await getZapSplits(relay, adminPubkey); + const dittoZapSplit = await getZapSplits(adminPubkey, c.var); if (!dittoZapSplit) { return c.json({ error: 'Zap split not activated, restart the server.' }, 404); } @@ -253,9 +157,9 @@ export const deleteZapSplitsController: AppController = async (c) => { }; export const getZapSplitsController: AppController = async (c) => { - const { conf, relay } = c.var; + const { conf } = c.var; - const dittoZapSplit: DittoZapSplits | undefined = await getZapSplits(relay, await conf.signer.getPublicKey()) ?? {}; + const dittoZapSplit: DittoZapSplits | undefined = await getZapSplits(await conf.signer.getPublicKey(), c.var) ?? {}; if (!dittoZapSplit) { return c.json({ error: 'Zap split not activated, restart the server.' }, 404); } @@ -325,7 +229,7 @@ const updateInstanceSchema = z.object({ }); export const updateInstanceController: AppController = async (c) => { - const { conf, relay, signal } = c.var; + const { conf } = c.var; const body = await parseBody(c.req.raw); const result = updateInstanceSchema.safeParse(body); @@ -335,7 +239,7 @@ export const updateInstanceController: AppController = async (c) => { return c.json(result.error, 422); } - const meta = await getInstanceMetadata(relay, signal); + const meta = await getInstanceMetadata(c.var); await updateAdminEvent( { kinds: [0], authors: [pubkey], limit: 1 }, diff --git a/packages/ditto/controllers/api/instance.ts b/packages/ditto/controllers/api/instance.ts index 1fb742e5..a887fe4d 100644 --- a/packages/ditto/controllers/api/instance.ts +++ b/packages/ditto/controllers/api/instance.ts @@ -15,9 +15,9 @@ const features = [ ]; const instanceV1Controller: AppController = async (c) => { - const { conf, relay, signal } = c.var; + const { conf } = c.var; const { host, protocol } = conf.url; - const meta = await getInstanceMetadata(relay, signal); + const meta = await getInstanceMetadata(c.var); /** Protocol to use for WebSocket URLs, depending on the protocol of the `LOCAL_DOMAIN`. */ const wsProtocol = protocol === 'http:' ? 'ws:' : 'wss:'; @@ -75,9 +75,9 @@ const instanceV1Controller: AppController = async (c) => { }; const instanceV2Controller: AppController = async (c) => { - const { conf, relay, signal } = c.var; + const { conf } = c.var; const { host, protocol } = conf.url; - const meta = await getInstanceMetadata(relay, signal); + const meta = await getInstanceMetadata(c.var); /** Protocol to use for WebSocket URLs, depending on the protocol of the `LOCAL_DOMAIN`. */ const wsProtocol = protocol === 'http:' ? 'ws:' : 'wss:'; @@ -164,9 +164,7 @@ const instanceV2Controller: AppController = async (c) => { }; const instanceDescriptionController: AppController = async (c) => { - const { relay, signal } = c.var; - - const meta = await getInstanceMetadata(relay, signal); + const meta = await getInstanceMetadata(c.var); return c.json({ content: meta.about, diff --git a/packages/ditto/controllers/api/media.ts b/packages/ditto/controllers/api/media.ts index c6c6b062..65660e99 100644 --- a/packages/ditto/controllers/api/media.ts +++ b/packages/ditto/controllers/api/media.ts @@ -21,7 +21,7 @@ const mediaUpdateSchema = z.object({ }); const mediaController: AppController = async (c) => { - const { user, signal } = c.var; + const { user, signal, requestId } = c.var; const pubkey = await user!.signer.getPublicKey(); const result = mediaBodySchema.safeParse(await parseBody(c.req.raw)); @@ -35,7 +35,7 @@ const mediaController: AppController = async (c) => { const media = await uploadFile(c, file, { pubkey, description }, signal); return c.json(renderAttachment(media)); } catch (e) { - logi({ level: 'error', ns: 'ditto.api.media', error: errorJson(e) }); + logi({ level: 'error', ns: 'ditto.api.media', requestId, error: errorJson(e) }); return c.json({ error: 'Failed to upload file.' }, 500); } }; diff --git a/packages/ditto/controllers/api/pleroma.ts b/packages/ditto/controllers/api/pleroma.ts index ef27696d..b9a5b561 100644 --- a/packages/ditto/controllers/api/pleroma.ts +++ b/packages/ditto/controllers/api/pleroma.ts @@ -1,15 +1,13 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; -import { configSchema, elixirTupleSchema } from '@/schemas/pleroma-api.ts'; import { createAdminEvent, updateAdminEvent, updateUser } from '@/utils/api.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; import { getPleromaConfigs } from '@/utils/pleroma.ts'; +import { configSchema, elixirTupleSchema } from '@/schemas/pleroma-api.ts'; const frontendConfigController: AppController = async (c) => { - const { relay, signal } = c.var; - - const configDB = await getPleromaConfigs(relay, signal); + const configDB = await getPleromaConfigs(c.var); const frontendConfig = configDB.get(':pleroma', ':frontend_configurations'); if (frontendConfig) { @@ -25,17 +23,15 @@ const frontendConfigController: AppController = async (c) => { }; const configController: AppController = async (c) => { - const { relay, signal } = c.var; - - const configs = await getPleromaConfigs(relay, signal); + const configs = await getPleromaConfigs(c.var); return c.json({ configs, need_reboot: false }); }; /** Pleroma admin config controller. */ const updateConfigController: AppController = async (c) => { - const { conf, relay, signal } = c.var; + const { conf } = c.var; - const configs = await getPleromaConfigs(relay, signal); + const configs = await getPleromaConfigs(c.var); const { configs: newConfigs } = z.object({ configs: z.array(configSchema) }).parse(await c.req.json()); configs.merge(newConfigs); diff --git a/packages/ditto/controllers/api/statuses.ts b/packages/ditto/controllers/api/statuses.ts index 8bc04151..53fab57a 100644 --- a/packages/ditto/controllers/api/statuses.ts +++ b/packages/ditto/controllers/api/statuses.ts @@ -196,7 +196,7 @@ const createStatusController: AppController = async (c) => { if (conf.zapSplitsEnabled) { const meta = n.json().pipe(n.metadata()).catch({}).parse(author?.content); const lnurl = getLnurl(meta); - const dittoZapSplit = await getZapSplits(relay, await conf.signer.getPublicKey()); + const dittoZapSplit = await getZapSplits(await conf.signer.getPublicKey(), c.var); if (lnurl && dittoZapSplit) { const totalSplit = Object.values(dittoZapSplit).reduce((total, { weight }) => total + weight, 0); for (const zapPubkey in dittoZapSplit) { diff --git a/packages/ditto/controllers/api/streaming.ts b/packages/ditto/controllers/api/streaming.ts index e6924641..4eebe6b3 100644 --- a/packages/ditto/controllers/api/streaming.ts +++ b/packages/ditto/controllers/api/streaming.ts @@ -65,7 +65,8 @@ const limiter = new TTLCache(); const connections = new Set(); const streamingController: AppController = async (c) => { - const { conf, relay, user } = c.var; + const { conf, relay, user, requestId } = c.var; + const upgrade = c.req.header('upgrade'); const token = c.req.header('sec-websocket-protocol'); const stream = streamSchema.optional().catch(undefined).parse(c.req.query('stream')); @@ -122,7 +123,7 @@ const streamingController: AppController = async (c) => { } } } catch (e) { - logi({ level: 'error', ns: 'ditto.streaming', msg: 'Error in streaming', error: errorJson(e) }); + logi({ level: 'error', ns: 'ditto.streaming', msg: 'Error in streaming', requestId, error: errorJson(e) }); } } diff --git a/packages/ditto/controllers/api/translate.ts b/packages/ditto/controllers/api/translate.ts index 65a95a26..ac62ddbe 100644 --- a/packages/ditto/controllers/api/translate.ts +++ b/packages/ditto/controllers/api/translate.ts @@ -17,7 +17,7 @@ const translateSchema = z.object({ }); const translateController: AppController = async (c) => { - const { relay, user, signal } = c.var; + const { relay, user, signal, requestId } = c.var; const result = translateSchema.safeParse(await parseBody(c.req.raw)); @@ -143,7 +143,7 @@ const translateController: AppController = async (c) => { if (e instanceof Error && e.message.includes('not supported')) { return c.json({ error: `Translation of source language '${event.language}' not supported` }, 422); } - logi({ level: 'error', ns: 'ditto.translate', error: errorJson(e) }); + logi({ level: 'error', ns: 'ditto.translate', requestId, error: errorJson(e) }); return c.json({ error: 'Service Unavailable' }, 503); } }; diff --git a/packages/ditto/controllers/error.ts b/packages/ditto/controllers/error.ts index a00a530b..42c2088f 100644 --- a/packages/ditto/controllers/error.ts +++ b/packages/ditto/controllers/error.ts @@ -4,7 +4,10 @@ import { logi } from '@soapbox/logi'; import { errorJson } from '@/utils/log.ts'; -export const errorHandler: ErrorHandler = (err, c) => { +import type { DittoEnv } from '@ditto/mastoapi/router'; + +export const errorHandler: ErrorHandler = (err, c) => { + const { requestId } = c.var; const { method } = c.req; const { pathname } = new URL(c.req.url); @@ -22,7 +25,15 @@ export const errorHandler: ErrorHandler = (err, c) => { return c.json({ error: 'The server was unable to respond in a timely manner' }, 500); } - logi({ level: 'error', ns: 'ditto.http', msg: 'Unhandled error', method, pathname, error: errorJson(err) }); + logi({ + level: 'error', + ns: 'ditto.http', + msg: 'Unhandled error', + method, + pathname, + requestId, + error: errorJson(err), + }); return c.json({ error: 'Something went wrong' }, 500); }; diff --git a/packages/ditto/controllers/frontend.ts b/packages/ditto/controllers/frontend.ts index ad98a9aa..bd2b4de3 100644 --- a/packages/ditto/controllers/frontend.ts +++ b/packages/ditto/controllers/frontend.ts @@ -14,6 +14,8 @@ import { renderAccount } from '@/views/mastodon/accounts.ts'; const META_PLACEHOLDER = '' as const; export const frontendController: AppMiddleware = async (c) => { + const { requestId } = c.var; + c.header('Cache-Control', 'max-age=86400, s-maxage=30, public, stale-if-error=604800'); try { @@ -26,7 +28,7 @@ export const frontendController: AppMiddleware = async (c) => { const meta = renderMetadata(c.req.url, entities); return c.html(content.replace(META_PLACEHOLDER, meta)); } catch (e) { - logi({ level: 'error', ns: 'ditto.frontend', msg: 'Error building meta tags', error: errorJson(e) }); + logi({ level: 'error', ns: 'ditto.frontend', msg: 'Error building meta tags', requestId, error: errorJson(e) }); return c.html(content); } } @@ -40,7 +42,7 @@ async function getEntities(c: AppContext, params: { acct?: string; statusId?: st const { relay } = c.var; const entities: MetadataEntities = { - instance: await getInstanceMetadata(relay), + instance: await getInstanceMetadata(c.var), }; if (params.statusId) { diff --git a/packages/ditto/controllers/manifest.ts b/packages/ditto/controllers/manifest.ts index 70d42dea..7d49ba1e 100644 --- a/packages/ditto/controllers/manifest.ts +++ b/packages/ditto/controllers/manifest.ts @@ -3,9 +3,7 @@ import { WebManifestCombined } from '@/types/webmanifest.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; export const manifestController: AppController = async (c) => { - const { relay, signal } = c.var; - - const meta = await getInstanceMetadata(relay, signal); + const meta = await getInstanceMetadata(c.var); const manifest: WebManifestCombined = { description: meta.about, diff --git a/packages/ditto/controllers/nostr/relay-info.ts b/packages/ditto/controllers/nostr/relay-info.ts index 50702c23..945e311b 100644 --- a/packages/ditto/controllers/nostr/relay-info.ts +++ b/packages/ditto/controllers/nostr/relay-info.ts @@ -4,9 +4,9 @@ import { AppController } from '@/app.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; const relayInfoController: AppController = async (c) => { - const { conf, relay, signal } = c.var; + const { conf } = c.var; - const meta = await getInstanceMetadata(relay, signal); + const meta = await getInstanceMetadata(c.var); c.res.headers.set('access-control-allow-origin', '*'); diff --git a/packages/ditto/controllers/nostr/relay.ts b/packages/ditto/controllers/nostr/relay.ts index f6641549..dea341e3 100644 --- a/packages/ditto/controllers/nostr/relay.ts +++ b/packages/ditto/controllers/nostr/relay.ts @@ -11,6 +11,7 @@ import { NostrClientMsg, NostrClientREQ, NostrRelayMsg, + NRelay, NSchema as n, } from '@nostrify/nostrify'; @@ -40,8 +41,17 @@ const limiters = { /** Connections for metrics purposes. */ const connections = new Set(); +interface ConnectStreamOpts { + conf: DittoConf; + relay: NRelay; + requestId: string; +} + /** Set up the Websocket connection. */ -function connectStream(conf: DittoConf, relay: DittoPgStore, socket: WebSocket, ip: string | undefined) { +function connectStream(socket: WebSocket, ip: string | undefined, opts: ConnectStreamOpts): void { + const { conf, requestId } = opts; + const relay = opts.relay as DittoPgStore; + const controllers = new Map(); if (ip) { @@ -74,7 +84,7 @@ function connectStream(conf: DittoConf, relay: DittoPgStore, socket: WebSocket, const msg = result.data; const verb = msg[0]; - logi({ level: 'trace', ns: 'ditto.relay.msg', verb, msg: msg as JsonValue, ip }); + logi({ level: 'trace', ns: 'ditto.relay.msg', verb, msg: msg as JsonValue, ip, requestId }); relayMessagesCounter.inc({ verb }); handleMsg(result.data); @@ -165,7 +175,7 @@ function connectStream(conf: DittoConf, relay: DittoPgStore, socket: WebSocket, send(['OK', event.id, false, e.message]); } else { send(['OK', event.id, false, 'error: something went wrong']); - logi({ level: 'error', ns: 'ditto.relay', msg: 'Error in relay', error: errorJson(e), ip }); + logi({ level: 'error', ns: 'ditto.relay', msg: 'Error in relay', error: errorJson(e), ip, requestId }); } } } @@ -195,7 +205,8 @@ function connectStream(conf: DittoConf, relay: DittoPgStore, socket: WebSocket, } const relayController: AppController = (c, next) => { - const { conf, relay } = c.var; + const { conf } = c.var; + const upgrade = c.req.header('upgrade'); // NIP-11: https://github.com/nostr-protocol/nips/blob/master/11.md @@ -214,7 +225,7 @@ const relayController: AppController = (c, next) => { } const { socket, response } = Deno.upgradeWebSocket(c.req.raw); - connectStream(conf, relay as DittoPgStore, socket, ip); + connectStream(socket, ip, c.var); return response; }; diff --git a/packages/ditto/controllers/well-known/nostr.ts b/packages/ditto/controllers/well-known/nostr.ts index 7c27aa70..7054d7cc 100644 --- a/packages/ditto/controllers/well-known/nostr.ts +++ b/packages/ditto/controllers/well-known/nostr.ts @@ -5,27 +5,23 @@ import { AppController } from '@/app.ts'; import { localNip05Lookup } from '@/utils/nip05.ts'; const nameSchema = z.string().min(1).regex(/^[\w.-]+$/); -const emptyResult: NostrJson = { names: {}, relays: {} }; /** * Serves NIP-05's nostr.json. * https://github.com/nostr-protocol/nips/blob/master/05.md */ const nostrController: AppController = async (c) => { - // If there are no query parameters, this will always return an empty result. - if (!Object.entries(c.req.queries()).length) { - c.header('Cache-Control', 'max-age=31536000, public, immutable, stale-while-revalidate=86400'); - return c.json(emptyResult); + const result = nameSchema.safeParse(c.req.query('name')); + + if (!result.success) { + return c.json({ error: 'Invalid name parameter' }, { status: 422 }); } - const result = nameSchema.safeParse(c.req.query('name')); - const name = result.success ? result.data : undefined; + const name = result.data; const pointer = name ? await localNip05Lookup(name, c.var) : undefined; - if (!name || !pointer) { - // Not found, cache for 5 minutes. - c.header('Cache-Control', 'max-age=300, public, stale-while-revalidate=30'); - return c.json(emptyResult); + if (!pointer) { + return c.json({ names: {}, relays: {} } satisfies NostrJson, { status: 404 }); } const { pubkey, relays = [] } = pointer; diff --git a/packages/ditto/deno.json b/packages/ditto/deno.json index 82d28139..fc75366b 100644 --- a/packages/ditto/deno.json +++ b/packages/ditto/deno.json @@ -1,5 +1,6 @@ { "name": "@ditto/ditto", + "version": "1.1.0", "exports": {}, "imports": { "@/": "./", diff --git a/packages/ditto/middleware/cspMiddleware.ts b/packages/ditto/middleware/cspMiddleware.ts index 8e890101..4417a3a1 100644 --- a/packages/ditto/middleware/cspMiddleware.ts +++ b/packages/ditto/middleware/cspMiddleware.ts @@ -9,7 +9,7 @@ export const cspMiddleware = (): AppMiddleware => { const { conf, relay } = c.var; if (!configDBCache) { - configDBCache = getPleromaConfigs(relay); + configDBCache = getPleromaConfigs({ conf, relay }); } const { host, protocol, origin } = conf.url; diff --git a/packages/ditto/middleware/logiMiddleware.ts b/packages/ditto/middleware/logiMiddleware.ts index be17e3bb..ad683022 100644 --- a/packages/ditto/middleware/logiMiddleware.ts +++ b/packages/ditto/middleware/logiMiddleware.ts @@ -1,11 +1,15 @@ -import { MiddlewareHandler } from '@hono/hono'; import { logi } from '@soapbox/logi'; -export const logiMiddleware: MiddlewareHandler = async (c, next) => { +import type { DittoMiddleware } from '@ditto/mastoapi/router'; + +export const logiMiddleware: DittoMiddleware = async (c, next) => { + const { requestId } = c.var; const { method } = c.req; const { pathname } = new URL(c.req.url); - logi({ level: 'info', ns: 'ditto.http.request', method, pathname }); + const ip = c.req.header('x-real-ip'); + + logi({ level: 'info', ns: 'ditto.http.request', method, pathname, ip, requestId }); const start = new Date(); @@ -15,5 +19,5 @@ export const logiMiddleware: MiddlewareHandler = async (c, next) => { const duration = (end.getTime() - start.getTime()) / 1000; const level = c.res.status >= 500 ? 'error' : 'info'; - logi({ level, ns: 'ditto.http.response', method, pathname, status: c.res.status, duration }); + logi({ level, ns: 'ditto.http.response', method, pathname, status: c.res.status, duration, ip, requestId }); }; diff --git a/packages/ditto/routes/dittoNamesRoute.test.ts b/packages/ditto/routes/dittoNamesRoute.test.ts new file mode 100644 index 00000000..9974b4a4 --- /dev/null +++ b/packages/ditto/routes/dittoNamesRoute.test.ts @@ -0,0 +1,59 @@ +import { TestApp } from '@ditto/mastoapi/test'; +import { assertEquals } from '@std/assert'; + +import route from './dittoNamesRoute.ts'; + +Deno.test('POST / creates a name request event', async () => { + await using app = new TestApp(route); + const { conf, relay } = app.var; + + const user = app.user(); + + const response = await app.api.post('/', { name: 'Alex@Ditto.pub', reason: 'for testing' }); + + assertEquals(response.status, 200); + + const [event] = await relay.query([{ kinds: [3036], authors: [await user.signer.getPublicKey()] }]); + + assertEquals(event?.tags, [ + ['r', 'Alex@Ditto.pub'], + ['r', 'alex@ditto.pub'], + ['L', 'nip05.domain'], + ['l', 'ditto.pub', 'nip05.domain'], + ['p', await conf.signer.getPublicKey()], + ]); + + assertEquals(event?.content, 'for testing'); +}); + +Deno.test('POST / can be called multiple times with the same name', async () => { + await using app = new TestApp(route); + + app.user(); + + const response1 = await app.api.post('/', { name: 'alex@ditto.pub' }); + const response2 = await app.api.post('/', { name: 'alex@ditto.pub' }); + + assertEquals(response1.status, 200); + assertEquals(response2.status, 200); +}); + +Deno.test('POST / returns 400 if the name has already been granted', async () => { + await using app = new TestApp(route); + const { conf, relay } = app.var; + + app.user(); + + const grant = await conf.signer.signEvent({ + kind: 30360, + tags: [['d', 'alex@ditto.pub']], + content: '', + created_at: 0, + }); + + await relay.event(grant); + + const response = await app.api.post('/', { name: 'alex@ditto.pub' }); + + assertEquals(response.status, 400); +}); diff --git a/packages/ditto/routes/dittoNamesRoute.ts b/packages/ditto/routes/dittoNamesRoute.ts new file mode 100644 index 00000000..8351be81 --- /dev/null +++ b/packages/ditto/routes/dittoNamesRoute.ts @@ -0,0 +1,130 @@ +import { paginationMiddleware, userMiddleware } from '@ditto/mastoapi/middleware'; +import { DittoRoute } from '@ditto/mastoapi/router'; +import { z } from 'zod'; + +import { createEvent } from '@/utils/api.ts'; +import { hydrateEvents } from '@/storages/hydrate.ts'; +import { renderNameRequest } from '@/views/ditto.ts'; +import { booleanParamSchema } from '@/schema.ts'; +import { NostrFilter } from '@nostrify/nostrify'; + +const nameRequestSchema = z.object({ + name: z.string().email(), + reason: z.string().max(500).optional(), +}); + +const route = new DittoRoute(); + +route.post('/', userMiddleware(), async (c) => { + const { conf, relay, user } = c.var; + + const result = nameRequestSchema.safeParse(await c.req.json()); + + if (!result.success) { + return c.json({ error: 'Invalid username', schema: result.error }, 422); + } + + const pubkey = await user.signer.getPublicKey(); + const adminPubkey = await conf.signer.getPublicKey(); + + const { name, reason } = result.data; + const [_localpart, domain] = name.split('@'); + + if (domain.toLowerCase() !== conf.url.host.toLowerCase()) { + return c.json({ error: 'Unsupported domain' }, 422); + } + + const d = name.toLowerCase(); + + const [grant] = await relay.query([{ kinds: [30360], authors: [adminPubkey], '#d': [d] }]); + if (grant) { + return c.json({ error: 'Name has already been granted' }, 400); + } + + const [pending] = await relay.query([{ + kinds: [30383], + authors: [adminPubkey], + '#p': [pubkey], + '#k': ['3036'], + '#r': [d], + '#n': ['pending'], + limit: 1, + }]); + if (pending) { + return c.json({ error: 'You have already requested that name, and it is pending approval by staff' }, 400); + } + + const tags: string[][] = [['r', name]]; + + if (name !== name.toLowerCase()) { + tags.push(['r', name.toLowerCase()]); + } + + const event = await createEvent({ + kind: 3036, + content: reason, + tags: [ + ...tags, + ['L', 'nip05.domain'], + ['l', domain.toLowerCase(), 'nip05.domain'], + ['p', await conf.signer.getPublicKey()], + ], + }, c); + + await hydrateEvents({ ...c.var, events: [event] }); + + const nameRequest = await renderNameRequest(event); + return c.json(nameRequest); +}); + +const nameRequestsSchema = z.object({ + approved: booleanParamSchema.optional(), + rejected: booleanParamSchema.optional(), +}); + +route.get('/', paginationMiddleware(), userMiddleware(), async (c) => { + const { conf, relay, user, pagination } = c.var; + const pubkey = await user!.signer.getPublicKey(); + + const { approved, rejected } = nameRequestsSchema.parse(c.req.query()); + + const filter: NostrFilter = { + kinds: [30383], + authors: [await conf.signer.getPublicKey()], + '#k': ['3036'], + '#p': [pubkey], + ...pagination, + }; + + if (approved) { + filter['#n'] = ['approved']; + } + if (rejected) { + filter['#n'] = ['rejected']; + } + + const orig = await relay.query([filter]); + const ids = new Set(); + + for (const event of orig) { + const d = event.tags.find(([name]) => name === 'd')?.[1]; + if (d) { + ids.add(d); + } + } + + if (!ids.size) { + return c.json([]); + } + + const events = await relay.query([{ kinds: [3036], ids: [...ids], authors: [pubkey] }]) + .then((events) => hydrateEvents({ ...c.var, events })); + + const nameRequests = await Promise.all( + events.map((event) => renderNameRequest(event)), + ); + + return c.var.paginate(orig, nameRequests); +}); + +export default route; diff --git a/packages/ditto/routes/pleromaAdminPermissionGroupsRoute.test.ts b/packages/ditto/routes/pleromaAdminPermissionGroupsRoute.test.ts new file mode 100644 index 00000000..84ad2e02 --- /dev/null +++ b/packages/ditto/routes/pleromaAdminPermissionGroupsRoute.test.ts @@ -0,0 +1,68 @@ +import { TestApp } from '@ditto/mastoapi/test'; +import { assertEquals } from '@std/assert'; +import { nip19 } from 'nostr-tools'; + +import route from './pleromaAdminPermissionGroupsRoute.ts'; + +Deno.test('POST /admin returns 403 if user is not an admin', async () => { + await using app = new TestApp(route); + + app.user(); + + const response = await app.api.post('/admin', { nicknames: ['alex@ditto.pub'] }); + + assertEquals(response.status, 403); +}); + +Deno.test('POST /admin promotes to admin', async () => { + await using app = new TestApp(route); + const { conf, relay } = app.var; + + await app.admin(); + + const pawn = app.createUser(); + const pubkey = await pawn.signer.getPublicKey(); + + const response = await app.api.post('/admin', { nicknames: [nip19.npubEncode(pubkey)] }); + const json = await response.json(); + + assertEquals(response.status, 200); + assertEquals(json, { is_admin: true }); + + const [event] = await relay.query([{ kinds: [30382], authors: [await conf.signer.getPublicKey()], '#d': [pubkey] }]); + + assertEquals(event.tags, [['d', pubkey], ['n', 'admin']]); +}); + +Deno.test('POST /moderator promotes to moderator', async () => { + await using app = new TestApp(route); + const { conf, relay } = app.var; + + await app.admin(); + + const pawn = app.createUser(); + const pubkey = await pawn.signer.getPublicKey(); + + const response = await app.api.post('/moderator', { nicknames: [nip19.npubEncode(pubkey)] }); + const json = await response.json(); + + assertEquals(response.status, 200); + assertEquals(json, { is_moderator: true }); + + const [event] = await relay.query([{ kinds: [30382], authors: [await conf.signer.getPublicKey()], '#d': [pubkey] }]); + + assertEquals(event.tags, [['d', pubkey], ['n', 'moderator']]); +}); + +Deno.test('POST /:group with an invalid group returns 422', async () => { + await using app = new TestApp(route); + + await app.admin(); + + const pawn = app.createUser(); + const pubkey = await pawn.signer.getPublicKey(); + + const response = await app.api.post('/yolo', { nicknames: [nip19.npubEncode(pubkey)] }); + + assertEquals(response.status, 422); +}); diff --git a/packages/ditto/routes/pleromaAdminPermissionGroupsRoute.ts b/packages/ditto/routes/pleromaAdminPermissionGroupsRoute.ts new file mode 100644 index 00000000..1e7665d0 --- /dev/null +++ b/packages/ditto/routes/pleromaAdminPermissionGroupsRoute.ts @@ -0,0 +1,40 @@ +import { userMiddleware } from '@ditto/mastoapi/middleware'; +import { DittoRoute } from '@ditto/mastoapi/router'; +import { z } from 'zod'; + +import { parseBody, updateUser } from '@/utils/api.ts'; +import { lookupPubkey } from '@/utils/lookup.ts'; + +const route = new DittoRoute(); + +const pleromaPromoteAdminSchema = z.object({ + nicknames: z.string().array(), +}); + +route.post('/:group', userMiddleware({ role: 'admin' }), async (c) => { + const body = await parseBody(c.req.raw); + const result = pleromaPromoteAdminSchema.safeParse(body); + const group = c.req.param('group'); + + if (!result.success) { + return c.json({ error: 'Bad request', schema: result.error }, 422); + } + + if (!['admin', 'moderator'].includes(group)) { + return c.json({ error: 'Bad request', schema: 'Invalid group' }, 422); + } + + const { data } = result; + const { nicknames } = data; + + for (const nickname of nicknames) { + const pubkey = await lookupPubkey(nickname, c.var); + if (pubkey) { + await updateUser(pubkey, { [group]: true }, c); + } + } + + return c.json({ [`is_${group}`]: true }, 200); +}); + +export default route; diff --git a/packages/ditto/sentry.ts b/packages/ditto/sentry.ts index 4875a12e..4e8c3c2b 100644 --- a/packages/ditto/sentry.ts +++ b/packages/ditto/sentry.ts @@ -1,15 +1,14 @@ import * as Sentry from '@sentry/deno'; import { logi } from '@soapbox/logi'; -import { Conf } from '@/config.ts'; +import type { DittoConf } from '@ditto/conf'; -// Sentry -if (Conf.sentryDsn) { - logi({ level: 'info', ns: 'ditto.sentry', msg: 'Sentry enabled.', enabled: true }); - Sentry.init({ - dsn: Conf.sentryDsn, - tracesSampleRate: 1.0, - }); -} else { - logi({ level: 'info', ns: 'ditto.sentry', msg: 'Sentry not configured. Skipping.', enabled: false }); +/** Start Sentry, if configured. */ +export function startSentry(conf: DittoConf): void { + if (conf.sentryDsn) { + logi({ level: 'info', ns: 'ditto.sentry', msg: 'Sentry enabled.', enabled: true }); + Sentry.init({ dsn: conf.sentryDsn }); + } else { + logi({ level: 'info', ns: 'ditto.sentry', msg: 'Sentry not configured. Skipping.', enabled: false }); + } } diff --git a/packages/ditto/server.ts b/packages/ditto/server.ts index 69dc1e02..0dad9fba 100644 --- a/packages/ditto/server.ts +++ b/packages/ditto/server.ts @@ -1,12 +1,12 @@ +import { DittoConf } from '@ditto/conf'; import { logi } from '@soapbox/logi'; -import '@/sentry.ts'; -import '@/nostr-wasm.ts'; import app from '@/app.ts'; -import { Conf } from '@/config.ts'; + +const conf = new DittoConf(Deno.env); Deno.serve({ - port: Conf.port, + port: conf.port, onListen({ hostname, port }): void { logi({ level: 'info', ns: 'ditto.server', msg: `Listening on http://${hostname}:${port}`, hostname, port }); }, diff --git a/packages/ditto/storages/DittoPgStore.test.ts b/packages/ditto/storages/DittoPgStore.test.ts index 405229dd..d243d0e4 100644 --- a/packages/ditto/storages/DittoPgStore.test.ts +++ b/packages/ditto/storages/DittoPgStore.test.ts @@ -5,7 +5,6 @@ import { generateSecretKey } from 'nostr-tools'; import { RelayError } from '@/RelayError.ts'; import { eventFixture } from '@/test.ts'; -import { Conf } from '@/config.ts'; import { DittoPgStore } from '@/storages/DittoPgStore.ts'; import { createTestDB } from '@/test.ts'; @@ -152,7 +151,7 @@ Deno.test("user cannot delete another user's event", async () => { Deno.test('admin can delete any event', async () => { await using db = await createTestDB({ pure: true }); - const { store } = db; + const { conf, store } = db; const sk = generateSecretKey(); @@ -168,7 +167,7 @@ Deno.test('admin can delete any event', async () => { assertEquals(await store.query([{ kinds: [1] }]), [two, one]); await store.event( - genEvent({ kind: 5, tags: [['e', one.id]] }, Conf.seckey), // admin sk + genEvent({ kind: 5, tags: [['e', one.id]] }, conf.seckey), // admin sk ); assertEquals(await store.query([{ kinds: [1] }]), [two]); @@ -176,12 +175,12 @@ Deno.test('admin can delete any event', async () => { Deno.test('throws a RelayError when inserting an event deleted by the admin', async () => { await using db = await createTestDB({ pure: true }); - const { store } = db; + const { conf, store } = db; const event = genEvent(); await store.event(event); - const deletion = genEvent({ kind: 5, tags: [['e', event.id]] }, Conf.seckey); + const deletion = genEvent({ kind: 5, tags: [['e', event.id]] }, conf.seckey); await store.event(deletion); await assertRejects( diff --git a/packages/ditto/storages/DittoPgStore.ts b/packages/ditto/storages/DittoPgStore.ts index ea3e864c..f73d1703 100644 --- a/packages/ditto/storages/DittoPgStore.ts +++ b/packages/ditto/storages/DittoPgStore.ts @@ -1,5 +1,6 @@ // deno-lint-ignore-file require-await +import { type DittoConf } from '@ditto/conf'; import { type DittoDB, type DittoTables } from '@ditto/db'; import { detectLanguage } from '@ditto/lang'; import { NPostgres, NPostgresSchema } from '@nostrify/db'; @@ -52,8 +53,8 @@ interface TagConditionOpts { interface DittoPgStoreOpts { /** Kysely instance to use. */ db: DittoDB; - /** Pubkey of the admin account. */ - pubkey: string; + /** Ditto configuration. */ + conf: DittoConf; /** Timeout in milliseconds for database queries. */ timeout?: number; /** Whether the event returned should be a Nostr event or a Ditto event. Defaults to false. */ @@ -169,9 +170,10 @@ export class DittoPgStore extends NPostgres { event: NostrEvent, opts: { signal?: AbortSignal; timeout?: number } = {}, ): Promise { + const { conf } = this.opts; try { await super.transaction(async (relay, kysely) => { - await updateStats({ event, relay, kysely: kysely as unknown as Kysely }); + await updateStats({ conf, relay, kysely: kysely as unknown as Kysely, event }); await relay.event(event, opts); }); } catch (e) { @@ -229,8 +231,11 @@ export class DittoPgStore extends NPostgres { /** Check if an event has been deleted by the admin. */ private async isDeletedAdmin(event: NostrEvent): Promise { + const { conf } = this.opts; + const adminPubkey = await conf.signer.getPublicKey(); + const filters: NostrFilter[] = [ - { kinds: [5], authors: [this.opts.pubkey], '#e': [event.id], limit: 1 }, + { kinds: [5], authors: [adminPubkey], '#e': [event.id], limit: 1 }, ]; if (NKinds.replaceable(event.kind) || NKinds.parameterizedReplaceable(event.kind)) { @@ -238,7 +243,7 @@ export class DittoPgStore extends NPostgres { filters.push({ kinds: [5], - authors: [this.opts.pubkey], + authors: [adminPubkey], '#a': [`${event.kind}:${event.pubkey}:${d}`], since: event.created_at, limit: 1, @@ -251,7 +256,10 @@ export class DittoPgStore extends NPostgres { /** The DITTO_NSEC can delete any event from the database. NDatabase already handles user deletions. */ private async deleteEventsAdmin(event: NostrEvent): Promise { - if (event.kind === 5 && event.pubkey === this.opts.pubkey) { + const { conf } = this.opts; + const adminPubkey = await conf.signer.getPublicKey(); + + if (event.kind === 5 && event.pubkey === adminPubkey) { const ids = new Set(event.tags.filter(([name]) => name === 'e').map(([_name, value]) => value)); const addrs = new Set(event.tags.filter(([name]) => name === 'a').map(([_name, value]) => value)); diff --git a/packages/ditto/storages/DittoRelayStore.test.ts b/packages/ditto/storages/DittoRelayStore.test.ts index b1a4cf10..cba7e164 100644 --- a/packages/ditto/storages/DittoRelayStore.test.ts +++ b/packages/ditto/storages/DittoRelayStore.test.ts @@ -2,6 +2,7 @@ import { DittoPolyPg } from '@ditto/db'; import { DittoConf } from '@ditto/conf'; import { genEvent, MockRelay } from '@nostrify/nostrify/test'; import { assertEquals } from '@std/assert'; +import { waitFor } from '@std/async/unstable-wait-for'; import { generateSecretKey, getPublicKey } from 'nostr-tools'; import { DittoRelayStore } from '@/storages/DittoRelayStore.ts'; @@ -9,6 +10,32 @@ import { DittoRelayStore } from '@/storages/DittoRelayStore.ts'; import type { NostrMetadata } from '@nostrify/types'; import { nostrNow } from '@/utils.ts'; +Deno.test('generates set event for nip05 request', async () => { + await using test = setupTest(); + + const admin = await test.conf.signer.getPublicKey(); + const event = genEvent({ kind: 3036, tags: [['r', 'alex@gleasonator.dev'], ['p', admin]] }); + + await test.store.event(event); + + const filter = { kinds: [30383], authors: [admin], '#d': [event.id] }; + + await waitFor(async () => { + const { count } = await test.store.count([filter]); + return count > 0; + }, 3000); + + const [result] = await test.store.query([filter]); + + assertEquals(result?.tags, [ + ['d', event.id], + ['p', event.pubkey], + ['k', '3036'], + ['r', 'alex@gleasonator.dev'], + ['n', 'pending'], + ]); +}); + Deno.test('updateAuthorData sets nip05', async () => { const alex = generateSecretKey(); @@ -81,14 +108,18 @@ Deno.test('Admin revokes nip05 grant and nip05 column gets null', async () => { assertEquals(row?.nip05_hostname, null); }); -function setupTest(cb: (req: Request) => Response | Promise) { +function setupTest(cb?: (req: Request) => Response | Promise) { const conf = new DittoConf(Deno.env); const db = new DittoPolyPg(conf.databaseUrl); const relay = new MockRelay(); const mockFetch: typeof fetch = async (input, init) => { const req = new Request(input, init); - return await cb(req); + if (cb) { + return await cb(req); + } else { + return new Response('Not mocked', { status: 404 }); + } }; const store = new DittoRelayStore({ conf, db, relay, fetch: mockFetch }); diff --git a/packages/ditto/storages/DittoRelayStore.ts b/packages/ditto/storages/DittoRelayStore.ts index e5fbb051..594a6fa4 100644 --- a/packages/ditto/storages/DittoRelayStore.ts +++ b/packages/ditto/storages/DittoRelayStore.ts @@ -366,7 +366,8 @@ export class DittoRelayStore implements NRelay { } private async prewarmLinkPreview(event: NostrEvent, signal?: AbortSignal): Promise { - const { firstUrl } = parseNoteContent(stripimeta(event.content, event.tags), []); + const { firstUrl } = parseNoteContent(stripimeta(event.content, event.tags), [], this.opts); + if (firstUrl) { await unfurlCardCached(firstUrl, signal); } @@ -399,19 +400,24 @@ export class DittoRelayStore implements NRelay { } if (event.kind === 3036 && tagsAdmin) { - const rel = await signer.signEvent({ - kind: 30383, - content: '', - tags: [ - ['d', event.id], - ['p', event.pubkey], - ['k', '3036'], - ['n', 'pending'], - ], - created_at: Math.floor(Date.now() / 1000), - }); + const r = event.tags.find(([name]) => name === 'r')?.[1]; - await this.event(rel, { signal: AbortSignal.timeout(1000) }); + if (r) { + const rel = await signer.signEvent({ + kind: 30383, + content: '', + tags: [ + ['d', event.id], + ['p', event.pubkey], + ['k', '3036'], + ['r', r.toLowerCase()], + ['n', 'pending'], + ], + created_at: Math.floor(Date.now() / 1000), + }); + + await this.event(rel, { signal: AbortSignal.timeout(1000) }); + } } } diff --git a/packages/ditto/storages/hydrate.ts b/packages/ditto/storages/hydrate.ts index a4dfe7ab..42d8b601 100644 --- a/packages/ditto/storages/hydrate.ts +++ b/packages/ditto/storages/hydrate.ts @@ -58,17 +58,19 @@ async function hydrateEvents(opts: HydrateOpts): Promise { return result; }, new Set()); - const favicons = ( - await db.kysely - .selectFrom('domain_favicons') - .select(['domain', 'favicon']) - .where('domain', 'in', [...domains]) - .execute() - ) - .reduce((result, { domain, favicon }) => { - result[domain] = favicon; - return result; - }, {} as Record); + const favicons: Record = domains.size + ? ( + await db.kysely + .selectFrom('domain_favicons') + .select(['domain', 'favicon']) + .where('domain', 'in', [...domains]) + .execute() + ) + .reduce((result, { domain, favicon }) => { + result[domain] = favicon; + return result; + }, {} as Record) + : {}; const stats = { authors: authorStats, diff --git a/packages/ditto/test.ts b/packages/ditto/test.ts index f8fd08d8..31b31e37 100644 --- a/packages/ditto/test.ts +++ b/packages/ditto/test.ts @@ -1,7 +1,7 @@ +import { DittoConf } from '@ditto/conf'; import { DittoPolyPg } from '@ditto/db'; import { NostrEvent } from '@nostrify/nostrify'; -import { Conf } from '@/config.ts'; import { DittoPgStore } from '@/storages/DittoPgStore.ts'; import { sql } from 'kysely'; @@ -13,13 +13,14 @@ export async function eventFixture(name: string): Promise { /** Create a database for testing. It uses `DATABASE_URL`, or creates an in-memory database by default. */ export async function createTestDB(opts?: { pure?: boolean }) { - const db = new DittoPolyPg(Conf.databaseUrl, { poolSize: 1 }); + const conf = new DittoConf(Deno.env); + const db = new DittoPolyPg(conf.databaseUrl, { poolSize: 1 }); await db.migrate(); const store = new DittoPgStore({ db, - timeout: Conf.db.timeouts.default, - pubkey: await Conf.signer.getPublicKey(), + conf, + timeout: conf.db.timeouts.default, pure: opts?.pure ?? false, notify: true, }); @@ -28,6 +29,7 @@ export async function createTestDB(opts?: { pure?: boolean }) { db, ...db, store, + conf, kysely: db.kysely, [Symbol.asyncDispose]: async () => { const { rows } = await sql< diff --git a/packages/ditto/utils/api.ts b/packages/ditto/utils/api.ts index b5d4fc3b..14a5bed5 100644 --- a/packages/ditto/utils/api.ts +++ b/packages/ditto/utils/api.ts @@ -27,10 +27,10 @@ async function createEvent( return createAdminEvent(fn(prev), c); } -function updateUser(pubkey: string, n: Record, c: AppContext): Promise { +function updateUser(pubkey: string, n: Record, c: Context): Promise { return updateNames(30382, pubkey, n, c); } diff --git a/packages/ditto/utils/instance.ts b/packages/ditto/utils/instance.ts index 3f746e07..52fe7358 100644 --- a/packages/ditto/utils/instance.ts +++ b/packages/ditto/utils/instance.ts @@ -1,9 +1,10 @@ import { NostrEvent, NostrMetadata, NSchema as n, NStore } from '@nostrify/nostrify'; import { z } from 'zod'; -import { Conf } from '@/config.ts'; import { screenshotsSchema, serverMetaSchema } from '@/schemas/nostr.ts'; +import type { DittoConf } from '@ditto/conf'; + /** Like NostrMetadata, but some fields are required and also contains some extra fields. */ export interface InstanceMetadata extends NostrMetadata { about: string; @@ -15,10 +16,18 @@ export interface InstanceMetadata extends NostrMetadata { screenshots: z.infer; } +interface GetInstanceMetadataOpts { + conf: DittoConf; + relay: NStore; + signal?: AbortSignal; +} + /** Get and parse instance metadata from the kind 0 of the admin user. */ -export async function getInstanceMetadata(store: NStore, signal?: AbortSignal): Promise { - const [event] = await store.query( - [{ kinds: [0], authors: [await Conf.signer.getPublicKey()], limit: 1 }], +export async function getInstanceMetadata(opts: GetInstanceMetadataOpts): Promise { + const { conf, relay, signal } = opts; + + const [event] = await relay.query( + [{ kinds: [0], authors: [await conf.signer.getPublicKey()], limit: 1 }], { signal }, ); @@ -33,8 +42,8 @@ export async function getInstanceMetadata(store: NStore, signal?: AbortSignal): name: meta.name ?? 'Ditto', about: meta.about ?? 'Nostr community server', tagline: meta.tagline ?? meta.about ?? 'Nostr community server', - email: meta.email ?? `postmaster@${Conf.url.host}`, - picture: meta.picture ?? Conf.local('/images/thumbnail.png'), + email: meta.email ?? `postmaster@${conf.url.host}`, + picture: meta.picture ?? conf.local('/images/thumbnail.png'), event, screenshots: meta.screenshots ?? [], }; diff --git a/packages/ditto/utils/note.test.ts b/packages/ditto/utils/note.test.ts index 699c4c5e..cdf29314 100644 --- a/packages/ditto/utils/note.test.ts +++ b/packages/ditto/utils/note.test.ts @@ -1,26 +1,35 @@ +import { DittoConf } from '@ditto/conf'; import { assertEquals } from '@std/assert'; import { eventFixture } from '@/test.ts'; import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts'; Deno.test('parseNoteContent', () => { - const { html, links, firstUrl } = parseNoteContent('Hello, world!', []); + const conf = new DittoConf(new Map()); + const { html, links, firstUrl } = parseNoteContent('Hello, world!', [], { conf }); + assertEquals(html, 'Hello, world!'); assertEquals(links, []); assertEquals(firstUrl, undefined); }); Deno.test('parseNoteContent parses URLs', () => { - const { html } = parseNoteContent('check out my website: https://alexgleason.me', []); + const conf = new DittoConf(new Map()); + const { html } = parseNoteContent('check out my website: https://alexgleason.me', [], { conf }); + assertEquals(html, 'check out my website: https://alexgleason.me'); }); Deno.test('parseNoteContent parses bare URLs', () => { - const { html } = parseNoteContent('have you seen ditto.pub?', []); + const conf = new DittoConf(new Map()); + const { html } = parseNoteContent('have you seen ditto.pub?', [], { conf }); + assertEquals(html, 'have you seen ditto.pub?'); }); Deno.test('parseNoteContent parses mentions with apostrophes', () => { + const conf = new DittoConf(new Map()); + const { html } = parseNoteContent( `did you see nostr:nprofile1qqsqgc0uhmxycvm5gwvn944c7yfxnnxm0nyh8tt62zhrvtd3xkj8fhgprdmhxue69uhkwmr9v9ek7mnpw3hhytnyv4mz7un9d3shjqgcwaehxw309ahx7umywf5hvefwv9c8qtmjv4kxz7gpzemhxue69uhhyetvv9ujumt0wd68ytnsw43z7s3al0v's speech?`, [{ @@ -29,7 +38,9 @@ Deno.test('parseNoteContent parses mentions with apostrophes', () => { acct: 'alex@gleasonator.dev', url: 'https://gleasonator.dev/@alex', }], + { conf }, ); + assertEquals( html, 'did you see @alex@gleasonator.dev's speech?', @@ -37,6 +48,8 @@ Deno.test('parseNoteContent parses mentions with apostrophes', () => { }); Deno.test('parseNoteContent parses mentions with commas', () => { + const conf = new DittoConf(new Map()); + const { html } = parseNoteContent( `Sim. Hi nostr:npub1q3sle0kvfsehgsuexttt3ugjd8xdklxfwwkh559wxckmzddywnws6cd26p and nostr:npub1gujeqakgt7fyp6zjggxhyy7ft623qtcaay5lkc8n8gkry4cvnrzqd3f67z, any chance to have Cobrafuma as PWA?`, [{ @@ -50,7 +63,9 @@ Deno.test('parseNoteContent parses mentions with commas', () => { acct: 'patrick@patrickdosreis.com', url: 'https://gleasonator.dev/@patrick@patrickdosreis.com', }], + { conf }, ); + assertEquals( html, 'Sim. Hi @alex@gleasonator.dev and @patrick@patrickdosreis.com, any chance to have Cobrafuma as PWA?', @@ -58,19 +73,26 @@ Deno.test('parseNoteContent parses mentions with commas', () => { }); Deno.test("parseNoteContent doesn't parse invalid nostr URIs", () => { - const { html } = parseNoteContent('nip19 has URIs like nostr:npub and nostr:nevent, etc.', []); + const conf = new DittoConf(new Map()); + const { html } = parseNoteContent('nip19 has URIs like nostr:npub and nostr:nevent, etc.', [], { conf }); assertEquals(html, 'nip19 has URIs like nostr:npub and nostr:nevent, etc.'); }); Deno.test('parseNoteContent renders empty for non-profile nostr URIs', () => { + const conf = new DittoConf(new Map()); + const { html } = parseNoteContent( 'nostr:nevent1qgsr9cvzwc652r4m83d86ykplrnm9dg5gwdvzzn8ameanlvut35wy3gpz3mhxue69uhhztnnwashymtnw3ezucm0d5qzqru8mkz2q4gzsxg99q7pdneyx7n8p5u0afe3ntapj4sryxxmg4gpcdvgce', [], + { conf }, ); + assertEquals(html, ''); }); Deno.test("parseNoteContent doesn't fuck up links to my own post", () => { + const conf = new DittoConf(new Map()); + const { html } = parseNoteContent( 'Check this post: https://gleasonator.dev/@alex@gleasonator.dev/posts/a8badb480d88f9e7b6a090342279ef47ed0e0a3989ed85f898dfedc6be94225f', [{ @@ -79,7 +101,9 @@ Deno.test("parseNoteContent doesn't fuck up links to my own post", () => { acct: 'alex@gleasonator.dev', url: 'https://gleasonator.dev/@alex', }], + { conf }, ); + assertEquals( html, 'Check this post: https://gleasonator.dev/@alex@gleasonator.dev/posts/a8badb480d88f9e7b6a090342279ef47ed0e0a3989ed85f898dfedc6be94225f', diff --git a/packages/ditto/utils/note.ts b/packages/ditto/utils/note.ts index a17a0f15..c51595f1 100644 --- a/packages/ditto/utils/note.ts +++ b/packages/ditto/utils/note.ts @@ -3,11 +3,11 @@ import linkifyStr from 'linkify-string'; import linkify from 'linkifyjs'; import { nip19, nip27 } from 'nostr-tools'; -import { Conf } from '@/config.ts'; import { html } from '@/utils/html.ts'; import { getUrlMediaType, isPermittedMediaType } from '@/utils/media.ts'; -import { MastodonMention } from '@ditto/mastoapi/types'; +import type { DittoConf } from '@ditto/conf'; +import type { MastodonMention } from '@ditto/mastoapi/types'; linkify.registerCustomProtocol('nostr', true); linkify.registerCustomProtocol('wss'); @@ -21,8 +21,14 @@ interface ParsedNoteContent { firstUrl: string | undefined; } +interface ParseNoteContentOpts { + conf: DittoConf; +} + /** Convert Nostr content to Mastodon API HTML. Also return parsed data. */ -function parseNoteContent(content: string, mentions: MastodonMention[]): ParsedNoteContent { +function parseNoteContent(content: string, mentions: MastodonMention[], opts: ParseNoteContentOpts): ParsedNoteContent { + const { conf } = opts; + const links = linkify.find(content).filter(({ type }) => type === 'url'); const firstUrl = links.find(isNonMediaLink)?.href; @@ -30,7 +36,7 @@ function parseNoteContent(content: string, mentions: MastodonMention[]): ParsedN render: { hashtag: ({ content }) => { const tag = content.replace(/^#/, ''); - const href = Conf.local(`/tags/${tag}`); + const href = conf.local(`/tags/${tag}`); return html``; }, url: ({ attributes, content }) => { @@ -49,7 +55,7 @@ function parseNoteContent(content: string, mentions: MastodonMention[]): ParsedN const npub = nip19.npubEncode(pubkey); const acct = mention?.acct ?? npub; const name = mention?.acct ?? npub.substring(0, 8); - const href = mention?.url ?? Conf.local(`/@${acct}`); + const href = mention?.url ?? conf.local(`/@${acct}`); return html`@${name}${extra}`; } else { return ''; diff --git a/packages/ditto/utils/pleroma.ts b/packages/ditto/utils/pleroma.ts index db3ca6a1..ae7588e3 100644 --- a/packages/ditto/utils/pleroma.ts +++ b/packages/ditto/utils/pleroma.ts @@ -1,14 +1,23 @@ import { NSchema as n, NStore } from '@nostrify/nostrify'; -import { Conf } from '@/config.ts'; import { configSchema } from '@/schemas/pleroma-api.ts'; import { PleromaConfigDB } from '@/utils/PleromaConfigDB.ts'; -export async function getPleromaConfigs(store: NStore, signal?: AbortSignal): Promise { - const signer = Conf.signer; +import type { DittoConf } from '@ditto/conf'; + +interface GetPleromaConfigsOpts { + conf: DittoConf; + relay: NStore; + signal?: AbortSignal; +} + +export async function getPleromaConfigs(opts: GetPleromaConfigsOpts): Promise { + const { conf, relay, signal } = opts; + + const signer = conf.signer; const pubkey = await signer.getPublicKey(); - const [event] = await store.query([{ + const [event] = await relay.query([{ kinds: [30078], authors: [pubkey], '#d': ['pub.ditto.pleroma.config'], diff --git a/packages/ditto/utils/stats.test.ts b/packages/ditto/utils/stats.test.ts index 043e6f13..2ebcab94 100644 --- a/packages/ditto/utils/stats.test.ts +++ b/packages/ditto/utils/stats.test.ts @@ -23,7 +23,7 @@ Deno.test('updateStats with kind 1 increments notes count', async () => { Deno.test('updateStats with kind 1 increments replies count', async () => { await using test = await setupTest(); - const { relay, kysely } = test; + const { kysely, relay } = test; const sk = generateSecretKey(); @@ -42,7 +42,7 @@ Deno.test('updateStats with kind 1 increments replies count', async () => { Deno.test('updateStats with kind 5 decrements notes count', async () => { await using test = await setupTest(); - const { relay, kysely } = test; + const { kysely, relay } = test; const sk = generateSecretKey(); const pubkey = getPublicKey(sk); @@ -74,7 +74,7 @@ Deno.test('updateStats with kind 3 increments followers count', async () => { Deno.test('updateStats with kind 3 decrements followers count', async () => { await using test = await setupTest(); - const { relay, kysely } = test; + const { kysely, relay } = test; const sk = generateSecretKey(); const follow = genEvent({ kind: 3, tags: [['p', 'alex']], created_at: 0 }, sk); @@ -101,7 +101,7 @@ Deno.test('getFollowDiff returns added and removed followers', () => { Deno.test('updateStats with kind 6 increments reposts count', async () => { await using test = await setupTest(); - const { relay, kysely } = test; + const { kysely, relay } = test; const note = genEvent({ kind: 1 }); await updateStats({ ...test, event: note }); @@ -118,7 +118,7 @@ Deno.test('updateStats with kind 6 increments reposts count', async () => { Deno.test('updateStats with kind 5 decrements reposts count', async () => { await using test = await setupTest(); - const { relay, kysely } = test; + const { kysely, relay } = test; const note = genEvent({ kind: 1 }); await updateStats({ ...test, event: note }); @@ -138,7 +138,7 @@ Deno.test('updateStats with kind 5 decrements reposts count', async () => { Deno.test('updateStats with kind 7 increments reactions count', async () => { await using test = await setupTest(); - const { relay, kysely } = test; + const { kysely, relay } = test; const note = genEvent({ kind: 1 }); await updateStats({ ...test, event: note }); @@ -155,7 +155,7 @@ Deno.test('updateStats with kind 7 increments reactions count', async () => { Deno.test('updateStats with kind 5 decrements reactions count', async () => { await using test = await setupTest(); - const { relay, kysely } = test; + const { kysely, relay } = test; const note = genEvent({ kind: 1 }); await updateStats({ ...test, event: note }); @@ -175,7 +175,7 @@ Deno.test('updateStats with kind 5 decrements reactions count', async () => { Deno.test('countAuthorStats counts author stats from the database', async () => { await using test = await setupTest(); - const { relay } = test; + const { kysely, relay } = test; const sk = generateSecretKey(); const pubkey = getPublicKey(sk); @@ -184,7 +184,7 @@ Deno.test('countAuthorStats counts author stats from the database', async () => await relay.event(genEvent({ kind: 1, content: 'yolo' }, sk)); await relay.event(genEvent({ kind: 3, tags: [['p', pubkey]] })); - await test.kysely.insertInto('author_stats').values({ + await kysely.insertInto('author_stats').values({ pubkey, search: 'Yolo Lolo', notes_count: 0, @@ -193,7 +193,7 @@ Deno.test('countAuthorStats counts author stats from the database', async () => }).onConflict((oc) => oc.column('pubkey').doUpdateSet({ 'search': 'baka' })) .execute(); - const stats = await countAuthorStats({ ...test, pubkey }); + const stats = await countAuthorStats({ ...test, kysely, pubkey }); assertEquals(stats!.notes_count, 2); assertEquals(stats!.followers_count, 1); @@ -206,9 +206,10 @@ async function setupTest() { await db.migrate(); const { kysely } = db; - const relay = new NPostgres(kysely); + const relay = new NPostgres(db.kysely); return { + conf, relay, kysely, [Symbol.asyncDispose]: async () => { diff --git a/packages/ditto/utils/stats.ts b/packages/ditto/utils/stats.ts index 448ba241..922d5dca 100644 --- a/packages/ditto/utils/stats.ts +++ b/packages/ditto/utils/stats.ts @@ -4,40 +4,46 @@ import { Insertable, Kysely, UpdateObject } from 'kysely'; import { SetRequired } from 'type-fest'; import { z } from 'zod'; -import { Conf } from '@/config.ts'; import { findQuoteTag, findReplyTag, getTagSet } from '@/utils/tags.ts'; +import type { DittoConf } from '@ditto/conf'; + interface UpdateStatsOpts { - kysely: Kysely; + conf: DittoConf; relay: NStore; + kysely: Kysely; event: NostrEvent; x?: 1 | -1; } /** Handle one event at a time and update relevant stats for it. */ // deno-lint-ignore require-await -export async function updateStats({ event, kysely, relay, x = 1 }: UpdateStatsOpts): Promise { +export async function updateStats(opts: UpdateStatsOpts): Promise { + const { event } = opts; + switch (event.kind) { case 1: case 20: case 1111: case 30023: - return handleEvent1(kysely, event, x); + return handleEvent1(opts); case 3: - return handleEvent3(kysely, event, x, relay); + return handleEvent3(opts); case 5: - return handleEvent5(kysely, event, -1, relay); + return handleEvent5(opts); case 6: - return handleEvent6(kysely, event, x); + return handleEvent6(opts); case 7: - return handleEvent7(kysely, event, x); + return handleEvent7(opts); case 9735: - return handleEvent9735(kysely, event); + return handleEvent9735(opts); } } /** Update stats for kind 1 event. */ -async function handleEvent1(kysely: Kysely, event: NostrEvent, x: number): Promise { +async function handleEvent1(opts: UpdateStatsOpts): Promise { + const { conf, kysely, event, x = 1 } = opts; + await updateAuthorStats(kysely, event.pubkey, (prev) => { const now = event.created_at; @@ -47,7 +53,7 @@ async function handleEvent1(kysely: Kysely, event: NostrEvent, x: n if (start && end) { // Streak exists. if (now <= end) { // Streak cannot go backwards in time. Skip it. - } else if (now - end > Conf.streakWindow) { + } else if (now - end > conf.streakWindow) { // Streak is broken. Start a new streak. start = now; end = now; @@ -88,7 +94,9 @@ async function handleEvent1(kysely: Kysely, event: NostrEvent, x: n } /** Update stats for kind 3 event. */ -async function handleEvent3(kysely: Kysely, event: NostrEvent, x: number, relay: NStore): Promise { +async function handleEvent3(opts: UpdateStatsOpts): Promise { + const { relay, kysely, event, x = 1 } = opts; + const following = getTagSet(event.tags, 'p'); await updateAuthorStats(kysely, event.pubkey, () => ({ following_count: following.size })); @@ -117,26 +125,34 @@ async function handleEvent3(kysely: Kysely, event: NostrEvent, x: n } /** Update stats for kind 5 event. */ -async function handleEvent5(kysely: Kysely, event: NostrEvent, x: -1, relay: NStore): Promise { +async function handleEvent5(opts: UpdateStatsOpts): Promise { + const { relay, event, x = -1 } = opts; + const id = event.tags.find(([name]) => name === 'e')?.[1]; + if (id) { const [target] = await relay.query([{ ids: [id], authors: [event.pubkey], limit: 1 }]); if (target) { - await updateStats({ event: target, kysely, relay, x }); + await updateStats({ ...opts, event: target, x }); } } } /** Update stats for kind 6 event. */ -async function handleEvent6(kysely: Kysely, event: NostrEvent, x: number): Promise { +async function handleEvent6(opts: UpdateStatsOpts): Promise { + const { kysely, event, x = 1 } = opts; + const id = event.tags.find(([name]) => name === 'e')?.[1]; + if (id) { await updateEventStats(kysely, id, ({ reposts_count }) => ({ reposts_count: Math.max(0, reposts_count + x) })); } } /** Update stats for kind 7 event. */ -async function handleEvent7(kysely: Kysely, event: NostrEvent, x: number): Promise { +async function handleEvent7(opts: UpdateStatsOpts): Promise { + const { kysely, event, x = 1 } = opts; + const id = event.tags.findLast(([name]) => name === 'e')?.[1]; const emoji = event.content; @@ -166,12 +182,15 @@ async function handleEvent7(kysely: Kysely, event: NostrEvent, x: n } /** Update stats for kind 9735 event. */ -async function handleEvent9735(kysely: Kysely, event: NostrEvent): Promise { +async function handleEvent9735(opts: UpdateStatsOpts): Promise { + const { kysely, event } = opts; + // https://github.com/nostr-protocol/nips/blob/master/57.md#appendix-f-validating-zap-receipts const id = event.tags.find(([name]) => name === 'e')?.[1]; if (!id) return; const amountSchema = z.coerce.number().int().nonnegative().catch(0); + let amount = 0; try { const zapRequest = n.json().pipe(n.event()).parse(event.tags.find(([name]) => name === 'description')?.[1]); diff --git a/packages/ditto/utils/upload.ts b/packages/ditto/utils/upload.ts index 6c160bb4..fc0316e8 100644 --- a/packages/ditto/utils/upload.ts +++ b/packages/ditto/utils/upload.ts @@ -1,3 +1,5 @@ +import { analyzeFile, extractVideoFrame, transcodeVideo } from '@ditto/transcode'; +import { ScopedPerformance } from '@esroyo/scoped-performance'; import { HTTPException } from '@hono/hono/http-exception'; import { logi } from '@soapbox/logi'; import { crypto } from '@std/crypto'; @@ -6,7 +8,6 @@ import { encode } from 'blurhash'; import sharp from 'sharp'; import { AppContext } from '@/app.ts'; -import { Conf } from '@/config.ts'; import { DittoUpload, dittoUploads } from '@/DittoUploads.ts'; import { errorJson } from '@/utils/log.ts'; @@ -22,7 +23,12 @@ export async function uploadFile( meta: FileMeta, signal?: AbortSignal, ): Promise { - const uploader = c.get('uploader'); + using perf = new ScopedPerformance(); + perf.mark('start'); + + const { conf, uploader } = c.var; + const { ffmpegPath, ffprobePath, mediaAnalyze, mediaTranscode } = conf; + if (!uploader) { throw new HTTPException(500, { res: c.json({ error: 'No uploader configured.' }), @@ -31,11 +37,47 @@ export async function uploadFile( const { pubkey, description } = meta; - if (file.size > Conf.maxUploadSize) { + if (file.size > conf.maxUploadSize) { throw new Error('File size is too large.'); } + const [baseType] = file.type.split('/'); + + perf.mark('probe-start'); + const probe = mediaTranscode ? await analyzeFile(file.stream(), { ffprobePath }).catch(() => null) : null; + const video = probe?.streams.find((stream) => stream.codec_type === 'video'); + perf.mark('probe-end'); + + perf.mark('transcode-start'); + if (baseType === 'video' && mediaTranscode) { + let needsTranscode = false; + + for (const stream of probe?.streams ?? []) { + if (stream.codec_type === 'video' && stream.codec_name !== 'h264') { + needsTranscode = true; + break; + } + if (stream.codec_type === 'audio' && stream.codec_name !== 'aac') { + needsTranscode = true; + break; + } + } + + if (needsTranscode) { + const tmp = new URL('file://' + await Deno.makeTempFile()); + await Deno.writeFile(tmp, file.stream()); + const stream = transcodeVideo(tmp, { ffmpegPath }); + const transcoded = await new Response(stream).bytes(); + file = new File([transcoded], file.name, { type: 'video/mp4' }); + await Deno.remove(tmp); + } + } + perf.mark('transcode-end'); + + perf.mark('upload-start'); const tags = await uploader.upload(file, { signal }); + perf.mark('upload-end'); + const url = tags[0][1]; if (description) { @@ -46,6 +88,8 @@ export async function uploadFile( const m = tags.find(([key]) => key === 'm')?.[1]; const dim = tags.find(([key]) => key === 'dim')?.[1]; const size = tags.find(([key]) => key === 'size')?.[1]; + const image = tags.find(([key]) => key === 'image')?.[1]; + const thumb = tags.find(([key]) => key === 'thumb')?.[1]; const blurhash = tags.find(([key]) => key === 'blurhash')?.[1]; if (!x) { @@ -61,34 +105,50 @@ export async function uploadFile( tags.push(['size', file.size.toString()]); } - // If the uploader didn't already, try to get a blurhash and media dimensions. - // This requires `MEDIA_ANALYZE=true` to be configured because it comes with security tradeoffs. - if (Conf.mediaAnalyze && (!blurhash || !dim)) { + perf.mark('analyze-start'); + + if (baseType === 'video' && mediaAnalyze && mediaTranscode && video && (!image || !thumb)) { try { - const bytes = await new Response(file.stream()).bytes(); - const img = sharp(bytes); + const tmp = new URL('file://' + await Deno.makeTempFile()); + await Deno.writeFile(tmp, file.stream()); + const frame = await extractVideoFrame(tmp, '00:00:01', { ffmpegPath }); + await Deno.remove(tmp); + const [[, url]] = await uploader.upload(new File([frame], 'thumb.jpg', { type: 'image/jpeg' }), { signal }); - const { width, height } = await img.metadata(); - - if (!dim && (width && height)) { - tags.push(['dim', `${width}x${height}`]); + if (!image) { + tags.push(['image', url]); } - if (!blurhash && (width && height)) { - const pixels = await img - .raw() - .ensureAlpha() - .toBuffer({ resolveWithObject: false }) - .then((buffer) => new Uint8ClampedArray(buffer)); + if (!dim) { + tags.push(['dim', await getImageDim(frame)]); + } - const blurhash = encode(pixels, width, height, 4, 4); - tags.push(['blurhash', blurhash]); + if (!blurhash) { + tags.push(['blurhash', await getBlurhash(frame)]); } } catch (e) { logi({ level: 'error', ns: 'ditto.upload.analyze', error: errorJson(e) }); } } + if (baseType === 'image' && mediaAnalyze && (!blurhash || !dim)) { + try { + const bytes = await new Response(file.stream()).bytes(); + + if (!dim) { + tags.push(['dim', await getImageDim(bytes)]); + } + + if (!blurhash) { + tags.push(['blurhash', await getBlurhash(bytes)]); + } + } catch (e) { + logi({ level: 'error', ns: 'ditto.upload.analyze', error: errorJson(e) }); + } + } + + perf.mark('analyze-end'); + const upload = { id: crypto.randomUUID(), url, @@ -99,5 +159,62 @@ export async function uploadFile( dittoUploads.set(upload.id, upload); + const timing = [ + perf.measure('probe', 'probe-start', 'probe-end'), + perf.measure('transcode', 'transcode-start', 'transcode-end'), + perf.measure('upload', 'upload-start', 'upload-end'), + perf.measure('analyze', 'analyze-start', 'analyze-end'), + ].reduce>((acc, m) => { + const name = m.name.split('::')[1]; // ScopedPerformance uses `::` to separate the name. + acc[name] = m.duration / 1000; // Convert to seconds for logging. + return acc; + }, {}); + + perf.mark('end'); + + logi({ + level: 'info', + ns: 'ditto.upload', + upload: { ...upload, uploadedAt: upload.uploadedAt.toISOString() }, + timing, + duration: perf.measure('total', 'start', 'end').duration / 1000, + }); + return upload; } + +async function getImageDim(bytes: Uint8Array): Promise<`${number}x${number}`> { + const img = sharp(bytes); + const { width, height } = await img.metadata(); + + if (!width || !height) { + throw new Error('Image metadata is missing.'); + } + + return `${width}x${height}`; +} + +/** Get a blurhash from an image file. */ +async function getBlurhash(bytes: Uint8Array, maxDim = 64): Promise { + const img = sharp(bytes); + + const { width, height } = await img.metadata(); + + if (!width || !height) { + throw new Error('Image metadata is missing.'); + } + + const { data, info } = await img + .raw() + .ensureAlpha() + .resize({ + width: width > height ? undefined : maxDim, + height: height > width ? undefined : maxDim, + fit: 'inside', + }) + .toBuffer({ resolveWithObject: true }); + + const pixels = new Uint8ClampedArray(data); + + return encode(pixels, info.width, info.height, 4, 4); +} diff --git a/packages/ditto/utils/zap-split.ts b/packages/ditto/utils/zap-split.ts index 85b6f056..a9d5ba2b 100644 --- a/packages/ditto/utils/zap-split.ts +++ b/packages/ditto/utils/zap-split.ts @@ -1,8 +1,9 @@ -import { Conf } from '@/config.ts'; import { NSchema as n, NStore } from '@nostrify/nostrify'; import { nostrNow } from '@/utils.ts'; import { percentageSchema } from '@/schema.ts'; +import type { DittoConf } from '@ditto/conf'; + type Pubkey = string; type ExtraMessage = string; /** Number from 1 to 100, stringified. */ @@ -12,11 +13,18 @@ export type DittoZapSplits = { [key: Pubkey]: { weight: splitPercentages; message: ExtraMessage }; }; +interface GetZapSplitsOpts { + conf: DittoConf; + relay: NStore; +} + /** Gets zap splits from NIP-78 in DittoZapSplits format. */ -export async function getZapSplits(store: NStore, pubkey: string): Promise { +export async function getZapSplits(pubkey: string, opts: GetZapSplitsOpts): Promise { + const { relay } = opts; + const zapSplits: DittoZapSplits = {}; - const [event] = await store.query([{ + const [event] = await relay.query([{ authors: [pubkey], kinds: [30078], '#d': ['pub.ditto.zapSplits'], @@ -36,15 +44,17 @@ export async function getZapSplits(store: NStore, pubkey: string): Promise { + const { conf, relay } = opts; + + const pubkey = await conf.signer.getPublicKey(); + const zapSplit: DittoZapSplits | undefined = await getZapSplits(pubkey, opts); if (!zapSplit) { const dittoPubkey = '781a1527055f74c1f70230f10384609b34548f8ab6a0a6caa74025827f9fdae5'; const dittoMsg = 'Official Ditto Account'; - const signer = Conf.signer; - const event = await signer.signEvent({ + const event = await conf.signer.signEvent({ content: '', created_at: nostrNow(), kind: 30078, @@ -54,6 +64,6 @@ export async function seedZapSplits(store: NStore) { ], }); - await store.event(event); + await relay.event(event); } } diff --git a/packages/ditto/views/mastodon/accounts.ts b/packages/ditto/views/mastodon/accounts.ts index 827b7921..4639ade3 100644 --- a/packages/ditto/views/mastodon/accounts.ts +++ b/packages/ditto/views/mastodon/accounts.ts @@ -48,7 +48,7 @@ function renderAccount(event: Omit, opts: ToAccountOpt const parsed05 = stats?.nip05 ? parseNip05(stats.nip05) : undefined; const acct = parsed05?.handle || npub; - const { html } = parseNoteContent(about || '', []); + const { html } = parseNoteContent(about || '', [], { conf: Conf }); const fields = _fields ?.slice(0, Conf.profileFields.maxFields) @@ -84,7 +84,7 @@ function renderAccount(event: Omit, opts: ToAccountOpt discoverable: true, display_name: name ?? '', emojis: renderEmojis(event), - fields: fields.map((field) => ({ ...field, value: parseNoteContent(field.value, []).html })), + fields: fields.map((field) => ({ ...field, value: parseNoteContent(field.value, [], { conf: Conf }).html })), follow_requests_count: 0, followers_count: stats?.followers_count ?? 0, following_count: stats?.following_count ?? 0, diff --git a/packages/ditto/views/mastodon/attachments.ts b/packages/ditto/views/mastodon/attachments.ts index b0d2e49c..1e24e794 100644 --- a/packages/ditto/views/mastodon/attachments.ts +++ b/packages/ditto/views/mastodon/attachments.ts @@ -14,6 +14,8 @@ function renderAttachment( const alt = tags.find(([name]) => name === 'alt')?.[1]; const cid = tags.find(([name]) => name === 'cid')?.[1]; const dim = tags.find(([name]) => name === 'dim')?.[1]; + const image = tags.find(([key]) => key === 'image')?.[1]; + const thumb = tags.find(([key]) => key === 'thumb')?.[1]; const blurhash = tags.find(([name]) => name === 'blurhash')?.[1]; if (!url) return; @@ -34,7 +36,7 @@ function renderAttachment( id: id ?? url, type: getAttachmentType(m ?? ''), url, - preview_url: url, + preview_url: image ?? thumb ?? url, remote_url: null, description: alt ?? '', blurhash: blurhash || null, diff --git a/packages/ditto/views/mastodon/statuses.ts b/packages/ditto/views/mastodon/statuses.ts index 55aa0808..065ac798 100644 --- a/packages/ditto/views/mastodon/statuses.ts +++ b/packages/ditto/views/mastodon/statuses.ts @@ -42,7 +42,7 @@ async function renderStatus( const mentions = event.mentions?.map((event) => renderMention(event)) ?? []; - const { html, links, firstUrl } = parseNoteContent(stripimeta(event.content, event.tags), mentions); + const { html, links, firstUrl } = parseNoteContent(stripimeta(event.content, event.tags), mentions, { conf: Conf }); const [card, relatedEvents] = await Promise .all([ diff --git a/packages/ditto/workers/deno-env.ts b/packages/ditto/workers/deno-env.ts deleted file mode 100644 index 1500eb57..00000000 --- a/packages/ditto/workers/deno-env.ts +++ /dev/null @@ -1,2 +0,0 @@ -// @ts-ignore Don't try to access the env from this worker. -Deno.env = new Map(); diff --git a/packages/ditto/workers/policy.test.ts b/packages/ditto/workers/policy.test.ts new file mode 100644 index 00000000..623e7c98 --- /dev/null +++ b/packages/ditto/workers/policy.test.ts @@ -0,0 +1,14 @@ +import { DittoConf } from '@ditto/conf'; +import { generateSecretKey, nip19 } from 'nostr-tools'; + +import { PolicyWorker } from './policy.ts'; + +Deno.test('PolicyWorker', () => { + const conf = new DittoConf( + new Map([ + ['DITTO_NSEC', nip19.nsecEncode(generateSecretKey())], + ]), + ); + + new PolicyWorker(conf); +}); diff --git a/packages/ditto/workers/policy.ts b/packages/ditto/workers/policy.ts index e2617f72..6cddd0a0 100644 --- a/packages/ditto/workers/policy.ts +++ b/packages/ditto/workers/policy.ts @@ -3,6 +3,8 @@ import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import * as Comlink from 'comlink'; +import { errorJson } from '@/utils/log.ts'; + import type { CustomPolicy } from '@/workers/policy.worker.ts'; export class PolicyWorker implements NPolicy { @@ -85,6 +87,15 @@ export class PolicyWorker implements NPolicy { return; } + logi({ + level: 'error', + ns: 'ditto.system.policy', + msg: 'Failed to load custom policy', + path: conf.policy, + error: errorJson(e), + enabled: false, + }); + throw new Error(`DITTO_POLICY (error importing policy): ${conf.policy}`); } } diff --git a/packages/ditto/workers/policy.worker.ts b/packages/ditto/workers/policy.worker.ts index b2ca3720..80966c2e 100644 --- a/packages/ditto/workers/policy.worker.ts +++ b/packages/ditto/workers/policy.worker.ts @@ -1,13 +1,16 @@ -import './deno-env.ts'; // HACK should be removed when `@/config.ts` is removed. - +import { DittoConf } from '@ditto/conf'; import { DittoPolyPg } from '@ditto/db'; import '@soapbox/safe-fetch/load'; import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify'; import { ReadOnlyPolicy } from '@nostrify/policies'; import * as Comlink from 'comlink'; +import { ReadOnlySigner } from '@/signers/ReadOnlySigner.ts'; import { DittoPgStore } from '@/storages/DittoPgStore.ts'; +// @ts-ignore Don't try to access the env from this worker. +Deno.env = new Map(); + /** Serializable object the worker can use to set up the state. */ interface PolicyInit { /** Path to the policy module (https, jsr, file, etc) */ @@ -31,9 +34,18 @@ export class CustomPolicy implements NPolicy { const db = new DittoPolyPg(databaseUrl, { poolSize: 1 }); + const conf = new Proxy(new DittoConf(new Map()), { + get(target, prop) { + if (prop === 'signer') { + return new ReadOnlySigner(pubkey); + } + return Reflect.get(target, prop); + }, + }); + const store = new DittoPgStore({ db, - pubkey, + conf, timeout: 5_000, }); diff --git a/packages/ditto/workers/verify.worker.ts b/packages/ditto/workers/verify.worker.ts index 3e71215d..e218474e 100644 --- a/packages/ditto/workers/verify.worker.ts +++ b/packages/ditto/workers/verify.worker.ts @@ -3,7 +3,6 @@ import * as Comlink from 'comlink'; import { VerifiedEvent, verifyEvent } from 'nostr-tools'; import '@/nostr-wasm.ts'; -import '@/sentry.ts'; export const VerifyWorker = { verifyEvent(event: NostrEvent): event is VerifiedEvent { diff --git a/packages/lang/deno.json b/packages/lang/deno.json index f192fb0f..09476a03 100644 --- a/packages/lang/deno.json +++ b/packages/lang/deno.json @@ -1,6 +1,6 @@ { "name": "@ditto/lang", - "version": "1.1.0", + "version": "0.1.0", "exports": { ".": "./language.ts" } diff --git a/packages/mastoapi/deno.json b/packages/mastoapi/deno.json index fc976655..d1451cf7 100644 --- a/packages/mastoapi/deno.json +++ b/packages/mastoapi/deno.json @@ -1,6 +1,6 @@ { "name": "@ditto/mastoapi", - "version": "1.1.0", + "version": "0.1.0", "exports": { "./middleware": "./middleware/mod.ts", "./pagination": "./pagination/mod.ts", diff --git a/packages/mastoapi/middleware/userMiddleware.test.ts b/packages/mastoapi/middleware/userMiddleware.test.ts index 2d30b0dc..fdd0a09a 100644 --- a/packages/mastoapi/middleware/userMiddleware.test.ts +++ b/packages/mastoapi/middleware/userMiddleware.test.ts @@ -1,21 +1,26 @@ -import { setUser, testApp } from '@ditto/mastoapi/test'; +import { TestApp } from '@ditto/mastoapi/test'; import { assertEquals } from '@std/assert'; import { userMiddleware } from './userMiddleware.ts'; import { ReadOnlySigner } from '../signers/ReadOnlySigner.ts'; Deno.test('no user 401', async () => { - const { app } = testApp(); + await using app = new TestApp(); const response = await app.use(userMiddleware()).request('/'); assertEquals(response.status, 401); }); Deno.test('unsupported signer 400', async () => { - const { app, relay } = testApp(); - const signer = new ReadOnlySigner('0461fcbecc4c3374439932d6b8f11269ccdb7cc973ad7a50ae362db135a474dd'); + await using app = new TestApp(); + + const user = { + signer: new ReadOnlySigner('0461fcbecc4c3374439932d6b8f11269ccdb7cc973ad7a50ae362db135a474dd'), + relay: app.var.relay, + }; + + app.user(user); const response = await app - .use(setUser({ signer, relay })) .use(userMiddleware({ enc: 'nip44' })) .use((c, next) => { c.var.user.signer.nip44.encrypt; // test that the type is set @@ -27,10 +32,11 @@ Deno.test('unsupported signer 400', async () => { }); Deno.test('with user 200', async () => { - const { app, user } = testApp(); + await using app = new TestApp(); + + app.user(); const response = await app - .use(setUser(user)) .use(userMiddleware()) .get('/', (c) => c.text('ok')) .request('/'); @@ -39,10 +45,11 @@ Deno.test('with user 200', async () => { }); Deno.test('user and role 403', async () => { - const { app, user } = testApp(); + await using app = new TestApp(); + + app.user(); const response = await app - .use(setUser(user)) .use(userMiddleware({ role: 'admin' })) .request('/'); @@ -50,7 +57,10 @@ Deno.test('user and role 403', async () => { }); Deno.test('admin role 200', async () => { - const { conf, app, user, relay } = testApp(); + await using app = new TestApp(); + const { conf, relay } = app.var; + + const user = app.user(); const event = await conf.signer.signEvent({ kind: 30382, @@ -65,7 +75,6 @@ Deno.test('admin role 200', async () => { await relay.event(event); const response = await app - .use(setUser(user)) .use(userMiddleware({ role: 'admin' })) .get('/', (c) => c.text('ok')) .request('/'); diff --git a/packages/mastoapi/router/DittoApp.test.ts b/packages/mastoapi/router/DittoApp.test.ts index c828d68a..f4a2bd44 100644 --- a/packages/mastoapi/router/DittoApp.test.ts +++ b/packages/mastoapi/router/DittoApp.test.ts @@ -1,13 +1,14 @@ import { DittoConf } from '@ditto/conf'; -import { DittoPolyPg } from '@ditto/db'; +import { DummyDB } from '@ditto/db'; import { Hono } from '@hono/hono'; import { MockRelay } from '@nostrify/nostrify/test'; +import { assertEquals } from '@std/assert'; import { DittoApp } from './DittoApp.ts'; import { DittoRoute } from './DittoRoute.ts'; Deno.test('DittoApp', async () => { - await using db = new DittoPolyPg('memory://'); + await using db = new DummyDB(); const conf = new DittoConf(new Map()); const relay = new MockRelay(); @@ -20,4 +21,11 @@ Deno.test('DittoApp', async () => { // @ts-expect-error Passing a non-DittoRoute to route. app.route('/', hono); + + app.get('/error', () => { + throw new Error('test error'); + }); + + const response = await app.request('/error'); + assertEquals(response.status, 500); }); diff --git a/packages/mastoapi/router/DittoApp.ts b/packages/mastoapi/router/DittoApp.ts index 2d3c0107..c541d46b 100644 --- a/packages/mastoapi/router/DittoApp.ts +++ b/packages/mastoapi/router/DittoApp.ts @@ -3,11 +3,13 @@ import { Hono } from '@hono/hono'; import type { HonoOptions } from '@hono/hono/hono-base'; import type { DittoEnv } from './DittoEnv.ts'; +export type DittoAppOpts = Omit & HonoOptions; + export class DittoApp extends Hono { // @ts-ignore Require a DittoRoute for type safety. declare route: (path: string, app: Hono) => Hono; - constructor(opts: Omit & HonoOptions) { + constructor(protected opts: DittoAppOpts) { super(opts); this.use((c, next) => { @@ -15,6 +17,7 @@ export class DittoApp extends Hono { c.set('conf', opts.conf); c.set('relay', opts.relay); c.set('signal', c.req.raw.signal); + c.set('requestId', c.req.header('X-Request-Id') ?? crypto.randomUUID()); return next(); }); } diff --git a/packages/mastoapi/router/DittoEnv.ts b/packages/mastoapi/router/DittoEnv.ts index 7f399e62..35fb0118 100644 --- a/packages/mastoapi/router/DittoEnv.ts +++ b/packages/mastoapi/router/DittoEnv.ts @@ -16,5 +16,7 @@ export interface DittoEnv extends Env { db: DittoDB; /** Abort signal for the request. */ signal: AbortSignal; + /** Unique ID for the request. */ + requestId: string; }; } diff --git a/packages/mastoapi/router/DittoMiddleware.ts b/packages/mastoapi/router/DittoMiddleware.ts index 1483ca90..91afd533 100644 --- a/packages/mastoapi/router/DittoMiddleware.ts +++ b/packages/mastoapi/router/DittoMiddleware.ts @@ -2,4 +2,4 @@ import type { MiddlewareHandler } from '@hono/hono'; import type { DittoEnv } from './DittoEnv.ts'; // deno-lint-ignore ban-types -export type DittoMiddleware = MiddlewareHandler; +export type DittoMiddleware = MiddlewareHandler; diff --git a/packages/mastoapi/router/DittoRoute.test.ts b/packages/mastoapi/router/DittoRoute.test.ts index 737019c4..7e48c8e2 100644 --- a/packages/mastoapi/router/DittoRoute.test.ts +++ b/packages/mastoapi/router/DittoRoute.test.ts @@ -1,12 +1,15 @@ -import { assertEquals } from '@std/assert'; +import { assertRejects } from '@std/assert'; import { DittoRoute } from './DittoRoute.ts'; Deno.test('DittoRoute', async () => { const route = new DittoRoute(); - const response = await route.request('/'); - const body = await response.json(); - assertEquals(response.status, 500); - assertEquals(body, { error: 'Missing required variable: db' }); + await assertRejects( + async () => { + await route.request('/'); + }, + Error, + 'Missing required variable: db', + ); }); diff --git a/packages/mastoapi/router/DittoRoute.ts b/packages/mastoapi/router/DittoRoute.ts index 369fb858..d4abb20c 100644 --- a/packages/mastoapi/router/DittoRoute.ts +++ b/packages/mastoapi/router/DittoRoute.ts @@ -25,6 +25,7 @@ export class DittoRoute extends Hono { if (!vars.conf) this.throwMissingVar('conf'); if (!vars.relay) this.throwMissingVar('relay'); if (!vars.signal) this.throwMissingVar('signal'); + if (!vars.requestId) this.throwMissingVar('requestId'); return { ...vars, @@ -32,11 +33,12 @@ export class DittoRoute extends Hono { conf: vars.conf, relay: vars.relay, signal: vars.signal, + requestId: vars.requestId, }; } private throwMissingVar(name: string): never { - throw new HTTPException(500, { message: `Missing required variable: ${name}` }); + throw new Error(`Missing required variable: ${name}`); } private _errorHandler: ErrorHandler = (error, c) => { @@ -48,6 +50,6 @@ export class DittoRoute extends Hono { } } - return c.json({ error: 'Something went wrong' }, 500); + throw error; }; } diff --git a/packages/mastoapi/test.ts b/packages/mastoapi/test.ts index 41e35c2c..7eef22b0 100644 --- a/packages/mastoapi/test.ts +++ b/packages/mastoapi/test.ts @@ -1,41 +1 @@ -import { DittoConf } from '@ditto/conf'; -import { type DittoDB, DummyDB } from '@ditto/db'; -import { DittoApp, type DittoMiddleware } from '@ditto/mastoapi/router'; -import { type NostrSigner, type NRelay, NSecSigner } from '@nostrify/nostrify'; -import { MockRelay } from '@nostrify/nostrify/test'; -import { generateSecretKey, nip19 } from 'nostr-tools'; - -import type { User } from '@ditto/mastoapi/middleware'; - -export function testApp(): { - app: DittoApp; - relay: NRelay; - conf: DittoConf; - db: DittoDB; - user: { - signer: NostrSigner; - relay: NRelay; - }; -} { - const db = new DummyDB(); - - const nsec = nip19.nsecEncode(generateSecretKey()); - const conf = new DittoConf(new Map([['DITTO_NSEC', nsec]])); - - const relay = new MockRelay(); - const app = new DittoApp({ conf, relay, db }); - - const user = { - signer: new NSecSigner(generateSecretKey()), - relay, - }; - - return { app, relay, conf, db, user }; -} - -export function setUser(user: User): DittoMiddleware<{ user: User }> { - return async (c, next) => { - c.set('user', user); - await next(); - }; -} +export { TestApp } from './test/TestApp.ts'; diff --git a/packages/mastoapi/test/TestApp.ts b/packages/mastoapi/test/TestApp.ts new file mode 100644 index 00000000..a12f48a4 --- /dev/null +++ b/packages/mastoapi/test/TestApp.ts @@ -0,0 +1,121 @@ +import { DittoConf } from '@ditto/conf'; +import { type DittoDB, DummyDB } from '@ditto/db'; +import { HTTPException } from '@hono/hono/http-exception'; +import { type NRelay, NSecSigner } from '@nostrify/nostrify'; +import { MockRelay } from '@nostrify/nostrify/test'; +import { generateSecretKey, nip19 } from 'nostr-tools'; + +import { DittoApp, type DittoAppOpts } from '../router/DittoApp.ts'; + +import type { Context } from '@hono/hono'; +import type { User } from '../middleware/User.ts'; +import type { DittoRoute } from '../router/DittoRoute.ts'; + +interface DittoVars { + db: DittoDB; + conf: DittoConf; + relay: NRelay; +} + +export class TestApp extends DittoApp implements AsyncDisposable { + private _user?: User; + + constructor(route?: DittoRoute, opts?: Partial) { + const nsec = nip19.nsecEncode(generateSecretKey()); + + const conf = opts?.conf ?? new DittoConf( + new Map([ + ['DITTO_NSEC', nsec], + ['LOCAL_DOMAIN', 'https://ditto.pub'], + ]), + ); + + const db = opts?.db ?? new DummyDB(); + const relay = opts?.relay ?? new MockRelay(); + + super({ + db, + conf, + relay, + ...opts, + }); + + this.use(async (c: Context<{ Variables: { user?: User } }>, next) => { + c.set('user', this._user); + await next(); + }); + + if (route) { + this.route('/', route); + } + + this.onError((err, c) => { + if (err instanceof HTTPException) { + if (err.res) { + return err.res; + } else { + return c.json({ error: err.message }, err.status); + } + } + + throw err; + }); + } + + get var(): DittoVars { + return { + db: this.opts.db, + conf: this.opts.conf, + relay: this.opts.relay, + }; + } + + async admin(user?: User): Promise { + const { conf, relay } = this.opts; + user ??= this.createUser(); + + const event = await conf.signer.signEvent({ + kind: 30382, + content: '', + tags: [ + ['d', await user.signer.getPublicKey()], + ['n', 'admin'], + ], + created_at: Math.floor(Date.now() / 1000), + }); + + await relay.event(event); + + return this.user(user); + } + + user(user?: User): User { + user ??= this.createUser(); + this._user = user; + return user; + } + + createUser(sk?: Uint8Array): User { + return { + relay: this.opts.relay, + signer: new NSecSigner(sk ?? generateSecretKey()), + }; + } + + api = { + get: async (path: string): Promise => { + return await this.request(path); + }, + post: async (path: string, body: unknown): Promise => { + return await this.request(path, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + }, + }; + + async [Symbol.asyncDispose](): Promise { + await this.opts.db[Symbol.asyncDispose](); + } +} diff --git a/packages/metrics/deno.json b/packages/metrics/deno.json index 12524c18..78733b38 100644 --- a/packages/metrics/deno.json +++ b/packages/metrics/deno.json @@ -1,5 +1,6 @@ { "name": "@ditto/metrics", + "version": "0.1.0", "exports": { ".": "./metrics.ts" } diff --git a/packages/nip98/deno.json b/packages/nip98/deno.json index 108e1bb8..b0a64f25 100644 --- a/packages/nip98/deno.json +++ b/packages/nip98/deno.json @@ -1,6 +1,6 @@ { "name": "@ditto/nip98", - "version": "1.0.0", + "version": "0.1.0", "exports": { ".": "./nip98.ts" } diff --git a/packages/policies/deno.json b/packages/policies/deno.json index ca190883..f27263fa 100644 --- a/packages/policies/deno.json +++ b/packages/policies/deno.json @@ -1,6 +1,6 @@ { "name": "@ditto/policies", - "version": "1.1.0", + "version": "0.1.0", "exports": { ".": "./mod.ts" } diff --git a/packages/ratelimiter/deno.json b/packages/ratelimiter/deno.json index 66e97171..4ba714ba 100644 --- a/packages/ratelimiter/deno.json +++ b/packages/ratelimiter/deno.json @@ -1,6 +1,6 @@ { "name": "@ditto/ratelimiter", - "version": "1.1.0", + "version": "0.1.0", "exports": { ".": "./mod.ts" } diff --git a/packages/transcode/.gitignore b/packages/transcode/.gitignore new file mode 100644 index 00000000..c0363794 --- /dev/null +++ b/packages/transcode/.gitignore @@ -0,0 +1 @@ +tmp/ \ No newline at end of file diff --git a/packages/transcode/analyze.test.ts b/packages/transcode/analyze.test.ts new file mode 100644 index 00000000..c1a23f5e --- /dev/null +++ b/packages/transcode/analyze.test.ts @@ -0,0 +1,13 @@ +import { assertObjectMatch } from '@std/assert'; + +import { analyzeFile } from './analyze.ts'; + +Deno.test('analyzeFile', async () => { + const uri = new URL('./buckbunny.mp4', import.meta.url); + + const { streams } = await analyzeFile(uri); + + const videoStream = streams.find((stream) => stream.codec_type === 'video')!; + + assertObjectMatch(videoStream, { width: 1920, height: 1080 }); +}); diff --git a/packages/transcode/analyze.ts b/packages/transcode/analyze.ts new file mode 100644 index 00000000..06f866f4 --- /dev/null +++ b/packages/transcode/analyze.ts @@ -0,0 +1,102 @@ +import { ffprobe } from './ffprobe.ts'; + +interface AnalyzeResult { + streams: Stream[]; + format: Format; +} + +interface Stream { + index: number; + codec_tag_string: string; + codec_tag: string; + codec_name?: string; + codec_long_name?: string; + profile?: string; + codec_type?: string; + width?: number; + height?: number; + coded_width?: number; + coded_height?: number; + closed_captions?: number; + has_b_frames?: number; + sample_aspect_ratio?: string; + display_aspect_ratio?: string; + pix_fmt?: string; + level?: number; + color_range?: string; + color_space?: string; + color_transfer?: string; + color_primaries?: string; + chroma_location?: string; + field_order?: string; + refs?: number; + sample_fmt?: string; + sample_rate?: string; + channels?: number; + channel_layout?: string; + bits_per_sample?: number; + id?: string; + r_frame_rate?: string; + avg_frame_rate?: string; + time_base?: string; + start_pts?: number; + start_time?: string; + duration_ts?: number; + duration?: string; + bit_rate?: string; + max_bit_rate?: string; + bits_per_raw_sample?: string; + nb_frames?: string; + nb_read_frames?: string; + nb_read_packets?: string; + disposition?: Disposition; + tags?: Record; +} + +interface Format { + filename: string; + nb_streams: number; + nb_programs: number; + format_name: string; + probe_score: number; + format_long_name?: string; + start_time?: string; + duration?: string; + size?: string; + bit_rate?: string; + tags?: Record; +} + +interface Disposition { + default: number; + dub: number; + original: number; + comment: number; + lyrics: number; + karaoke: number; + forced: number; + hearing_impaired: number; + visual_impaired: number; + clean_effects: number; + attached_pic: number; + timed_thumbnails: number; + captions: number; + descriptions: number; + metadata: number; + dependent: number; + still_image: number; +} + +export function analyzeFile( + input: URL | ReadableStream, + opts?: { ffprobePath?: string | URL }, +): Promise { + const stream = ffprobe(input, { + 'loglevel': 'fatal', + 'show_streams': '', + 'show_format': '', + 'of': 'json', + }, opts); + + return new Response(stream).json(); +} diff --git a/packages/transcode/buckbunny.mp4 b/packages/transcode/buckbunny.mp4 new file mode 100644 index 00000000..91fdbb8a Binary files /dev/null and b/packages/transcode/buckbunny.mp4 differ diff --git a/packages/transcode/deno.json b/packages/transcode/deno.json new file mode 100644 index 00000000..e302f9f1 --- /dev/null +++ b/packages/transcode/deno.json @@ -0,0 +1,7 @@ +{ + "name": "@ditto/transcode", + "version": "0.1.0", + "exports": { + ".": "./mod.ts" + } +} diff --git a/packages/transcode/ffmpeg.test.ts b/packages/transcode/ffmpeg.test.ts new file mode 100644 index 00000000..d93be547 --- /dev/null +++ b/packages/transcode/ffmpeg.test.ts @@ -0,0 +1,31 @@ +import { ffmpeg } from './ffmpeg.ts'; + +const uri = new URL('./buckbunny.mp4', import.meta.url); + +Deno.test('ffmpeg', async () => { + await using file = await Deno.open(uri); + + const output = ffmpeg(file.readable, { + 'c:v': 'libx264', + 'preset': 'veryfast', + 'loglevel': 'fatal', + 'movflags': 'frag_keyframe+empty_moov', + 'f': 'mp4', + }); + + await Deno.mkdir(new URL('./tmp', import.meta.url), { recursive: true }); + await Deno.writeFile(new URL('./tmp/transcoded-1.mp4', import.meta.url), output); +}); + +Deno.test('ffmpeg from file URI', async () => { + const output = ffmpeg(uri, { + 'c:v': 'libx264', + 'preset': 'veryfast', + 'loglevel': 'fatal', + 'movflags': 'frag_keyframe+empty_moov', + 'f': 'mp4', + }); + + await Deno.mkdir(new URL('./tmp', import.meta.url), { recursive: true }); + await Deno.writeFile(new URL('./tmp/transcoded-2.mp4', import.meta.url), output); +}); diff --git a/packages/transcode/ffmpeg.ts b/packages/transcode/ffmpeg.ts new file mode 100644 index 00000000..f2ebd5a5 --- /dev/null +++ b/packages/transcode/ffmpeg.ts @@ -0,0 +1,58 @@ +export interface FFmpegFlags { + 'safe'?: string; + 'nostdin'?: string; + 'c:v'?: string; + 'preset'?: string; + 'loglevel'?: string; + 'crf'?: string; + 'c:a'?: string; + 'b:a'?: string; + 'movflags'?: string; + 'f'?: string; + [key: string]: string | undefined; +} + +export function ffmpeg( + input: URL | ReadableStream, + flags: FFmpegFlags, + opts?: { ffmpegPath?: string | URL }, +): ReadableStream { + const { ffmpegPath = 'ffmpeg' } = opts ?? {}; + + const args = ['-i', input instanceof URL ? input.href : 'pipe:0']; + + for (const [key, value] of Object.entries(flags)) { + if (typeof value === 'string') { + if (value) { + args.push(`-${key}`, value); + } else { + args.push(`-${key}`); + } + } + } + + args.push('pipe:1'); // Output to stdout + + // Spawn the FFmpeg process + const command = new Deno.Command(ffmpegPath, { + args, + stdin: input instanceof ReadableStream ? 'piped' : 'null', + stdout: 'piped', + }); + + const child = command.spawn(); + + // Pipe the input stream into FFmpeg stdin and ensure completion + if (input instanceof ReadableStream) { + input.pipeTo(child.stdin).catch((e: unknown) => { + if (e instanceof Error && e.name === 'BrokenPipe') { + // Ignore. ffprobe closes the pipe once it has read the metadata. + } else { + throw e; + } + }); + } + + // Return the FFmpeg stdout stream + return child.stdout; +} diff --git a/packages/transcode/ffprobe.test.ts b/packages/transcode/ffprobe.test.ts new file mode 100644 index 00000000..953c6271 --- /dev/null +++ b/packages/transcode/ffprobe.test.ts @@ -0,0 +1,33 @@ +import { assertObjectMatch } from '@std/assert'; + +import { ffprobe } from './ffprobe.ts'; + +const uri = new URL('./buckbunny.mp4', import.meta.url); + +Deno.test('ffprobe from ReadableStream', async () => { + await using file = await Deno.open(uri); + + const stream = ffprobe(file.readable, { + 'v': 'error', + 'select_streams': 'v:0', + 'show_entries': 'stream=width,height', + 'of': 'json', + }); + + const { streams: [dimensions] } = await new Response(stream).json(); + + assertObjectMatch(dimensions, { width: 1920, height: 1080 }); +}); + +Deno.test('ffprobe from file URI', async () => { + const stream = ffprobe(uri, { + 'v': 'error', + 'select_streams': 'v:0', + 'show_entries': 'stream=width,height', + 'of': 'json', + }); + + const { streams: [dimensions] } = await new Response(stream).json(); + + assertObjectMatch(dimensions, { width: 1920, height: 1080 }); +}); diff --git a/packages/transcode/ffprobe.ts b/packages/transcode/ffprobe.ts new file mode 100644 index 00000000..7605cbe5 --- /dev/null +++ b/packages/transcode/ffprobe.ts @@ -0,0 +1,56 @@ +export interface FFprobeFlags { + 'v'?: string; + 'select_streams'?: string; + 'show_entries'?: string; + 'of'?: string; + [key: string]: string | undefined; +} + +export function ffprobe( + input: URL | ReadableStream, + flags: FFprobeFlags, + opts?: { ffprobePath?: string | URL }, +): ReadableStream { + const { ffprobePath = 'ffprobe' } = opts ?? {}; + + const args = []; + + for (const [key, value] of Object.entries(flags)) { + if (typeof value === 'string') { + if (value) { + args.push(`-${key}`, value); + } else { + args.push(`-${key}`); + } + } + } + + if (input instanceof URL) { + args.push('-i', input.href); + } else { + args.push('-i', 'pipe:0'); + } + + // Spawn the FFprobe process + const command = new Deno.Command(ffprobePath, { + args, + stdin: input instanceof ReadableStream ? 'piped' : 'null', + stdout: 'piped', + }); + + const child = command.spawn(); + + // Pipe the input stream into FFmpeg stdin and ensure completion + if (input instanceof ReadableStream) { + input.pipeTo(child.stdin).catch((e: unknown) => { + if (e instanceof Error && e.name === 'BrokenPipe') { + // Ignore. ffprobe closes the pipe once it has read the metadata. + } else { + throw e; + } + }); + } + + // Return the FFmpeg stdout stream + return child.stdout; +} diff --git a/packages/transcode/frame.test.ts b/packages/transcode/frame.test.ts new file mode 100644 index 00000000..c0710cfc --- /dev/null +++ b/packages/transcode/frame.test.ts @@ -0,0 +1,12 @@ +import { extractVideoFrame } from './frame.ts'; + +const uri = new URL('./buckbunny.mp4', import.meta.url); + +Deno.test('extractVideoFrame', async () => { + await using file = await Deno.open(uri); + + const result = await extractVideoFrame(file.readable); + + await Deno.mkdir(new URL('./tmp', import.meta.url), { recursive: true }); + await Deno.writeFile(new URL('./tmp/poster.jpg', import.meta.url), result); +}); diff --git a/packages/transcode/frame.ts b/packages/transcode/frame.ts new file mode 100644 index 00000000..d03ea63b --- /dev/null +++ b/packages/transcode/frame.ts @@ -0,0 +1,17 @@ +import { ffmpeg } from './ffmpeg.ts'; + +export function extractVideoFrame( + input: URL | ReadableStream, + ss: string = '00:00:01', + opts?: { ffmpegPath?: string | URL }, +): Promise { + const output = ffmpeg(input, { + 'ss': ss, // Seek to timestamp + 'frames:v': '1', // Extract only 1 frame + 'q:v': '2', // High-quality JPEG (lower = better quality) + 'f': 'image2', // Force image format + 'loglevel': 'fatal', + }, opts); + + return new Response(output).bytes(); +} diff --git a/packages/transcode/mod.ts b/packages/transcode/mod.ts new file mode 100644 index 00000000..8da45b0e --- /dev/null +++ b/packages/transcode/mod.ts @@ -0,0 +1,5 @@ +export { analyzeFile } from './analyze.ts'; +export { ffmpeg, type FFmpegFlags } from './ffmpeg.ts'; +export { ffprobe, type FFprobeFlags } from './ffprobe.ts'; +export { extractVideoFrame } from './frame.ts'; +export { transcodeVideo } from './transcode.ts'; diff --git a/packages/transcode/transcode.test.ts b/packages/transcode/transcode.test.ts new file mode 100644 index 00000000..971b4fb9 --- /dev/null +++ b/packages/transcode/transcode.test.ts @@ -0,0 +1,9 @@ +import { transcodeVideo } from './transcode.ts'; + +Deno.test('transcodeVideo', async () => { + await using file = await Deno.open(new URL('./buckbunny.mp4', import.meta.url)); + const output = transcodeVideo(file.readable); + + await Deno.mkdir(new URL('./tmp', import.meta.url), { recursive: true }); + await Deno.writeFile(new URL('./tmp/buckbunny-transcoded.mp4', import.meta.url), output); +}); diff --git a/packages/transcode/transcode.ts b/packages/transcode/transcode.ts new file mode 100644 index 00000000..d31cacb1 --- /dev/null +++ b/packages/transcode/transcode.ts @@ -0,0 +1,19 @@ +import { ffmpeg } from './ffmpeg.ts'; + +export function transcodeVideo( + input: URL | ReadableStream, + opts?: { ffmpegPath?: string | URL }, +): ReadableStream { + return ffmpeg(input, { + 'safe': '1', // Safe mode + 'nostdin': '', // Disable stdin + 'c:v': 'libx264', // Convert to H.264 + 'preset': 'veryfast', // Encoding speed + 'loglevel': 'fatal', // Suppress logs + 'crf': '23', // Compression level (lower = better quality) + 'c:a': 'aac', // Convert to AAC audio + 'b:a': '128k', // Audio bitrate + 'movflags': 'frag_keyframe+empty_moov', // Ensures MP4 streaming compatibility + 'f': 'mp4', // Force MP4 format + }, opts); +} diff --git a/packages/translators/LibreTranslateTranslator.ts b/packages/translators/LibreTranslateTranslator.ts index cc978e90..62c39e1c 100644 --- a/packages/translators/LibreTranslateTranslator.ts +++ b/packages/translators/LibreTranslateTranslator.ts @@ -72,8 +72,6 @@ export class LibreTranslateTranslator implements DittoTranslator { const response = await this.fetch(request); const json = await response.json(); - console.log(json); - if (!response.ok) { const result = LibreTranslateTranslator.errorSchema().safeParse(json); diff --git a/packages/translators/deno.json b/packages/translators/deno.json index 5d603f3a..c6fda101 100644 --- a/packages/translators/deno.json +++ b/packages/translators/deno.json @@ -1,6 +1,6 @@ { "name": "@ditto/translators", - "version": "1.1.0", + "version": "0.1.0", "exports": { ".": "./mod.ts" } diff --git a/packages/uploaders/deno.json b/packages/uploaders/deno.json index b37b8aa7..506e08f1 100644 --- a/packages/uploaders/deno.json +++ b/packages/uploaders/deno.json @@ -1,6 +1,6 @@ { "name": "@ditto/uploaders", - "version": "1.1.0", + "version": "0.1.0", "exports": { ".": "./mod.ts" } diff --git a/scripts/admin-event.ts b/scripts/admin-event.ts index bec49460..750057ec 100644 --- a/scripts/admin-event.ts +++ b/scripts/admin-event.ts @@ -9,7 +9,7 @@ import { nostrNow } from '../packages/ditto/utils.ts'; const conf = new DittoConf(Deno.env); const db = new DittoPolyPg(conf.databaseUrl); -const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); +const relay = new DittoPgStore({ db, conf }); const { signer } = conf; diff --git a/scripts/admin-role.ts b/scripts/admin-role.ts index 59b95878..ba2059f6 100644 --- a/scripts/admin-role.ts +++ b/scripts/admin-role.ts @@ -8,7 +8,7 @@ import { nostrNow } from '../packages/ditto/utils.ts'; const conf = new DittoConf(Deno.env); const db = new DittoPolyPg(conf.databaseUrl); -const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); +const relay = new DittoPgStore({ db, conf }); const [pubkeyOrNpub, role] = Deno.args; const pubkey = pubkeyOrNpub.startsWith('npub1') ? nip19.decode(pubkeyOrNpub as `npub1${string}`).data : pubkeyOrNpub; diff --git a/scripts/db-export.ts b/scripts/db-export.ts index d9295420..215e9d6f 100644 --- a/scripts/db-export.ts +++ b/scripts/db-export.ts @@ -7,7 +7,7 @@ import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; const conf = new DittoConf(Deno.env); const db = new DittoPolyPg(conf.databaseUrl); -const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); +const relay = new DittoPgStore({ db, conf }); interface ExportFilter { authors?: string[]; diff --git a/scripts/db-import.ts b/scripts/db-import.ts index 4d27e54a..3a51ddb0 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -9,7 +9,7 @@ import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; const conf = new DittoConf(Deno.env); const db = new DittoPolyPg(conf.databaseUrl); -const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); +const relay = new DittoPgStore({ db, conf }); const sem = new Semaphore(conf.pg.poolSize); console.warn('Importing events...'); diff --git a/scripts/db-policy.ts b/scripts/db-policy.ts index b7ceee96..9dcf8986 100644 --- a/scripts/db-policy.ts +++ b/scripts/db-policy.ts @@ -6,7 +6,7 @@ import { PolicyWorker } from '../packages/ditto/workers/policy.ts'; const conf = new DittoConf(Deno.env); const db = new DittoPolyPg(conf.databaseUrl); -const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); +const relay = new DittoPgStore({ db, conf }); const policyWorker = new PolicyWorker(conf); let count = 0; diff --git a/scripts/db-populate-nip05.ts b/scripts/db-populate-nip05.ts index c1015f9f..49866579 100644 --- a/scripts/db-populate-nip05.ts +++ b/scripts/db-populate-nip05.ts @@ -10,7 +10,7 @@ import { DittoRelayStore } from '../packages/ditto/storages/DittoRelayStore.ts'; const conf = new DittoConf(Deno.env); const db = new DittoPolyPg(conf.databaseUrl); -const pgstore = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); +const pgstore = new DittoPgStore({ db, conf }); const relaystore = new DittoRelayStore({ conf, db, relay: pgstore }); const sem = new Semaphore(5); diff --git a/scripts/db-populate-search.ts b/scripts/db-populate-search.ts index 7189b30c..54b28737 100644 --- a/scripts/db-populate-search.ts +++ b/scripts/db-populate-search.ts @@ -6,7 +6,7 @@ import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; const conf = new DittoConf(Deno.env); const db = new DittoPolyPg(conf.databaseUrl); -const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); +const relay = new DittoPgStore({ db, conf }); for await (const msg of relay.req([{ kinds: [0] }])) { if (msg[0] === 'EVENT') { diff --git a/scripts/nostr-pull.ts b/scripts/nostr-pull.ts index d8a4513a..b8204f94 100644 --- a/scripts/nostr-pull.ts +++ b/scripts/nostr-pull.ts @@ -12,7 +12,7 @@ import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; const conf = new DittoConf(Deno.env); const db = new DittoPolyPg(conf.databaseUrl); -const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); +const relay = new DittoPgStore({ db, conf }); interface ImportEventsOpts { profilesOnly: boolean; diff --git a/scripts/setup-kind0.ts b/scripts/setup-kind0.ts index b3dd0682..5428fdcd 100644 --- a/scripts/setup-kind0.ts +++ b/scripts/setup-kind0.ts @@ -7,7 +7,7 @@ import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; const conf = new DittoConf(Deno.env); const db = new DittoPolyPg(conf.databaseUrl); -const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); +const relay = new DittoPgStore({ db, conf }); function die(code: number, ...args: unknown[]) { console.error(...args); diff --git a/scripts/stats-recompute.ts b/scripts/stats-recompute.ts index 16614e45..432e6f92 100644 --- a/scripts/stats-recompute.ts +++ b/scripts/stats-recompute.ts @@ -7,7 +7,7 @@ import { refreshAuthorStats } from '../packages/ditto/utils/stats.ts'; const conf = new DittoConf(Deno.env); const db = new DittoPolyPg(conf.databaseUrl); -const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); +const relay = new DittoPgStore({ db, conf }); const { kysely } = db; diff --git a/scripts/trends.ts b/scripts/trends.ts index 2a878a12..067643d9 100644 --- a/scripts/trends.ts +++ b/scripts/trends.ts @@ -13,7 +13,7 @@ import { const conf = new DittoConf(Deno.env); const db = new DittoPolyPg(conf.databaseUrl); -const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); +const relay = new DittoPgStore({ db, conf }); const ctx = { conf, db, relay }; const trendSchema = z.enum(['pubkeys', 'zapped_events', 'events', 'hashtags', 'links']);