diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 68d0f790..99636568 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,6 +8,8 @@ stages: test: stage: test + before_script: + - apt-get update && apt-get install -y ffmpeg script: - deno fmt --check - deno task lint diff --git a/deno.json b/deno.json index c8a226af..dfeea2f6 100644 --- a/deno.json +++ b/deno.json @@ -11,6 +11,7 @@ "./packages/nip98", "./packages/policies", "./packages/ratelimiter", + "./packages/transcode", "./packages/translators", "./packages/uploaders" ], diff --git a/packages/conf/DittoConf.ts b/packages/conf/DittoConf.ts index f775a861..576cc7e8 100644 --- a/packages/conf/DittoConf.ts +++ b/packages/conf/DittoConf.ts @@ -279,6 +279,11 @@ export class DittoConf { return optionalBooleanSchema.parse(this.env.get('MEDIA_ANALYZE')) ?? false; } + /** Whether to transcode uploaded video files with ffmpeg. */ + get mediaTranscode(): boolean { + return optionalBooleanSchema.parse(this.env.get('MEDIA_TRANSCODE')) ?? false; + } + /** Max upload size for files in number of bytes. Default 100MiB. */ get maxUploadSize(): number { return Number(this.env.get('MAX_UPLOAD_SIZE') || 100 * 1024 * 1024); @@ -480,4 +485,14 @@ export class DittoConf { get precheck(): boolean { return optionalBooleanSchema.parse(this.env.get('DITTO_PRECHECK')) ?? true; } + + /** Path to `ffmpeg` executable. */ + get ffmpegPath(): string { + return this.env.get('FFMPEG_PATH') || 'ffmpeg'; + } + + /** Path to `ffprobe` executable. */ + get ffprobePath(): string { + return this.env.get('FFPROBE_PATH') || 'ffprobe'; + } } diff --git a/packages/ditto/utils/upload.ts b/packages/ditto/utils/upload.ts index 1dcce807..fc0316e8 100644 --- a/packages/ditto/utils/upload.ts +++ b/packages/ditto/utils/upload.ts @@ -1,3 +1,5 @@ +import { analyzeFile, extractVideoFrame, transcodeVideo } from '@ditto/transcode'; +import { ScopedPerformance } from '@esroyo/scoped-performance'; import { HTTPException } from '@hono/hono/http-exception'; import { logi } from '@soapbox/logi'; import { crypto } from '@std/crypto'; @@ -21,7 +23,11 @@ export async function uploadFile( meta: FileMeta, signal?: AbortSignal, ): Promise { + using perf = new ScopedPerformance(); + perf.mark('start'); + const { conf, uploader } = c.var; + const { ffmpegPath, ffprobePath, mediaAnalyze, mediaTranscode } = conf; if (!uploader) { throw new HTTPException(500, { @@ -35,7 +41,43 @@ export async function uploadFile( throw new Error('File size is too large.'); } + const [baseType] = file.type.split('/'); + + perf.mark('probe-start'); + const probe = mediaTranscode ? await analyzeFile(file.stream(), { ffprobePath }).catch(() => null) : null; + const video = probe?.streams.find((stream) => stream.codec_type === 'video'); + perf.mark('probe-end'); + + perf.mark('transcode-start'); + if (baseType === 'video' && mediaTranscode) { + let needsTranscode = false; + + for (const stream of probe?.streams ?? []) { + if (stream.codec_type === 'video' && stream.codec_name !== 'h264') { + needsTranscode = true; + break; + } + if (stream.codec_type === 'audio' && stream.codec_name !== 'aac') { + needsTranscode = true; + break; + } + } + + if (needsTranscode) { + const tmp = new URL('file://' + await Deno.makeTempFile()); + await Deno.writeFile(tmp, file.stream()); + const stream = transcodeVideo(tmp, { ffmpegPath }); + const transcoded = await new Response(stream).bytes(); + file = new File([transcoded], file.name, { type: 'video/mp4' }); + await Deno.remove(tmp); + } + } + perf.mark('transcode-end'); + + perf.mark('upload-start'); const tags = await uploader.upload(file, { signal }); + perf.mark('upload-end'); + const url = tags[0][1]; if (description) { @@ -46,6 +88,8 @@ export async function uploadFile( const m = tags.find(([key]) => key === 'm')?.[1]; const dim = tags.find(([key]) => key === 'dim')?.[1]; const size = tags.find(([key]) => key === 'size')?.[1]; + const image = tags.find(([key]) => key === 'image')?.[1]; + const thumb = tags.find(([key]) => key === 'thumb')?.[1]; const blurhash = tags.find(([key]) => key === 'blurhash')?.[1]; if (!x) { @@ -61,34 +105,50 @@ export async function uploadFile( tags.push(['size', file.size.toString()]); } - // If the uploader didn't already, try to get a blurhash and media dimensions. - // This requires `MEDIA_ANALYZE=true` to be configured because it comes with security tradeoffs. - if (conf.mediaAnalyze && (!blurhash || !dim)) { + perf.mark('analyze-start'); + + if (baseType === 'video' && mediaAnalyze && mediaTranscode && video && (!image || !thumb)) { try { - const bytes = await new Response(file.stream()).bytes(); - const img = sharp(bytes); + const tmp = new URL('file://' + await Deno.makeTempFile()); + await Deno.writeFile(tmp, file.stream()); + const frame = await extractVideoFrame(tmp, '00:00:01', { ffmpegPath }); + await Deno.remove(tmp); + const [[, url]] = await uploader.upload(new File([frame], 'thumb.jpg', { type: 'image/jpeg' }), { signal }); - const { width, height } = await img.metadata(); - - if (!dim && (width && height)) { - tags.push(['dim', `${width}x${height}`]); + if (!image) { + tags.push(['image', url]); } - if (!blurhash && (width && height)) { - const pixels = await img - .raw() - .ensureAlpha() - .toBuffer({ resolveWithObject: false }) - .then((buffer) => new Uint8ClampedArray(buffer)); + if (!dim) { + tags.push(['dim', await getImageDim(frame)]); + } - const blurhash = encode(pixels, width, height, 4, 4); - tags.push(['blurhash', blurhash]); + if (!blurhash) { + tags.push(['blurhash', await getBlurhash(frame)]); } } catch (e) { logi({ level: 'error', ns: 'ditto.upload.analyze', error: errorJson(e) }); } } + if (baseType === 'image' && mediaAnalyze && (!blurhash || !dim)) { + try { + const bytes = await new Response(file.stream()).bytes(); + + if (!dim) { + tags.push(['dim', await getImageDim(bytes)]); + } + + if (!blurhash) { + tags.push(['blurhash', await getBlurhash(bytes)]); + } + } catch (e) { + logi({ level: 'error', ns: 'ditto.upload.analyze', error: errorJson(e) }); + } + } + + perf.mark('analyze-end'); + const upload = { id: crypto.randomUUID(), url, @@ -99,5 +159,62 @@ export async function uploadFile( dittoUploads.set(upload.id, upload); + const timing = [ + perf.measure('probe', 'probe-start', 'probe-end'), + perf.measure('transcode', 'transcode-start', 'transcode-end'), + perf.measure('upload', 'upload-start', 'upload-end'), + perf.measure('analyze', 'analyze-start', 'analyze-end'), + ].reduce>((acc, m) => { + const name = m.name.split('::')[1]; // ScopedPerformance uses `::` to separate the name. + acc[name] = m.duration / 1000; // Convert to seconds for logging. + return acc; + }, {}); + + perf.mark('end'); + + logi({ + level: 'info', + ns: 'ditto.upload', + upload: { ...upload, uploadedAt: upload.uploadedAt.toISOString() }, + timing, + duration: perf.measure('total', 'start', 'end').duration / 1000, + }); + return upload; } + +async function getImageDim(bytes: Uint8Array): Promise<`${number}x${number}`> { + const img = sharp(bytes); + const { width, height } = await img.metadata(); + + if (!width || !height) { + throw new Error('Image metadata is missing.'); + } + + return `${width}x${height}`; +} + +/** Get a blurhash from an image file. */ +async function getBlurhash(bytes: Uint8Array, maxDim = 64): Promise { + const img = sharp(bytes); + + const { width, height } = await img.metadata(); + + if (!width || !height) { + throw new Error('Image metadata is missing.'); + } + + const { data, info } = await img + .raw() + .ensureAlpha() + .resize({ + width: width > height ? undefined : maxDim, + height: height > width ? undefined : maxDim, + fit: 'inside', + }) + .toBuffer({ resolveWithObject: true }); + + const pixels = new Uint8ClampedArray(data); + + return encode(pixels, info.width, info.height, 4, 4); +} diff --git a/packages/ditto/views/mastodon/attachments.ts b/packages/ditto/views/mastodon/attachments.ts index b0d2e49c..1e24e794 100644 --- a/packages/ditto/views/mastodon/attachments.ts +++ b/packages/ditto/views/mastodon/attachments.ts @@ -14,6 +14,8 @@ function renderAttachment( const alt = tags.find(([name]) => name === 'alt')?.[1]; const cid = tags.find(([name]) => name === 'cid')?.[1]; const dim = tags.find(([name]) => name === 'dim')?.[1]; + const image = tags.find(([key]) => key === 'image')?.[1]; + const thumb = tags.find(([key]) => key === 'thumb')?.[1]; const blurhash = tags.find(([name]) => name === 'blurhash')?.[1]; if (!url) return; @@ -34,7 +36,7 @@ function renderAttachment( id: id ?? url, type: getAttachmentType(m ?? ''), url, - preview_url: url, + preview_url: image ?? thumb ?? url, remote_url: null, description: alt ?? '', blurhash: blurhash || null, diff --git a/packages/transcode/.gitignore b/packages/transcode/.gitignore new file mode 100644 index 00000000..c0363794 --- /dev/null +++ b/packages/transcode/.gitignore @@ -0,0 +1 @@ +tmp/ \ No newline at end of file diff --git a/packages/transcode/analyze.test.ts b/packages/transcode/analyze.test.ts new file mode 100644 index 00000000..c1a23f5e --- /dev/null +++ b/packages/transcode/analyze.test.ts @@ -0,0 +1,13 @@ +import { assertObjectMatch } from '@std/assert'; + +import { analyzeFile } from './analyze.ts'; + +Deno.test('analyzeFile', async () => { + const uri = new URL('./buckbunny.mp4', import.meta.url); + + const { streams } = await analyzeFile(uri); + + const videoStream = streams.find((stream) => stream.codec_type === 'video')!; + + assertObjectMatch(videoStream, { width: 1920, height: 1080 }); +}); diff --git a/packages/transcode/analyze.ts b/packages/transcode/analyze.ts new file mode 100644 index 00000000..06f866f4 --- /dev/null +++ b/packages/transcode/analyze.ts @@ -0,0 +1,102 @@ +import { ffprobe } from './ffprobe.ts'; + +interface AnalyzeResult { + streams: Stream[]; + format: Format; +} + +interface Stream { + index: number; + codec_tag_string: string; + codec_tag: string; + codec_name?: string; + codec_long_name?: string; + profile?: string; + codec_type?: string; + width?: number; + height?: number; + coded_width?: number; + coded_height?: number; + closed_captions?: number; + has_b_frames?: number; + sample_aspect_ratio?: string; + display_aspect_ratio?: string; + pix_fmt?: string; + level?: number; + color_range?: string; + color_space?: string; + color_transfer?: string; + color_primaries?: string; + chroma_location?: string; + field_order?: string; + refs?: number; + sample_fmt?: string; + sample_rate?: string; + channels?: number; + channel_layout?: string; + bits_per_sample?: number; + id?: string; + r_frame_rate?: string; + avg_frame_rate?: string; + time_base?: string; + start_pts?: number; + start_time?: string; + duration_ts?: number; + duration?: string; + bit_rate?: string; + max_bit_rate?: string; + bits_per_raw_sample?: string; + nb_frames?: string; + nb_read_frames?: string; + nb_read_packets?: string; + disposition?: Disposition; + tags?: Record; +} + +interface Format { + filename: string; + nb_streams: number; + nb_programs: number; + format_name: string; + probe_score: number; + format_long_name?: string; + start_time?: string; + duration?: string; + size?: string; + bit_rate?: string; + tags?: Record; +} + +interface Disposition { + default: number; + dub: number; + original: number; + comment: number; + lyrics: number; + karaoke: number; + forced: number; + hearing_impaired: number; + visual_impaired: number; + clean_effects: number; + attached_pic: number; + timed_thumbnails: number; + captions: number; + descriptions: number; + metadata: number; + dependent: number; + still_image: number; +} + +export function analyzeFile( + input: URL | ReadableStream, + opts?: { ffprobePath?: string | URL }, +): Promise { + const stream = ffprobe(input, { + 'loglevel': 'fatal', + 'show_streams': '', + 'show_format': '', + 'of': 'json', + }, opts); + + return new Response(stream).json(); +} diff --git a/packages/transcode/buckbunny.mp4 b/packages/transcode/buckbunny.mp4 new file mode 100644 index 00000000..91fdbb8a Binary files /dev/null and b/packages/transcode/buckbunny.mp4 differ diff --git a/packages/transcode/deno.json b/packages/transcode/deno.json new file mode 100644 index 00000000..e4cdd6bf --- /dev/null +++ b/packages/transcode/deno.json @@ -0,0 +1,7 @@ +{ + "name": "@ditto/transcode", + "version": "1.0.0", + "exports": { + ".": "./mod.ts" + } +} diff --git a/packages/transcode/ffmpeg.test.ts b/packages/transcode/ffmpeg.test.ts new file mode 100644 index 00000000..d93be547 --- /dev/null +++ b/packages/transcode/ffmpeg.test.ts @@ -0,0 +1,31 @@ +import { ffmpeg } from './ffmpeg.ts'; + +const uri = new URL('./buckbunny.mp4', import.meta.url); + +Deno.test('ffmpeg', async () => { + await using file = await Deno.open(uri); + + const output = ffmpeg(file.readable, { + 'c:v': 'libx264', + 'preset': 'veryfast', + 'loglevel': 'fatal', + 'movflags': 'frag_keyframe+empty_moov', + 'f': 'mp4', + }); + + await Deno.mkdir(new URL('./tmp', import.meta.url), { recursive: true }); + await Deno.writeFile(new URL('./tmp/transcoded-1.mp4', import.meta.url), output); +}); + +Deno.test('ffmpeg from file URI', async () => { + const output = ffmpeg(uri, { + 'c:v': 'libx264', + 'preset': 'veryfast', + 'loglevel': 'fatal', + 'movflags': 'frag_keyframe+empty_moov', + 'f': 'mp4', + }); + + await Deno.mkdir(new URL('./tmp', import.meta.url), { recursive: true }); + await Deno.writeFile(new URL('./tmp/transcoded-2.mp4', import.meta.url), output); +}); diff --git a/packages/transcode/ffmpeg.ts b/packages/transcode/ffmpeg.ts new file mode 100644 index 00000000..f2ebd5a5 --- /dev/null +++ b/packages/transcode/ffmpeg.ts @@ -0,0 +1,58 @@ +export interface FFmpegFlags { + 'safe'?: string; + 'nostdin'?: string; + 'c:v'?: string; + 'preset'?: string; + 'loglevel'?: string; + 'crf'?: string; + 'c:a'?: string; + 'b:a'?: string; + 'movflags'?: string; + 'f'?: string; + [key: string]: string | undefined; +} + +export function ffmpeg( + input: URL | ReadableStream, + flags: FFmpegFlags, + opts?: { ffmpegPath?: string | URL }, +): ReadableStream { + const { ffmpegPath = 'ffmpeg' } = opts ?? {}; + + const args = ['-i', input instanceof URL ? input.href : 'pipe:0']; + + for (const [key, value] of Object.entries(flags)) { + if (typeof value === 'string') { + if (value) { + args.push(`-${key}`, value); + } else { + args.push(`-${key}`); + } + } + } + + args.push('pipe:1'); // Output to stdout + + // Spawn the FFmpeg process + const command = new Deno.Command(ffmpegPath, { + args, + stdin: input instanceof ReadableStream ? 'piped' : 'null', + stdout: 'piped', + }); + + const child = command.spawn(); + + // Pipe the input stream into FFmpeg stdin and ensure completion + if (input instanceof ReadableStream) { + input.pipeTo(child.stdin).catch((e: unknown) => { + if (e instanceof Error && e.name === 'BrokenPipe') { + // Ignore. ffprobe closes the pipe once it has read the metadata. + } else { + throw e; + } + }); + } + + // Return the FFmpeg stdout stream + return child.stdout; +} diff --git a/packages/transcode/ffprobe.test.ts b/packages/transcode/ffprobe.test.ts new file mode 100644 index 00000000..953c6271 --- /dev/null +++ b/packages/transcode/ffprobe.test.ts @@ -0,0 +1,33 @@ +import { assertObjectMatch } from '@std/assert'; + +import { ffprobe } from './ffprobe.ts'; + +const uri = new URL('./buckbunny.mp4', import.meta.url); + +Deno.test('ffprobe from ReadableStream', async () => { + await using file = await Deno.open(uri); + + const stream = ffprobe(file.readable, { + 'v': 'error', + 'select_streams': 'v:0', + 'show_entries': 'stream=width,height', + 'of': 'json', + }); + + const { streams: [dimensions] } = await new Response(stream).json(); + + assertObjectMatch(dimensions, { width: 1920, height: 1080 }); +}); + +Deno.test('ffprobe from file URI', async () => { + const stream = ffprobe(uri, { + 'v': 'error', + 'select_streams': 'v:0', + 'show_entries': 'stream=width,height', + 'of': 'json', + }); + + const { streams: [dimensions] } = await new Response(stream).json(); + + assertObjectMatch(dimensions, { width: 1920, height: 1080 }); +}); diff --git a/packages/transcode/ffprobe.ts b/packages/transcode/ffprobe.ts new file mode 100644 index 00000000..7605cbe5 --- /dev/null +++ b/packages/transcode/ffprobe.ts @@ -0,0 +1,56 @@ +export interface FFprobeFlags { + 'v'?: string; + 'select_streams'?: string; + 'show_entries'?: string; + 'of'?: string; + [key: string]: string | undefined; +} + +export function ffprobe( + input: URL | ReadableStream, + flags: FFprobeFlags, + opts?: { ffprobePath?: string | URL }, +): ReadableStream { + const { ffprobePath = 'ffprobe' } = opts ?? {}; + + const args = []; + + for (const [key, value] of Object.entries(flags)) { + if (typeof value === 'string') { + if (value) { + args.push(`-${key}`, value); + } else { + args.push(`-${key}`); + } + } + } + + if (input instanceof URL) { + args.push('-i', input.href); + } else { + args.push('-i', 'pipe:0'); + } + + // Spawn the FFprobe process + const command = new Deno.Command(ffprobePath, { + args, + stdin: input instanceof ReadableStream ? 'piped' : 'null', + stdout: 'piped', + }); + + const child = command.spawn(); + + // Pipe the input stream into FFmpeg stdin and ensure completion + if (input instanceof ReadableStream) { + input.pipeTo(child.stdin).catch((e: unknown) => { + if (e instanceof Error && e.name === 'BrokenPipe') { + // Ignore. ffprobe closes the pipe once it has read the metadata. + } else { + throw e; + } + }); + } + + // Return the FFmpeg stdout stream + return child.stdout; +} diff --git a/packages/transcode/frame.test.ts b/packages/transcode/frame.test.ts new file mode 100644 index 00000000..c0710cfc --- /dev/null +++ b/packages/transcode/frame.test.ts @@ -0,0 +1,12 @@ +import { extractVideoFrame } from './frame.ts'; + +const uri = new URL('./buckbunny.mp4', import.meta.url); + +Deno.test('extractVideoFrame', async () => { + await using file = await Deno.open(uri); + + const result = await extractVideoFrame(file.readable); + + await Deno.mkdir(new URL('./tmp', import.meta.url), { recursive: true }); + await Deno.writeFile(new URL('./tmp/poster.jpg', import.meta.url), result); +}); diff --git a/packages/transcode/frame.ts b/packages/transcode/frame.ts new file mode 100644 index 00000000..d03ea63b --- /dev/null +++ b/packages/transcode/frame.ts @@ -0,0 +1,17 @@ +import { ffmpeg } from './ffmpeg.ts'; + +export function extractVideoFrame( + input: URL | ReadableStream, + ss: string = '00:00:01', + opts?: { ffmpegPath?: string | URL }, +): Promise { + const output = ffmpeg(input, { + 'ss': ss, // Seek to timestamp + 'frames:v': '1', // Extract only 1 frame + 'q:v': '2', // High-quality JPEG (lower = better quality) + 'f': 'image2', // Force image format + 'loglevel': 'fatal', + }, opts); + + return new Response(output).bytes(); +} diff --git a/packages/transcode/mod.ts b/packages/transcode/mod.ts new file mode 100644 index 00000000..8da45b0e --- /dev/null +++ b/packages/transcode/mod.ts @@ -0,0 +1,5 @@ +export { analyzeFile } from './analyze.ts'; +export { ffmpeg, type FFmpegFlags } from './ffmpeg.ts'; +export { ffprobe, type FFprobeFlags } from './ffprobe.ts'; +export { extractVideoFrame } from './frame.ts'; +export { transcodeVideo } from './transcode.ts'; diff --git a/packages/transcode/transcode.test.ts b/packages/transcode/transcode.test.ts new file mode 100644 index 00000000..971b4fb9 --- /dev/null +++ b/packages/transcode/transcode.test.ts @@ -0,0 +1,9 @@ +import { transcodeVideo } from './transcode.ts'; + +Deno.test('transcodeVideo', async () => { + await using file = await Deno.open(new URL('./buckbunny.mp4', import.meta.url)); + const output = transcodeVideo(file.readable); + + await Deno.mkdir(new URL('./tmp', import.meta.url), { recursive: true }); + await Deno.writeFile(new URL('./tmp/buckbunny-transcoded.mp4', import.meta.url), output); +}); diff --git a/packages/transcode/transcode.ts b/packages/transcode/transcode.ts new file mode 100644 index 00000000..d31cacb1 --- /dev/null +++ b/packages/transcode/transcode.ts @@ -0,0 +1,19 @@ +import { ffmpeg } from './ffmpeg.ts'; + +export function transcodeVideo( + input: URL | ReadableStream, + opts?: { ffmpegPath?: string | URL }, +): ReadableStream { + return ffmpeg(input, { + 'safe': '1', // Safe mode + 'nostdin': '', // Disable stdin + 'c:v': 'libx264', // Convert to H.264 + 'preset': 'veryfast', // Encoding speed + 'loglevel': 'fatal', // Suppress logs + 'crf': '23', // Compression level (lower = better quality) + 'c:a': 'aac', // Convert to AAC audio + 'b:a': '128k', // Audio bitrate + 'movflags': 'frag_keyframe+empty_moov', // Ensures MP4 streaming compatibility + 'f': 'mp4', // Force MP4 format + }, opts); +}