Merge branch 'transcode' into 'main'

Transcode uploaded videos with ffmpeg

See merge request soapbox-pub/ditto!706

.gitlab-ci.yml

@@ -8,6 +8,8 @@ stages:
 
 test:
   stage: test
+  before_script:
+    - apt-get update && apt-get install -y ffmpeg
   script:
     - deno fmt --check
     - deno task lint

deno.json

@@ -11,6 +11,7 @@
     "./packages/nip98",
     "./packages/policies",
     "./packages/ratelimiter",
+    "./packages/transcode",
     "./packages/translators",
     "./packages/uploaders"
   ],

packages/conf/DittoConf.ts

@@ -279,6 +279,11 @@ export class DittoConf {
     return optionalBooleanSchema.parse(this.env.get('MEDIA_ANALYZE')) ?? false;
   }
 
+  /** Whether to transcode uploaded video files with ffmpeg. */
+  get mediaTranscode(): boolean {
+    return optionalBooleanSchema.parse(this.env.get('MEDIA_TRANSCODE')) ?? false;
+  }
+
   /** Max upload size for files in number of bytes. Default 100MiB. */
   get maxUploadSize(): number {
     return Number(this.env.get('MAX_UPLOAD_SIZE') || 100 * 1024 * 1024);
@@ -480,4 +485,14 @@ export class DittoConf {
   get precheck(): boolean {
     return optionalBooleanSchema.parse(this.env.get('DITTO_PRECHECK')) ?? true;
   }
+
+  /** Path to `ffmpeg` executable. */
+  get ffmpegPath(): string {
+    return this.env.get('FFMPEG_PATH') || 'ffmpeg';
+  }
+
+  /** Path to `ffprobe` executable. */
+  get ffprobePath(): string {
+    return this.env.get('FFPROBE_PATH') || 'ffprobe';
+  }
 }

packages/ditto/utils/upload.ts

@@ -1,3 +1,5 @@
+import { analyzeFile, extractVideoFrame, transcodeVideo } from '@ditto/transcode';
+import { ScopedPerformance } from '@esroyo/scoped-performance';
 import { HTTPException } from '@hono/hono/http-exception';
 import { logi } from '@soapbox/logi';
 import { crypto } from '@std/crypto';
@@ -21,7 +23,11 @@ export async function uploadFile(
   meta: FileMeta,
   signal?: AbortSignal,
 ): Promise<DittoUpload> {
+  using perf = new ScopedPerformance();
+  perf.mark('start');
+
   const { conf, uploader } = c.var;
+  const { ffmpegPath, ffprobePath, mediaAnalyze, mediaTranscode } = conf;
 
   if (!uploader) {
     throw new HTTPException(500, {
@@ -35,7 +41,43 @@ export async function uploadFile(
     throw new Error('File size is too large.');
   }
 
+  const [baseType] = file.type.split('/');
+
+  perf.mark('probe-start');
+  const probe = mediaTranscode ? await analyzeFile(file.stream(), { ffprobePath }).catch(() => null) : null;
+  const video = probe?.streams.find((stream) => stream.codec_type === 'video');
+  perf.mark('probe-end');
+
+  perf.mark('transcode-start');
+  if (baseType === 'video' && mediaTranscode) {
+    let needsTranscode = false;
+
+    for (const stream of probe?.streams ?? []) {
+      if (stream.codec_type === 'video' && stream.codec_name !== 'h264') {
+        needsTranscode = true;
+        break;
+      }
+      if (stream.codec_type === 'audio' && stream.codec_name !== 'aac') {
+        needsTranscode = true;
+        break;
+      }
+    }
+
+    if (needsTranscode) {
+      const tmp = new URL('file://' + await Deno.makeTempFile());
+      await Deno.writeFile(tmp, file.stream());
+      const stream = transcodeVideo(tmp, { ffmpegPath });
+      const transcoded = await new Response(stream).bytes();
+      file = new File([transcoded], file.name, { type: 'video/mp4' });
+      await Deno.remove(tmp);
+    }
+  }
+  perf.mark('transcode-end');
+
+  perf.mark('upload-start');
   const tags = await uploader.upload(file, { signal });
+  perf.mark('upload-end');
+
   const url = tags[0][1];
 
   if (description) {
@@ -46,6 +88,8 @@ export async function uploadFile(
   const m = tags.find(([key]) => key === 'm')?.[1];
   const dim = tags.find(([key]) => key === 'dim')?.[1];
   const size = tags.find(([key]) => key === 'size')?.[1];
+  const image = tags.find(([key]) => key === 'image')?.[1];
+  const thumb = tags.find(([key]) => key === 'thumb')?.[1];
   const blurhash = tags.find(([key]) => key === 'blurhash')?.[1];
 
   if (!x) {
@@ -61,34 +105,50 @@ export async function uploadFile(
     tags.push(['size', file.size.toString()]);
   }
 
-  // If the uploader didn't already, try to get a blurhash and media dimensions.
+  perf.mark('analyze-start');
-  // This requires `MEDIA_ANALYZE=true` to be configured because it comes with security tradeoffs.
+
-  if (conf.mediaAnalyze && (!blurhash || !dim)) {
+  if (baseType === 'video' && mediaAnalyze && mediaTranscode && video && (!image || !thumb)) {
     try {
-      const bytes = await new Response(file.stream()).bytes();
+      const tmp = new URL('file://' + await Deno.makeTempFile());
-      const img = sharp(bytes);
+      await Deno.writeFile(tmp, file.stream());
+      const frame = await extractVideoFrame(tmp, '00:00:01', { ffmpegPath });
+      await Deno.remove(tmp);
+      const [[, url]] = await uploader.upload(new File([frame], 'thumb.jpg', { type: 'image/jpeg' }), { signal });
 
-      const { width, height } = await img.metadata();
+      if (!image) {
-
+        tags.push(['image', url]);
-      if (!dim && (width && height)) {
-        tags.push(['dim', `${width}x${height}`]);
       }
 
-      if (!blurhash && (width && height)) {
+      if (!dim) {
-        const pixels = await img
+        tags.push(['dim', await getImageDim(frame)]);
-          .raw()
+      }
-          .ensureAlpha()
-          .toBuffer({ resolveWithObject: false })
-          .then((buffer) => new Uint8ClampedArray(buffer));
 
-        const blurhash = encode(pixels, width, height, 4, 4);
+      if (!blurhash) {
-        tags.push(['blurhash', blurhash]);
+        tags.push(['blurhash', await getBlurhash(frame)]);
       }
     } catch (e) {
       logi({ level: 'error', ns: 'ditto.upload.analyze', error: errorJson(e) });
     }
   }
 
+  if (baseType === 'image' && mediaAnalyze && (!blurhash || !dim)) {
+    try {
+      const bytes = await new Response(file.stream()).bytes();
+
+      if (!dim) {
+        tags.push(['dim', await getImageDim(bytes)]);
+      }
+
+      if (!blurhash) {
+        tags.push(['blurhash', await getBlurhash(bytes)]);
+      }
+    } catch (e) {
+      logi({ level: 'error', ns: 'ditto.upload.analyze', error: errorJson(e) });
+    }
+  }
+
+  perf.mark('analyze-end');
+
   const upload = {
     id: crypto.randomUUID(),
     url,
@@ -99,5 +159,62 @@ export async function uploadFile(
 
   dittoUploads.set(upload.id, upload);
 
+  const timing = [
+    perf.measure('probe', 'probe-start', 'probe-end'),
+    perf.measure('transcode', 'transcode-start', 'transcode-end'),
+    perf.measure('upload', 'upload-start', 'upload-end'),
+    perf.measure('analyze', 'analyze-start', 'analyze-end'),
+  ].reduce<Record<string, number>>((acc, m) => {
+    const name = m.name.split('::')[1]; // ScopedPerformance uses `::` to separate the name.
+    acc[name] = m.duration / 1000; // Convert to seconds for logging.
+    return acc;
+  }, {});
+
+  perf.mark('end');
+
+  logi({
+    level: 'info',
+    ns: 'ditto.upload',
+    upload: { ...upload, uploadedAt: upload.uploadedAt.toISOString() },
+    timing,
+    duration: perf.measure('total', 'start', 'end').duration / 1000,
+  });
+
   return upload;
 }
+
+async function getImageDim(bytes: Uint8Array): Promise<`${number}x${number}`> {
+  const img = sharp(bytes);
+  const { width, height } = await img.metadata();
+
+  if (!width || !height) {
+    throw new Error('Image metadata is missing.');
+  }
+
+  return `${width}x${height}`;
+}
+
+/** Get a blurhash from an image file. */
+async function getBlurhash(bytes: Uint8Array, maxDim = 64): Promise<string> {
+  const img = sharp(bytes);
+
+  const { width, height } = await img.metadata();
+
+  if (!width || !height) {
+    throw new Error('Image metadata is missing.');
+  }
+
+  const { data, info } = await img
+    .raw()
+    .ensureAlpha()
+    .resize({
+      width: width > height ? undefined : maxDim,
+      height: height > width ? undefined : maxDim,
+      fit: 'inside',
+    })
+    .toBuffer({ resolveWithObject: true });
+
+  const pixels = new Uint8ClampedArray(data);
+
+  return encode(pixels, info.width, info.height, 4, 4);
+}

packages/ditto/views/mastodon/attachments.ts

@@ -14,6 +14,8 @@ function renderAttachment(
   const alt = tags.find(([name]) => name === 'alt')?.[1];
   const cid = tags.find(([name]) => name === 'cid')?.[1];
   const dim = tags.find(([name]) => name === 'dim')?.[1];
+  const image = tags.find(([key]) => key === 'image')?.[1];
+  const thumb = tags.find(([key]) => key === 'thumb')?.[1];
   const blurhash = tags.find(([name]) => name === 'blurhash')?.[1];
 
   if (!url) return;
@@ -34,7 +36,7 @@ function renderAttachment(
     id: id ?? url,
     type: getAttachmentType(m ?? ''),
     url,
-    preview_url: url,
+    preview_url: image ?? thumb ?? url,
     remote_url: null,
     description: alt ?? '',
     blurhash: blurhash || null,

packages/transcode/.gitignore

@@ -0,0 +1 @@
+tmp/

packages/transcode/analyze.test.ts

@@ -0,0 +1,13 @@
+import { assertObjectMatch } from '@std/assert';
+
+import { analyzeFile } from './analyze.ts';
+
+Deno.test('analyzeFile', async () => {
+  const uri = new URL('./buckbunny.mp4', import.meta.url);
+
+  const { streams } = await analyzeFile(uri);
+
+  const videoStream = streams.find((stream) => stream.codec_type === 'video')!;
+
+  assertObjectMatch(videoStream, { width: 1920, height: 1080 });
+});

packages/transcode/analyze.ts

@@ -0,0 +1,102 @@
+import { ffprobe } from './ffprobe.ts';
+
+interface AnalyzeResult {
+  streams: Stream[];
+  format: Format;
+}
+
+interface Stream {
+  index: number;
+  codec_tag_string: string;
+  codec_tag: string;
+  codec_name?: string;
+  codec_long_name?: string;
+  profile?: string;
+  codec_type?: string;
+  width?: number;
+  height?: number;
+  coded_width?: number;
+  coded_height?: number;
+  closed_captions?: number;
+  has_b_frames?: number;
+  sample_aspect_ratio?: string;
+  display_aspect_ratio?: string;
+  pix_fmt?: string;
+  level?: number;
+  color_range?: string;
+  color_space?: string;
+  color_transfer?: string;
+  color_primaries?: string;
+  chroma_location?: string;
+  field_order?: string;
+  refs?: number;
+  sample_fmt?: string;
+  sample_rate?: string;
+  channels?: number;
+  channel_layout?: string;
+  bits_per_sample?: number;
+  id?: string;
+  r_frame_rate?: string;
+  avg_frame_rate?: string;
+  time_base?: string;
+  start_pts?: number;
+  start_time?: string;
+  duration_ts?: number;
+  duration?: string;
+  bit_rate?: string;
+  max_bit_rate?: string;
+  bits_per_raw_sample?: string;
+  nb_frames?: string;
+  nb_read_frames?: string;
+  nb_read_packets?: string;
+  disposition?: Disposition;
+  tags?: Record<string, string>;
+}
+
+interface Format {
+  filename: string;
+  nb_streams: number;
+  nb_programs: number;
+  format_name: string;
+  probe_score: number;
+  format_long_name?: string;
+  start_time?: string;
+  duration?: string;
+  size?: string;
+  bit_rate?: string;
+  tags?: Record<string, string>;
+}
+
+interface Disposition {
+  default: number;
+  dub: number;
+  original: number;
+  comment: number;
+  lyrics: number;
+  karaoke: number;
+  forced: number;
+  hearing_impaired: number;
+  visual_impaired: number;
+  clean_effects: number;
+  attached_pic: number;
+  timed_thumbnails: number;
+  captions: number;
+  descriptions: number;
+  metadata: number;
+  dependent: number;
+  still_image: number;
+}
+
+export function analyzeFile(
+  input: URL | ReadableStream<Uint8Array>,
+  opts?: { ffprobePath?: string | URL },
+): Promise<AnalyzeResult> {
+  const stream = ffprobe(input, {
+    'loglevel': 'fatal',
+    'show_streams': '',
+    'show_format': '',
+    'of': 'json',
+  }, opts);
+
+  return new Response(stream).json();
+}

packages/transcode/deno.json

@@ -0,0 +1,7 @@
+{
+  "name": "@ditto/transcode",
+  "version": "1.0.0",
+  "exports": {
+    ".": "./mod.ts"
+  }
+}

packages/transcode/ffmpeg.test.ts

@@ -0,0 +1,31 @@
+import { ffmpeg } from './ffmpeg.ts';
+
+const uri = new URL('./buckbunny.mp4', import.meta.url);
+
+Deno.test('ffmpeg', async () => {
+  await using file = await Deno.open(uri);
+
+  const output = ffmpeg(file.readable, {
+    'c:v': 'libx264',
+    'preset': 'veryfast',
+    'loglevel': 'fatal',
+    'movflags': 'frag_keyframe+empty_moov',
+    'f': 'mp4',
+  });
+
+  await Deno.mkdir(new URL('./tmp', import.meta.url), { recursive: true });
+  await Deno.writeFile(new URL('./tmp/transcoded-1.mp4', import.meta.url), output);
+});
+
+Deno.test('ffmpeg from file URI', async () => {
+  const output = ffmpeg(uri, {
+    'c:v': 'libx264',
+    'preset': 'veryfast',
+    'loglevel': 'fatal',
+    'movflags': 'frag_keyframe+empty_moov',
+    'f': 'mp4',
+  });
+
+  await Deno.mkdir(new URL('./tmp', import.meta.url), { recursive: true });
+  await Deno.writeFile(new URL('./tmp/transcoded-2.mp4', import.meta.url), output);
+});

packages/transcode/ffmpeg.ts

@@ -0,0 +1,58 @@
+export interface FFmpegFlags {
+  'safe'?: string;
+  'nostdin'?: string;
+  'c:v'?: string;
+  'preset'?: string;
+  'loglevel'?: string;
+  'crf'?: string;
+  'c:a'?: string;
+  'b:a'?: string;
+  'movflags'?: string;
+  'f'?: string;
+  [key: string]: string | undefined;
+}
+
+export function ffmpeg(
+  input: URL | ReadableStream<Uint8Array>,
+  flags: FFmpegFlags,
+  opts?: { ffmpegPath?: string | URL },
+): ReadableStream<Uint8Array> {
+  const { ffmpegPath = 'ffmpeg' } = opts ?? {};
+
+  const args = ['-i', input instanceof URL ? input.href : 'pipe:0'];
+
+  for (const [key, value] of Object.entries(flags)) {
+    if (typeof value === 'string') {
+      if (value) {
+        args.push(`-${key}`, value);
+      } else {
+        args.push(`-${key}`);
+      }
+    }
+  }
+
+  args.push('pipe:1'); // Output to stdout
+
+  // Spawn the FFmpeg process
+  const command = new Deno.Command(ffmpegPath, {
+    args,
+    stdin: input instanceof ReadableStream ? 'piped' : 'null',
+    stdout: 'piped',
+  });
+
+  const child = command.spawn();
+
+  // Pipe the input stream into FFmpeg stdin and ensure completion
+  if (input instanceof ReadableStream) {
+    input.pipeTo(child.stdin).catch((e: unknown) => {
+      if (e instanceof Error && e.name === 'BrokenPipe') {
+        // Ignore. ffprobe closes the pipe once it has read the metadata.
+      } else {
+        throw e;
+      }
+    });
+  }
+
+  // Return the FFmpeg stdout stream
+  return child.stdout;
+}

packages/transcode/ffprobe.test.ts

@@ -0,0 +1,33 @@
+import { assertObjectMatch } from '@std/assert';
+
+import { ffprobe } from './ffprobe.ts';
+
+const uri = new URL('./buckbunny.mp4', import.meta.url);
+
+Deno.test('ffprobe from ReadableStream', async () => {
+  await using file = await Deno.open(uri);
+
+  const stream = ffprobe(file.readable, {
+    'v': 'error',
+    'select_streams': 'v:0',
+    'show_entries': 'stream=width,height',
+    'of': 'json',
+  });
+
+  const { streams: [dimensions] } = await new Response(stream).json();
+
+  assertObjectMatch(dimensions, { width: 1920, height: 1080 });
+});
+
+Deno.test('ffprobe from file URI', async () => {
+  const stream = ffprobe(uri, {
+    'v': 'error',
+    'select_streams': 'v:0',
+    'show_entries': 'stream=width,height',
+    'of': 'json',
+  });
+
+  const { streams: [dimensions] } = await new Response(stream).json();
+
+  assertObjectMatch(dimensions, { width: 1920, height: 1080 });
+});

packages/transcode/ffprobe.ts

@@ -0,0 +1,56 @@
+export interface FFprobeFlags {
+  'v'?: string;
+  'select_streams'?: string;
+  'show_entries'?: string;
+  'of'?: string;
+  [key: string]: string | undefined;
+}
+
+export function ffprobe(
+  input: URL | ReadableStream<Uint8Array>,
+  flags: FFprobeFlags,
+  opts?: { ffprobePath?: string | URL },
+): ReadableStream<Uint8Array> {
+  const { ffprobePath = 'ffprobe' } = opts ?? {};
+
+  const args = [];
+
+  for (const [key, value] of Object.entries(flags)) {
+    if (typeof value === 'string') {
+      if (value) {
+        args.push(`-${key}`, value);
+      } else {
+        args.push(`-${key}`);
+      }
+    }
+  }
+
+  if (input instanceof URL) {
+    args.push('-i', input.href);
+  } else {
+    args.push('-i', 'pipe:0');
+  }
+
+  // Spawn the FFprobe process
+  const command = new Deno.Command(ffprobePath, {
+    args,
+    stdin: input instanceof ReadableStream ? 'piped' : 'null',
+    stdout: 'piped',
+  });
+
+  const child = command.spawn();
+
+  // Pipe the input stream into FFmpeg stdin and ensure completion
+  if (input instanceof ReadableStream) {
+    input.pipeTo(child.stdin).catch((e: unknown) => {
+      if (e instanceof Error && e.name === 'BrokenPipe') {
+        // Ignore. ffprobe closes the pipe once it has read the metadata.
+      } else {
+        throw e;
+      }
+    });
+  }
+
+  // Return the FFmpeg stdout stream
+  return child.stdout;
+}

packages/transcode/frame.test.ts

@@ -0,0 +1,12 @@
+import { extractVideoFrame } from './frame.ts';
+
+const uri = new URL('./buckbunny.mp4', import.meta.url);
+
+Deno.test('extractVideoFrame', async () => {
+  await using file = await Deno.open(uri);
+
+  const result = await extractVideoFrame(file.readable);
+
+  await Deno.mkdir(new URL('./tmp', import.meta.url), { recursive: true });
+  await Deno.writeFile(new URL('./tmp/poster.jpg', import.meta.url), result);
+});

packages/transcode/frame.ts

@@ -0,0 +1,17 @@
+import { ffmpeg } from './ffmpeg.ts';
+
+export function extractVideoFrame(
+  input: URL | ReadableStream<Uint8Array>,
+  ss: string = '00:00:01',
+  opts?: { ffmpegPath?: string | URL },
+): Promise<Uint8Array> {
+  const output = ffmpeg(input, {
+    'ss': ss, // Seek to timestamp
+    'frames:v': '1', // Extract only 1 frame
+    'q:v': '2', // High-quality JPEG (lower = better quality)
+    'f': 'image2', // Force image format
+    'loglevel': 'fatal',
+  }, opts);
+
+  return new Response(output).bytes();
+}

packages/transcode/mod.ts

@@ -0,0 +1,5 @@
+export { analyzeFile } from './analyze.ts';
+export { ffmpeg, type FFmpegFlags } from './ffmpeg.ts';
+export { ffprobe, type FFprobeFlags } from './ffprobe.ts';
+export { extractVideoFrame } from './frame.ts';
+export { transcodeVideo } from './transcode.ts';

packages/transcode/transcode.test.ts

@@ -0,0 +1,9 @@
+import { transcodeVideo } from './transcode.ts';
+
+Deno.test('transcodeVideo', async () => {
+  await using file = await Deno.open(new URL('./buckbunny.mp4', import.meta.url));
+  const output = transcodeVideo(file.readable);
+
+  await Deno.mkdir(new URL('./tmp', import.meta.url), { recursive: true });
+  await Deno.writeFile(new URL('./tmp/buckbunny-transcoded.mp4', import.meta.url), output);
+});

packages/transcode/transcode.ts

@@ -0,0 +1,19 @@
+import { ffmpeg } from './ffmpeg.ts';
+
+export function transcodeVideo(
+  input: URL | ReadableStream<Uint8Array>,
+  opts?: { ffmpegPath?: string | URL },
+): ReadableStream<Uint8Array> {
+  return ffmpeg(input, {
+    'safe': '1', // Safe mode
+    'nostdin': '', // Disable stdin
+    'c:v': 'libx264', // Convert to H.264
+    'preset': 'veryfast', // Encoding speed
+    'loglevel': 'fatal', // Suppress logs
+    'crf': '23', // Compression level (lower = better quality)
+    'c:a': 'aac', // Convert to AAC audio
+    'b:a': '128k', // Audio bitrate
+    'movflags': 'frag_keyframe+empty_moov', // Ensures MP4 streaming compatibility
+    'f': 'mp4', // Force MP4 format
+  }, opts);
+}