Merge branch 'deepl-fix' into 'main'

Fix DeepL Response parsing, mock DeepL tests so they can always run without API keys

See merge request soapbox-pub/ditto!680

packages/ditto/controllers/api/translate.ts

@@ -132,7 +132,7 @@ const translateController: AppController = async (c) => {
       }
     }
 
-    mastodonTranslation.detected_source_language = data.source_lang;
+    mastodonTranslation.detected_source_language = data.sourceLang;
 
     translationCache.set(cacheKey, mastodonTranslation);
     cachedTranslationsSizeGauge.set(translationCache.size);

packages/translators/DeepLTranslator.test.ts

@@ -1,21 +1,20 @@
-import { DittoConf } from '@ditto/conf';
 import { detectLanguage } from '@ditto/lang';
 import { assert, assertEquals } from '@std/assert';
 
 import { DeepLTranslator } from './DeepLTranslator.ts';
 
-const {
+Deno.test('DeepL translation with source language omitted', async () => {
-  deeplBaseUrl: baseUrl,
+  const translator = mockDeepL({
-  deeplApiKey: apiKey,
+    translations: [
-  translationProvider,
+      { detected_source_language: 'PT', text: 'Good morning friends' },
-} = new DittoConf(Deno.env);
+      { detected_source_language: 'PT', text: 'My name is Patrick' },
-
+      {
-const deepl = 'deepl';
+        detected_source_language: 'PT',
-
+        text:
-Deno.test('DeepL translation with source language omitted', {
+          'I will live in America, I promise. But first, I should mention that lande is interpreting this text as Italian, how strange.',
-  ignore: !(translationProvider === deepl && apiKey),
+      },
-}, async () => {
+    ],
-  const translator = new DeepLTranslator({ fetch: fetch, baseUrl, apiKey: apiKey! });
+  });
 
   const data = await translator.translate(
     [
@@ -27,16 +26,24 @@ Deno.test('DeepL translation with source language omitted', {
     'en',
   );
 
-  assertEquals(data.source_lang, 'pt');
+  assertEquals(data.sourceLang, 'pt');
   assertEquals(detectLanguage(data.results[0], 0), 'en');
   assertEquals(detectLanguage(data.results[1], 0), 'en');
   assertEquals(detectLanguage(data.results[2], 0), 'en');
 });
 
-Deno.test('DeepL translation with source language set', {
+Deno.test('DeepL translation with source language set', async () => {
-  ignore: !(translationProvider === deepl && apiKey),
+  const translator = mockDeepL({
-}, async () => {
+    translations: [
-  const translator = new DeepLTranslator({ fetch: fetch, baseUrl, apiKey: apiKey as string });
+      { detected_source_language: 'PT', text: 'Good morning friends' },
+      { detected_source_language: 'PT', text: 'My name is Patrick' },
+      {
+        detected_source_language: 'PT',
+        text:
+          'I will live in America, I promise. But first, I should mention that lande is interpreting this text as Italian, how strange.',
+      },
+    ],
+  });
 
   const data = await translator.translate(
     [
@@ -48,16 +55,22 @@ Deno.test('DeepL translation with source language set', {
     'en',
   );
 
-  assertEquals(data.source_lang, 'pt');
+  assertEquals(data.sourceLang, 'pt');
   assertEquals(detectLanguage(data.results[0], 0), 'en');
   assertEquals(detectLanguage(data.results[1], 0), 'en');
   assertEquals(detectLanguage(data.results[2], 0), 'en');
 });
 
-Deno.test("DeepL translation doesn't alter Nostr URIs", {
+Deno.test("DeepL translation doesn't alter Nostr URIs", async () => {
-  ignore: !(translationProvider === deepl && apiKey),
+  const translator = mockDeepL({
-}, async () => {
+    translations: [
-  const translator = new DeepLTranslator({ fetch: fetch, baseUrl, apiKey: apiKey as string });
+      {
+        detected_source_language: 'EN',
+        text:
+          'Graças ao trabalho de nostr:nprofile1qy2hwumn8ghj7erfw36x7tnsw43z7un9d3shjqpqgujeqakgt7fyp6zjggxhyy7ft623qtcaay5lkc8n8gkry4cvnrzqep59se e nostr:nprofile1qy2hwumn8ghj7erfw36x7tnsw43z7un9d3shjqpqe6tnvlr46lv3lwdu80r07kanhk6jcxy5r07w9umgv9kuhu9dl5hsz44l8s , agora é possível filtrar o feed global por idioma no #Ditto!',
+      },
+    ],
+  });
 
   const patrick =
     'nostr:nprofile1qy2hwumn8ghj7erfw36x7tnsw43z7un9d3shjqpqgujeqakgt7fyp6zjggxhyy7ft623qtcaay5lkc8n8gkry4cvnrzqep59se';
@@ -72,3 +85,17 @@ Deno.test("DeepL translation doesn't alter Nostr URIs", {
   assert(output.includes(patrick));
   assert(output.includes(danidfra));
 });
+
+interface DeepLResponse {
+  translations: {
+    detected_source_language: string;
+    text: string;
+  }[];
+}
+
+function mockDeepL(json: DeepLResponse): DeepLTranslator {
+  return new DeepLTranslator({
+    apiKey: 'deepl',
+    fetch: () => Promise.resolve(new Response(JSON.stringify(json))),
+  });
+}

packages/translators/DeepLTranslator.ts

@@ -32,12 +32,12 @@ export class DeepLTranslator implements DittoTranslator {
     source: LanguageCode | undefined,
     dest: LanguageCode,
     opts?: { signal?: AbortSignal },
-  ): Promise<{ results: string[]; source_lang: LanguageCode }> {
+  ): Promise<{ results: string[]; sourceLang: LanguageCode }> {
     const { translations } = await this.translateMany(texts, source, dest, opts);
 
     return {
       results: translations.map((value) => value.text),
-      source_lang: translations[0]?.detected_source_language,
+      sourceLang: translations[0]?.detected_source_language,
     };
   }
 
@@ -72,7 +72,13 @@ export class DeepLTranslator implements DittoTranslator {
     const json = await response.json();
 
     if (!response.ok) {
-      throw new Error(json['message']);
+      const result = DeepLTranslator.errorSchema().safeParse(json);
+
+      if (result.success) {
+        throw new Error(result.data.message);
+      } else {
+        throw new Error(`Unexpected DeepL error: ${response.statusText} (${response.status})`);
+      }
     }
 
     return DeepLTranslator.schema().parse(json);
@@ -84,10 +90,17 @@ export class DeepLTranslator implements DittoTranslator {
     return z.object({
       translations: z.array(
         z.object({
-          detected_source_language: languageSchema,
+          detected_source_language: z.string().transform((val) => val.toLowerCase()).pipe(languageSchema),
           text: z.string(),
         }),
       ),
     });
   }
+
+  /** DeepL error response schema. */
+  private static errorSchema() {
+    return z.object({
+      message: z.string(),
+    });
+  }
 }

packages/translators/DittoTranslator.ts

@@ -14,5 +14,5 @@ export interface DittoTranslator {
     targetLanguage: LanguageCode,
     /** Custom options. */
     opts?: { signal?: AbortSignal },
-  ): Promise<{ results: string[]; source_lang: LanguageCode }>;
+  ): Promise<{ results: string[]; sourceLang: LanguageCode }>;
 }

packages/translators/LibreTranslateTranslator.test.ts

@@ -1,21 +1,10 @@
-import { DittoConf } from '@ditto/conf';
 import { detectLanguage } from '@ditto/lang';
 import { assertEquals } from '@std/assert';
 
 import { LibreTranslateTranslator } from './LibreTranslateTranslator.ts';
 
-const {
+Deno.test('LibreTranslate translation with source language omitted', async () => {
-  libretranslateBaseUrl: baseUrl,
+  const translator = mockLibreTranslate();
-  libretranslateApiKey: apiKey,
-  translationProvider,
-} = new DittoConf(Deno.env);
-
-const libretranslate = 'libretranslate';
-
-Deno.test('LibreTranslate translation with source language omitted', {
-  ignore: !(translationProvider === libretranslate && apiKey),
-}, async () => {
-  const translator = new LibreTranslateTranslator({ fetch: fetch, baseUrl, apiKey: apiKey! });
 
   const data = await translator.translate(
     [
@@ -27,16 +16,14 @@ Deno.test('LibreTranslate translation with source language omitted', {
     'ca',
   );
 
-  assertEquals(data.source_lang, 'pt');
+  assertEquals(data.sourceLang, 'pt');
   assertEquals(detectLanguage(data.results[0], 0), 'ca');
   assertEquals(detectLanguage(data.results[1], 0), 'ca');
   assertEquals(detectLanguage(data.results[2], 0), 'ca');
 });
 
-Deno.test('LibreTranslate translation with source language set', {
+Deno.test('LibreTranslate translation with source language set', async () => {
-  ignore: !(translationProvider === libretranslate && apiKey),
+  const translator = mockLibreTranslate();
-}, async () => {
-  const translator = new LibreTranslateTranslator({ fetch: fetch, baseUrl, apiKey: apiKey! });
 
   const data = await translator.translate(
     [
@@ -48,8 +35,55 @@ Deno.test('LibreTranslate translation with source language set', {
     'ca',
   );
 
-  assertEquals(data.source_lang, 'pt');
+  assertEquals(data.sourceLang, 'pt');
   assertEquals(detectLanguage(data.results[0], 0), 'ca');
   assertEquals(detectLanguage(data.results[1], 0), 'ca');
   assertEquals(detectLanguage(data.results[2], 0), 'ca');
 });
+
+function mockLibreTranslate(): LibreTranslateTranslator {
+  return new LibreTranslateTranslator({
+    apiKey: 'libretranslate',
+    fetch: async (input, init) => {
+      const req = new Request(input, init);
+      const body = await req.json();
+
+      switch (body.q) {
+        case 'Bom dia amigos':
+          return jsonResponse({
+            detectedLanguage: { language: 'pt' },
+            translatedText: 'Bon dia, amics.',
+          });
+        case 'Meu nome é Patrick, um nome belo ou feio? A questão é mais profunda do que parece.':
+          return jsonResponse({
+            detectedLanguage: { language: 'pt' },
+            translatedText: 'Em dic Patrick, un nom molt o lleig? La pregunta és més profunda del que sembla.',
+          });
+        case 'A respiração é mais importante do que comer e tomar agua.':
+          return jsonResponse({
+            detectedLanguage: { language: 'pt' },
+            translatedText: 'La respiració és més important que menjar i prendre aigua.',
+          });
+      }
+
+      return new Response(JSON.stringify({ error: 'Not found' }), { status: 404 });
+    },
+  });
+}
+
+interface LibreTranslateResponse {
+  translatedText: string;
+  detectedLanguage?: {
+    language: string;
+  };
+}
+
+function jsonResponse(json: LibreTranslateResponse): Response {
+  const body = JSON.stringify(json);
+
+  return new Response(body, {
+    headers: {
+      'Content-Type': 'application/json',
+    },
+  });
+}

packages/translators/LibreTranslateTranslator.ts

@@ -32,14 +32,14 @@ export class LibreTranslateTranslator implements DittoTranslator {
     source: LanguageCode | undefined,
     dest: LanguageCode,
     opts?: { signal?: AbortSignal },
-  ): Promise<{ results: string[]; source_lang: LanguageCode }> {
+  ): Promise<{ results: string[]; sourceLang: LanguageCode }> {
     const translations = await Promise.all(
       texts.map((text) => this.translateOne(text, source, dest, 'html', { signal: opts?.signal })),
     );
 
     return {
       results: translations.map((value) => value.translatedText),
-      source_lang: (translations[0]?.detectedLanguage?.language ?? source) as LanguageCode, // cast is ok
+      sourceLang: (translations[0]?.detectedLanguage?.language ?? source) as LanguageCode, // cast is ok
     };
   }
 
@@ -71,12 +71,20 @@ export class LibreTranslateTranslator implements DittoTranslator {
 
     const response = await this.fetch(request);
     const json = await response.json();
-    if (!response.ok) {
-      throw new Error(json['error']);
-    }
-    const data = LibreTranslateTranslator.schema().parse(json);
 
-    return data;
+    console.log(json);
+
+    if (!response.ok) {
+      const result = LibreTranslateTranslator.errorSchema().safeParse(json);
+
+      if (result.success) {
+        throw new Error(result.data.error);
+      } else {
+        throw new Error(`Unexpected LibreTranslate error: ${response.statusText} (${response.status})`);
+      }
+    }
+
+    return LibreTranslateTranslator.schema().parse(json);
   }
 
   /** Libretranslate response schema.
@@ -90,4 +98,11 @@ export class LibreTranslateTranslator implements DittoTranslator {
       }).optional(),
     });
   }
+
+  /** Libretranslate error response schema. */
+  private static errorSchema() {
+    return z.object({
+      error: z.string(),
+    });
+  }
 }

packages/translators/schema.test.ts

@@ -0,0 +1,8 @@
+import { assertEquals } from '@std/assert';
+
+import { languageSchema } from './schema.ts';
+
+Deno.test('languageSchema', () => {
+  assertEquals(languageSchema.safeParse('pt').success, true);
+  assertEquals(languageSchema.safeParse('PT').success, false);
+});