From d791a9b35079be5d25acfcd214ab0917bdd35dfe Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 10:35:27 -0600 Subject: [PATCH 1/4] Fix DeepL Response parsing, mock DeepL tests so they can always run without API keys --- packages/translators/DeepLTranslator.test.ts | 69 ++++++++++++++------ packages/translators/DeepLTranslator.ts | 17 ++++- packages/translators/schema.test.ts | 7 ++ 3 files changed, 70 insertions(+), 23 deletions(-) create mode 100644 packages/translators/schema.test.ts diff --git a/packages/translators/DeepLTranslator.test.ts b/packages/translators/DeepLTranslator.test.ts index ae1565c9..8e37e44b 100644 --- a/packages/translators/DeepLTranslator.test.ts +++ b/packages/translators/DeepLTranslator.test.ts @@ -1,21 +1,20 @@ -import { DittoConf } from '@ditto/conf'; import { detectLanguage } from '@ditto/lang'; import { assert, assertEquals } from '@std/assert'; import { DeepLTranslator } from './DeepLTranslator.ts'; -const { - deeplBaseUrl: baseUrl, - deeplApiKey: apiKey, - translationProvider, -} = new DittoConf(Deno.env); - -const deepl = 'deepl'; - -Deno.test('DeepL translation with source language omitted', { - ignore: !(translationProvider === deepl && apiKey), -}, async () => { - const translator = new DeepLTranslator({ fetch: fetch, baseUrl, apiKey: apiKey! }); +Deno.test('DeepL translation with source language omitted', async () => { + const translator = mockDeepL({ + translations: [ + { detected_source_language: 'PT', text: 'Good morning friends' }, + { detected_source_language: 'PT', text: 'My name is Patrick' }, + { + detected_source_language: 'PT', + text: + 'I will live in America, I promise. But first, I should mention that lande is interpreting this text as Italian, how strange.', + }, + ], + }); const data = await translator.translate( [ @@ -33,10 +32,18 @@ Deno.test('DeepL translation with source language omitted', { assertEquals(detectLanguage(data.results[2], 0), 'en'); }); -Deno.test('DeepL translation with source language set', { - ignore: !(translationProvider === deepl && apiKey), -}, async () => { - const translator = new DeepLTranslator({ fetch: fetch, baseUrl, apiKey: apiKey as string }); +Deno.test('DeepL translation with source language set', async () => { + const translator = mockDeepL({ + translations: [ + { detected_source_language: 'PT', text: 'Good morning friends' }, + { detected_source_language: 'PT', text: 'My name is Patrick' }, + { + detected_source_language: 'PT', + text: + 'I will live in America, I promise. But first, I should mention that lande is interpreting this text as Italian, how strange.', + }, + ], + }); const data = await translator.translate( [ @@ -54,10 +61,16 @@ Deno.test('DeepL translation with source language set', { assertEquals(detectLanguage(data.results[2], 0), 'en'); }); -Deno.test("DeepL translation doesn't alter Nostr URIs", { - ignore: !(translationProvider === deepl && apiKey), -}, async () => { - const translator = new DeepLTranslator({ fetch: fetch, baseUrl, apiKey: apiKey as string }); +Deno.test("DeepL translation doesn't alter Nostr URIs", async () => { + const translator = mockDeepL({ + translations: [ + { + detected_source_language: 'EN', + text: + 'Graças ao trabalho de nostr:nprofile1qy2hwumn8ghj7erfw36x7tnsw43z7un9d3shjqpqgujeqakgt7fyp6zjggxhyy7ft623qtcaay5lkc8n8gkry4cvnrzqep59se e nostr:nprofile1qy2hwumn8ghj7erfw36x7tnsw43z7un9d3shjqpqe6tnvlr46lv3lwdu80r07kanhk6jcxy5r07w9umgv9kuhu9dl5hsz44l8s , agora é possível filtrar o feed global por idioma no #Ditto!', + }, + ], + }); const patrick = 'nostr:nprofile1qy2hwumn8ghj7erfw36x7tnsw43z7un9d3shjqpqgujeqakgt7fyp6zjggxhyy7ft623qtcaay5lkc8n8gkry4cvnrzqep59se'; @@ -72,3 +85,17 @@ Deno.test("DeepL translation doesn't alter Nostr URIs", { assert(output.includes(patrick)); assert(output.includes(danidfra)); }); + +interface DeepLResponse { + translations: { + detected_source_language: string; + text: string; + }[]; +} + +function mockDeepL(json: DeepLResponse): DeepLTranslator { + return new DeepLTranslator({ + apiKey: 'deepl', + fetch: () => Promise.resolve(new Response(JSON.stringify(json))), + }); +} diff --git a/packages/translators/DeepLTranslator.ts b/packages/translators/DeepLTranslator.ts index f4b6f918..93da8ad7 100644 --- a/packages/translators/DeepLTranslator.ts +++ b/packages/translators/DeepLTranslator.ts @@ -72,7 +72,13 @@ export class DeepLTranslator implements DittoTranslator { const json = await response.json(); if (!response.ok) { - throw new Error(json['message']); + const result = DeepLTranslator.errorSchema().safeParse(json); + + if (result.success) { + throw new Error(result.data.message); + } else { + throw new Error(`Unexpected DeepL error: ${response.statusText} (${response.status})`); + } } return DeepLTranslator.schema().parse(json); @@ -84,10 +90,17 @@ export class DeepLTranslator implements DittoTranslator { return z.object({ translations: z.array( z.object({ - detected_source_language: languageSchema, + detected_source_language: z.string().transform((val) => val.toLowerCase()).pipe(languageSchema), text: z.string(), }), ), }); } + + /** DeepL error response schema. */ + private static errorSchema() { + return z.object({ + message: z.string(), + }); + } } diff --git a/packages/translators/schema.test.ts b/packages/translators/schema.test.ts new file mode 100644 index 00000000..6d37992c --- /dev/null +++ b/packages/translators/schema.test.ts @@ -0,0 +1,7 @@ +import { assertEquals } from '@std/assert'; + +import { languageSchema } from './schema.ts'; + +Deno.test('languageSchema', () => { + assertEquals(languageSchema.safeParse('en').success, true); +}); From 2150259abad023234cf8a9aa92ee8c59f37eff74 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 10:36:43 -0600 Subject: [PATCH 2/4] languageSchema does not lowercase the code --- packages/translators/schema.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/translators/schema.test.ts b/packages/translators/schema.test.ts index 6d37992c..4ca84adc 100644 --- a/packages/translators/schema.test.ts +++ b/packages/translators/schema.test.ts @@ -3,5 +3,6 @@ import { assertEquals } from '@std/assert'; import { languageSchema } from './schema.ts'; Deno.test('languageSchema', () => { - assertEquals(languageSchema.safeParse('en').success, true); + assertEquals(languageSchema.safeParse('pt').success, true); + assertEquals(languageSchema.safeParse('PT').success, false); }); From 1afb09e60495477425373203bb7f3cf3662b6b71 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 10:39:31 -0600 Subject: [PATCH 3/4] DittoTranslator: source_lang -> sourceLang --- packages/ditto/controllers/api/translate.ts | 2 +- packages/translators/DeepLTranslator.test.ts | 4 ++-- packages/translators/DeepLTranslator.ts | 4 ++-- packages/translators/DittoTranslator.ts | 2 +- packages/translators/LibreTranslateTranslator.test.ts | 4 ++-- packages/translators/LibreTranslateTranslator.ts | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/ditto/controllers/api/translate.ts b/packages/ditto/controllers/api/translate.ts index 8a99edde..de183e23 100644 --- a/packages/ditto/controllers/api/translate.ts +++ b/packages/ditto/controllers/api/translate.ts @@ -132,7 +132,7 @@ const translateController: AppController = async (c) => { } } - mastodonTranslation.detected_source_language = data.source_lang; + mastodonTranslation.detected_source_language = data.sourceLang; translationCache.set(cacheKey, mastodonTranslation); cachedTranslationsSizeGauge.set(translationCache.size); diff --git a/packages/translators/DeepLTranslator.test.ts b/packages/translators/DeepLTranslator.test.ts index 8e37e44b..a688f135 100644 --- a/packages/translators/DeepLTranslator.test.ts +++ b/packages/translators/DeepLTranslator.test.ts @@ -26,7 +26,7 @@ Deno.test('DeepL translation with source language omitted', async () => { 'en', ); - assertEquals(data.source_lang, 'pt'); + assertEquals(data.sourceLang, 'pt'); assertEquals(detectLanguage(data.results[0], 0), 'en'); assertEquals(detectLanguage(data.results[1], 0), 'en'); assertEquals(detectLanguage(data.results[2], 0), 'en'); @@ -55,7 +55,7 @@ Deno.test('DeepL translation with source language set', async () => { 'en', ); - assertEquals(data.source_lang, 'pt'); + assertEquals(data.sourceLang, 'pt'); assertEquals(detectLanguage(data.results[0], 0), 'en'); assertEquals(detectLanguage(data.results[1], 0), 'en'); assertEquals(detectLanguage(data.results[2], 0), 'en'); diff --git a/packages/translators/DeepLTranslator.ts b/packages/translators/DeepLTranslator.ts index 93da8ad7..673c6e07 100644 --- a/packages/translators/DeepLTranslator.ts +++ b/packages/translators/DeepLTranslator.ts @@ -32,12 +32,12 @@ export class DeepLTranslator implements DittoTranslator { source: LanguageCode | undefined, dest: LanguageCode, opts?: { signal?: AbortSignal }, - ): Promise<{ results: string[]; source_lang: LanguageCode }> { + ): Promise<{ results: string[]; sourceLang: LanguageCode }> { const { translations } = await this.translateMany(texts, source, dest, opts); return { results: translations.map((value) => value.text), - source_lang: translations[0]?.detected_source_language, + sourceLang: translations[0]?.detected_source_language, }; } diff --git a/packages/translators/DittoTranslator.ts b/packages/translators/DittoTranslator.ts index 7e5e1d50..2a9fb7db 100644 --- a/packages/translators/DittoTranslator.ts +++ b/packages/translators/DittoTranslator.ts @@ -14,5 +14,5 @@ export interface DittoTranslator { targetLanguage: LanguageCode, /** Custom options. */ opts?: { signal?: AbortSignal }, - ): Promise<{ results: string[]; source_lang: LanguageCode }>; + ): Promise<{ results: string[]; sourceLang: LanguageCode }>; } diff --git a/packages/translators/LibreTranslateTranslator.test.ts b/packages/translators/LibreTranslateTranslator.test.ts index fc6c0a55..ca8c1d79 100644 --- a/packages/translators/LibreTranslateTranslator.test.ts +++ b/packages/translators/LibreTranslateTranslator.test.ts @@ -27,7 +27,7 @@ Deno.test('LibreTranslate translation with source language omitted', { 'ca', ); - assertEquals(data.source_lang, 'pt'); + assertEquals(data.sourceLang, 'pt'); assertEquals(detectLanguage(data.results[0], 0), 'ca'); assertEquals(detectLanguage(data.results[1], 0), 'ca'); assertEquals(detectLanguage(data.results[2], 0), 'ca'); @@ -48,7 +48,7 @@ Deno.test('LibreTranslate translation with source language set', { 'ca', ); - assertEquals(data.source_lang, 'pt'); + assertEquals(data.sourceLang, 'pt'); assertEquals(detectLanguage(data.results[0], 0), 'ca'); assertEquals(detectLanguage(data.results[1], 0), 'ca'); assertEquals(detectLanguage(data.results[2], 0), 'ca'); diff --git a/packages/translators/LibreTranslateTranslator.ts b/packages/translators/LibreTranslateTranslator.ts index b75f9b54..a8145223 100644 --- a/packages/translators/LibreTranslateTranslator.ts +++ b/packages/translators/LibreTranslateTranslator.ts @@ -32,14 +32,14 @@ export class LibreTranslateTranslator implements DittoTranslator { source: LanguageCode | undefined, dest: LanguageCode, opts?: { signal?: AbortSignal }, - ): Promise<{ results: string[]; source_lang: LanguageCode }> { + ): Promise<{ results: string[]; sourceLang: LanguageCode }> { const translations = await Promise.all( texts.map((text) => this.translateOne(text, source, dest, 'html', { signal: opts?.signal })), ); return { results: translations.map((value) => value.translatedText), - source_lang: (translations[0]?.detectedLanguage?.language ?? source) as LanguageCode, // cast is ok + sourceLang: (translations[0]?.detectedLanguage?.language ?? source) as LanguageCode, // cast is ok }; } From 91f9bd944210afe4d6869cae3c6c20d8db0bf3c7 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 20 Feb 2025 11:04:41 -0600 Subject: [PATCH 4/4] Add mock LibreTranslate tests --- .../LibreTranslateTranslator.test.ts | 68 ++++++++++++++----- .../translators/LibreTranslateTranslator.ts | 25 +++++-- 2 files changed, 71 insertions(+), 22 deletions(-) diff --git a/packages/translators/LibreTranslateTranslator.test.ts b/packages/translators/LibreTranslateTranslator.test.ts index ca8c1d79..94da0ec0 100644 --- a/packages/translators/LibreTranslateTranslator.test.ts +++ b/packages/translators/LibreTranslateTranslator.test.ts @@ -1,21 +1,10 @@ -import { DittoConf } from '@ditto/conf'; import { detectLanguage } from '@ditto/lang'; import { assertEquals } from '@std/assert'; import { LibreTranslateTranslator } from './LibreTranslateTranslator.ts'; -const { - libretranslateBaseUrl: baseUrl, - libretranslateApiKey: apiKey, - translationProvider, -} = new DittoConf(Deno.env); - -const libretranslate = 'libretranslate'; - -Deno.test('LibreTranslate translation with source language omitted', { - ignore: !(translationProvider === libretranslate && apiKey), -}, async () => { - const translator = new LibreTranslateTranslator({ fetch: fetch, baseUrl, apiKey: apiKey! }); +Deno.test('LibreTranslate translation with source language omitted', async () => { + const translator = mockLibreTranslate(); const data = await translator.translate( [ @@ -33,10 +22,8 @@ Deno.test('LibreTranslate translation with source language omitted', { assertEquals(detectLanguage(data.results[2], 0), 'ca'); }); -Deno.test('LibreTranslate translation with source language set', { - ignore: !(translationProvider === libretranslate && apiKey), -}, async () => { - const translator = new LibreTranslateTranslator({ fetch: fetch, baseUrl, apiKey: apiKey! }); +Deno.test('LibreTranslate translation with source language set', async () => { + const translator = mockLibreTranslate(); const data = await translator.translate( [ @@ -53,3 +40,50 @@ Deno.test('LibreTranslate translation with source language set', { assertEquals(detectLanguage(data.results[1], 0), 'ca'); assertEquals(detectLanguage(data.results[2], 0), 'ca'); }); + +function mockLibreTranslate(): LibreTranslateTranslator { + return new LibreTranslateTranslator({ + apiKey: 'libretranslate', + fetch: async (input, init) => { + const req = new Request(input, init); + const body = await req.json(); + + switch (body.q) { + case 'Bom dia amigos': + return jsonResponse({ + detectedLanguage: { language: 'pt' }, + translatedText: 'Bon dia, amics.', + }); + case 'Meu nome é Patrick, um nome belo ou feio? A questão é mais profunda do que parece.': + return jsonResponse({ + detectedLanguage: { language: 'pt' }, + translatedText: 'Em dic Patrick, un nom molt o lleig? La pregunta és més profunda del que sembla.', + }); + case 'A respiração é mais importante do que comer e tomar agua.': + return jsonResponse({ + detectedLanguage: { language: 'pt' }, + translatedText: 'La respiració és més important que menjar i prendre aigua.', + }); + } + + return new Response(JSON.stringify({ error: 'Not found' }), { status: 404 }); + }, + }); +} + +interface LibreTranslateResponse { + translatedText: string; + detectedLanguage?: { + language: string; + }; +} + +function jsonResponse(json: LibreTranslateResponse): Response { + const body = JSON.stringify(json); + + return new Response(body, { + headers: { + 'Content-Type': 'application/json', + }, + }); +} diff --git a/packages/translators/LibreTranslateTranslator.ts b/packages/translators/LibreTranslateTranslator.ts index a8145223..cc978e90 100644 --- a/packages/translators/LibreTranslateTranslator.ts +++ b/packages/translators/LibreTranslateTranslator.ts @@ -71,12 +71,20 @@ export class LibreTranslateTranslator implements DittoTranslator { const response = await this.fetch(request); const json = await response.json(); - if (!response.ok) { - throw new Error(json['error']); - } - const data = LibreTranslateTranslator.schema().parse(json); - return data; + console.log(json); + + if (!response.ok) { + const result = LibreTranslateTranslator.errorSchema().safeParse(json); + + if (result.success) { + throw new Error(result.data.error); + } else { + throw new Error(`Unexpected LibreTranslate error: ${response.statusText} (${response.status})`); + } + } + + return LibreTranslateTranslator.schema().parse(json); } /** Libretranslate response schema. @@ -90,4 +98,11 @@ export class LibreTranslateTranslator implements DittoTranslator { }).optional(), }); } + + /** Libretranslate error response schema. */ + private static errorSchema() { + return z.object({ + error: z.string(), + }); + } }