mirror/ditto
mirror/ditto
1
0
Fork 0
mirror of https://gitlab.com/soapbox-pub/ditto.git synced 2025-12-06 11:29:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'deepl-fix' into 'main'

Browse source 
Fix DeepL Response parsing, mock DeepL tests so they can always run without API keys

See merge request soapbox-pub/ditto!680
This commit is contained in:
Alex Gleason 2025-02-20 17:08:10 +00:00
commit 9a4746e199
7 changed files with 152 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/ditto/controllers/api/translate.ts
View file

@ -132,7 +132,7 @@ const translateController: AppController = async (c) => {
}
}
mastodonTranslation.detected_source_language = data.source_lang;
mastodonTranslation.detected_source_language = data.sourceLang;
translationCache.set(cacheKey, mastodonTranslation);
cachedTranslationsSizeGauge.set(translationCache.size);

73
packages/translators/DeepLTranslator.test.ts
View file

@ -1,21 +1,20 @@
import { DittoConf } from '@ditto/conf';
import { detectLanguage } from '@ditto/lang';
import { assert, assertEquals } from '@std/assert';
import { DeepLTranslator } from './DeepLTranslator.ts';
const {
deeplBaseUrl: baseUrl,
deeplApiKey: apiKey,
translationProvider,
} = new DittoConf(Deno.env);
const deepl = 'deepl';
Deno.test('DeepL translation with source language omitted', {
ignore: !(translationProvider === deepl && apiKey),
}, async () => {
const translator = new DeepLTranslator({ fetch: fetch, baseUrl, apiKey: apiKey! });
Deno.test('DeepL translation with source language omitted', async () => {
const translator = mockDeepL({
translations: [
{ detected_source_language: 'PT', text: 'Good morning friends' },
{ detected_source_language: 'PT', text: 'My name is Patrick' },
{
detected_source_language: 'PT',
text:
'I will live in America, I promise. But first, I should mention that lande is interpreting this text as Italian, how strange.',
},
],
});
const data = await translator.translate(
[
@ -27,16 +26,24 @@ Deno.test('DeepL translation with source language omitted', {
'en',
);
assertEquals(data.source_lang, 'pt');
assertEquals(data.sourceLang, 'pt');
assertEquals(detectLanguage(data.results[0], 0), 'en');
assertEquals(detectLanguage(data.results[1], 0), 'en');
assertEquals(detectLanguage(data.results[2], 0), 'en');
});
Deno.test('DeepL translation with source language set', {
ignore: !(translationProvider === deepl && apiKey),
}, async () => {
const translator = new DeepLTranslator({ fetch: fetch, baseUrl, apiKey: apiKey as string });
Deno.test('DeepL translation with source language set', async () => {
const translator = mockDeepL({
translations: [
{ detected_source_language: 'PT', text: 'Good morning friends' },
{ detected_source_language: 'PT', text: 'My name is Patrick' },
{
detected_source_language: 'PT',
text:
'I will live in America, I promise. But first, I should mention that lande is interpreting this text as Italian, how strange.',
},
],
});
const data = await translator.translate(
[
@ -48,16 +55,22 @@ Deno.test('DeepL translation with source language set', {
'en',
);
assertEquals(data.source_lang, 'pt');
assertEquals(data.sourceLang, 'pt');
assertEquals(detectLanguage(data.results[0], 0), 'en');
assertEquals(detectLanguage(data.results[1], 0), 'en');
assertEquals(detectLanguage(data.results[2], 0), 'en');
});
Deno.test("DeepL translation doesn't alter Nostr URIs", {
ignore: !(translationProvider === deepl && apiKey),
}, async () => {
const translator = new DeepLTranslator({ fetch: fetch, baseUrl, apiKey: apiKey as string });
Deno.test("DeepL translation doesn't alter Nostr URIs", async () => {
const translator = mockDeepL({
translations: [
{
detected_source_language: 'EN',
text:
'Graças ao trabalho de nostr:nprofile1qy2hwumn8ghj7erfw36x7tnsw43z7un9d3shjqpqgujeqakgt7fyp6zjggxhyy7ft623qtcaay5lkc8n8gkry4cvnrzqep59se e nostr:nprofile1qy2hwumn8ghj7erfw36x7tnsw43z7un9d3shjqpqe6tnvlr46lv3lwdu80r07kanhk6jcxy5r07w9umgv9kuhu9dl5hsz44l8s , agora é possível filtrar o feed global por idioma no #Ditto!',
},
],
});
const patrick =
'nostr:nprofile1qy2hwumn8ghj7erfw36x7tnsw43z7un9d3shjqpqgujeqakgt7fyp6zjggxhyy7ft623qtcaay5lkc8n8gkry4cvnrzqep59se';
@ -72,3 +85,17 @@ Deno.test("DeepL translation doesn't alter Nostr URIs", {
assert(output.includes(patrick));
assert(output.includes(danidfra));
});
interface DeepLResponse {
translations: {
detected_source_language: string;
text: string;
}[];
}
function mockDeepL(json: DeepLResponse): DeepLTranslator {
return new DeepLTranslator({
apiKey: 'deepl',
fetch: () => Promise.resolve(new Response(JSON.stringify(json))),
});
}

21
packages/translators/DeepLTranslator.ts
View file

@ -32,12 +32,12 @@ export class DeepLTranslator implements DittoTranslator {
source: LanguageCode | undefined,
dest: LanguageCode,
opts?: { signal?: AbortSignal },
): Promise<{ results: string[]; source_lang: LanguageCode }> {
): Promise<{ results: string[]; sourceLang: LanguageCode }> {
const { translations } = await this.translateMany(texts, source, dest, opts);
return {
results: translations.map((value) => value.text),
source_lang: translations[0]?.detected_source_language,
sourceLang: translations[0]?.detected_source_language,
};
}
@ -72,7 +72,13 @@ export class DeepLTranslator implements DittoTranslator {
const json = await response.json();
if (!response.ok) {
throw new Error(json['message']);
const result = DeepLTranslator.errorSchema().safeParse(json);
if (result.success) {
throw new Error(result.data.message);
} else {
throw new Error(`Unexpected DeepL error: ${response.statusText} (${response.status})`);
}
}
return DeepLTranslator.schema().parse(json);
@ -84,10 +90,17 @@ export class DeepLTranslator implements DittoTranslator {
return z.object({
translations: z.array(
z.object({
detected_source_language: languageSchema,
detected_source_language: z.string().transform((val) => val.toLowerCase()).pipe(languageSchema),
text: z.string(),
}),
),
});
}
/** DeepL error response schema. */
private static errorSchema() {
return z.object({
message: z.string(),
});
}
}

2
packages/translators/DittoTranslator.ts
View file

@ -14,5 +14,5 @@ export interface DittoTranslator {
targetLanguage: LanguageCode,
/** Custom options. */
opts?: { signal?: AbortSignal },
): Promise<{ results: string[]; source_lang: LanguageCode }>;
): Promise<{ results: string[]; sourceLang: LanguageCode }>;
}

72
packages/translators/LibreTranslateTranslator.test.ts
View file

@ -1,21 +1,10 @@
import { DittoConf } from '@ditto/conf';
import { detectLanguage } from '@ditto/lang';
import { assertEquals } from '@std/assert';
import { LibreTranslateTranslator } from './LibreTranslateTranslator.ts';
const {
libretranslateBaseUrl: baseUrl,
libretranslateApiKey: apiKey,
translationProvider,
} = new DittoConf(Deno.env);
const libretranslate = 'libretranslate';
Deno.test('LibreTranslate translation with source language omitted', {
ignore: !(translationProvider === libretranslate && apiKey),
}, async () => {
const translator = new LibreTranslateTranslator({ fetch: fetch, baseUrl, apiKey: apiKey! });
Deno.test('LibreTranslate translation with source language omitted', async () => {
const translator = mockLibreTranslate();
const data = await translator.translate(
[
@ -27,16 +16,14 @@ Deno.test('LibreTranslate translation with source language omitted', {
'ca',
);
assertEquals(data.source_lang, 'pt');
assertEquals(data.sourceLang, 'pt');
assertEquals(detectLanguage(data.results[0], 0), 'ca');
assertEquals(detectLanguage(data.results[1], 0), 'ca');
assertEquals(detectLanguage(data.results[2], 0), 'ca');
});
Deno.test('LibreTranslate translation with source language set', {
ignore: !(translationProvider === libretranslate && apiKey),
}, async () => {
const translator = new LibreTranslateTranslator({ fetch: fetch, baseUrl, apiKey: apiKey! });
Deno.test('LibreTranslate translation with source language set', async () => {
const translator = mockLibreTranslate();
const data = await translator.translate(
[
@ -48,8 +35,55 @@ Deno.test('LibreTranslate translation with source language set', {
'ca',
);
assertEquals(data.source_lang, 'pt');
assertEquals(data.sourceLang, 'pt');
assertEquals(detectLanguage(data.results[0], 0), 'ca');
assertEquals(detectLanguage(data.results[1], 0), 'ca');
assertEquals(detectLanguage(data.results[2], 0), 'ca');
});
function mockLibreTranslate(): LibreTranslateTranslator {
return new LibreTranslateTranslator({
apiKey: 'libretranslate',
fetch: async (input, init) => {
const req = new Request(input, init);
const body = await req.json();
switch (body.q) {
case 'Bom dia amigos':
return jsonResponse({
detectedLanguage: { language: 'pt' },
translatedText: 'Bon dia, amics.',
});
case 'Meu nome é Patrick, um nome belo ou feio? A questão é mais profunda do que parece.':
return jsonResponse({
detectedLanguage: { language: 'pt' },
translatedText: 'Em dic Patrick, un nom molt o lleig? La pregunta és més profunda del que sembla.',
});
case 'A respiração é mais importante do que comer e tomar agua.':
return jsonResponse({
detectedLanguage: { language: 'pt' },
translatedText: 'La respiració és més important que menjar i prendre aigua.',
});
}
return new Response(JSON.stringify({ error: 'Not found' }), { status: 404 });
},
});
}
interface LibreTranslateResponse {
translatedText: string;
detectedLanguage?: {
language: string;
};
}
function jsonResponse(json: LibreTranslateResponse): Response {
const body = JSON.stringify(json);
return new Response(body, {
headers: {
'Content-Type': 'application/json',
},
});
}

29
packages/translators/LibreTranslateTranslator.ts
View file

@ -32,14 +32,14 @@ export class LibreTranslateTranslator implements DittoTranslator {
source: LanguageCode | undefined,
dest: LanguageCode,
opts?: { signal?: AbortSignal },
): Promise<{ results: string[]; source_lang: LanguageCode }> {
): Promise<{ results: string[]; sourceLang: LanguageCode }> {
const translations = await Promise.all(
texts.map((text) => this.translateOne(text, source, dest, 'html', { signal: opts?.signal })),
);
return {
results: translations.map((value) => value.translatedText),
source_lang: (translations[0]?.detectedLanguage?.language ?? source) as LanguageCode, // cast is ok
sourceLang: (translations[0]?.detectedLanguage?.language ?? source) as LanguageCode, // cast is ok
};
}
@ -71,12 +71,20 @@ export class LibreTranslateTranslator implements DittoTranslator {
const response = await this.fetch(request);
const json = await response.json();
if (!response.ok) {
throw new Error(json['error']);
}
const data = LibreTranslateTranslator.schema().parse(json);
return data;
console.log(json);
if (!response.ok) {
const result = LibreTranslateTranslator.errorSchema().safeParse(json);
if (result.success) {
throw new Error(result.data.error);
} else {
throw new Error(`Unexpected LibreTranslate error: ${response.statusText} (${response.status})`);
}
}
return LibreTranslateTranslator.schema().parse(json);
}
/** Libretranslate response schema.
@ -90,4 +98,11 @@ export class LibreTranslateTranslator implements DittoTranslator {
}).optional(),
});
}
/** Libretranslate error response schema. */
private static errorSchema() {
return z.object({
error: z.string(),
});
}
}

8
packages/translators/schema.test.ts Normal file
View file

@ -0,0 +1,8 @@
import { assertEquals } from '@std/assert';
import { languageSchema } from './schema.ts';
Deno.test('languageSchema', () => {
assertEquals(languageSchema.safeParse('pt').success, true);
assertEquals(languageSchema.safeParse('PT').success, false);
});