From ab85360d2ff5de1e3aa42e4a35aa4b640cccaf31 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 3 Oct 2024 11:17:21 -0300 Subject: [PATCH 01/34] refactor: move getConfigs() function and frontendConfig logic to 'src/utils/frontendConfig.ts' --- src/controllers/api/pleroma.ts | 32 ++++----------------------- src/utils/frontendConfig.ts | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 28 deletions(-) create mode 100644 src/utils/frontendConfig.ts diff --git a/src/controllers/api/pleroma.ts b/src/controllers/api/pleroma.ts index 31d8545f..2c025b8e 100644 --- a/src/controllers/api/pleroma.ts +++ b/src/controllers/api/pleroma.ts @@ -1,26 +1,20 @@ -import { NSchema as n, NStore } from '@nostrify/nostrify'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { configSchema, elixirTupleSchema, type PleromaConfig } from '@/schemas/pleroma-api.ts'; +import { configSchema } from '@/schemas/pleroma-api.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Storages } from '@/storages.ts'; import { createAdminEvent, updateAdminEvent, updateUser } from '@/utils/api.ts'; +import { getConfigs, getPleromaConfig } from '@/utils/frontendConfig.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; const frontendConfigController: AppController = async (c) => { const store = await Storages.db(); - const configs = await getConfigs(store, c.req.raw.signal); - const frontendConfig = configs.find(({ group, key }) => group === ':pleroma' && key === ':frontend_configurations'); + const frontendConfig = await getPleromaConfig(store, c.req.raw.signal); if (frontendConfig) { - const schema = elixirTupleSchema.transform(({ tuple }) => tuple).array(); - const data = schema.parse(frontendConfig.value).reduce>((result, [name, data]) => { - result[name.replace(/^:/, '')] = data; - return result; - }, {}); - return c.json(data); + return c.json(frontendConfig); } else { return c.json({}); } @@ -70,24 +64,6 @@ const pleromaAdminDeleteStatusController: AppController = async (c) => { return c.json({}); }; -async function getConfigs(store: NStore, signal: AbortSignal): Promise { - const { pubkey } = Conf; - - const [event] = await store.query([{ - kinds: [30078], - authors: [pubkey], - '#d': ['pub.ditto.pleroma.config'], - limit: 1, - }], { signal }); - - try { - const decrypted = await new AdminSigner().nip44.decrypt(Conf.pubkey, event.content); - return n.json().pipe(configSchema.array()).catch([]).parse(decrypted); - } catch (_e) { - return []; - } -} - const pleromaAdminTagSchema = z.object({ nicknames: z.string().array(), tags: z.string().array(), diff --git a/src/utils/frontendConfig.ts b/src/utils/frontendConfig.ts new file mode 100644 index 00000000..a4f3f592 --- /dev/null +++ b/src/utils/frontendConfig.ts @@ -0,0 +1,40 @@ +import { NSchema as n, NStore } from '@nostrify/nostrify'; + +import { AdminSigner } from '@/signers/AdminSigner.ts'; +import { Conf } from '@/config.ts'; +import { configSchema, elixirTupleSchema, type PleromaConfig } from '@/schemas/pleroma-api.ts'; + +export async function getPleromaConfig( + store: NStore, + signal?: AbortSignal, +): Promise> { + const configs = await getConfigs(store, signal ?? AbortSignal.timeout(1000)); + const frontendConfig = configs.find(({ group, key }) => group === ':pleroma' && key === ':frontend_configurations'); + if (frontendConfig) { + const schema = elixirTupleSchema.transform(({ tuple }) => tuple).array(); + const data = schema.parse(frontendConfig.value).reduce>((result, [name, data]) => { + result[name.replace(/^:/, '')] = data; + return result; + }, {}); + return data; + } + return undefined; +} + +export async function getConfigs(store: NStore, signal: AbortSignal): Promise { + const { pubkey } = Conf; + + const [event] = await store.query([{ + kinds: [30078], + authors: [pubkey], + '#d': ['pub.ditto.pleroma.config'], + limit: 1, + }], { signal }); + + try { + const decrypted = await new AdminSigner().nip44.decrypt(Conf.pubkey, event.content); + return n.json().pipe(configSchema.array()).catch([]).parse(decrypted); + } catch (_e) { + return []; + } +} From f3b7f91a07707adff13c2941a646bd5c022675d7 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 4 Oct 2024 23:37:01 -0300 Subject: [PATCH 02/34] feat: languageSchema converts value to lowercase and returns type LanguageCode --- src/schema.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/schema.ts b/src/schema.ts index 5efa7769..edaba0b4 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -1,4 +1,4 @@ -import ISO6391 from 'iso-639-1'; +import ISO6391, { LanguageCode } from 'iso-639-1'; import { z } from 'zod'; /** Validates individual items in an array, dropping any that aren't valid. */ @@ -42,6 +42,7 @@ const fileSchema = z.custom((value) => value instanceof File); const percentageSchema = z.coerce.number().int().gte(1).lte(100); const languageSchema = z.string().transform((val, ctx) => { + val = val.toLowerCase(); if (!ISO6391.validate(val)) { ctx.addIssue({ code: z.ZodIssueCode.custom, @@ -49,7 +50,7 @@ const languageSchema = z.string().transform((val, ctx) => { }); return z.NEVER; } - return val; + return val as LanguageCode; }); export { From 0d126ad3b7157c0dc2dfd45dc03dd1b54edcad40 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 7 Oct 2024 14:36:26 -0300 Subject: [PATCH 03/34] feat(languageSchema): split value to extract only language and not country code pt-BR becomes pt en-US becomes en --- src/schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/schema.ts b/src/schema.ts index edaba0b4..f21128d3 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -42,7 +42,7 @@ const fileSchema = z.custom((value) => value instanceof File); const percentageSchema = z.coerce.number().int().gte(1).lte(100); const languageSchema = z.string().transform((val, ctx) => { - val = val.toLowerCase(); + val = (val.toLowerCase()).split('-')[0]; // pt-BR -> pt if (!ISO6391.validate(val)) { ctx.addIssue({ code: z.ZodIssueCode.custom, From de8eba40790b3c985c01ace87aa966da161d84d2 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 7 Oct 2024 14:37:53 -0300 Subject: [PATCH 04/34] feat: create getLanguage() function, used for testing purposes --- src/test.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/test.ts b/src/test.ts index f4e720e1..c8fcfe6b 100644 --- a/src/test.ts +++ b/src/test.ts @@ -1,3 +1,5 @@ +import ISO6391, { LanguageCode } from 'iso-639-1'; +import lande from 'lande'; import { NostrEvent } from '@nostrify/nostrify'; import { finalizeEvent, generateSecretKey } from 'nostr-tools'; @@ -65,3 +67,15 @@ export async function createTestDB() { export function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } + +export function getLanguage(text: string): LanguageCode | undefined { + const [topResult] = lande(text); + if (topResult) { + const [iso6393] = topResult; + const locale = new Intl.Locale(iso6393); + if (ISO6391.validate(locale.language)) { + return locale.language as LanguageCode; + } + } + return; +} From c6626313bc544baded47f7ea1e878d9726bd0b62 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 7 Oct 2024 14:38:55 -0300 Subject: [PATCH 05/34] feat: get TRANSLATION_PROVIDER, TRANSLATION_PROVIDER_ENDPOINT & TRANSLATION_PROVIDER_API_KEY enviornment variables --- src/config.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/config.ts b/src/config.ts index ae841997..733ecc44 100644 --- a/src/config.ts +++ b/src/config.ts @@ -252,6 +252,18 @@ class Conf { static get preferredLanguages(): LanguageCode[] | undefined { return Deno.env.get('DITTO_LANGUAGES')?.split(',')?.filter(ISO6391.validate) as LanguageCode[]; } + /** Translation provider used to translate posts. */ + static get translationProvider(): string | undefined { + return Deno.env.get('TRANSLATION_PROVIDER')?.toLowerCase(); + } + /** Translation provider URL endpoint. */ + static get translationProviderEndpoint(): string | undefined { + return Deno.env.get('TRANSLATION_PROVIDER_ENDPOINT'); + } + /** Translation provider API KEY. */ + static get translationProviderApiKey(): string | undefined { + return Deno.env.get('TRANSLATION_PROVIDER_API_KEY'); + } /** Cache settings. */ static caches = { /** NIP-05 cache settings. */ From f434f875844504096497a4f64f2a33400ca65a31 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 7 Oct 2024 14:39:42 -0300 Subject: [PATCH 06/34] feat(instanceV2Controller): enable translation --- src/controllers/api/instance.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/api/instance.ts b/src/controllers/api/instance.ts index b350963d..09e50632 100644 --- a/src/controllers/api/instance.ts +++ b/src/controllers/api/instance.ts @@ -128,7 +128,7 @@ const instanceV2Controller: AppController = async (c) => { max_expiration: 2629746, }, translation: { - enabled: false, + enabled: true, }, }, registrations: { From ea4d0f002afae725ceff0c49349aadb2348bd2f2 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 7 Oct 2024 14:42:10 -0300 Subject: [PATCH 07/34] feat: create dittoTranslations LRUCache, create DittoTranslator interface, create MastodonTranslation type, create DittoTranslation type, create Provider type and some other minor ones --- src/translators/translator.ts | 65 +++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 src/translators/translator.ts diff --git a/src/translators/translator.ts b/src/translators/translator.ts new file mode 100644 index 00000000..515b335f --- /dev/null +++ b/src/translators/translator.ts @@ -0,0 +1,65 @@ +import { LanguageCode } from 'iso-639-1'; +import { LRUCache } from 'lru-cache'; + +import { Time } from '@/utils/time.ts'; + +/** Supported providers. */ +export type Provider = 'DeepL.com' | 'libretranslate.com'; + +/** Original language of the post */ +export type SourceLanguage = LanguageCode; + +/** Content will be translated to this language */ +export type TargetLanguage = LanguageCode; + +/** Entity returned by DittoTranslator and LRUCache */ +type DittoTranslation = { + data: MastodonTranslation; +}; + +export type MastodonTranslation = { + /** HTML-encoded translated content of the status. */ + content: string; + /** The translated spoiler warning of the status. */ + spoiler_text: string; + /** The translated media descriptions of the status. */ + media_attachments: { id: string; description: string }[]; + /** The translated poll of the status. */ + poll: { id: string; options: { title: string }[] } | null; + //** The language of the source text, as auto-detected by the machine translation provider. */ + detected_source_language: SourceLanguage; + /** The service that provided the machine translation. */ + provider: Provider; +}; + +/** DittoTranslator class, used for status translation. */ +export interface DittoTranslator { + /** Translate the 'content' into 'targetLanguage'. */ + translate( + /** HTML-encoded content of the status. */ + content: string, + /** Spoiler warning of the status. */ + spoilerText: string, + /** Media descriptions of the status. */ + mediaAttachments: { id: string; description: string }[], + /** Poll of the status. */ + poll: { id: string; options: { title: string }[] } | null, + /** The language of the source text/status. */ + sourceLanguage: SourceLanguage | undefined, + /** The status content will be translated into this language. */ + targetLanguage: TargetLanguage, + /** Custom options. */ + opts?: { signal?: AbortSignal }, + ): Promise; +} + +/** Includes the TARGET language and the status id. + * Example: en-390f5b01b49a8ee6e13fe917420c023d889b3da8e983a14c9e84587e43d12c15 + * The example above means: + * I want the status 390f5b01b49a8ee6e13fe917420c023d889b3da8e983a14c9e84587e43d12c15 translated to english (if it exists in the LRUCache). */ +export type dittoTranslationsKey = `${TargetLanguage}-${string}`; + +export const dittoTranslations = new LRUCache({ + max: 1000, + ttl: Time.hours(6), +}); From 27f435a93cd68ea2e5974f610cb875b664a329b7 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 7 Oct 2024 14:54:10 -0300 Subject: [PATCH 08/34] feat: create DeepLTranslator class that implements DittoTranslator --- src/translators/DeepLTranslator.ts | 142 +++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 src/translators/DeepLTranslator.ts diff --git a/src/translators/DeepLTranslator.ts b/src/translators/DeepLTranslator.ts new file mode 100644 index 00000000..fd8788dd --- /dev/null +++ b/src/translators/DeepLTranslator.ts @@ -0,0 +1,142 @@ +import { z } from 'zod'; + +import { + DittoTranslator, + MastodonTranslation, + Provider, + SourceLanguage, + TargetLanguage, +} from '@/translators/translator.ts'; +import { languageSchema } from '@/schema.ts'; + +interface DeepLTranslatorOpts { + /** DeepL endpoint to use. Default: 'https://api.deepl.com '*/ + endpoint?: string; + /** DeepL API key. */ + apiKey: string; + /** Custom fetch implementation. */ + fetch?: typeof fetch; +} + +export class DeepLTranslator implements DittoTranslator { + private readonly endpoint: string; + private readonly apiKey: string; + private readonly fetch: typeof fetch; + private readonly provider: Provider; + + constructor(opts: DeepLTranslatorOpts) { + this.endpoint = opts.endpoint ?? 'https://api.deepl.com'; + this.fetch = opts.fetch ?? globalThis.fetch; + this.provider = 'DeepL.com'; + this.apiKey = opts.apiKey; + } + + async translate( + contentHTMLencoded: string, + spoilerText: string, + mediaAttachments: { id: string; description: string }[], + poll: { id: string; options: { title: string }[] } | null, + sourceLanguage: SourceLanguage | undefined, + targetLanguage: TargetLanguage, + opts?: { signal?: AbortSignal }, + ) { + // --------------------- START explanation + // Order of texts: + // 1 - contentHTMLencoded + // 2 - spoilerText + // 3 - mediaAttachments descriptions + // 4 - poll title options + const medias = mediaAttachments.map((value) => value.description); + + const polls = poll?.options.map((value) => value.title) ?? []; + + const text = [contentHTMLencoded, spoilerText].concat(medias, polls); + // --------------------- END explanation + + const body: any = { + text, + target_lang: targetLanguage.toUpperCase(), + tag_handling: 'html', + split_sentences: '1', + }; + if (sourceLanguage) { + body.source_lang = sourceLanguage.toUpperCase(); + } + + const headers = new Headers(); + headers.append('Authorization', 'DeepL-Auth-Key' + ' ' + this.apiKey); + headers.append('Content-Type', 'application/json'); + + const request = new Request(this.endpoint + '/v2/translate', { + method: 'POST', + body: JSON.stringify(body), + headers, + signal: opts?.signal, + }); + + const response = await this.fetch(request); + const json = await response.json(); + const data = DeepLTranslator.schema().parse(json).translations; + + const mastodonTranslation: MastodonTranslation = { + content: '', + spoiler_text: '', + media_attachments: [], + poll: null, + detected_source_language: 'en', + provider: this.provider, + }; + + /** Used to keep track of the offset. When slicing, should be used as the start value. */ + let startIndex = 0; + mastodonTranslation.content = data[0].text; + startIndex++; + + mastodonTranslation.spoiler_text = data[1].text; + startIndex++; + + if (medias.length) { + const mediasTranslated = data.slice(startIndex, startIndex + medias.length); + for (let i = 0; i < mediasTranslated.length; i++) { + mastodonTranslation.media_attachments.push({ + id: mediaAttachments[i].id, + description: mediasTranslated[i].text, + }); + } + startIndex += mediasTranslated.length; + } + + if (polls.length && poll) { + const pollsTranslated = data.slice(startIndex); + mastodonTranslation.poll = { + id: poll.id, + options: [], + }; + for (let i = 0; i < pollsTranslated.length; i++) { + mastodonTranslation.poll.options.push({ + title: pollsTranslated[i].text, + }); + } + startIndex += pollsTranslated.length; + } + + mastodonTranslation.detected_source_language = data[0].detected_source_language; + + return { + data: mastodonTranslation, + }; + } + + /** DeepL response schema. + * https://developers.deepl.com/docs/api-reference/translate/openapi-spec-for-text-translation */ + private static schema() { + return z.object({ + translations: z.array( + z.object({ + detected_source_language: languageSchema, + text: z.string(), + }), + ), + }); + } +} From 321d26b47093df75fd6b9ee0bbb5dd8194b2f2a1 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 7 Oct 2024 14:55:12 -0300 Subject: [PATCH 09/34] test(DeepLTranslator): create multiple tests --- src/translators/DeepLTranslator.test.ts | 139 ++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 src/translators/DeepLTranslator.test.ts diff --git a/src/translators/DeepLTranslator.test.ts b/src/translators/DeepLTranslator.test.ts new file mode 100644 index 00000000..e261a9b6 --- /dev/null +++ b/src/translators/DeepLTranslator.test.ts @@ -0,0 +1,139 @@ +import { assertEquals } from '@std/assert'; + +import { Conf } from '@/config.ts'; +import { DeepLTranslator } from '@/translators/DeepLTranslator.ts'; +import { getLanguage } from '@/test.ts'; + +const endpoint = Conf.translationProviderEndpoint; +const apiKey = Conf.translationProviderApiKey; +const translationProvider = Conf.translationProvider; +const deepL = 'DeepL'.toLowerCase(); + +Deno.test('Translate status with EMPTY media_attachments and WITHOUT poll', { + ignore: !(translationProvider === deepL && apiKey), +}, async () => { + const translator = new DeepLTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); + + const mastodonTranslation = await translator.translate( + 'Bom dia amigos do Element, meu nome é Patrick', + '', + [], + null, + 'pt', + 'en', + ); + + assertEquals(getLanguage(mastodonTranslation.data.content), 'en'); + assertEquals(mastodonTranslation.data.spoiler_text, ''); + assertEquals(mastodonTranslation.data.media_attachments, []); + assertEquals(mastodonTranslation.data.poll, null); + assertEquals(mastodonTranslation.data.provider, 'DeepL.com'); +}); + +Deno.test('Translate status WITH auto detect and with EMPTY media_attachments and WITHOUT poll', { + ignore: !(translationProvider === deepL && apiKey), +}, async () => { + const translator = new DeepLTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); + + const mastodonTranslation = await translator.translate( + 'Bom dia amigos do Element, meu nome é Patrick', + '', + [], + null, + undefined, + 'en', + ); + + assertEquals(getLanguage(mastodonTranslation.data.content), 'en'); + assertEquals(mastodonTranslation.data.spoiler_text, ''); + assertEquals(mastodonTranslation.data.media_attachments, []); + assertEquals(mastodonTranslation.data.poll, null); + assertEquals(mastodonTranslation.data.provider, 'DeepL.com'); +}); + +Deno.test('Translate status WITH media_attachments and WITHOUT poll', { + ignore: !(translationProvider === deepL && apiKey), +}, async () => { + const translator = new DeepLTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); + + const mastodonTranslation = await translator.translate( + 'Hello my friends, my name is Alex and I am american.', + "That is spoiler isn't it", + [{ id: 'game', description: 'I should be playing Miles Edgeworth with my wife' }], + null, + 'en', + 'pt', + ); + + assertEquals(getLanguage(mastodonTranslation.data.content), 'pt'); + assertEquals(getLanguage(mastodonTranslation.data.spoiler_text), 'pt'); + assertEquals(mastodonTranslation.data.media_attachments.map((value) => getLanguage(value.description)), ['pt']); + assertEquals(mastodonTranslation.data.poll, null); + assertEquals(mastodonTranslation.data.provider, 'DeepL.com'); +}); + +Deno.test('Translate status WITHOUT media_attachments and WITH poll', { + ignore: !(translationProvider === deepL && apiKey), +}, async () => { + const translator = new DeepLTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); + + const poll = { + 'id': '34858', + 'options': [ + { + 'title': 'Kill him right now', + }, + { + 'title': 'Save him right now', + }, + ], + }; + + const mastodonTranslation = await translator.translate( + 'Hello my friends, my name is Alex and I am american.', + '', + [], + poll, + 'en', + 'pt', + ); + + assertEquals(getLanguage(mastodonTranslation.data.content), 'pt'); + assertEquals(mastodonTranslation.data.spoiler_text, ''); + assertEquals(mastodonTranslation.data.media_attachments, []); + assertEquals(mastodonTranslation.data.poll?.options.map((value) => getLanguage(value.title)), ['pt', 'pt']); + assertEquals(mastodonTranslation.data.provider, 'DeepL.com'); +}); + +Deno.test('Translate status WITH media_attachments and WITH poll', { + ignore: !(translationProvider === deepL && apiKey), +}, async () => { + const translator = new DeepLTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); + + const poll = { + 'id': '34858', + 'options': [ + { + 'title': 'Kill him right now', + }, + { + 'title': 'Save him right now', + }, + ], + }; + + const mastodonTranslation = await translator.translate( + 'Hello my friends, my name is Alex and I am american.', + '', + [{ id: 'game', description: 'I should be playing Miles Edgeworth with my wife' }], + poll, + 'en', + 'pt', + ); + + assertEquals(getLanguage(mastodonTranslation.data.content), 'pt'); + assertEquals(mastodonTranslation.data.spoiler_text, ''); + assertEquals(mastodonTranslation.data.media_attachments.map((value) => getLanguage(value.description)), ['pt']); + assertEquals(mastodonTranslation.data.poll?.options.map((value) => getLanguage(value.title)), ['pt', 'pt']); + assertEquals(mastodonTranslation.data.provider, 'DeepL.com'); +}); From c23ddb7d843388eb9f7f6cbb85f2d97ffd4ad436 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 7 Oct 2024 14:58:33 -0300 Subject: [PATCH 10/34] feat: create LibreTranslateTranslator class that implements DittoTranslator --- src/translators/LibreTranslateTranslator.ts | 147 ++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 src/translators/LibreTranslateTranslator.ts diff --git a/src/translators/LibreTranslateTranslator.ts b/src/translators/LibreTranslateTranslator.ts new file mode 100644 index 00000000..80f44479 --- /dev/null +++ b/src/translators/LibreTranslateTranslator.ts @@ -0,0 +1,147 @@ +import { z } from 'zod'; + +import { + DittoTranslator, + MastodonTranslation, + Provider, + SourceLanguage, + TargetLanguage, +} from '@/translators/translator.ts'; + +interface LibreTranslateTranslatorOpts { + /** Libretranslate endpoint to use. Default: 'https://libretranslate.com' */ + endpoint?: string; + /** Libretranslate API key. */ + apiKey: string; + /** Custom fetch implementation. */ + fetch?: typeof fetch; +} + +export class LibreTranslateTranslator implements DittoTranslator { + private readonly endpoint: string; + private readonly apiKey: string; + private readonly fetch: typeof fetch; + private readonly provider: Provider; + + constructor(opts: LibreTranslateTranslatorOpts) { + this.endpoint = opts.endpoint ?? 'https://libretranslate.com'; + this.fetch = opts.fetch ?? globalThis.fetch; + this.provider = 'libretranslate.com'; + this.apiKey = opts.apiKey; + } + + async translate( + contentHTMLencoded: string, + spoilerText: string, + mediaAttachments: { id: string; description: string }[], + poll: { id: string; options: { title: string }[] } | null, + sourceLanguage: SourceLanguage | undefined, + targetLanguage: TargetLanguage, + opts?: { signal?: AbortSignal }, + ) { + const mastodonTranslation: MastodonTranslation = { + content: '', + spoiler_text: '', + media_attachments: [], + poll: null, + detected_source_language: 'en', + provider: this.provider, + }; + + const translatedContent = await this.makeRequest(contentHTMLencoded, sourceLanguage, targetLanguage, 'html', { + signal: opts?.signal, + }); + mastodonTranslation.content = translatedContent; + + if (spoilerText.length) { + const translatedSpoilerText = await this.makeRequest(spoilerText, sourceLanguage, targetLanguage, 'text', { + signal: opts?.signal, + }); + mastodonTranslation.spoiler_text = translatedSpoilerText; + } + + if (mediaAttachments) { + for (const media of mediaAttachments) { + const translatedDescription = await this.makeRequest( + media.description, + sourceLanguage, + targetLanguage, + 'text', + { + signal: opts?.signal, + }, + ); + mastodonTranslation.media_attachments.push({ + id: media.id, + description: translatedDescription, + }); + } + } + + if (poll) { + mastodonTranslation.poll = { + id: poll.id, + options: [], + }; + + for (const option of poll.options) { + const translatedTitle = await this.makeRequest( + option.title, + sourceLanguage, + targetLanguage, + 'text', + { + signal: opts?.signal, + }, + ); + mastodonTranslation.poll.options.push({ + title: translatedTitle, + }); + } + } + + return { + data: mastodonTranslation, + }; + } + + private async makeRequest( + q: string, + sourceLanguage: string | undefined, + targetLanguage: string, + format: 'html' | 'text', + opts?: { signal?: AbortSignal }, + ): Promise { + const body = { + q, + source: sourceLanguage?.toLowerCase() ?? 'auto', + target: targetLanguage.toLowerCase(), + format, + api_key: this.apiKey, + }; + + const headers = new Headers(); + headers.append('Content-Type', 'application/json'); + + const request = new Request(this.endpoint + '/translate', { + method: 'POST', + body: JSON.stringify(body), + headers, + signal: opts?.signal, + }); + + const response = await this.fetch(request); + const json = await response.json(); + const data = LibreTranslateTranslator.schema().parse(json).translatedText; + + return data; + } + + /** Libretranslate response schema. + * https://libretranslate.com/docs/#/translate/post_translate */ + private static schema() { + return z.object({ + translatedText: z.string(), + }); + } +} From a2d8478e1a4f0c20463c6c23f1dd73c1488b01a3 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 7 Oct 2024 14:59:05 -0300 Subject: [PATCH 11/34] test(LibreTranslateTranslator): create multiple tests --- .../LibreTranslateTranslator.test.ts | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 src/translators/LibreTranslateTranslator.test.ts diff --git a/src/translators/LibreTranslateTranslator.test.ts b/src/translators/LibreTranslateTranslator.test.ts new file mode 100644 index 00000000..11f57d09 --- /dev/null +++ b/src/translators/LibreTranslateTranslator.test.ts @@ -0,0 +1,139 @@ +import { assertEquals } from '@std/assert'; + +import { Conf } from '@/config.ts'; +import { LibreTranslateTranslator } from '@/translators/LibreTranslateTranslator.ts'; +import { getLanguage } from '@/test.ts'; + +const endpoint = Conf.translationProviderEndpoint; +const apiKey = Conf.translationProviderApiKey; +const translationProvider = Conf.translationProvider; +const libreTranslate = 'Libretranslate'.toLowerCase(); + +Deno.test('Translate status with EMPTY media_attachments and WITHOUT poll', { + ignore: !(translationProvider === libreTranslate && apiKey), +}, async () => { + const translator = new LibreTranslateTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); + + const mastodonTranslation = await translator.translate( + 'Bom dia amigos do Element, meu nome é Patrick', + '', + [], + null, + 'pt', + 'en', + ); + + assertEquals(getLanguage(mastodonTranslation.data.content), 'en'); + assertEquals(mastodonTranslation.data.spoiler_text, ''); + assertEquals(mastodonTranslation.data.media_attachments, []); + assertEquals(mastodonTranslation.data.poll, null); + assertEquals(mastodonTranslation.data.provider, 'libretranslate.com'); +}); + +Deno.test('Translate status WITH auto detect and with EMPTY media_attachments and WITHOUT poll', { + ignore: !(translationProvider === libreTranslate && apiKey), +}, async () => { + const translator = new LibreTranslateTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); + + const mastodonTranslation = await translator.translate( + 'Bom dia amigos do Element, meu nome é Patrick', + '', + [], + null, + undefined, + 'en', + ); + + assertEquals(getLanguage(mastodonTranslation.data.content), 'en'); + assertEquals(mastodonTranslation.data.spoiler_text, ''); + assertEquals(mastodonTranslation.data.media_attachments, []); + assertEquals(mastodonTranslation.data.poll, null); + assertEquals(mastodonTranslation.data.provider, 'libretranslate.com'); +}); + +Deno.test('Translate status WITH media_attachments and WITHOUT poll', { + ignore: !(translationProvider === libreTranslate && apiKey), +}, async () => { + const translator = new LibreTranslateTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); + + const mastodonTranslation = await translator.translate( + 'Hello my friends, my name is Alex and I am american.', + "That is spoiler isn't it", + [{ id: 'game', description: 'I should be playing Miles Edgeworth with my wife' }], + null, + 'en', + 'pt', + ); + + assertEquals(getLanguage(mastodonTranslation.data.content), 'pt'); + assertEquals(getLanguage(mastodonTranslation.data.spoiler_text), 'pt'); + assertEquals(mastodonTranslation.data.media_attachments.map((value) => getLanguage(value.description)), ['pt']); + assertEquals(mastodonTranslation.data.poll, null); + assertEquals(mastodonTranslation.data.provider, 'libretranslate.com'); +}); + +Deno.test('Translate status WITHOUT media_attachments and WITH poll', { + ignore: !(translationProvider === libreTranslate && apiKey), +}, async () => { + const translator = new LibreTranslateTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); + + const poll = { + 'id': '34858', + 'options': [ + { + 'title': 'Kill him right now', + }, + { + 'title': 'Save him right now', + }, + ], + }; + + const mastodonTranslation = await translator.translate( + 'Hello my friends, my name is Alex and I am american.', + '', + [], + poll, + 'en', + 'pt', + ); + + assertEquals(getLanguage(mastodonTranslation.data.content), 'pt'); + assertEquals(mastodonTranslation.data.spoiler_text, ''); + assertEquals(mastodonTranslation.data.media_attachments, []); + assertEquals(mastodonTranslation.data.poll?.options.map((value) => getLanguage(value.title)), ['pt', 'pt']); + assertEquals(mastodonTranslation.data.provider, 'libretranslate.com'); +}); + +Deno.test('Translate status WITH media_attachments and WITH poll', { + ignore: !(translationProvider === libreTranslate && apiKey), +}, async () => { + const translator = new LibreTranslateTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); + + const poll = { + 'id': '34858', + 'options': [ + { + 'title': 'Kill him right now', + }, + { + 'title': 'Save him right now', + }, + ], + }; + + const mastodonTranslation = await translator.translate( + 'Hello my friends, my name is Alex and I am american.', + '', + [{ id: 'game', description: 'I should be playing Miles Edgeworth with my wife' }], + poll, + 'en', + 'pt', + ); + + assertEquals(getLanguage(mastodonTranslation.data.content), 'pt'); + assertEquals(mastodonTranslation.data.spoiler_text, ''); + assertEquals(mastodonTranslation.data.media_attachments.map((value) => getLanguage(value.description)), ['pt']); + assertEquals(mastodonTranslation.data.poll?.options.map((value) => getLanguage(value.title)), ['pt', 'pt']); + assertEquals(mastodonTranslation.data.provider, 'libretranslate.com'); +}); From 8e58b1a7d46d1ed8f65da5c3a8a7d9af605dfd9f Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 7 Oct 2024 15:00:46 -0300 Subject: [PATCH 12/34] feat: create translatorMiddleware --- src/middleware/translatorMiddleware.ts | 33 ++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 src/middleware/translatorMiddleware.ts diff --git a/src/middleware/translatorMiddleware.ts b/src/middleware/translatorMiddleware.ts new file mode 100644 index 00000000..6e336ad5 --- /dev/null +++ b/src/middleware/translatorMiddleware.ts @@ -0,0 +1,33 @@ +import { AppMiddleware } from '@/app.ts'; +import { Conf } from '@/config.ts'; +import { fetchWorker } from '@/workers/fetch.ts'; +import { DeepLTranslator } from '@/translators/DeepLTranslator.ts'; +import { LibreTranslateTranslator } from '@/translators/LibreTranslateTranslator.ts'; + +/** Set the translator used for translating posts. */ +export const translatorMiddleware: AppMiddleware = async (c, next) => { + const endpoint = Conf.translationProviderEndpoint; + const apiKey = Conf.translationProviderApiKey; + const translationProvider = Conf.translationProvider; + + switch (translationProvider) { + case 'DeepL'.toLowerCase(): + if (apiKey) { + c.set( + 'translator', + new DeepLTranslator({ endpoint, apiKey, fetch: fetchWorker }), + ); + } + break; + case 'Libretranslate'.toLowerCase(): + if (apiKey) { + c.set( + 'translator', + new LibreTranslateTranslator({ endpoint, apiKey, fetch: fetchWorker }), + ); + } + break; + } + + await next(); +}; From b369b2171d38fe16b4d6277030595ba71bf5af7f Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 7 Oct 2024 15:02:04 -0300 Subject: [PATCH 13/34] feat: create translateController - /api/v1/statuses/:id/translate --- src/app.ts | 6 +++ src/controllers/api/translate.ts | 89 ++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 src/controllers/api/translate.ts diff --git a/src/app.ts b/src/app.ts index 9fb67ced..8831f188 100644 --- a/src/app.ts +++ b/src/app.ts @@ -109,6 +109,7 @@ import { trendingStatusesController, trendingTagsController, } from '@/controllers/api/trends.ts'; +import { translateController } from '@/controllers/api/translate.ts'; import { errorHandler } from '@/controllers/error.ts'; import { frontendController } from '@/controllers/frontend.ts'; import { metricsController } from '@/controllers/metrics.ts'; @@ -126,6 +127,8 @@ import { requireSigner } from '@/middleware/requireSigner.ts'; import { signerMiddleware } from '@/middleware/signerMiddleware.ts'; import { storeMiddleware } from '@/middleware/storeMiddleware.ts'; import { uploaderMiddleware } from '@/middleware/uploaderMiddleware.ts'; +import { DittoTranslator } from '@/translators/translator.ts'; +import { translatorMiddleware } from '@/middleware/translatorMiddleware.ts'; interface AppEnv extends HonoEnv { Variables: { @@ -141,6 +144,8 @@ interface AppEnv extends HonoEnv { pagination: { since?: number; until?: number; limit: number }; /** Normalized list pagination params. */ listPagination: { offset: number; limit: number }; + /** Translation service. */ + translator?: DittoTranslator; }; } @@ -220,6 +225,7 @@ app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/bookmark', requireSigner, bookmarkC app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unbookmark', requireSigner, unbookmarkController); app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/pin', requireSigner, pinController); app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unpin', requireSigner, unpinController); +app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/translate', requireSigner, translatorMiddleware, translateController); app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/reblog', requireSigner, reblogStatusController); app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unreblog', requireSigner, unreblogStatusController); app.post('/api/v1/statuses', requireSigner, createStatusController); diff --git a/src/controllers/api/translate.ts b/src/controllers/api/translate.ts new file mode 100644 index 00000000..86aaa1f4 --- /dev/null +++ b/src/controllers/api/translate.ts @@ -0,0 +1,89 @@ +import { LanguageCode } from 'iso-639-1'; +import { z } from 'zod'; + +import { AppController } from '@/app.ts'; +import { languageSchema } from '@/schema.ts'; +import { Storages } from '@/storages.ts'; +import { dittoTranslations, dittoTranslationsKey } from '@/translators/translator.ts'; +import { parseBody } from '@/utils/api.ts'; +import { getEvent } from '@/queries.ts'; +import { renderStatus } from '@/views/mastodon/statuses.ts'; + +const translateSchema = z.object({ + //lang: languageSchema, // Correct property name, as stated by Mastodon docs + target_language: languageSchema, // Property name soapbox sends +}); + +const translateController: AppController = async (c) => { + const result = translateSchema.safeParse(await parseBody(c.req.raw)); + const { signal } = c.req.raw; + + if (!result.success) { + return c.json({ error: 'Bad request.', schema: result.error }, 422); + } + + const translator = c.get('translator'); + if (!translator) { + return c.json({ error: 'No translator configured.' }, 500); + } + + const { target_language } = result.data; + const targetLang = target_language; + const id = c.req.param('id'); + + const event = await getEvent(id, { signal }); + if (!event) { + return c.json({ error: 'Record not found' }, 400); + } + + const viewerPubkey = await c.get('signer')?.getPublicKey(); + + const kysely = await Storages.kysely(); + + let sourceLang = (await kysely + .selectFrom('nostr_events') + .select('language').where('id', '=', id) + .limit(1) + .execute())[0]?.language as LanguageCode | undefined; + if (!sourceLang) { + sourceLang = undefined; + } + + if (targetLang.toLowerCase() === sourceLang?.toLowerCase()) { + return c.json({ error: 'Source and target languages are the same. No translation needed.' }, 400); + } + + const status = await renderStatus(event, { viewerPubkey }); + + const translatedId = `${target_language}-${id}` as dittoTranslationsKey; + const translationCache = dittoTranslations.get(translatedId); + + if (translationCache) { + return c.json(translationCache.data, 200); + } + + const mediaAttachments = status?.media_attachments.map((value) => { + return { + id: value.id, + description: value.description ?? '', + }; + }) ?? []; + + try { + const translation = await translator.translate( + status?.content ?? '', + status?.spoiler_text ?? '', + mediaAttachments, + null, + sourceLang, + targetLang, + { signal }, + ); + dittoTranslations.set(translatedId, translation); + return c.json(translation.data, 200); + } catch (_) { + return c.json({ error: 'Service Unavailable' }, 503); + } +}; + +export { translateController }; From bbbce958d9201c6af408cf5c5083160699608885 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 7 Oct 2024 16:04:59 -0300 Subject: [PATCH 14/34] chore: update nostrify:db --- deno.json | 2 +- deno.lock | 23 ++++++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/deno.json b/deno.json index f97d4fa7..06952b75 100644 --- a/deno.json +++ b/deno.json @@ -41,7 +41,7 @@ "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/db": "jsr:@nostrify/db@^0.35.0", + "@nostrify/db": "jsr:@nostrify/db@^0.36.1", "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.36.0", "@nostrify/policies": "jsr:@nostrify/policies@^0.35.0", "@scure/base": "npm:@scure/base@^1.1.6", diff --git a/deno.lock b/deno.lock index 84969bb2..37507e0d 100644 --- a/deno.lock +++ b/deno.lock @@ -20,9 +20,9 @@ "jsr:@gleasonator/policy@0.6.4": "jsr:@gleasonator/policy@0.6.4", "jsr:@gleasonator/policy@0.7.0": "jsr:@gleasonator/policy@0.7.0", "jsr:@gleasonator/policy@0.7.1": "jsr:@gleasonator/policy@0.7.1", - "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.6.2", + "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.6.3", "jsr:@lambdalisue/async@^2.1.1": "jsr:@lambdalisue/async@2.1.1", - "jsr:@nostrify/db@^0.35.0": "jsr:@nostrify/db@0.35.0", + "jsr:@nostrify/db@^0.36.1": "jsr:@nostrify/db@0.36.1", "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", @@ -62,7 +62,7 @@ "jsr:@std/fs@^0.229.3": "jsr:@std/fs@0.229.3", "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.4", "jsr:@std/io@^0.223.0": "jsr:@std/io@0.223.0", - "jsr:@std/io@^0.224": "jsr:@std/io@0.224.8", + "jsr:@std/io@^0.224": "jsr:@std/io@0.224.9", "jsr:@std/json@^0.223.0": "jsr:@std/json@0.223.0", "jsr:@std/media-types@^0.224.1": "jsr:@std/media-types@0.224.1", "jsr:@std/path@0.213.1": "jsr:@std/path@0.213.1", @@ -260,13 +260,16 @@ "@hono/hono@4.6.2": { "integrity": "35fcf3be4687825080b01bed7bbe2ac66f8d8b8939f0bad459661bf3b46d916f" }, + "@hono/hono@4.6.3": { + "integrity": "a1f5a18cd12a0db54755b0461dd5a4e2d93a6f85403073eb710103eacc42daf3" + }, "@lambdalisue/async@2.1.1": { "integrity": "1fc9bc6f4ed50215cd2f7217842b18cea80f81c25744f88f8c5eb4be5a1c9ab4" }, - "@nostrify/db@0.35.0": { - "integrity": "637191c41812544e361b7997dc44ea098f8bd7efebb28f37a8a7142a0ecada8d", + "@nostrify/db@0.36.1": { + "integrity": "b65b89ca6fe98d9dbcc0402b5c9c07b8430c2c91f84ba4128ff2eeed70c3d49f", "dependencies": [ - "jsr:@nostrify/nostrify@^0.35.0", + "jsr:@nostrify/nostrify@^0.36.0", "jsr:@nostrify/types@^0.35.0", "npm:kysely@^0.27.3", "npm:nostr-tools@^2.7.0" @@ -558,6 +561,12 @@ "jsr:@std/bytes@^1.0.2" ] }, + "@std/io@0.224.9": { + "integrity": "4414664b6926f665102e73c969cfda06d2c4c59bd5d0c603fd4f1b1c840d6ee3", + "dependencies": [ + "jsr:@std/bytes@^1.0.2" + ] + }, "@std/json@0.223.0": { "integrity": "9a4a255931dd0397924c6b10bb6a72fe3e28ddd876b981ada2e3b8dd0764163f", "dependencies": [ @@ -2149,7 +2158,7 @@ "jsr:@gfx/canvas-wasm@^0.4.2", "jsr:@hono/hono@^4.4.6", "jsr:@lambdalisue/async@^2.1.1", - "jsr:@nostrify/db@^0.35.0", + "jsr:@nostrify/db@^0.36.1", "jsr:@nostrify/nostrify@^0.36.0", "jsr:@nostrify/policies@^0.35.0", "jsr:@soapbox/kysely-pglite@^1.0.0", From 4712cb1d80aa2bd82bf6e08103e2697d84da1069 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 7 Oct 2024 16:05:22 -0300 Subject: [PATCH 15/34] fix: fix language property in the Mastodon API --- src/interfaces/DittoEvent.ts | 3 +++ src/storages/EventsDB.ts | 18 +++++++++++++++++- src/views/mastodon/statuses.ts | 2 +- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/interfaces/DittoEvent.ts b/src/interfaces/DittoEvent.ts index dcaec6ae..cca7c0ca 100644 --- a/src/interfaces/DittoEvent.ts +++ b/src/interfaces/DittoEvent.ts @@ -1,4 +1,5 @@ import { NostrEvent } from '@nostrify/nostrify'; +import { LanguageCode } from 'iso-639-1'; /** Ditto internal stats for the event's author. */ export interface AuthorStats { @@ -43,4 +44,6 @@ export interface DittoEvent extends NostrEvent { zap_sender?: DittoEvent | string; zap_amount?: number; zap_message?: string; + /** Language of the event (kind 1s are more accurate). */ + language?: LanguageCode; } diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 1bf3cd86..dfff6394 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -1,5 +1,6 @@ // deno-lint-ignore-file require-await +import { LanguageCode } from 'iso-639-1'; import { NPostgres, NPostgresSchema } from '@nostrify/db'; import { NIP50, NKinds, NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { Stickynotes } from '@soapbox/stickynotes'; @@ -12,6 +13,7 @@ import { RelayError } from '@/RelayError.ts'; import { isNostrId, isURL } from '@/utils.ts'; import { abortError } from '@/utils/abort.ts'; import { purifyEvent } from '@/utils/purify.ts'; +import { DittoEvent } from '@/interfaces/DittoEvent.ts'; /** Function to decide whether or not to index a tag. */ type TagCondition = ({ event, count, value }: { @@ -175,7 +177,7 @@ class EventsDB extends NPostgres { override async query( filters: NostrFilter[], opts: { signal?: AbortSignal; timeout?: number; limit?: number } = {}, - ): Promise { + ): Promise { filters = await this.expandFilters(filters); for (const filter of filters) { @@ -199,6 +201,20 @@ class EventsDB extends NPostgres { return super.query(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } + /** Parse an event row from the database. */ + protected override parseEventRow(row: Pick): DittoEvent { + return { + id: row.id, + kind: row.kind, + pubkey: row.pubkey, + content: row.content, + created_at: Number(row.created_at), + tags: row.tags, + sig: row.sig, + language: (row.language || undefined) as LanguageCode, + }; + } + /** Delete events based on filters from the database. */ override async remove(filters: NostrFilter[], opts: { signal?: AbortSignal; timeout?: number } = {}): Promise { this.console.debug('DELETE', JSON.stringify(filters)); diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index e21c9e1c..48d8e099 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -113,7 +113,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< sensitive: !!cw, spoiler_text: (cw ? cw[1] : subject?.[1]) || '', visibility: 'public', - language: event.tags.find((tag) => tag[0] === 'l' && tag[2] === 'ISO-639-1')?.[1] || null, + language: event.language ?? null, replies_count: event.event_stats?.replies_count ?? 0, reblogs_count: event.event_stats?.reposts_count ?? 0, favourites_count: event.event_stats?.reactions['+'] ?? 0, From bfab84d9376e61258617b126e67281e5247a2003 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 7 Oct 2024 17:54:03 -0300 Subject: [PATCH 16/34] refactor: make provider lowercase because supporting case insensitive is allegedly protocol bloat --- src/config.ts | 2 +- src/middleware/translatorMiddleware.ts | 4 ++-- src/translators/DeepLTranslator.test.ts | 2 +- src/translators/LibreTranslateTranslator.test.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/config.ts b/src/config.ts index 8148ab3b..848f9836 100644 --- a/src/config.ts +++ b/src/config.ts @@ -273,7 +273,7 @@ class Conf { } /** Translation provider used to translate posts. */ static get translationProvider(): string | undefined { - return Deno.env.get('TRANSLATION_PROVIDER')?.toLowerCase(); + return Deno.env.get('TRANSLATION_PROVIDER'); } /** Translation provider URL endpoint. */ static get translationProviderEndpoint(): string | undefined { diff --git a/src/middleware/translatorMiddleware.ts b/src/middleware/translatorMiddleware.ts index 6e336ad5..2d3971dd 100644 --- a/src/middleware/translatorMiddleware.ts +++ b/src/middleware/translatorMiddleware.ts @@ -11,7 +11,7 @@ export const translatorMiddleware: AppMiddleware = async (c, next) => { const translationProvider = Conf.translationProvider; switch (translationProvider) { - case 'DeepL'.toLowerCase(): + case 'deepl': if (apiKey) { c.set( 'translator', @@ -19,7 +19,7 @@ export const translatorMiddleware: AppMiddleware = async (c, next) => { ); } break; - case 'Libretranslate'.toLowerCase(): + case 'libretranslate': if (apiKey) { c.set( 'translator', diff --git a/src/translators/DeepLTranslator.test.ts b/src/translators/DeepLTranslator.test.ts index e261a9b6..8335670e 100644 --- a/src/translators/DeepLTranslator.test.ts +++ b/src/translators/DeepLTranslator.test.ts @@ -7,7 +7,7 @@ import { getLanguage } from '@/test.ts'; const endpoint = Conf.translationProviderEndpoint; const apiKey = Conf.translationProviderApiKey; const translationProvider = Conf.translationProvider; -const deepL = 'DeepL'.toLowerCase(); +const deepL = 'deepl'; Deno.test('Translate status with EMPTY media_attachments and WITHOUT poll', { ignore: !(translationProvider === deepL && apiKey), diff --git a/src/translators/LibreTranslateTranslator.test.ts b/src/translators/LibreTranslateTranslator.test.ts index 11f57d09..8d1fc24d 100644 --- a/src/translators/LibreTranslateTranslator.test.ts +++ b/src/translators/LibreTranslateTranslator.test.ts @@ -7,7 +7,7 @@ import { getLanguage } from '@/test.ts'; const endpoint = Conf.translationProviderEndpoint; const apiKey = Conf.translationProviderApiKey; const translationProvider = Conf.translationProvider; -const libreTranslate = 'Libretranslate'.toLowerCase(); +const libreTranslate = 'libretranslate'; Deno.test('Translate status with EMPTY media_attachments and WITHOUT poll', { ignore: !(translationProvider === libreTranslate && apiKey), From acbdae29ae71ee3f569eba7da9cd29d35f0244c4 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 7 Oct 2024 17:54:41 -0300 Subject: [PATCH 17/34] fix(EventsDB): type is correct, ignore type complaint --- src/storages/EventsDB.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index dfff6394..957e996c 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -146,10 +146,12 @@ class EventsDB extends NPostgres { } } + // @ts-ignore The type is correct, but NPostgres doesn't realize it. I don't think it's solvable without modifying NPostgres again, which I don't think is worth it for this. protected override getFilterQuery(trx: Kysely, filter: NostrFilter) { if (filter.search) { const tokens = NIP50.parseInput(filter.search); + // @ts-ignore The type is correct, but NPostgres doesn't realize it. I don't think it's solvable without modifying NPostgres again, which I don't think is worth it for this. let query = super.getFilterQuery(trx, { ...filter, search: tokens.filter((t) => typeof t === 'string').join(' '), From 6c931531176de804c8391759b59b8d3a4e399d8b Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 7 Oct 2024 17:55:50 -0300 Subject: [PATCH 18/34] refactor: get language from event itself --- src/controllers/api/translate.ts | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/controllers/api/translate.ts b/src/controllers/api/translate.ts index 86aaa1f4..3e8bf13c 100644 --- a/src/controllers/api/translate.ts +++ b/src/controllers/api/translate.ts @@ -38,18 +38,7 @@ const translateController: AppController = async (c) => { const viewerPubkey = await c.get('signer')?.getPublicKey(); - const kysely = await Storages.kysely(); - - let sourceLang = (await kysely - .selectFrom('nostr_events') - .select('language').where('id', '=', id) - .limit(1) - .execute())[0]?.language as LanguageCode | undefined; - if (!sourceLang) { - sourceLang = undefined; - } - - if (targetLang.toLowerCase() === sourceLang?.toLowerCase()) { + if (targetLang.toLowerCase() === event?.language?.toLowerCase()) { return c.json({ error: 'Source and target languages are the same. No translation needed.' }, 400); } @@ -75,7 +64,7 @@ const translateController: AppController = async (c) => { status?.spoiler_text ?? '', mediaAttachments, null, - sourceLang, + event.language, targetLang, { signal }, ); From 3f00f255a5e817a3149e1157fc215f33ffc48987 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 8 Oct 2024 11:01:34 -0300 Subject: [PATCH 19/34] fix: type assertions in EventsDB --- src/db/DittoTables.ts | 4 +--- src/storages/EventsDB.ts | 6 ++---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index b6fa93f4..46eeeab9 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -1,5 +1,3 @@ -import { Nullable } from 'kysely'; - import { NPostgresSchema } from '@nostrify/db'; export interface DittoTables extends NPostgresSchema { @@ -12,7 +10,7 @@ export interface DittoTables extends NPostgresSchema { } type NostrEventsRow = NPostgresSchema['nostr_events'] & { - language: Nullable; + language: string | null; }; interface AuthorStatsRow { diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 957e996c..905883b7 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -146,16 +146,14 @@ class EventsDB extends NPostgres { } } - // @ts-ignore The type is correct, but NPostgres doesn't realize it. I don't think it's solvable without modifying NPostgres again, which I don't think is worth it for this. protected override getFilterQuery(trx: Kysely, filter: NostrFilter) { if (filter.search) { const tokens = NIP50.parseInput(filter.search); - // @ts-ignore The type is correct, but NPostgres doesn't realize it. I don't think it's solvable without modifying NPostgres again, which I don't think is worth it for this. let query = super.getFilterQuery(trx, { ...filter, search: tokens.filter((t) => typeof t === 'string').join(' '), - }) as SelectQueryBuilder>; + }) as SelectQueryBuilder; const languages = new Set(); @@ -204,7 +202,7 @@ class EventsDB extends NPostgres { } /** Parse an event row from the database. */ - protected override parseEventRow(row: Pick): DittoEvent { + protected override parseEventRow(row: DittoTables['nostr_events']): DittoEvent { return { id: row.id, kind: row.kind, From 17be4ab48fde243190fabdf6234bbd88d0f2ac28 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 8 Oct 2024 11:11:29 -0300 Subject: [PATCH 20/34] fix(instanceV1Controller): add translation field --- src/controllers/api/instance.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/controllers/api/instance.ts b/src/controllers/api/instance.ts index 09e50632..ae31bdc2 100644 --- a/src/controllers/api/instance.ts +++ b/src/controllers/api/instance.ts @@ -36,6 +36,9 @@ const instanceV1Controller: AppController = async (c) => { max_characters: Conf.postCharLimit, max_media_attachments: 20, }, + translation: { + enabled: true, + }, }, pleroma: { metadata: { From df27959d35a6c045acefb0593be60bc9d9c4c008 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 8 Oct 2024 14:15:48 -0300 Subject: [PATCH 21/34] fix(relay.ts): purify event --- src/controllers/nostr/relay.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 2a38e751..f62ad76b 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -18,6 +18,7 @@ import * as pipeline from '@/pipeline.ts'; import { RelayError } from '@/RelayError.ts'; import { Storages } from '@/storages.ts'; import { Time } from '@/utils/time.ts'; +import { purifyEvent } from '@/utils/purify.ts'; /** Limit of initial events returned for a subscription. */ const FILTER_LIMIT = 100; @@ -105,7 +106,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) { try { for (const event of await store.query(filters, { limit: FILTER_LIMIT, timeout: Conf.db.timeouts.relay })) { - send(['EVENT', subId, event]); + send(['EVENT', subId, purifyEvent(event)]); } } catch (e: any) { if (e instanceof RelayError) { @@ -137,7 +138,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) { relayEventsCounter.inc({ kind: event.kind.toString() }); try { // This will store it (if eligible) and run other side-effects. - await pipeline.handleEvent(event, AbortSignal.timeout(1000)); + await pipeline.handleEvent(purifyEvent(event), AbortSignal.timeout(1000)); send(['OK', event.id, true, '']); } catch (e) { if (e instanceof RelayError) { From d4a8ec21fef73bdf600ee56cd17f8bd904381e65 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 8 Oct 2024 14:17:27 -0300 Subject: [PATCH 22/34] fix: add 'pure' option in EventsDB if pure is true, EventsDB will return a Nostr event, otherwise it will return a Ditto event --- src/storages/EventsDB.test.ts | 26 +++++++++++++------------- src/storages/EventsDB.ts | 15 +++++++++++++-- src/test.ts | 3 ++- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index b24032aa..e19fe775 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -7,7 +7,7 @@ import { Conf } from '@/config.ts'; import { createTestDB } from '@/test.ts'; Deno.test('count filters', async () => { - await using db = await createTestDB(); + await using db = await createTestDB({ pure: true }); const { store } = db; const event1 = await eventFixture('event-1'); @@ -18,7 +18,7 @@ Deno.test('count filters', async () => { }); Deno.test('insert and filter events', async () => { - await using db = await createTestDB(); + await using db = await createTestDB({ pure: true }); const { store } = db; const event1 = await eventFixture('event-1'); @@ -35,7 +35,7 @@ Deno.test('insert and filter events', async () => { }); Deno.test('query events with domain search filter', async () => { - await using db = await createTestDB(); + await using db = await createTestDB({ pure: true }); const { store, kysely } = db; const event1 = await eventFixture('event-1'); @@ -55,7 +55,7 @@ Deno.test('query events with domain search filter', async () => { }); Deno.test('query events with language search filter', async () => { - await using db = await createTestDB(); + await using db = await createTestDB({ pure: true }); const { store, kysely } = db; const en = genEvent({ kind: 1, content: 'hello world!' }); @@ -72,7 +72,7 @@ Deno.test('query events with language search filter', async () => { }); Deno.test('delete events', async () => { - await using db = await createTestDB(); + await using db = await createTestDB({ pure: true }); const { store } = db; const sk = generateSecretKey(); @@ -96,7 +96,7 @@ Deno.test('delete events', async () => { }); Deno.test("user cannot delete another user's event", async () => { - await using db = await createTestDB(); + await using db = await createTestDB({ pure: true }); const { store } = db; const event = genEvent({ kind: 1, content: 'hello world', created_at: 1 }); @@ -113,7 +113,7 @@ Deno.test("user cannot delete another user's event", async () => { }); Deno.test('admin can delete any event', async () => { - await using db = await createTestDB(); + await using db = await createTestDB({ pure: true }); const { store } = db; const sk = generateSecretKey(); @@ -137,7 +137,7 @@ Deno.test('admin can delete any event', async () => { }); Deno.test('throws a RelayError when inserting an event deleted by the admin', async () => { - await using db = await createTestDB(); + await using db = await createTestDB({ pure: true }); const { store } = db; const event = genEvent(); @@ -154,7 +154,7 @@ Deno.test('throws a RelayError when inserting an event deleted by the admin', as }); Deno.test('throws a RelayError when inserting an event deleted by a user', async () => { - await using db = await createTestDB(); + await using db = await createTestDB({ pure: true }); const { store } = db; const sk = generateSecretKey(); @@ -173,7 +173,7 @@ Deno.test('throws a RelayError when inserting an event deleted by a user', async }); Deno.test('inserting replaceable events', async () => { - await using db = await createTestDB(); + await using db = await createTestDB({ pure: true }); const { store } = db; const sk = generateSecretKey(); @@ -190,7 +190,7 @@ Deno.test('inserting replaceable events', async () => { }); Deno.test("throws a RelayError when querying an event with a large 'since'", async () => { - await using db = await createTestDB(); + await using db = await createTestDB({ pure: true }); const { store } = db; await assertRejects( @@ -201,7 +201,7 @@ Deno.test("throws a RelayError when querying an event with a large 'since'", asy }); Deno.test("throws a RelayError when querying an event with a large 'until'", async () => { - await using db = await createTestDB(); + await using db = await createTestDB({ pure: true }); const { store } = db; await assertRejects( @@ -212,7 +212,7 @@ Deno.test("throws a RelayError when querying an event with a large 'until'", asy }); Deno.test("throws a RelayError when querying an event with a large 'kind'", async () => { - await using db = await createTestDB(); + await using db = await createTestDB({ pure: true }); const { store } = db; await assertRejects( diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 905883b7..b303dad0 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -30,6 +30,8 @@ interface EventsDBOpts { pubkey: string; /** Timeout in milliseconds for database queries. */ timeout: number; + /** Whether the event returned should be a Nostr event or a Ditto event. Defaults to false. */ + pure?: boolean; } /** SQL database storage adapter for Nostr events. */ @@ -203,7 +205,7 @@ class EventsDB extends NPostgres { /** Parse an event row from the database. */ protected override parseEventRow(row: DittoTables['nostr_events']): DittoEvent { - return { + const event: DittoEvent = { id: row.id, kind: row.kind, pubkey: row.pubkey, @@ -211,8 +213,17 @@ class EventsDB extends NPostgres { created_at: Number(row.created_at), tags: row.tags, sig: row.sig, - language: (row.language || undefined) as LanguageCode, }; + + if (this.opts.pure) { + return event; + } + + if (row.language) { + event.language = row.language as LanguageCode; + } + + return event; } /** Delete events based on filters from the database. */ diff --git a/src/test.ts b/src/test.ts index c8fcfe6b..3f2d1c38 100644 --- a/src/test.ts +++ b/src/test.ts @@ -35,7 +35,7 @@ export function genEvent(t: Partial = {}, sk: Uint8Array = generateS } /** Create a database for testing. It uses `TEST_DATABASE_URL`, or creates an in-memory database by default. */ -export async function createTestDB() { +export async function createTestDB(opts?: { pure?: boolean }) { const { testDatabaseUrl } = Conf; const { kysely } = DittoDB.create(testDatabaseUrl, { poolSize: 1 }); @@ -45,6 +45,7 @@ export async function createTestDB() { kysely, timeout: Conf.db.timeouts.default, pubkey: Conf.pubkey, + pure: opts?.pure ?? false, }); return { From ba2393172706a90b767d146c86111e9b0ed7cd68 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 8 Oct 2024 14:25:25 -0300 Subject: [PATCH 23/34] refactor: remove unused imports --- src/controllers/api/translate.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/controllers/api/translate.ts b/src/controllers/api/translate.ts index 3e8bf13c..629571ff 100644 --- a/src/controllers/api/translate.ts +++ b/src/controllers/api/translate.ts @@ -1,9 +1,7 @@ -import { LanguageCode } from 'iso-639-1'; import { z } from 'zod'; import { AppController } from '@/app.ts'; import { languageSchema } from '@/schema.ts'; -import { Storages } from '@/storages.ts'; import { dittoTranslations, dittoTranslationsKey } from '@/translators/translator.ts'; import { parseBody } from '@/utils/api.ts'; import { getEvent } from '@/queries.ts'; From a3bc5ec5c3bd9e4baeda4dd3f9cba9160641b5ae Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 8 Oct 2024 14:27:00 -0300 Subject: [PATCH 24/34] refactor: remove translation enabled in instanceV1Controller --- src/controllers/api/instance.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/controllers/api/instance.ts b/src/controllers/api/instance.ts index ae31bdc2..09e50632 100644 --- a/src/controllers/api/instance.ts +++ b/src/controllers/api/instance.ts @@ -36,9 +36,6 @@ const instanceV1Controller: AppController = async (c) => { max_characters: Conf.postCharLimit, max_media_attachments: 20, }, - translation: { - enabled: true, - }, }, pleroma: { metadata: { From f76ee000b098abb00acbbfe912ac43bdb1890a58 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 8 Oct 2024 14:35:51 -0300 Subject: [PATCH 25/34] refactor: use 'lang' instead of 'target_language' --- src/controllers/api/translate.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/controllers/api/translate.ts b/src/controllers/api/translate.ts index 629571ff..37b0bcea 100644 --- a/src/controllers/api/translate.ts +++ b/src/controllers/api/translate.ts @@ -8,8 +8,7 @@ import { getEvent } from '@/queries.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; const translateSchema = z.object({ - //lang: languageSchema, // Correct property name, as stated by Mastodon docs - target_language: languageSchema, // Property name soapbox sends + lang: languageSchema, }); const translateController: AppController = async (c) => { @@ -25,8 +24,7 @@ const translateController: AppController = async (c) => { return c.json({ error: 'No translator configured.' }, 500); } - const { target_language } = result.data; - const targetLang = target_language; + const { lang } = result.data; const id = c.req.param('id'); const event = await getEvent(id, { signal }); @@ -36,13 +34,13 @@ const translateController: AppController = async (c) => { const viewerPubkey = await c.get('signer')?.getPublicKey(); - if (targetLang.toLowerCase() === event?.language?.toLowerCase()) { + if (lang.toLowerCase() === event?.language?.toLowerCase()) { return c.json({ error: 'Source and target languages are the same. No translation needed.' }, 400); } const status = await renderStatus(event, { viewerPubkey }); - const translatedId = `${target_language}-${id}` as dittoTranslationsKey; + const translatedId = `${lang}-${id}` as dittoTranslationsKey; const translationCache = dittoTranslations.get(translatedId); if (translationCache) { @@ -63,7 +61,7 @@ const translateController: AppController = async (c) => { mediaAttachments, null, event.language, - targetLang, + lang, { signal }, ); dittoTranslations.set(translatedId, translation); From dbd590228d7128353ae026bcb2cddba3e00adb46 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 8 Oct 2024 14:39:59 -0300 Subject: [PATCH 26/34] fix: typo --- src/translators/DeepLTranslator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/translators/DeepLTranslator.ts b/src/translators/DeepLTranslator.ts index fd8788dd..b340a715 100644 --- a/src/translators/DeepLTranslator.ts +++ b/src/translators/DeepLTranslator.ts @@ -10,7 +10,7 @@ import { import { languageSchema } from '@/schema.ts'; interface DeepLTranslatorOpts { - /** DeepL endpoint to use. Default: 'https://api.deepl.com '*/ + /** DeepL endpoint to use. Default: 'https://api.deepl.com' */ endpoint?: string; /** DeepL API key. */ apiKey: string; From 20caaa9ebdc6b1ac412349313384f91256fc4080 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 8 Oct 2024 16:53:30 -0300 Subject: [PATCH 27/34] refactor: LibreTranslate and DeepL with separate environment variables for their configuration --- src/config.ts | 20 +++++++++++++------ src/middleware/translatorMiddleware.ts | 18 +++++++++++------ src/translators/DeepLTranslator.test.ts | 4 ++-- .../LibreTranslateTranslator.test.ts | 4 ++-- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/config.ts b/src/config.ts index 848f9836..0e4fc816 100644 --- a/src/config.ts +++ b/src/config.ts @@ -275,13 +275,21 @@ class Conf { static get translationProvider(): string | undefined { return Deno.env.get('TRANSLATION_PROVIDER'); } - /** Translation provider URL endpoint. */ - static get translationProviderEndpoint(): string | undefined { - return Deno.env.get('TRANSLATION_PROVIDER_ENDPOINT'); + /** DeepL URL endpoint. */ + static get deepLendpoint(): string | undefined { + return Deno.env.get('DEEPL_ENDPOINT'); } - /** Translation provider API KEY. */ - static get translationProviderApiKey(): string | undefined { - return Deno.env.get('TRANSLATION_PROVIDER_API_KEY'); + /** DeepL API KEY. */ + static get deepLapiKey(): string | undefined { + return Deno.env.get('DEEPL_API_KEY'); + } + /** LibreTranslate URL endpoint. */ + static get libreTranslateEndpoint(): string | undefined { + return Deno.env.get('LIBRETRANSLATE_ENDPOINT'); + } + /** LibreTranslate API KEY. */ + static get libreTranslateApiKey(): string | undefined { + return Deno.env.get('LIBRETRANSLATE_API_KEY'); } /** Cache settings. */ static caches = { diff --git a/src/middleware/translatorMiddleware.ts b/src/middleware/translatorMiddleware.ts index 2d3971dd..b8a07686 100644 --- a/src/middleware/translatorMiddleware.ts +++ b/src/middleware/translatorMiddleware.ts @@ -6,24 +6,30 @@ import { LibreTranslateTranslator } from '@/translators/LibreTranslateTranslator /** Set the translator used for translating posts. */ export const translatorMiddleware: AppMiddleware = async (c, next) => { - const endpoint = Conf.translationProviderEndpoint; - const apiKey = Conf.translationProviderApiKey; + const deepLendpoint = Conf.deepLendpoint; + const deepLapiKey = Conf.deepLapiKey; + const libreTranslateEndpoint = Conf.libreTranslateEndpoint; + const libreTranslateApiKey = Conf.libreTranslateApiKey; const translationProvider = Conf.translationProvider; switch (translationProvider) { case 'deepl': - if (apiKey) { + if (deepLapiKey) { c.set( 'translator', - new DeepLTranslator({ endpoint, apiKey, fetch: fetchWorker }), + new DeepLTranslator({ endpoint: deepLendpoint, apiKey: deepLapiKey, fetch: fetchWorker }), ); } break; case 'libretranslate': - if (apiKey) { + if (libreTranslateApiKey) { c.set( 'translator', - new LibreTranslateTranslator({ endpoint, apiKey, fetch: fetchWorker }), + new LibreTranslateTranslator({ + endpoint: libreTranslateEndpoint, + apiKey: libreTranslateApiKey, + fetch: fetchWorker, + }), ); } break; diff --git a/src/translators/DeepLTranslator.test.ts b/src/translators/DeepLTranslator.test.ts index 8335670e..f8a12ede 100644 --- a/src/translators/DeepLTranslator.test.ts +++ b/src/translators/DeepLTranslator.test.ts @@ -4,8 +4,8 @@ import { Conf } from '@/config.ts'; import { DeepLTranslator } from '@/translators/DeepLTranslator.ts'; import { getLanguage } from '@/test.ts'; -const endpoint = Conf.translationProviderEndpoint; -const apiKey = Conf.translationProviderApiKey; +const endpoint = Conf.deepLendpoint; +const apiKey = Conf.deepLapiKey; const translationProvider = Conf.translationProvider; const deepL = 'deepl'; diff --git a/src/translators/LibreTranslateTranslator.test.ts b/src/translators/LibreTranslateTranslator.test.ts index 8d1fc24d..f7e89b30 100644 --- a/src/translators/LibreTranslateTranslator.test.ts +++ b/src/translators/LibreTranslateTranslator.test.ts @@ -4,8 +4,8 @@ import { Conf } from '@/config.ts'; import { LibreTranslateTranslator } from '@/translators/LibreTranslateTranslator.ts'; import { getLanguage } from '@/test.ts'; -const endpoint = Conf.translationProviderEndpoint; -const apiKey = Conf.translationProviderApiKey; +const endpoint = Conf.libreTranslateEndpoint; +const apiKey = Conf.libreTranslateApiKey; const translationProvider = Conf.translationProvider; const libreTranslate = 'libretranslate'; From fc5e9b29902efdb4d188673b671e734af6046107 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 8 Oct 2024 16:58:42 -0300 Subject: [PATCH 28/34] Revert "refactor: move getConfigs() function and frontendConfig logic to 'src/utils/frontendConfig.ts'" This reverts commit ab85360d2ff5de1e3aa42e4a35aa4b640cccaf31. The discussion started publicly in Gitlab: https://gitlab.com/soapbox-pub/ditto/-/merge_requests/537#note_2148430111 Then it kept going in Element, basically the purpose of the commit is correct, but the way Patrick did it is not good. --- src/controllers/api/pleroma.ts | 32 +++++++++++++++++++++++---- src/utils/frontendConfig.ts | 40 ---------------------------------- 2 files changed, 28 insertions(+), 44 deletions(-) delete mode 100644 src/utils/frontendConfig.ts diff --git a/src/controllers/api/pleroma.ts b/src/controllers/api/pleroma.ts index 2c025b8e..31d8545f 100644 --- a/src/controllers/api/pleroma.ts +++ b/src/controllers/api/pleroma.ts @@ -1,20 +1,26 @@ +import { NSchema as n, NStore } from '@nostrify/nostrify'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { configSchema } from '@/schemas/pleroma-api.ts'; +import { configSchema, elixirTupleSchema, type PleromaConfig } from '@/schemas/pleroma-api.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Storages } from '@/storages.ts'; import { createAdminEvent, updateAdminEvent, updateUser } from '@/utils/api.ts'; -import { getConfigs, getPleromaConfig } from '@/utils/frontendConfig.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; const frontendConfigController: AppController = async (c) => { const store = await Storages.db(); - const frontendConfig = await getPleromaConfig(store, c.req.raw.signal); + const configs = await getConfigs(store, c.req.raw.signal); + const frontendConfig = configs.find(({ group, key }) => group === ':pleroma' && key === ':frontend_configurations'); if (frontendConfig) { - return c.json(frontendConfig); + const schema = elixirTupleSchema.transform(({ tuple }) => tuple).array(); + const data = schema.parse(frontendConfig.value).reduce>((result, [name, data]) => { + result[name.replace(/^:/, '')] = data; + return result; + }, {}); + return c.json(data); } else { return c.json({}); } @@ -64,6 +70,24 @@ const pleromaAdminDeleteStatusController: AppController = async (c) => { return c.json({}); }; +async function getConfigs(store: NStore, signal: AbortSignal): Promise { + const { pubkey } = Conf; + + const [event] = await store.query([{ + kinds: [30078], + authors: [pubkey], + '#d': ['pub.ditto.pleroma.config'], + limit: 1, + }], { signal }); + + try { + const decrypted = await new AdminSigner().nip44.decrypt(Conf.pubkey, event.content); + return n.json().pipe(configSchema.array()).catch([]).parse(decrypted); + } catch (_e) { + return []; + } +} + const pleromaAdminTagSchema = z.object({ nicknames: z.string().array(), tags: z.string().array(), diff --git a/src/utils/frontendConfig.ts b/src/utils/frontendConfig.ts deleted file mode 100644 index a4f3f592..00000000 --- a/src/utils/frontendConfig.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { NSchema as n, NStore } from '@nostrify/nostrify'; - -import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { Conf } from '@/config.ts'; -import { configSchema, elixirTupleSchema, type PleromaConfig } from '@/schemas/pleroma-api.ts'; - -export async function getPleromaConfig( - store: NStore, - signal?: AbortSignal, -): Promise> { - const configs = await getConfigs(store, signal ?? AbortSignal.timeout(1000)); - const frontendConfig = configs.find(({ group, key }) => group === ':pleroma' && key === ':frontend_configurations'); - if (frontendConfig) { - const schema = elixirTupleSchema.transform(({ tuple }) => tuple).array(); - const data = schema.parse(frontendConfig.value).reduce>((result, [name, data]) => { - result[name.replace(/^:/, '')] = data; - return result; - }, {}); - return data; - } - return undefined; -} - -export async function getConfigs(store: NStore, signal: AbortSignal): Promise { - const { pubkey } = Conf; - - const [event] = await store.query([{ - kinds: [30078], - authors: [pubkey], - '#d': ['pub.ditto.pleroma.config'], - limit: 1, - }], { signal }); - - try { - const decrypted = await new AdminSigner().nip44.decrypt(Conf.pubkey, event.content); - return n.json().pipe(configSchema.array()).catch([]).parse(decrypted); - } catch (_e) { - return []; - } -} From 49d815826cc3b1bb7b6b1852fd19ad9e991879b0 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 8 Oct 2024 17:07:05 -0300 Subject: [PATCH 29/34] refactor(languageSchema): enforce return type --- src/schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/schema.ts b/src/schema.ts index f21128d3..9adcffdd 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -41,7 +41,7 @@ const fileSchema = z.custom((value) => value instanceof File); const percentageSchema = z.coerce.number().int().gte(1).lte(100); -const languageSchema = z.string().transform((val, ctx) => { +const languageSchema = z.string().transform((val, ctx) => { val = (val.toLowerCase()).split('-')[0]; // pt-BR -> pt if (!ISO6391.validate(val)) { ctx.addIssue({ From c1c25d7c08c9dad73c95d7200b96fbeb347e14b0 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Wed, 9 Oct 2024 14:57:28 -0300 Subject: [PATCH 30/34] feat: create localeSchema --- src/schema.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/schema.ts b/src/schema.ts index 9adcffdd..a9dd56e3 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -42,7 +42,7 @@ const fileSchema = z.custom((value) => value instanceof File); const percentageSchema = z.coerce.number().int().gte(1).lte(100); const languageSchema = z.string().transform((val, ctx) => { - val = (val.toLowerCase()).split('-')[0]; // pt-BR -> pt + val = val.toLowerCase(); if (!ISO6391.validate(val)) { ctx.addIssue({ code: z.ZodIssueCode.custom, @@ -53,6 +53,18 @@ const languageSchema = z.string().transform((val, ctx) => { return val as LanguageCode; }); +const localeSchema = z.string().transform((val, ctx) => { + try { + return new Intl.Locale(val); + } catch { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Invalid locale', + }); + return z.NEVER; + } +}); + export { booleanParamSchema, decode64Schema, @@ -60,6 +72,7 @@ export { filteredArray, hashtagSchema, languageSchema, + localeSchema, percentageSchema, safeUrlSchema, }; From 4f8c8fd1de7f59f606adce25fc9f4a8f34d34ca0 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Wed, 9 Oct 2024 15:03:11 -0300 Subject: [PATCH 31/34] refactor: simply DittoTranslator interface and classes that implement it --- src/controllers/api/translate.ts | 98 ++++++++++--- src/translators/DeepLTranslator.test.ts | 131 +++-------------- src/translators/DeepLTranslator.ts | 107 ++++---------- .../LibreTranslateTranslator.test.ts | 133 +++--------------- src/translators/LibreTranslateTranslator.ts | 103 ++++---------- src/translators/translator.ts | 12 +- 6 files changed, 183 insertions(+), 401 deletions(-) diff --git a/src/controllers/api/translate.ts b/src/controllers/api/translate.ts index 37b0bcea..f2ca9eae 100644 --- a/src/controllers/api/translate.ts +++ b/src/controllers/api/translate.ts @@ -1,14 +1,15 @@ +import { LanguageCode } from 'iso-639-1'; import { z } from 'zod'; import { AppController } from '@/app.ts'; -import { languageSchema } from '@/schema.ts'; -import { dittoTranslations, dittoTranslationsKey } from '@/translators/translator.ts'; +import { localeSchema } from '@/schema.ts'; +import { dittoTranslations, dittoTranslationsKey, MastodonTranslation } from '@/translators/translator.ts'; import { parseBody } from '@/utils/api.ts'; import { getEvent } from '@/queries.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; const translateSchema = z.object({ - lang: languageSchema, + lang: localeSchema, }); const translateController: AppController = async (c) => { @@ -24,7 +25,8 @@ const translateController: AppController = async (c) => { return c.json({ error: 'No translator configured.' }, 500); } - const { lang } = result.data; + const lang = result.data.lang.language.slice(0, 2) as LanguageCode; + const id = c.req.param('id'); const event = await getEvent(id, { signal }); @@ -39,6 +41,9 @@ const translateController: AppController = async (c) => { } const status = await renderStatus(event, { viewerPubkey }); + if (!status?.content) { + return c.json({ error: 'Bad request.', schema: result.error }, 400); + } const translatedId = `${lang}-${id}` as dittoTranslationsKey; const translationCache = dittoTranslations.get(translatedId); @@ -55,18 +60,79 @@ const translateController: AppController = async (c) => { }) ?? []; try { - const translation = await translator.translate( - status?.content ?? '', - status?.spoiler_text ?? '', - mediaAttachments, - null, - event.language, - lang, - { signal }, - ); - dittoTranslations.set(translatedId, translation); - return c.json(translation.data, 200); - } catch (_) { + const texts: string[] = []; + + const mastodonTranslation: MastodonTranslation = { + content: '', + spoiler_text: '', + media_attachments: [], + poll: null, + detected_source_language: event.language ?? 'en', + provider: translator.getProvider(), + }; + + if ((status?.poll as MastodonTranslation['poll'])?.options) { + mastodonTranslation.poll = { id: (status?.poll as MastodonTranslation['poll'])?.id!, options: [] }; + } + + type TranslationIndex = { + [key: number]: 'content' | 'spoilerText' | 'poll' | { type: 'media'; id: string }; + }; + const translationIndex: TranslationIndex = {}; + let index = 0; + + // Content + translationIndex[index] = 'content'; + texts.push(status.content); + index++; + + // Spoiler text + if (status.spoiler_text) { + translationIndex[index] = 'spoilerText'; + texts.push(status.spoiler_text); + index++; + } + + // Media description + for (const [mediaIndex, value] of mediaAttachments.entries()) { + translationIndex[index + mediaIndex] = { type: 'media', id: value.id }; + texts.push(mediaAttachments[mediaIndex].description); + index += mediaIndex; + } + + // Poll title + if (status?.poll) { + for (const [pollIndex] of (status?.poll as MastodonTranslation['poll'])!.options.entries()) { + translationIndex[index + pollIndex] = 'poll'; + texts.push((status.poll as MastodonTranslation['poll'])!.options[pollIndex].title); + index += pollIndex; + } + } + + const data = await translator.translate(texts, event.language, lang, { signal }); + const translatedTexts = data.results; + + for (let i = 0; i < texts.length; i++) { + if (translationIndex[i] === 'content') { + mastodonTranslation.content = translatedTexts[i]; + } else if (translationIndex[i] === 'spoilerText') { + mastodonTranslation.spoiler_text = translatedTexts[i]; + } else if (translationIndex[i] === 'poll') { + mastodonTranslation.poll?.options.push({ title: translatedTexts[i] }); + } else { + const media = translationIndex[i] as { type: 'media'; id: string }; + mastodonTranslation.media_attachments.push({ + id: media.id, + description: translatedTexts[i], + }); + } + } + + mastodonTranslation.detected_source_language = data.source_lang; + + dittoTranslations.set(translatedId, { data: mastodonTranslation }); + return c.json(mastodonTranslation, 200); + } catch { return c.json({ error: 'Service Unavailable' }, 503); } }; diff --git a/src/translators/DeepLTranslator.test.ts b/src/translators/DeepLTranslator.test.ts index f8a12ede..385c10fc 100644 --- a/src/translators/DeepLTranslator.test.ts +++ b/src/translators/DeepLTranslator.test.ts @@ -9,131 +9,44 @@ const apiKey = Conf.deepLapiKey; const translationProvider = Conf.translationProvider; const deepL = 'deepl'; -Deno.test('Translate status with EMPTY media_attachments and WITHOUT poll', { +Deno.test('DeepL translation with source language omitted', { ignore: !(translationProvider === deepL && apiKey), }, async () => { const translator = new DeepLTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); - const mastodonTranslation = await translator.translate( - 'Bom dia amigos do Element, meu nome é Patrick', - '', - [], - null, - 'pt', - 'en', - ); - - assertEquals(getLanguage(mastodonTranslation.data.content), 'en'); - assertEquals(mastodonTranslation.data.spoiler_text, ''); - assertEquals(mastodonTranslation.data.media_attachments, []); - assertEquals(mastodonTranslation.data.poll, null); - assertEquals(mastodonTranslation.data.provider, 'DeepL.com'); -}); - -Deno.test('Translate status WITH auto detect and with EMPTY media_attachments and WITHOUT poll', { - ignore: !(translationProvider === deepL && apiKey), -}, async () => { - const translator = new DeepLTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); - - const mastodonTranslation = await translator.translate( - 'Bom dia amigos do Element, meu nome é Patrick', - '', - [], - null, + const data = await translator.translate( + [ + 'Bom dia amigos', + 'Meu nome é Patrick', + 'Eu irei morar na America, eu prometo. Mas antes, eu devo mencionar que o lande está interpretando este texto como italiano, que estranho.', + ], undefined, 'en', ); - assertEquals(getLanguage(mastodonTranslation.data.content), 'en'); - assertEquals(mastodonTranslation.data.spoiler_text, ''); - assertEquals(mastodonTranslation.data.media_attachments, []); - assertEquals(mastodonTranslation.data.poll, null); - assertEquals(mastodonTranslation.data.provider, 'DeepL.com'); + assertEquals(data.source_lang, 'pt'); + assertEquals(getLanguage(data.results[0]), 'en'); + assertEquals(getLanguage(data.results[1]), 'en'); + assertEquals(getLanguage(data.results[2]), 'en'); }); -Deno.test('Translate status WITH media_attachments and WITHOUT poll', { +Deno.test('DeepL translation with source language set', { ignore: !(translationProvider === deepL && apiKey), }, async () => { const translator = new DeepLTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); - const mastodonTranslation = await translator.translate( - 'Hello my friends, my name is Alex and I am american.', - "That is spoiler isn't it", - [{ id: 'game', description: 'I should be playing Miles Edgeworth with my wife' }], - null, - 'en', - 'pt', - ); - - assertEquals(getLanguage(mastodonTranslation.data.content), 'pt'); - assertEquals(getLanguage(mastodonTranslation.data.spoiler_text), 'pt'); - assertEquals(mastodonTranslation.data.media_attachments.map((value) => getLanguage(value.description)), ['pt']); - assertEquals(mastodonTranslation.data.poll, null); - assertEquals(mastodonTranslation.data.provider, 'DeepL.com'); -}); - -Deno.test('Translate status WITHOUT media_attachments and WITH poll', { - ignore: !(translationProvider === deepL && apiKey), -}, async () => { - const translator = new DeepLTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); - - const poll = { - 'id': '34858', - 'options': [ - { - 'title': 'Kill him right now', - }, - { - 'title': 'Save him right now', - }, + const data = await translator.translate( + [ + 'Bom dia amigos', + 'Meu nome é Patrick', + 'Eu irei morar na America, eu prometo. Mas antes, eu devo mencionar que o lande está interpretando este texto como italiano, que estranho.', ], - }; - - const mastodonTranslation = await translator.translate( - 'Hello my friends, my name is Alex and I am american.', - '', - [], - poll, - 'en', 'pt', + 'en', ); - assertEquals(getLanguage(mastodonTranslation.data.content), 'pt'); - assertEquals(mastodonTranslation.data.spoiler_text, ''); - assertEquals(mastodonTranslation.data.media_attachments, []); - assertEquals(mastodonTranslation.data.poll?.options.map((value) => getLanguage(value.title)), ['pt', 'pt']); - assertEquals(mastodonTranslation.data.provider, 'DeepL.com'); -}); - -Deno.test('Translate status WITH media_attachments and WITH poll', { - ignore: !(translationProvider === deepL && apiKey), -}, async () => { - const translator = new DeepLTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); - - const poll = { - 'id': '34858', - 'options': [ - { - 'title': 'Kill him right now', - }, - { - 'title': 'Save him right now', - }, - ], - }; - - const mastodonTranslation = await translator.translate( - 'Hello my friends, my name is Alex and I am american.', - '', - [{ id: 'game', description: 'I should be playing Miles Edgeworth with my wife' }], - poll, - 'en', - 'pt', - ); - - assertEquals(getLanguage(mastodonTranslation.data.content), 'pt'); - assertEquals(mastodonTranslation.data.spoiler_text, ''); - assertEquals(mastodonTranslation.data.media_attachments.map((value) => getLanguage(value.description)), ['pt']); - assertEquals(mastodonTranslation.data.poll?.options.map((value) => getLanguage(value.title)), ['pt', 'pt']); - assertEquals(mastodonTranslation.data.provider, 'DeepL.com'); + assertEquals(data.source_lang, 'pt'); + assertEquals(getLanguage(data.results[0]), 'en'); + assertEquals(getLanguage(data.results[1]), 'en'); + assertEquals(getLanguage(data.results[2]), 'en'); }); diff --git a/src/translators/DeepLTranslator.ts b/src/translators/DeepLTranslator.ts index b340a715..d97c59a1 100644 --- a/src/translators/DeepLTranslator.ts +++ b/src/translators/DeepLTranslator.ts @@ -1,12 +1,6 @@ import { z } from 'zod'; -import { - DittoTranslator, - MastodonTranslation, - Provider, - SourceLanguage, - TargetLanguage, -} from '@/translators/translator.ts'; +import { DittoTranslator, Provider, SourceLanguage, TargetLanguage } from '@/translators/translator.ts'; import { languageSchema } from '@/schema.ts'; interface DeepLTranslatorOpts { @@ -22,45 +16,43 @@ export class DeepLTranslator implements DittoTranslator { private readonly endpoint: string; private readonly apiKey: string; private readonly fetch: typeof fetch; - private readonly provider: Provider; + private static provider: Provider = 'DeepL.com'; constructor(opts: DeepLTranslatorOpts) { this.endpoint = opts.endpoint ?? 'https://api.deepl.com'; this.fetch = opts.fetch ?? globalThis.fetch; - this.provider = 'DeepL.com'; this.apiKey = opts.apiKey; } async translate( - contentHTMLencoded: string, - spoilerText: string, - mediaAttachments: { id: string; description: string }[], - poll: { id: string; options: { title: string }[] } | null, - sourceLanguage: SourceLanguage | undefined, + texts: string[], + source: SourceLanguage | undefined, + dest: TargetLanguage, + opts?: { signal?: AbortSignal }, + ) { + const data = (await this.translateMany(texts, source, dest, opts)).translations; + + return { + results: data.map((value) => value.text), + source_lang: data[0].detected_source_language, + }; + } + + /** DeepL translate request. */ + private async translateMany( + texts: string[], + source: SourceLanguage | undefined, targetLanguage: TargetLanguage, opts?: { signal?: AbortSignal }, ) { - // --------------------- START explanation - // Order of texts: - // 1 - contentHTMLencoded - // 2 - spoilerText - // 3 - mediaAttachments descriptions - // 4 - poll title options - const medias = mediaAttachments.map((value) => value.description); - - const polls = poll?.options.map((value) => value.title) ?? []; - - const text = [contentHTMLencoded, spoilerText].concat(medias, polls); - // --------------------- END explanation - const body: any = { - text, + text: texts, target_lang: targetLanguage.toUpperCase(), tag_handling: 'html', split_sentences: '1', }; - if (sourceLanguage) { - body.source_lang = sourceLanguage.toUpperCase(); + if (source) { + body.source_lang = source.toUpperCase(); } const headers = new Headers(); @@ -76,55 +68,9 @@ export class DeepLTranslator implements DittoTranslator { const response = await this.fetch(request); const json = await response.json(); - const data = DeepLTranslator.schema().parse(json).translations; + const data = DeepLTranslator.schema().parse(json); - const mastodonTranslation: MastodonTranslation = { - content: '', - spoiler_text: '', - media_attachments: [], - poll: null, - detected_source_language: 'en', - provider: this.provider, - }; - - /** Used to keep track of the offset. When slicing, should be used as the start value. */ - let startIndex = 0; - mastodonTranslation.content = data[0].text; - startIndex++; - - mastodonTranslation.spoiler_text = data[1].text; - startIndex++; - - if (medias.length) { - const mediasTranslated = data.slice(startIndex, startIndex + medias.length); - for (let i = 0; i < mediasTranslated.length; i++) { - mastodonTranslation.media_attachments.push({ - id: mediaAttachments[i].id, - description: mediasTranslated[i].text, - }); - } - startIndex += mediasTranslated.length; - } - - if (polls.length && poll) { - const pollsTranslated = data.slice(startIndex); - mastodonTranslation.poll = { - id: poll.id, - options: [], - }; - for (let i = 0; i < pollsTranslated.length; i++) { - mastodonTranslation.poll.options.push({ - title: pollsTranslated[i].text, - }); - } - startIndex += pollsTranslated.length; - } - - mastodonTranslation.detected_source_language = data[0].detected_source_language; - - return { - data: mastodonTranslation, - }; + return data; } /** DeepL response schema. @@ -139,4 +85,9 @@ export class DeepLTranslator implements DittoTranslator { ), }); } + + /** DeepL provider. */ + getProvider(): Provider { + return DeepLTranslator.provider; + } } diff --git a/src/translators/LibreTranslateTranslator.test.ts b/src/translators/LibreTranslateTranslator.test.ts index f7e89b30..6b87cc91 100644 --- a/src/translators/LibreTranslateTranslator.test.ts +++ b/src/translators/LibreTranslateTranslator.test.ts @@ -9,131 +9,44 @@ const apiKey = Conf.libreTranslateApiKey; const translationProvider = Conf.translationProvider; const libreTranslate = 'libretranslate'; -Deno.test('Translate status with EMPTY media_attachments and WITHOUT poll', { +Deno.test('LibreTranslate translation with source language omitted', { ignore: !(translationProvider === libreTranslate && apiKey), }, async () => { const translator = new LibreTranslateTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); - const mastodonTranslation = await translator.translate( - 'Bom dia amigos do Element, meu nome é Patrick', - '', - [], - null, - 'pt', - 'en', - ); - - assertEquals(getLanguage(mastodonTranslation.data.content), 'en'); - assertEquals(mastodonTranslation.data.spoiler_text, ''); - assertEquals(mastodonTranslation.data.media_attachments, []); - assertEquals(mastodonTranslation.data.poll, null); - assertEquals(mastodonTranslation.data.provider, 'libretranslate.com'); -}); - -Deno.test('Translate status WITH auto detect and with EMPTY media_attachments and WITHOUT poll', { - ignore: !(translationProvider === libreTranslate && apiKey), -}, async () => { - const translator = new LibreTranslateTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); - - const mastodonTranslation = await translator.translate( - 'Bom dia amigos do Element, meu nome é Patrick', - '', - [], - null, + const data = await translator.translate( + [ + 'Bom dia amigos', + 'Meu nome é Patrick, um nome belo ou feio? A questão é mais profunda do que parece.', + 'A respiração é mais importante do que comer e tomar agua.', + ], undefined, - 'en', + 'ca', ); - assertEquals(getLanguage(mastodonTranslation.data.content), 'en'); - assertEquals(mastodonTranslation.data.spoiler_text, ''); - assertEquals(mastodonTranslation.data.media_attachments, []); - assertEquals(mastodonTranslation.data.poll, null); - assertEquals(mastodonTranslation.data.provider, 'libretranslate.com'); + assertEquals(data.source_lang, 'pt'); + assertEquals(getLanguage(data.results[0]), 'ca'); + assertEquals(getLanguage(data.results[1]), 'ca'); + assertEquals(getLanguage(data.results[2]), 'ca'); }); -Deno.test('Translate status WITH media_attachments and WITHOUT poll', { +Deno.test('LibreTranslate translation with source language set', { ignore: !(translationProvider === libreTranslate && apiKey), }, async () => { const translator = new LibreTranslateTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); - const mastodonTranslation = await translator.translate( - 'Hello my friends, my name is Alex and I am american.', - "That is spoiler isn't it", - [{ id: 'game', description: 'I should be playing Miles Edgeworth with my wife' }], - null, - 'en', - 'pt', - ); - - assertEquals(getLanguage(mastodonTranslation.data.content), 'pt'); - assertEquals(getLanguage(mastodonTranslation.data.spoiler_text), 'pt'); - assertEquals(mastodonTranslation.data.media_attachments.map((value) => getLanguage(value.description)), ['pt']); - assertEquals(mastodonTranslation.data.poll, null); - assertEquals(mastodonTranslation.data.provider, 'libretranslate.com'); -}); - -Deno.test('Translate status WITHOUT media_attachments and WITH poll', { - ignore: !(translationProvider === libreTranslate && apiKey), -}, async () => { - const translator = new LibreTranslateTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); - - const poll = { - 'id': '34858', - 'options': [ - { - 'title': 'Kill him right now', - }, - { - 'title': 'Save him right now', - }, + const data = await translator.translate( + [ + 'Bom dia amigos', + 'Meu nome é Patrick, um nome belo ou feio? A questão é mais profunda do que parece.', + 'A respiração é mais importante do que comer e tomar agua.', ], - }; - - const mastodonTranslation = await translator.translate( - 'Hello my friends, my name is Alex and I am american.', - '', - [], - poll, - 'en', 'pt', + 'ca', ); - assertEquals(getLanguage(mastodonTranslation.data.content), 'pt'); - assertEquals(mastodonTranslation.data.spoiler_text, ''); - assertEquals(mastodonTranslation.data.media_attachments, []); - assertEquals(mastodonTranslation.data.poll?.options.map((value) => getLanguage(value.title)), ['pt', 'pt']); - assertEquals(mastodonTranslation.data.provider, 'libretranslate.com'); -}); - -Deno.test('Translate status WITH media_attachments and WITH poll', { - ignore: !(translationProvider === libreTranslate && apiKey), -}, async () => { - const translator = new LibreTranslateTranslator({ fetch: fetch, endpoint, apiKey: apiKey as string }); - - const poll = { - 'id': '34858', - 'options': [ - { - 'title': 'Kill him right now', - }, - { - 'title': 'Save him right now', - }, - ], - }; - - const mastodonTranslation = await translator.translate( - 'Hello my friends, my name is Alex and I am american.', - '', - [{ id: 'game', description: 'I should be playing Miles Edgeworth with my wife' }], - poll, - 'en', - 'pt', - ); - - assertEquals(getLanguage(mastodonTranslation.data.content), 'pt'); - assertEquals(mastodonTranslation.data.spoiler_text, ''); - assertEquals(mastodonTranslation.data.media_attachments.map((value) => getLanguage(value.description)), ['pt']); - assertEquals(mastodonTranslation.data.poll?.options.map((value) => getLanguage(value.title)), ['pt', 'pt']); - assertEquals(mastodonTranslation.data.provider, 'libretranslate.com'); + assertEquals(data.source_lang, 'pt'); + assertEquals(getLanguage(data.results[0]), 'ca'); + assertEquals(getLanguage(data.results[1]), 'ca'); + assertEquals(getLanguage(data.results[2]), 'ca'); }); diff --git a/src/translators/LibreTranslateTranslator.ts b/src/translators/LibreTranslateTranslator.ts index 80f44479..d632c71e 100644 --- a/src/translators/LibreTranslateTranslator.ts +++ b/src/translators/LibreTranslateTranslator.ts @@ -1,12 +1,8 @@ +import { LanguageCode } from 'iso-639-1'; import { z } from 'zod'; -import { - DittoTranslator, - MastodonTranslation, - Provider, - SourceLanguage, - TargetLanguage, -} from '@/translators/translator.ts'; +import { DittoTranslator, Provider, SourceLanguage, TargetLanguage } from '@/translators/translator.ts'; +import { languageSchema } from '@/schema.ts'; interface LibreTranslateTranslatorOpts { /** Libretranslate endpoint to use. Default: 'https://libretranslate.com' */ @@ -21,97 +17,37 @@ export class LibreTranslateTranslator implements DittoTranslator { private readonly endpoint: string; private readonly apiKey: string; private readonly fetch: typeof fetch; - private readonly provider: Provider; + private static provider: Provider = 'libretranslate.com'; constructor(opts: LibreTranslateTranslatorOpts) { this.endpoint = opts.endpoint ?? 'https://libretranslate.com'; this.fetch = opts.fetch ?? globalThis.fetch; - this.provider = 'libretranslate.com'; this.apiKey = opts.apiKey; } async translate( - contentHTMLencoded: string, - spoilerText: string, - mediaAttachments: { id: string; description: string }[], - poll: { id: string; options: { title: string }[] } | null, - sourceLanguage: SourceLanguage | undefined, - targetLanguage: TargetLanguage, + texts: string[], + source: SourceLanguage | undefined, + dest: TargetLanguage, opts?: { signal?: AbortSignal }, ) { - const mastodonTranslation: MastodonTranslation = { - content: '', - spoiler_text: '', - media_attachments: [], - poll: null, - detected_source_language: 'en', - provider: this.provider, - }; - - const translatedContent = await this.makeRequest(contentHTMLencoded, sourceLanguage, targetLanguage, 'html', { - signal: opts?.signal, - }); - mastodonTranslation.content = translatedContent; - - if (spoilerText.length) { - const translatedSpoilerText = await this.makeRequest(spoilerText, sourceLanguage, targetLanguage, 'text', { - signal: opts?.signal, - }); - mastodonTranslation.spoiler_text = translatedSpoilerText; - } - - if (mediaAttachments) { - for (const media of mediaAttachments) { - const translatedDescription = await this.makeRequest( - media.description, - sourceLanguage, - targetLanguage, - 'text', - { - signal: opts?.signal, - }, - ); - mastodonTranslation.media_attachments.push({ - id: media.id, - description: translatedDescription, - }); - } - } - - if (poll) { - mastodonTranslation.poll = { - id: poll.id, - options: [], - }; - - for (const option of poll.options) { - const translatedTitle = await this.makeRequest( - option.title, - sourceLanguage, - targetLanguage, - 'text', - { - signal: opts?.signal, - }, - ); - mastodonTranslation.poll.options.push({ - title: translatedTitle, - }); - } - } + const translations = await Promise.all( + texts.map((text) => this.translateOne(text, source, dest, 'html', { signal: opts?.signal })), + ); return { - data: mastodonTranslation, + results: translations.map((value) => value.translatedText), + source_lang: translations[0]?.detectedLanguage?.language ?? source as LanguageCode, // cast is ok }; } - private async makeRequest( + private async translateOne( q: string, sourceLanguage: string | undefined, targetLanguage: string, format: 'html' | 'text', opts?: { signal?: AbortSignal }, - ): Promise { + ) { const body = { q, source: sourceLanguage?.toLowerCase() ?? 'auto', @@ -132,7 +68,7 @@ export class LibreTranslateTranslator implements DittoTranslator { const response = await this.fetch(request); const json = await response.json(); - const data = LibreTranslateTranslator.schema().parse(json).translatedText; + const data = LibreTranslateTranslator.schema().parse(json); return data; } @@ -142,6 +78,15 @@ export class LibreTranslateTranslator implements DittoTranslator { private static schema() { return z.object({ translatedText: z.string(), + /** This field is only available if the 'source' is set to 'auto' */ + detectedLanguage: z.object({ + language: languageSchema, + }).optional(), }); } + + /** LibreTranslate provider. */ + getProvider(): Provider { + return LibreTranslateTranslator.provider; + } } diff --git a/src/translators/translator.ts b/src/translators/translator.ts index 515b335f..29874964 100644 --- a/src/translators/translator.ts +++ b/src/translators/translator.ts @@ -36,21 +36,15 @@ export type MastodonTranslation = { export interface DittoTranslator { /** Translate the 'content' into 'targetLanguage'. */ translate( - /** HTML-encoded content of the status. */ - content: string, - /** Spoiler warning of the status. */ - spoilerText: string, - /** Media descriptions of the status. */ - mediaAttachments: { id: string; description: string }[], - /** Poll of the status. */ - poll: { id: string; options: { title: string }[] } | null, + texts: string[], /** The language of the source text/status. */ sourceLanguage: SourceLanguage | undefined, /** The status content will be translated into this language. */ targetLanguage: TargetLanguage, /** Custom options. */ opts?: { signal?: AbortSignal }, - ): Promise; + ): Promise<{ results: string[]; source_lang: SourceLanguage }>; + getProvider(): Provider; } /** Includes the TARGET language and the status id. From 22fa3f944c77b7cc2183520500ded44dbd89595f Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Wed, 9 Oct 2024 15:15:32 -0300 Subject: [PATCH 32/34] chore: update nostrify:db --- deno.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deno.lock b/deno.lock index 0a0d9e31..a00d4d31 100644 --- a/deno.lock +++ b/deno.lock @@ -22,7 +22,7 @@ "jsr:@gleasonator/policy@0.8.0": "0.8.0", "jsr:@hono/hono@^4.4.6": "4.6.2", "jsr:@lambdalisue/async@^2.1.1": "2.1.1", - "jsr:@nostrify/db@0.35": "0.35.0", + "jsr:@nostrify/db@~0.36.1": "0.36.1", "jsr:@nostrify/nostrify@0.31": "0.31.0", "jsr:@nostrify/nostrify@0.32": "0.32.0", "jsr:@nostrify/nostrify@0.35": "0.35.0", @@ -270,10 +270,10 @@ "@lambdalisue/async@2.1.1": { "integrity": "1fc9bc6f4ed50215cd2f7217842b18cea80f81c25744f88f8c5eb4be5a1c9ab4" }, - "@nostrify/db@0.35.0": { - "integrity": "637191c41812544e361b7997dc44ea098f8bd7efebb28f37a8a7142a0ecada8d", + "@nostrify/db@0.36.1": { + "integrity": "b65b89ca6fe98d9dbcc0402b5c9c07b8430c2c91f84ba4128ff2eeed70c3d49f", "dependencies": [ - "jsr:@nostrify/nostrify@0.35", + "jsr:@nostrify/nostrify@0.36", "jsr:@nostrify/types@0.35", "npm:kysely@~0.27.3", "npm:nostr-tools@^2.7.0" @@ -2048,7 +2048,7 @@ "jsr:@gfx/canvas-wasm@~0.4.2", "jsr:@hono/hono@^4.4.6", "jsr:@lambdalisue/async@^2.1.1", - "jsr:@nostrify/db@0.35", + "jsr:@nostrify/db@~0.36.1", "jsr:@nostrify/nostrify@0.36", "jsr:@nostrify/policies@0.35", "jsr:@soapbox/kysely-pglite@1", From 57bbbb289b1a91a6d2af52fdddf2b477373d402a Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Wed, 9 Oct 2024 15:22:09 -0300 Subject: [PATCH 33/34] fix: types must have the type prefix apparently happens in Deno 2.0? --- src/app.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.ts b/src/app.ts index e58535c0..39d29a04 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,4 +1,4 @@ -import { Context, Env as HonoEnv, Handler, Hono, Input as HonoInput, MiddlewareHandler } from '@hono/hono'; +import { type Context, Env as HonoEnv, Handler, Hono, Input as HonoInput, MiddlewareHandler } from '@hono/hono'; import { cors } from '@hono/hono/cors'; import { serveStatic } from '@hono/hono/deno'; import { logger } from '@hono/hono/logger'; From cad0da27320bbc0c3f0f29b7405083eb895b9cc9 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Wed, 9 Oct 2024 15:24:01 -0300 Subject: [PATCH 34/34] feat: rateLimit translate endpoint --- src/app.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/app.ts b/src/app.ts index 39d29a04..6659169f 100644 --- a/src/app.ts +++ b/src/app.ts @@ -227,7 +227,13 @@ app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/bookmark', requireSigner, bookmarkC app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unbookmark', requireSigner, unbookmarkController); app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/pin', requireSigner, pinController); app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unpin', requireSigner, unpinController); -app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/translate', requireSigner, translatorMiddleware, translateController); +app.post( + '/api/v1/statuses/:id{[0-9a-f]{64}}/translate', + requireSigner, + rateLimitMiddleware(30, Time.minutes(1)), + translatorMiddleware, + translateController, +); app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/reblog', requireSigner, reblogStatusController); app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unreblog', requireSigner, unreblogStatusController); app.post('/api/v1/statuses', requireSigner, createStatusController);