Merge remote-tracking branch 'origin/main' into push

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-image: denoland/deno:2.0.0-rc.3
+image: denoland/deno:2.0.0
 
 default:
   interruptible: true

.tool-versions

@@ -1 +1 @@
-deno 1.46.3
+deno 2.0.0

Dockerfile

@@ -1,4 +1,4 @@
-FROM denoland/deno:1.44.2
+FROM denoland/deno:2.0.0
 ENV PORT 5000
 
 WORKDIR /app

deno.json

@@ -10,7 +10,7 @@
     "nostr:pull": "deno run -A scripts/nostr-pull.ts",
     "debug": "deno run -A --inspect src/server.ts",
     "test": "deno test -A --junit-path=./deno-test.xml",
-    "check": "deno check src/server.ts",
+    "check": "deno check --allow-import src/server.ts",
     "nsec": "deno run scripts/nsec.ts",
     "admin:event": "deno run -A scripts/admin-event.ts",
     "admin:role": "deno run -A scripts/admin-role.ts",
@@ -37,13 +37,14 @@
     "@b-fuze/deno-dom": "jsr:@b-fuze/deno-dom@^0.1.47",
     "@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/s3-lite-client@^0.7.4",
     "@electric-sql/pglite": "npm:@electric-sql/pglite@^0.2.8",
+    "@esroyo/scoped-performance": "jsr:@esroyo/scoped-performance@^3.1.0",
     "@gfx/canvas-wasm": "jsr:@gfx/canvas-wasm@^0.4.2",
     "@hono/hono": "jsr:@hono/hono@^4.4.6",
     "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1",
     "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1",
     "@negrel/webpush": "jsr:@negrel/webpush@^0.3.0",
     "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0",
-    "@nostrify/db": "jsr:@nostrify/db@^0.35.0",
+    "@nostrify/db": "jsr:@nostrify/db@^0.36.1",
     "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.36.0",
     "@nostrify/policies": "jsr:@nostrify/policies@^0.35.0",
     "@scure/base": "npm:@scure/base@^1.1.6",
@@ -69,7 +70,7 @@
     "formdata-helper": "npm:formdata-helper@^0.3.0",
     "hono-rate-limiter": "npm:hono-rate-limiter@^0.3.0",
     "iso-639-1": "npm:iso-639-1@2.1.15",
-    "isomorphic-dompurify": "npm:isomorphic-dompurify@^2.11.0",
+    "isomorphic-dompurify": "npm:isomorphic-dompurify@^2.16.0",
     "kysely": "npm:kysely@^0.27.4",
     "kysely-postgres-js": "npm:kysely-postgres-js@2.0.0",
     "lande": "npm:lande@^1.0.10",

grafana/Ditto-Dashboard.json

@@ -879,7 +879,7 @@
             "uid": "${prometheus}"
           },
           "editorMode": "code",
-          "expr": "increase(ditto_db_query_duration_ms_sum[$__rate_interval])",
+          "expr": "increase(ditto_db_query_duration_seconds_sum[$__rate_interval])",
           "instant": false,
           "legendFormat": "Time",
           "range": true,

src/app.ts

@@ -1,4 +1,4 @@
-import { Context, Env as HonoEnv, Handler, Hono, Input as HonoInput, MiddlewareHandler } from '@hono/hono';
+import { type Context, Env as HonoEnv, Handler, Hono, Input as HonoInput, MiddlewareHandler } from '@hono/hono';
 import { cors } from '@hono/hono/cors';
 import { serveStatic } from '@hono/hono/deno';
 import { logger } from '@hono/hono/logger';
@@ -113,6 +113,7 @@ import {
   trendingStatusesController,
   trendingTagsController,
 } from '@/controllers/api/trends.ts';
+import { translateController } from '@/controllers/api/translate.ts';
 import { errorHandler } from '@/controllers/error.ts';
 import { frontendController } from '@/controllers/frontend.ts';
 import { metricsController } from '@/controllers/metrics.ts';
@@ -120,6 +121,7 @@ import { indexController } from '@/controllers/site.ts';
 import { manifestController } from '@/controllers/manifest.ts';
 import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts';
 import { nostrController } from '@/controllers/well-known/nostr.ts';
+import { DittoTranslator } from '@/interfaces/DittoTranslator.ts';
 import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts';
 import { cspMiddleware } from '@/middleware/cspMiddleware.ts';
 import { metricsMiddleware } from '@/middleware/metricsMiddleware.ts';
@@ -129,6 +131,7 @@ import { requireSigner } from '@/middleware/requireSigner.ts';
 import { signerMiddleware } from '@/middleware/signerMiddleware.ts';
 import { storeMiddleware } from '@/middleware/storeMiddleware.ts';
 import { uploaderMiddleware } from '@/middleware/uploaderMiddleware.ts';
+import { translatorMiddleware } from '@/middleware/translatorMiddleware.ts';
 
 interface AppEnv extends HonoEnv {
   Variables: {
@@ -144,6 +147,8 @@ interface AppEnv extends HonoEnv {
     pagination: { since?: number; until?: number; limit: number };
     /** Normalized list pagination params. */
     listPagination: { offset: number; limit: number };
+    /** Translation service. */
+    translator?: DittoTranslator;
   };
 }
 
@@ -223,6 +228,13 @@ app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/bookmark', requireSigner, bookmarkC
 app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unbookmark', requireSigner, unbookmarkController);
 app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/pin', requireSigner, pinController);
 app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unpin', requireSigner, unpinController);
+app.post(
+  '/api/v1/statuses/:id{[0-9a-f]{64}}/translate',
+  requireSigner,
+  rateLimitMiddleware(15, Time.minutes(1)),
+  translatorMiddleware,
+  translateController,
+);
 app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/reblog', requireSigner, reblogStatusController);
 app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unreblog', requireSigner, unreblogStatusController);
 app.post('/api/v1/statuses', requireSigner, createStatusController);

src/caches/translationCache.ts

@@ -0,0 +1,11 @@
+import { LanguageCode } from 'iso-639-1';
+import { LRUCache } from 'lru-cache';
+
+import { Conf } from '@/config.ts';
+import { MastodonTranslation } from '@/entities/MastodonTranslation.ts';
+
+/** Translations LRU cache. */
+export const translationCache = new LRUCache<`${LanguageCode}-${string}`, MastodonTranslation>({
+  max: Conf.caches.translation.max,
+  ttl: Conf.caches.translation.ttl,
+});

src/config.ts

@@ -296,6 +296,26 @@ class Conf {
   static get preferredLanguages(): LanguageCode[] | undefined {
     return Deno.env.get('DITTO_LANGUAGES')?.split(',')?.filter(ISO6391.validate) as LanguageCode[];
   }
+  /** Translation provider used to translate posts. */
+  static get translationProvider(): string | undefined {
+    return Deno.env.get('TRANSLATION_PROVIDER');
+  }
+  /** DeepL URL endpoint. */
+  static get deeplBaseUrl(): string | undefined {
+    return Deno.env.get('DEEPL_BASE_URL');
+  }
+  /** DeepL API KEY. */
+  static get deeplApiKey(): string | undefined {
+    return Deno.env.get('DEEPL_API_KEY');
+  }
+  /** LibreTranslate URL endpoint. */
+  static get libretranslateBaseUrl(): string | undefined {
+    return Deno.env.get('LIBRETRANSLATE_BASE_URL');
+  }
+  /** LibreTranslate API KEY. */
+  static get libretranslateApiKey(): string | undefined {
+    return Deno.env.get('LIBRETRANSLATE_API_KEY');
+  }
   /** Cache settings. */
   static caches = {
     /** NIP-05 cache settings. */
@@ -319,6 +339,13 @@ class Conf {
         ttl: Number(Deno.env.get('DITTO_CACHE_LINK_PREVIEW_TTL') || 12 * 60 * 60 * 1000),
       };
     },
+    /** Translation cache settings. */
+    get translation(): { max: number; ttl: number } {
+      return {
+        max: Number(Deno.env.get('DITTO_CACHE_TRANSLATION_MAX') || 1000),
+        ttl: Number(Deno.env.get('DITTO_CACHE_TRANSLATION_TTL') || 6 * 60 * 60 * 1000),
+      };
+    },
   };
 }
 

src/controllers/api/accounts.ts

@@ -51,7 +51,7 @@ const verifyCredentialsController: AppController = async (c) => {
 
   const store = await Storages.db();
 
-  const [author, [settingsStore], [captcha]] = await Promise.all([
+  const [author, [settingsEvent]] = await Promise.all([
     getAuthor(pubkey, { signal: AbortSignal.timeout(5000) }),
 
     store.query([{
@@ -60,32 +60,18 @@ const verifyCredentialsController: AppController = async (c) => {
       '#d': ['pub.ditto.pleroma_settings_store'],
       limit: 1,
     }]),
-
-    store.query([{
-      kinds: [1985],
-      authors: [Conf.pubkey],
-      '#L': ['pub.ditto.captcha'],
-      '#l': ['solved'],
-      '#p': [pubkey],
-      limit: 1,
-    }]),
   ]);
 
-  const account = author
+  let settingsStore: Record<string, unknown> | undefined;
-    ? await renderAccount(author, { withSource: true })
-    : await accountFromPubkey(pubkey, { withSource: true });
-
-  if (settingsStore) {
   try {
-      account.pleroma.settings_store = JSON.parse(settingsStore.content);
+    settingsStore = n.json().pipe(z.record(z.string(), z.unknown())).parse(settingsEvent?.content);
   } catch {
-      // Ignore
+    // Do nothing
-    }
   }
 
-  if (captcha && account.source) {
+  const account = author
-    account.source.ditto.captcha_solved = true;
+    ? await renderAccount(author, { withSource: true, settingsStore })
-  }
+    : await accountFromPubkey(pubkey, { withSource: true, settingsStore });
 
   return c.json(account);
 };
@@ -280,7 +266,7 @@ const updateCredentialsSchema = z.object({
   bot: z.boolean().optional(),
   discoverable: z.boolean().optional(),
   nip05: z.string().email().or(z.literal('')).optional(),
-  pleroma_settings_store: z.unknown().optional(),
+  pleroma_settings_store: z.record(z.string(), z.unknown()).optional(),
   lud16: z.string().email().or(z.literal('')).optional(),
   website: z.string().url().or(z.literal('')).optional(),
 });
@@ -339,8 +325,8 @@ const updateCredentialsController: AppController = async (c) => {
     c,
   );
 
-  const account = await renderAccount(event, { withSource: true });
   const settingsStore = result.data.pleroma_settings_store;
+  const account = await renderAccount(event, { withSource: true, settingsStore });
 
   if (settingsStore) {
     await createEvent({
@@ -350,8 +336,6 @@ const updateCredentialsController: AppController = async (c) => {
     }, c);
   }
 
-  account.pleroma.settings_store = settingsStore;
-
   return c.json(account);
 };
 

src/controllers/api/captcha.ts

@@ -4,7 +4,7 @@ import { z } from 'zod';
 
 import { AppController } from '@/app.ts';
 import { Conf } from '@/config.ts';
-import { createAdminEvent } from '@/utils/api.ts';
+import { updateUser } from '@/utils/api.ts';
 
 interface Point {
   x: number;
@@ -169,16 +169,7 @@ export const captchaVerifyController: AppController = async (c) => {
 
   if (solved) {
     captchas.delete(id);
-
+    await updateUser(pubkey, { captcha_solved: true }, c);
-    await createAdminEvent({
-      kind: 1985,
-      tags: [
-        ['L', 'pub.ditto.captcha'],
-        ['l', 'solved', 'pub.ditto.captcha'],
-        ['p', pubkey, Conf.relay],
-      ],
-    }, c);
-
     return new Response(null, { status: 204 });
   }
 

src/controllers/api/instance.ts

@@ -129,7 +129,7 @@ const instanceV2Controller: AppController = async (c) => {
         max_expiration: 2629746,
       },
       translation: {
-        enabled: false,
+        enabled: Boolean(Conf.translationProvider),
       },
     },
     nostr: {
@@ -142,7 +142,7 @@ const instanceV2Controller: AppController = async (c) => {
       },
     },
     registrations: {
-      enabled: false,
+      enabled: true,
       approval_required: false,
       message: null,
       url: null,

src/controllers/api/translate.ts

@@ -0,0 +1,147 @@
+import { LanguageCode } from 'iso-639-1';
+import { z } from 'zod';
+
+import { AppController } from '@/app.ts';
+import { translationCache } from '@/caches/translationCache.ts';
+import { MastodonTranslation } from '@/entities/MastodonTranslation.ts';
+import { cachedTranslationsSizeGauge } from '@/metrics.ts';
+import { getEvent } from '@/queries.ts';
+import { localeSchema } from '@/schema.ts';
+import { parseBody } from '@/utils/api.ts';
+import { renderStatus } from '@/views/mastodon/statuses.ts';
+
+const translateSchema = z.object({
+  lang: localeSchema,
+});
+
+const translateController: AppController = async (c) => {
+  const result = translateSchema.safeParse(await parseBody(c.req.raw));
+  const { signal } = c.req.raw;
+
+  if (!result.success) {
+    return c.json({ error: 'Bad request.', schema: result.error }, 422);
+  }
+
+  const translator = c.get('translator');
+  if (!translator) {
+    return c.json({ error: 'No translator configured.' }, 500);
+  }
+
+  const lang = result.data.lang.language.slice(0, 2) as LanguageCode;
+
+  const id = c.req.param('id');
+
+  const event = await getEvent(id, { signal });
+  if (!event) {
+    return c.json({ error: 'Record not found' }, 400);
+  }
+
+  const viewerPubkey = await c.get('signer')?.getPublicKey();
+
+  if (lang.toLowerCase() === event?.language?.toLowerCase()) {
+    return c.json({ error: 'Source and target languages are the same. No translation needed.' }, 400);
+  }
+
+  const status = await renderStatus(event, { viewerPubkey });
+  if (!status?.content) {
+    return c.json({ error: 'Bad request.', schema: result.error }, 400);
+  }
+
+  const cacheKey: `${LanguageCode}-${string}` = `${lang}-${id}`;
+  const cached = translationCache.get(cacheKey);
+
+  if (cached) {
+    return c.json(cached, 200);
+  }
+
+  const mediaAttachments = status?.media_attachments.map((value) => {
+    return {
+      id: value.id,
+      description: value.description ?? '',
+    };
+  }) ?? [];
+
+  try {
+    const texts: string[] = [];
+
+    const mastodonTranslation: MastodonTranslation = {
+      content: '',
+      spoiler_text: '',
+      media_attachments: [],
+      poll: null,
+      detected_source_language: event.language ?? 'en',
+      provider: translator.provider,
+    };
+
+    if ((status?.poll as MastodonTranslation['poll'])?.options) {
+      mastodonTranslation.poll = { id: (status?.poll as MastodonTranslation['poll'])?.id!, options: [] };
+    }
+
+    type TranslationIndex = {
+      [key: number]: 'content' | 'spoilerText' | 'poll' | { type: 'media'; id: string };
+    };
+    const translationIndex: TranslationIndex = {};
+    let index = 0;
+
+    // Content
+    translationIndex[index] = 'content';
+    texts.push(status.content);
+    index++;
+
+    // Spoiler text
+    if (status.spoiler_text) {
+      translationIndex[index] = 'spoilerText';
+      texts.push(status.spoiler_text);
+      index++;
+    }
+
+    // Media description
+    for (const [mediaIndex, value] of mediaAttachments.entries()) {
+      translationIndex[index + mediaIndex] = { type: 'media', id: value.id };
+      texts.push(mediaAttachments[mediaIndex].description);
+      index += mediaIndex;
+    }
+
+    // Poll title
+    if (status?.poll) {
+      for (const [pollIndex] of (status?.poll as MastodonTranslation['poll'])!.options.entries()) {
+        translationIndex[index + pollIndex] = 'poll';
+        texts.push((status.poll as MastodonTranslation['poll'])!.options[pollIndex].title);
+        index += pollIndex;
+      }
+    }
+
+    const data = await translator.translate(texts, event.language, lang, { signal });
+    const translatedTexts = data.results;
+
+    for (let i = 0; i < texts.length; i++) {
+      if (translationIndex[i] === 'content') {
+        mastodonTranslation.content = translatedTexts[i];
+      } else if (translationIndex[i] === 'spoilerText') {
+        mastodonTranslation.spoiler_text = translatedTexts[i];
+      } else if (translationIndex[i] === 'poll') {
+        mastodonTranslation.poll?.options.push({ title: translatedTexts[i] });
+      } else {
+        const media = translationIndex[i] as { type: 'media'; id: string };
+        mastodonTranslation.media_attachments.push({
+          id: media.id,
+          description: translatedTexts[i],
+        });
+      }
+    }
+
+    mastodonTranslation.detected_source_language = data.source_lang;
+
+    translationCache.set(cacheKey, mastodonTranslation);
+    cachedTranslationsSizeGauge.set(translationCache.size);
+
+    return c.json(mastodonTranslation, 200);
+  } catch (e) {
+    if (e instanceof Error && e.message.includes('not supported')) {
+      return c.json({ error: `Translation of source language '${event.language}' not supported` }, 422);
+    }
+    return c.json({ error: 'Service Unavailable' }, 503);
+  }
+};
+
+export { translateController };

src/controllers/nostr/relay.ts

@@ -18,6 +18,7 @@ import * as pipeline from '@/pipeline.ts';
 import { RelayError } from '@/RelayError.ts';
 import { Storages } from '@/storages.ts';
 import { Time } from '@/utils/time.ts';
+import { purifyEvent } from '@/utils/purify.ts';
 
 /** Limit of initial events returned for a subscription. */
 const FILTER_LIMIT = 100;
@@ -105,7 +106,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) {
 
     try {
       for (const event of await store.query(filters, { limit: FILTER_LIMIT, timeout: Conf.db.timeouts.relay })) {
-        send(['EVENT', subId, event]);
+        send(['EVENT', subId, purifyEvent(event)]);
       }
     } catch (e: any) {
       if (e instanceof RelayError) {
@@ -137,7 +138,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) {
     relayEventsCounter.inc({ kind: event.kind.toString() });
     try {
       // This will store it (if eligible) and run other side-effects.
-      await pipeline.handleEvent(event, AbortSignal.timeout(1000));
+      await pipeline.handleEvent(purifyEvent(event), AbortSignal.timeout(1000));
       send(['OK', event.id, true, '']);
     } catch (e) {
       if (e instanceof RelayError) {

src/db/DittoTables.ts

@@ -1,4 +1,4 @@
-import { Generated, Nullable } from 'kysely';
+import { Generated } from 'kysely';
 
 import { NPostgresSchema } from '@nostrify/db';
 
@@ -13,7 +13,7 @@ export interface DittoTables extends NPostgresSchema {
 }
 
 type NostrEventsRow = NPostgresSchema['nostr_events'] & {
-  language: Nullable<string>;
+  language: string | null;
 };
 
 interface AuthorStatsRow {

src/db/KyselyLogger.ts

@@ -9,12 +9,14 @@ export const KyselyLogger: Logger = (event) => {
   const { query, queryDurationMillis } = event;
   const { sql, parameters } = query;
 
+  const queryDurationSeconds = queryDurationMillis / 1000;
+
   dbQueriesCounter.inc();
-  dbQueryDurationHistogram.observe(queryDurationMillis);
+  dbQueryDurationHistogram.observe(queryDurationSeconds);
 
   console.debug(
     sql,
     JSON.stringify(parameters),
-    `\x1b[90m(${(queryDurationMillis / 1000).toFixed(2)}s)\x1b[0m`,
+    `\x1b[90m(${(queryDurationSeconds / 1000).toFixed(2)}s)\x1b[0m`,
   );
 };

src/entities/MastodonAccount.ts

@@ -54,7 +54,7 @@ export interface MastodonAccount {
     is_moderator: boolean;
     is_suggested: boolean;
     is_local: boolean;
-    settings_store: unknown;
+    settings_store?: Record<string, unknown>;
     tags: string[];
   };
   nostr: {

src/entities/MastodonTranslation.ts

@@ -0,0 +1,17 @@
+import { LanguageCode } from 'iso-639-1';
+
+/** https://docs.joinmastodon.org/entities/Translation/ */
+export interface MastodonTranslation {
+  /** HTML-encoded translated content of the status. */
+  content: string;
+  /** The translated spoiler warning of the status. */
+  spoiler_text: string;
+  /** The translated media descriptions of the status. */
+  media_attachments: { id: string; description: string }[];
+  /** The translated poll of the status. */
+  poll: { id: string; options: { title: string }[] } | null;
+  //** The language of the source text, as auto-detected by the machine translation provider. */
+  detected_source_language: LanguageCode;
+  /** The service that provided the machine translation. */
+  provider: string;
+}

src/interfaces/DittoEvent.ts

@@ -1,4 +1,5 @@
 import { NostrEvent } from '@nostrify/nostrify';
+import { LanguageCode } from 'iso-639-1';
 
 /** Ditto internal stats for the event's author. */
 export interface AuthorStats {
@@ -43,4 +44,6 @@ export interface DittoEvent extends NostrEvent {
   zap_sender?: DittoEvent | string;
   zap_amount?: number;
   zap_message?: string;
+  /** Language of the event (kind 1s are more accurate). */
+  language?: LanguageCode;
 }

src/interfaces/DittoTranslator.ts

@@ -0,0 +1,18 @@
+import type { LanguageCode } from 'iso-639-1';
+
+/** DittoTranslator class, used for status translation. */
+export interface DittoTranslator {
+  /** Provider name, eg `DeepL.com` */
+  provider: string;
+  /** Translate the 'content' into 'targetLanguage'. */
+  translate(
+    /** Texts to translate. */
+    texts: string[],
+    /** The language of the source texts. */
+    sourceLanguage: LanguageCode | undefined,
+    /** The texts will be translated into this language. */
+    targetLanguage: LanguageCode,
+    /** Custom options. */
+    opts?: { signal?: AbortSignal },
+  ): Promise<{ results: string[]; source_lang: LanguageCode }>;
+}

src/metrics.ts

@@ -12,6 +12,12 @@ export const httpResponsesCounter = new Counter({
   labelNames: ['method', 'path', 'status'],
 });
 
+export const httpResponseDurationHistogram = new Histogram({
+  name: 'ditto_http_response_duration_seconds',
+  help: 'Histogram of HTTP response times in seconds',
+  labelNames: ['method', 'path', 'status'],
+});
+
 export const streamingConnectionsGauge = new Gauge({
   name: 'ditto_streaming_connections',
   help: 'Number of active connections to the streaming API',
@@ -91,7 +97,7 @@ export const dbAvailableConnectionsGauge = new Gauge({
 });
 
 export const dbQueryDurationHistogram = new Histogram({
-  name: 'ditto_db_query_duration_ms',
+  name: 'ditto_db_query_duration_seconds',
   help: 'Duration of database queries',
 });
 
@@ -115,6 +121,11 @@ export const cachedLinkPreviewSizeGauge = new Gauge({
   help: 'Number of link previews in cache',
 });
 
+export const cachedTranslationsSizeGauge = new Gauge({
+  name: 'ditto_cached_translations_size',
+  help: 'Number of translated statuses in cache',
+});
+
 export const internalSubscriptionsSizeGauge = new Gauge({
   name: 'ditto_internal_subscriptions_size',
   help: "Number of active subscriptions to Ditto's internal relay",

src/middleware/cacheMiddleware.ts

@@ -1,28 +0,0 @@
-import Debug from '@soapbox/stickynotes/debug';
-import { type MiddlewareHandler } from 'hono';
-
-import ExpiringCache from '@/utils/expiring-cache.ts';
-
-const debug = Debug('ditto:middleware:cache');
-
-export const cacheMiddleware = (options: {
-  cacheName: string;
-  expires?: number;
-}): MiddlewareHandler => {
-  return async (c, next) => {
-    const key = c.req.url.replace('http://', 'https://');
-    const cache = new ExpiringCache(await caches.open(options.cacheName));
-    const response = await cache.match(key);
-    if (!response) {
-      debug('Building cache for page', c.req.url);
-      await next();
-      const response = c.res.clone();
-      if (response.status < 500) {
-        await cache.putExpiring(key, response, options.expires ?? 0);
-      }
-    } else {
-      debug('Serving page from cache', c.req.url);
-      return response;
-    }
-  };
-};

src/middleware/metricsMiddleware.ts

@@ -1,9 +1,14 @@
+import { ScopedPerformance } from '@esroyo/scoped-performance';
 import { MiddlewareHandler } from '@hono/hono';
 
-import { httpRequestsCounter, httpResponsesCounter } from '@/metrics.ts';
+import { httpRequestsCounter, httpResponseDurationHistogram, httpResponsesCounter } from '@/metrics.ts';
 
 /** Prometheus metrics middleware that tracks HTTP requests by methods and responses by status code. */
 export const metricsMiddleware: MiddlewareHandler = async (c, next) => {
+  // Start a timer to measure the duration of the response.
+  using perf = new ScopedPerformance();
+  perf.mark('start');
+
   // HTTP Request.
   const { method } = c.req;
   httpRequestsCounter.inc({ method });
@@ -17,4 +22,8 @@ export const metricsMiddleware: MiddlewareHandler = async (c, next) => {
   // Tries to find actual route names first before falling back on potential middleware handlers like `app.use('*')`.
   const path = c.req.matchedRoutes.find((r) => r.method !== 'ALL')?.path ?? c.req.routePath;
   httpResponsesCounter.inc({ method, status, path });
+
+  // Measure the duration of the response.
+  const { duration } = perf.measure('total', 'start');
+  httpResponseDurationHistogram.observe({ method, status, path }, duration / 1000);
 };

src/middleware/translatorMiddleware.ts

@@ -0,0 +1,28 @@
+import { AppMiddleware } from '@/app.ts';
+import { Conf } from '@/config.ts';
+import { fetchWorker } from '@/workers/fetch.ts';
+import { DeepLTranslator } from '@/translators/DeepLTranslator.ts';
+import { LibreTranslateTranslator } from '@/translators/LibreTranslateTranslator.ts';
+
+/** Set the translator used for translating posts. */
+export const translatorMiddleware: AppMiddleware = async (c, next) => {
+  switch (Conf.translationProvider) {
+    case 'deepl': {
+      const { deeplApiKey: apiKey, deeplBaseUrl: baseUrl } = Conf;
+      if (apiKey) {
+        c.set('translator', new DeepLTranslator({ baseUrl, apiKey, fetch: fetchWorker }));
+      }
+      break;
+    }
+
+    case 'libretranslate': {
+      const { libretranslateApiKey: apiKey, libretranslateBaseUrl: baseUrl } = Conf;
+      if (apiKey) {
+        c.set('translator', new LibreTranslateTranslator({ baseUrl, apiKey, fetch: fetchWorker }));
+      }
+      break;
+    }
+  }
+
+  await next();
+};

src/pipeline.ts

@@ -1,8 +1,6 @@
 import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify';
 import { Stickynotes } from '@soapbox/stickynotes';
-import ISO6391 from 'iso-639-1';
 import { Kysely, sql } from 'kysely';
-import lande from 'lande';
 import { LRUCache } from 'lru-cache';
 import { nip19 } from 'nostr-tools';
 import { z } from 'zod';
@@ -19,6 +17,7 @@ import { Storages } from '@/storages.ts';
 import { MastodonPush } from '@/types/MastodonPush.ts';
 import { eventAge, parseNip05, Time } from '@/utils.ts';
 import { getAmount } from '@/utils/bolt11.ts';
+import { detectLanguage } from '@/utils/language.ts';
 import { nip05Cache } from '@/utils/nip05.ts';
 import { purifyEvent } from '@/utils/purify.ts';
 import { updateStats } from '@/utils/stats.ts';
@@ -205,25 +204,21 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise<vo
 
 /** Update the event in the database and set its language. */
 async function setLanguage(event: NostrEvent): Promise<void> {
-  const [topResult] = lande(event.content);
+  if (event.kind !== 1) return;
 
-  if (topResult) {
+  const language = detectLanguage(event.content, 0.90);
-    const [iso6393, confidence] = topResult;
+  if (!language) return;
-    const locale = new Intl.Locale(iso6393);
 
-    if (confidence >= 0.95 && ISO6391.validate(locale.language)) {
   const kysely = await Storages.kysely();
   try {
     await kysely.updateTable('nostr_events')
-          .set('language', locale.language)
+      .set('language', language)
       .where('id', '=', event.id)
       .execute();
   } catch {
     // do nothing
   }
 }
-  }
-}
 
 /** Determine if the event is being received in a timely manner. */
 function isFresh(event: NostrEvent): boolean {

src/schema.ts

@@ -1,4 +1,4 @@
-import ISO6391 from 'iso-639-1';
+import ISO6391, { LanguageCode } from 'iso-639-1';
 import { z } from 'zod';
 
 /** Validates individual items in an array, dropping any that aren't valid. */
@@ -41,7 +41,8 @@ const fileSchema = z.custom<File>((value) => value instanceof File);
 
 const percentageSchema = z.coerce.number().int().gte(1).lte(100);
 
-const languageSchema = z.string().transform((val, ctx) => {
+const languageSchema = z.string().transform<LanguageCode>((val, ctx) => {
+  val = val.toLowerCase();
   if (!ISO6391.validate(val)) {
     ctx.addIssue({
       code: z.ZodIssueCode.custom,
@@ -49,7 +50,19 @@ const languageSchema = z.string().transform((val, ctx) => {
     });
     return z.NEVER;
   }
-  return val;
+  return val as LanguageCode;
+});
+
+const localeSchema = z.string().transform<Intl.Locale>((val, ctx) => {
+  try {
+    return new Intl.Locale(val);
+  } catch {
+    ctx.addIssue({
+      code: z.ZodIssueCode.custom,
+      message: 'Invalid locale',
+    });
+    return z.NEVER;
+  }
 });
 
 export {
@@ -59,6 +72,7 @@ export {
   filteredArray,
   hashtagSchema,
   languageSchema,
+  localeSchema,
   percentageSchema,
   safeUrlSchema,
 };

src/storages/EventsDB.test.ts

@@ -7,7 +7,7 @@ import { Conf } from '@/config.ts';
 import { createTestDB } from '@/test.ts';
 
 Deno.test('count filters', async () => {
-  await using db = await createTestDB();
+  await using db = await createTestDB({ pure: true });
   const { store } = db;
 
   const event1 = await eventFixture('event-1');
@@ -18,7 +18,7 @@ Deno.test('count filters', async () => {
 });
 
 Deno.test('insert and filter events', async () => {
-  await using db = await createTestDB();
+  await using db = await createTestDB({ pure: true });
   const { store } = db;
 
   const event1 = await eventFixture('event-1');
@@ -35,7 +35,7 @@ Deno.test('insert and filter events', async () => {
 });
 
 Deno.test('query events with domain search filter', async () => {
-  await using db = await createTestDB();
+  await using db = await createTestDB({ pure: true });
   const { store, kysely } = db;
 
   const event1 = await eventFixture('event-1');
@@ -55,7 +55,7 @@ Deno.test('query events with domain search filter', async () => {
 });
 
 Deno.test('query events with language search filter', async () => {
-  await using db = await createTestDB();
+  await using db = await createTestDB({ pure: true });
   const { store, kysely } = db;
 
   const en = genEvent({ kind: 1, content: 'hello world!' });
@@ -72,7 +72,7 @@ Deno.test('query events with language search filter', async () => {
 });
 
 Deno.test('delete events', async () => {
-  await using db = await createTestDB();
+  await using db = await createTestDB({ pure: true });
   const { store } = db;
 
   const sk = generateSecretKey();
@@ -96,7 +96,7 @@ Deno.test('delete events', async () => {
 });
 
 Deno.test("user cannot delete another user's event", async () => {
-  await using db = await createTestDB();
+  await using db = await createTestDB({ pure: true });
   const { store } = db;
 
   const event = genEvent({ kind: 1, content: 'hello world', created_at: 1 });
@@ -113,7 +113,7 @@ Deno.test("user cannot delete another user's event", async () => {
 });
 
 Deno.test('admin can delete any event', async () => {
-  await using db = await createTestDB();
+  await using db = await createTestDB({ pure: true });
   const { store } = db;
 
   const sk = generateSecretKey();
@@ -137,7 +137,7 @@ Deno.test('admin can delete any event', async () => {
 });
 
 Deno.test('throws a RelayError when inserting an event deleted by the admin', async () => {
-  await using db = await createTestDB();
+  await using db = await createTestDB({ pure: true });
   const { store } = db;
 
   const event = genEvent();
@@ -154,7 +154,7 @@ Deno.test('throws a RelayError when inserting an event deleted by the admin', as
 });
 
 Deno.test('throws a RelayError when inserting an event deleted by a user', async () => {
-  await using db = await createTestDB();
+  await using db = await createTestDB({ pure: true });
   const { store } = db;
 
   const sk = generateSecretKey();
@@ -173,7 +173,7 @@ Deno.test('throws a RelayError when inserting an event deleted by a user', async
 });
 
 Deno.test('inserting replaceable events', async () => {
-  await using db = await createTestDB();
+  await using db = await createTestDB({ pure: true });
   const { store } = db;
 
   const sk = generateSecretKey();
@@ -190,7 +190,7 @@ Deno.test('inserting replaceable events', async () => {
 });
 
 Deno.test("throws a RelayError when querying an event with a large 'since'", async () => {
-  await using db = await createTestDB();
+  await using db = await createTestDB({ pure: true });
   const { store } = db;
 
   await assertRejects(
@@ -201,7 +201,7 @@ Deno.test("throws a RelayError when querying an event with a large 'since'", asy
 });
 
 Deno.test("throws a RelayError when querying an event with a large 'until'", async () => {
-  await using db = await createTestDB();
+  await using db = await createTestDB({ pure: true });
   const { store } = db;
 
   await assertRejects(
@@ -212,7 +212,7 @@ Deno.test("throws a RelayError when querying an event with a large 'until'", asy
 });
 
 Deno.test("throws a RelayError when querying an event with a large 'kind'", async () => {
-  await using db = await createTestDB();
+  await using db = await createTestDB({ pure: true });
   const { store } = db;
 
   await assertRejects(

src/storages/EventsDB.ts

@@ -1,5 +1,6 @@
 // deno-lint-ignore-file require-await
 
+import { LanguageCode } from 'iso-639-1';
 import { NPostgres, NPostgresSchema } from '@nostrify/db';
 import { NIP50, NKinds, NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify';
 import { Stickynotes } from '@soapbox/stickynotes';
@@ -12,6 +13,7 @@ import { RelayError } from '@/RelayError.ts';
 import { isNostrId, isURL } from '@/utils.ts';
 import { abortError } from '@/utils/abort.ts';
 import { purifyEvent } from '@/utils/purify.ts';
+import { DittoEvent } from '@/interfaces/DittoEvent.ts';
 
 /** Function to decide whether or not to index a tag. */
 type TagCondition = ({ event, count, value }: {
@@ -28,6 +30,8 @@ interface EventsDBOpts {
   pubkey: string;
   /** Timeout in milliseconds for database queries. */
   timeout: number;
+  /** Whether the event returned should be a Nostr event or a Ditto event. Defaults to false. */
+  pure?: boolean;
 }
 
 /** SQL database storage adapter for Nostr events. */
@@ -151,7 +155,7 @@ class EventsDB extends NPostgres {
       let query = super.getFilterQuery(trx, {
         ...filter,
         search: tokens.filter((t) => typeof t === 'string').join(' '),
-      }) as SelectQueryBuilder<DittoTables, 'nostr_events', Pick<DittoTables['nostr_events'], keyof NostrEvent>>;
+      }) as SelectQueryBuilder<DittoTables, 'nostr_events', DittoTables['nostr_events']>;
 
       const languages = new Set<string>();
 
@@ -175,7 +179,7 @@ class EventsDB extends NPostgres {
   override async query(
     filters: NostrFilter[],
     opts: { signal?: AbortSignal; timeout?: number; limit?: number } = {},
-  ): Promise<NostrEvent[]> {
+  ): Promise<DittoEvent[]> {
     filters = await this.expandFilters(filters);
 
     for (const filter of filters) {
@@ -199,6 +203,29 @@ class EventsDB extends NPostgres {
     return super.query(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout });
   }
 
+  /** Parse an event row from the database. */
+  protected override parseEventRow(row: DittoTables['nostr_events']): DittoEvent {
+    const event: DittoEvent = {
+      id: row.id,
+      kind: row.kind,
+      pubkey: row.pubkey,
+      content: row.content,
+      created_at: Number(row.created_at),
+      tags: row.tags,
+      sig: row.sig,
+    };
+
+    if (this.opts.pure) {
+      return event;
+    }
+
+    if (row.language) {
+      event.language = row.language as LanguageCode;
+    }
+
+    return event;
+  }
+
   /** Delete events based on filters from the database. */
   override async remove(filters: NostrFilter[], opts: { signal?: AbortSignal; timeout?: number } = {}): Promise<void> {
     this.console.debug('DELETE', JSON.stringify(filters));

src/test.ts

@@ -1,3 +1,5 @@
+import ISO6391, { LanguageCode } from 'iso-639-1';
+import lande from 'lande';
 import { NostrEvent } from '@nostrify/nostrify';
 import { finalizeEvent, generateSecretKey } from 'nostr-tools';
 
@@ -33,7 +35,7 @@ export function genEvent(t: Partial<NostrEvent> = {}, sk: Uint8Array = generateS
 }
 
 /** Create a database for testing. It uses `TEST_DATABASE_URL`, or creates an in-memory database by default. */
-export async function createTestDB() {
+export async function createTestDB(opts?: { pure?: boolean }) {
   const { testDatabaseUrl } = Conf;
   const { kysely } = DittoDB.create(testDatabaseUrl, { poolSize: 1 });
 
@@ -43,6 +45,7 @@ export async function createTestDB() {
     kysely,
     timeout: Conf.db.timeouts.default,
     pubkey: Conf.pubkey,
+    pure: opts?.pure ?? false,
   });
 
   return {
@@ -65,3 +68,15 @@ export async function createTestDB() {
 export function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
+
+export function getLanguage(text: string): LanguageCode | undefined {
+  const [topResult] = lande(text);
+  if (topResult) {
+    const [iso6393] = topResult;
+    const locale = new Intl.Locale(iso6393);
+    if (ISO6391.validate(locale.language)) {
+      return locale.language as LanguageCode;
+    }
+  }
+  return;
+}

src/translators/DeepLTranslator.test.ts

@@ -0,0 +1,55 @@
+import { assertEquals } from '@std/assert';
+
+import { Conf } from '@/config.ts';
+import { DeepLTranslator } from '@/translators/DeepLTranslator.ts';
+import { getLanguage } from '@/test.ts';
+
+const {
+  deeplBaseUrl: baseUrl,
+  deeplApiKey: apiKey,
+  translationProvider,
+} = Conf;
+
+const deepl = 'deepl';
+
+Deno.test('DeepL translation with source language omitted', {
+  ignore: !(translationProvider === deepl && apiKey),
+}, async () => {
+  const translator = new DeepLTranslator({ fetch: fetch, baseUrl, apiKey: apiKey! });
+
+  const data = await translator.translate(
+    [
+      'Bom dia amigos',
+      'Meu nome é Patrick',
+      'Eu irei morar na America, eu prometo. Mas antes, eu devo mencionar que o lande está interpretando este texto como italiano, que estranho.',
+    ],
+    undefined,
+    'en',
+  );
+
+  assertEquals(data.source_lang, 'pt');
+  assertEquals(getLanguage(data.results[0]), 'en');
+  assertEquals(getLanguage(data.results[1]), 'en');
+  assertEquals(getLanguage(data.results[2]), 'en');
+});
+
+Deno.test('DeepL translation with source language set', {
+  ignore: !(translationProvider === deepl && apiKey),
+}, async () => {
+  const translator = new DeepLTranslator({ fetch: fetch, baseUrl, apiKey: apiKey as string });
+
+  const data = await translator.translate(
+    [
+      'Bom dia amigos',
+      'Meu nome é Patrick',
+      'Eu irei morar na America, eu prometo. Mas antes, eu devo mencionar que o lande está interpretando este texto como italiano, que estranho.',
+    ],
+    'pt',
+    'en',
+  );
+
+  assertEquals(data.source_lang, 'pt');
+  assertEquals(getLanguage(data.results[0]), 'en');
+  assertEquals(getLanguage(data.results[1]), 'en');
+  assertEquals(getLanguage(data.results[2]), 'en');
+});

src/translators/DeepLTranslator.ts

@@ -0,0 +1,94 @@
+import { LanguageCode } from 'iso-639-1';
+import { z } from 'zod';
+
+import { DittoTranslator } from '@/interfaces/DittoTranslator.ts';
+import { languageSchema } from '@/schema.ts';
+
+interface DeepLTranslatorOpts {
+  /** DeepL base URL to use. Default: 'https://api.deepl.com' */
+  baseUrl?: string;
+  /** DeepL API key. */
+  apiKey: string;
+  /** Custom fetch implementation. */
+  fetch?: typeof fetch;
+}
+
+export class DeepLTranslator implements DittoTranslator {
+  private readonly baseUrl: string;
+  private readonly apiKey: string;
+  private readonly fetch: typeof fetch;
+
+  readonly provider = 'DeepL.com';
+
+  constructor(opts: DeepLTranslatorOpts) {
+    this.baseUrl = opts.baseUrl ?? 'https://api.deepl.com';
+    this.fetch = opts.fetch ?? globalThis.fetch;
+    this.apiKey = opts.apiKey;
+  }
+
+  async translate(
+    texts: string[],
+    source: LanguageCode | undefined,
+    dest: LanguageCode,
+    opts?: { signal?: AbortSignal },
+  ) {
+    const { translations } = await this.translateMany(texts, source, dest, opts);
+
+    return {
+      results: translations.map((value) => value.text),
+      source_lang: translations[0]?.detected_source_language as LanguageCode,
+    };
+  }
+
+  /** DeepL translate request. */
+  private async translateMany(
+    texts: string[],
+    source: LanguageCode | undefined,
+    targetLanguage: LanguageCode,
+    opts?: { signal?: AbortSignal },
+  ) {
+    const body: any = {
+      text: texts,
+      target_lang: targetLanguage.toUpperCase(),
+      tag_handling: 'html',
+      split_sentences: '1',
+    };
+    if (source) {
+      body.source_lang = source.toUpperCase();
+    }
+
+    const url = new URL('/v2/translate', this.baseUrl);
+
+    const request = new Request(url, {
+      method: 'POST',
+      body: JSON.stringify(body),
+      headers: {
+        'Authorization': `DeepL-Auth-Key ${this.apiKey}`,
+        'Content-Type': 'application/json',
+      },
+      signal: opts?.signal,
+    });
+
+    const response = await this.fetch(request);
+    const json = await response.json();
+
+    if (!response.ok) {
+      throw new Error(json['message']);
+    }
+
+    return DeepLTranslator.schema().parse(json);
+  }
+
+  /** DeepL response schema.
+   * https://developers.deepl.com/docs/api-reference/translate/openapi-spec-for-text-translation */
+  private static schema() {
+    return z.object({
+      translations: z.array(
+        z.object({
+          detected_source_language: languageSchema,
+          text: z.string(),
+        }),
+      ),
+    });
+  }
+}

src/translators/LibreTranslateTranslator.test.ts

@@ -0,0 +1,55 @@
+import { assertEquals } from '@std/assert';
+
+import { Conf } from '@/config.ts';
+import { LibreTranslateTranslator } from '@/translators/LibreTranslateTranslator.ts';
+import { getLanguage } from '@/test.ts';
+
+const {
+  libretranslateBaseUrl: baseUrl,
+  libretranslateApiKey: apiKey,
+  translationProvider,
+} = Conf;
+
+const libretranslate = 'libretranslate';
+
+Deno.test('LibreTranslate translation with source language omitted', {
+  ignore: !(translationProvider === libretranslate && apiKey),
+}, async () => {
+  const translator = new LibreTranslateTranslator({ fetch: fetch, baseUrl, apiKey: apiKey! });
+
+  const data = await translator.translate(
+    [
+      'Bom dia amigos',
+      'Meu nome é Patrick, um nome belo ou feio? A questão é mais profunda do que parece.',
+      'A respiração é mais importante do que comer e tomar agua.',
+    ],
+    undefined,
+    'ca',
+  );
+
+  assertEquals(data.source_lang, 'pt');
+  assertEquals(getLanguage(data.results[0]), 'ca');
+  assertEquals(getLanguage(data.results[1]), 'ca');
+  assertEquals(getLanguage(data.results[2]), 'ca');
+});
+
+Deno.test('LibreTranslate translation with source language set', {
+  ignore: !(translationProvider === libretranslate && apiKey),
+}, async () => {
+  const translator = new LibreTranslateTranslator({ fetch: fetch, baseUrl, apiKey: apiKey! });
+
+  const data = await translator.translate(
+    [
+      'Bom dia amigos',
+      'Meu nome é Patrick, um nome belo ou feio? A questão é mais profunda do que parece.',
+      'A respiração é mais importante do que comer e tomar agua.',
+    ],
+    'pt',
+    'ca',
+  );
+
+  assertEquals(data.source_lang, 'pt');
+  assertEquals(getLanguage(data.results[0]), 'ca');
+  assertEquals(getLanguage(data.results[1]), 'ca');
+  assertEquals(getLanguage(data.results[2]), 'ca');
+});

src/translators/LibreTranslateTranslator.ts

@@ -0,0 +1,92 @@
+import { LanguageCode } from 'iso-639-1';
+import { z } from 'zod';
+
+import { DittoTranslator } from '@/interfaces/DittoTranslator.ts';
+import { languageSchema } from '@/schema.ts';
+
+interface LibreTranslateTranslatorOpts {
+  /** Libretranslate endpoint to use. Default: 'https://libretranslate.com' */
+  baseUrl?: string;
+  /** Libretranslate API key. */
+  apiKey: string;
+  /** Custom fetch implementation. */
+  fetch?: typeof fetch;
+}
+
+export class LibreTranslateTranslator implements DittoTranslator {
+  private readonly baseUrl: string;
+  private readonly apiKey: string;
+  private readonly fetch: typeof fetch;
+
+  readonly provider = 'libretranslate.com';
+
+  constructor(opts: LibreTranslateTranslatorOpts) {
+    this.baseUrl = opts.baseUrl ?? 'https://libretranslate.com';
+    this.fetch = opts.fetch ?? globalThis.fetch;
+    this.apiKey = opts.apiKey;
+  }
+
+  async translate(
+    texts: string[],
+    source: LanguageCode | undefined,
+    dest: LanguageCode,
+    opts?: { signal?: AbortSignal },
+  ) {
+    const translations = await Promise.all(
+      texts.map((text) => this.translateOne(text, source, dest, 'html', { signal: opts?.signal })),
+    );
+
+    return {
+      results: translations.map((value) => value.translatedText),
+      source_lang: (translations[0]?.detectedLanguage?.language ?? source) as LanguageCode, // cast is ok
+    };
+  }
+
+  private async translateOne(
+    q: string,
+    sourceLanguage: string | undefined,
+    targetLanguage: string,
+    format: 'html' | 'text',
+    opts?: { signal?: AbortSignal },
+  ) {
+    const body = {
+      q,
+      source: sourceLanguage?.toLowerCase() ?? 'auto',
+      target: targetLanguage.toLowerCase(),
+      format,
+      api_key: this.apiKey,
+    };
+
+    const url = new URL('/translate', this.baseUrl);
+
+    const request = new Request(url, {
+      method: 'POST',
+      body: JSON.stringify(body),
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      signal: opts?.signal,
+    });
+
+    const response = await this.fetch(request);
+    const json = await response.json();
+    if (!response.ok) {
+      throw new Error(json['error']);
+    }
+    const data = LibreTranslateTranslator.schema().parse(json);
+
+    return data;
+  }
+
+  /** Libretranslate response schema.
+   *  https://libretranslate.com/docs/#/translate/post_translate */
+  private static schema() {
+    return z.object({
+      translatedText: z.string(),
+      /** This field is only available if the 'source' is set to 'auto' */
+      detectedLanguage: z.object({
+        language: languageSchema,
+      }).optional(),
+    });
+  }
+}

src/utils/api.ts

@@ -1,4 +1,4 @@
-import { Context } from '@hono/hono';
+import { type Context } from '@hono/hono';
 import { HTTPException } from '@hono/hono/http-exception';
 import { NostrEvent, NostrFilter } from '@nostrify/nostrify';
 import Debug from '@soapbox/stickynotes/debug';

src/utils/expiring-cache.test.ts

@@ -1,18 +0,0 @@
-import { assert } from '@std/assert';
-
-import ExpiringCache from './expiring-cache.ts';
-
-Deno.test('ExpiringCache', async () => {
-  const cache = new ExpiringCache(await caches.open('test'));
-
-  await cache.putExpiring('http://mostr.local/1', new Response('hello world'), 300);
-  await cache.putExpiring('http://mostr.local/2', new Response('hello world'), -1);
-
-  // const resp1 = await cache.match('http://mostr.local/1');
-  const resp2 = await cache.match('http://mostr.local/2');
-
-  // assert(resp1!.headers.get('Expires'));
-  assert(!resp2);
-
-  // await resp1!.text();
-});

src/utils/expiring-cache.ts

@@ -1,68 +0,0 @@
-class ExpiringCache implements Cache {
-  #cache: Cache;
-
-  constructor(cache: Cache) {
-    this.#cache = cache;
-  }
-
-  add(request: RequestInfo | URL): Promise<void> {
-    return this.#cache.add(request);
-  }
-
-  addAll(requests: RequestInfo[]): Promise<void> {
-    return this.#cache.addAll(requests);
-  }
-
-  keys(request?: RequestInfo | URL | undefined, options?: CacheQueryOptions | undefined): Promise<readonly Request[]> {
-    return this.#cache.keys(request, options);
-  }
-
-  matchAll(
-    request?: RequestInfo | URL | undefined,
-    options?: CacheQueryOptions | undefined,
-  ): Promise<readonly Response[]> {
-    return this.#cache.matchAll(request, options);
-  }
-
-  put(request: RequestInfo | URL, response: Response): Promise<void> {
-    return this.#cache.put(request, response);
-  }
-
-  putExpiring(request: RequestInfo | URL, response: Response, expiresIn: number): Promise<void> {
-    const expires = Date.now() + expiresIn;
-
-    const clone = new Response(response.body, {
-      status: response.status,
-      headers: {
-        expires: new Date(expires).toUTCString(),
-        ...Object.fromEntries(response.headers.entries()),
-      },
-    });
-
-    return this.#cache.put(request, clone);
-  }
-
-  async match(request: RequestInfo | URL, options?: CacheQueryOptions | undefined): Promise<Response | undefined> {
-    const response = await this.#cache.match(request, options);
-    const expires = response?.headers.get('Expires');
-
-    if (response && expires) {
-      if (new Date(expires).getTime() > Date.now()) {
-        return response;
-      } else {
-        await Promise.all([
-          this.delete(request),
-          response.text(), // Prevent memory leaks
-        ]);
-      }
-    } else if (response) {
-      return response;
-    }
-  }
-
-  delete(request: RequestInfo | URL, options?: CacheQueryOptions | undefined): Promise<boolean> {
-    return this.#cache.delete(request, options);
-  }
-}
-
-export default ExpiringCache;

src/utils/language.test.ts

@@ -0,0 +1,28 @@
+import { detectLanguage } from '@/utils/language.ts';
+import { assertEquals } from '@std/assert';
+
+Deno.test('Detect English language', () => {
+  assertEquals(detectLanguage(``, 0.90), undefined);
+  assertEquals(detectLanguage(`Good morning my fellow friends`, 0.90), 'en');
+  assertEquals(
+    detectLanguage(
+      `Would you listen to Michael Jackson's songs?\n\nnostr:nevent1qvzqqqqqqypzqprpljlvcnpnw3pejvkkhrc3y6wvmd7vjuad0fg2ud3dky66gaxaqyvhwumn8ghj7cm0vfexzen4d4sjucm0d5hhyetvv9usqg8htx8xcjq7ffrzxu7nrhlr8vljcv6gpmet0auy87mpj6djxk4myqha02kp`,
+      0.90,
+    ),
+    'en',
+  );
+  assertEquals(
+    detectLanguage(
+      `https://youtu.be/FxppefYTA2I?si=grgEpbEhFu_-3V_uhttps://youtu.be/FxppefYTA2I?si=grgEpbEhFu_-3V_uhttps://youtu.be/FxppefYTA2I?si=grgEpbEhFu_-3V_uhttps://youtu.be/FxppefYTA2I?si=grgEpbEhFu_-3V_uWould you listen to Michael Jackson's songs?\n\nnostr:nevent1qvzqqqqqqypzqprpljlvcnpnw3pejvkkhrc3y6wvmd7vjuad0fg2ud3dky66gaxaqyvhwumn8ghj7cm0vfexzen4d4sjucm0d5hhyetvv9usqg8htx8xcjq7ffrzxu7nrhlr8vljcv6gpmet0auy87mpj6djxk4myqha02kp`,
+      0.90,
+    ),
+    'en',
+  );
+  assertEquals(
+    detectLanguage(
+      `https://youtu.be/FxppefYTA2I?si=grgEpbEhFu_-3V_u 😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎😂💯♡⌨︎    https://youtu.be/FxppefYTA2I?si=grgEpbEhFu_-3V_uhttps://youtu.be/FxppefYTA2I?si=grgEpbEhFu_-3V_uhttps://youtu.be/FxppefYTA2I?si=grgEpbEhFu_-3V_u Would you listen to Michael Jackson's songs?\n\nnostr:nevent1qvzqqqqqqypzqprpljlvcnpnw3pejvkkhrc3y6wvmd7vjuad0fg2ud3dky66gaxaqyvhwumn8ghj7cm0vfexzen4d4sjucm0d5hhyetvv9usqg8htx8xcjq7ffrzxu7nrhlr8vljcv6gpmet0auy87mpj6djxk4myqha02kp`,
+      0.90,
+    ),
+    'en',
+  );
+});

src/utils/language.ts

@@ -0,0 +1,34 @@
+import ISO6391, { type LanguageCode } from 'iso-639-1';
+import lande from 'lande';
+import linkify from 'linkifyjs';
+
+linkify.registerCustomProtocol('nostr', true);
+
+/** Returns the detected language if the confidence is greater or equal than 'minConfidence'
+ * 'minConfidence' must be a number between 0 and 1, such as 0.95
+ */
+export function detectLanguage(text: string, minConfidence: number): LanguageCode | undefined {
+  // It's better to remove the emojis first
+  const sanitizedText = linkify.tokenize(
+    text
+      .replaceAll(/\p{Extended_Pictographic}/gu, '')
+      .replaceAll(/[\s\uFEFF\u00A0\u200B-\u200D\u{0FE0E}]+/gu, ' '),
+  ).reduce((acc, { t, v }) => t === 'text' ? acc + v : acc, '').trim();
+
+  if (sanitizedText.length < 10) { // heuristics
+    return;
+  }
+
+  const [topResult] = lande(
+    sanitizedText,
+  );
+  if (topResult) {
+    const [iso6393, confidence] = topResult;
+    const locale = new Intl.Locale(iso6393);
+
+    if (confidence >= minConfidence && ISO6391.validate(locale.language)) {
+      return locale.language as LanguageCode;
+    }
+  }
+  return;
+}

src/views/mastodon/accounts.ts

@@ -12,16 +12,18 @@ import { faviconCache } from '@/utils/favicon.ts';
 import { nostrDate, nostrNow } from '@/utils.ts';
 import { renderEmojis } from '@/views/mastodon/emojis.ts';
 
-interface ToAccountOpts {
+type ToAccountOpts = {
-  withSource?: boolean;
+  withSource: true;
-}
+  settingsStore: Record<string, unknown> | undefined;
+} | {
+  withSource?: false;
+};
 
 async function renderAccount(
   event: Omit<DittoEvent, 'id' | 'sig'>,
   opts: ToAccountOpts = {},
   signal = AbortSignal.timeout(3000),
 ): Promise<MastodonAccount> {
-  const { withSource = false } = opts;
   const { pubkey } = event;
 
   const names = getTagSet(event.user?.tags ?? [], 'n');
@@ -76,7 +78,7 @@ async function renderAccount(
     locked: false,
     note: about ? escape(about) : '',
     roles: [],
-    source: withSource
+    source: opts.withSource
       ? {
         fields: [],
         language: '',
@@ -88,7 +90,7 @@ async function renderAccount(
           nip05,
         },
         ditto: {
-          captcha_solved: false,
+          captcha_solved: names.has('captcha_solved'),
         },
       }
       : undefined,
@@ -107,7 +109,7 @@ async function renderAccount(
       is_moderator: names.has('admin') || names.has('moderator'),
       is_suggested: names.has('suggested'),
       is_local: parsed05?.domain === Conf.url.host,
-      settings_store: undefined as unknown,
+      settings_store: opts.withSource ? opts.settingsStore : undefined,
       tags: [...getTagSet(event.user?.tags ?? [], 't')],
       favicon: favicon?.toString(),
     },

src/views/mastodon/statuses.ts

@@ -113,7 +113,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
     sensitive: !!cw,
     spoiler_text: (cw ? cw[1] : subject?.[1]) || '',
     visibility: 'public',
-    language: event.tags.find((tag) => tag[0] === 'l' && tag[2] === 'ISO-639-1')?.[1] || null,
+    language: event.language ?? null,
     replies_count: event.event_stats?.replies_count ?? 0,
     reblogs_count: event.event_stats?.reposts_count ?? 0,
     favourites_count: event.event_stats?.reactions['+'] ?? 0,

src/workers/fetch.ts

@@ -5,7 +5,7 @@ import './handlers/abortsignal.ts';
 
 import { fetchResponsesCounter } from '@/metrics.ts';
 
-const worker = new Worker(new URL('./fetch.worker.ts', import.meta.url), { type: 'module' });
+const worker = new Worker(new URL('./fetch.worker.ts', import.meta.url), { type: 'module', name: 'fetchWorker' });
 const client = Comlink.wrap<typeof FetchWorker>(worker);
 
 // Wait for the worker to be ready before we start using it.

src/workers/policy.ts

@@ -20,14 +20,17 @@ class PolicyWorker implements NPolicy {
         new URL('./policy.worker.ts', import.meta.url),
         {
           type: 'module',
-          deno: {
+          name: 'PolicyWorker',
-            permissions: {
+          // FIXME: Disabled until Deno 2.0 adds support for `import` permission here.
-              read: [Conf.denoDir, Conf.policy, Conf.dataDir],
+          // https://github.com/denoland/deno/issues/26074
-              write: [Conf.dataDir],
+          // deno: {
-              net: 'inherit',
+          //   permissions: {
-              env: false,
+          //     read: [Conf.denoDir, Conf.policy, Conf.dataDir],
-            },
+          //     write: [Conf.dataDir],
-          },
+          //     net: 'inherit',
+          //     env: false,
+          //   },
+          // },
         },
       ),
     );

src/workers/policy.worker.ts

@@ -37,7 +37,7 @@ export class CustomPolicy implements NPolicy {
     const store = new EventsDB({
       kysely,
       pubkey,
-      timeout: 1_000,
+      timeout: 5_000,
     });
 
     this.policy = new Policy({ store, pubkey });

src/workers/verify.ts

@@ -4,7 +4,7 @@ import * as Comlink from 'comlink';
 import type { VerifyWorker } from './verify.worker.ts';
 
 const worker = Comlink.wrap<typeof VerifyWorker>(
-  new Worker(new URL('./verify.worker.ts', import.meta.url), { type: 'module' }),
+  new Worker(new URL('./verify.worker.ts', import.meta.url), { type: 'module', name: 'verifyEventWorker' }),
 );
 
 function verifyEventWorker(event: NostrEvent): Promise<boolean> {