Merge remote-tracking branch 'origin/main' into rewrite

packages/ditto/app.ts

@@ -139,7 +139,11 @@ import { metricsController } from '@/controllers/metrics.ts';
 import { manifestController } from '@/controllers/manifest.ts';
 import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts';
 import { nostrController } from '@/controllers/well-known/nostr.ts';
+<<<<<<< HEAD
 import { requireProof, requireRole } from '@/middleware/auth98Middleware.ts';
+=======
+import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts';
+>>>>>>> origin/main
 import { cacheControlMiddleware } from '@/middleware/cacheControlMiddleware.ts';
 import { cspMiddleware } from '@/middleware/cspMiddleware.ts';
 import { metricsMiddleware } from '@/middleware/metricsMiddleware.ts';

packages/ditto/controllers/api/streaming.ts

@@ -1,4 +1,5 @@
 import { DittoTables } from '@ditto/db';
+import { MuteListPolicy } from '@ditto/policies';
 import {
   streamingClientMessagesCounter,
   streamingConnectionsGauge,
@@ -11,7 +12,6 @@ import { Kysely } from 'kysely';
 import { z } from 'zod';
 
 import { type AppController } from '@/app.ts';
-import { MuteListPolicy } from '../../../policies/MuteListPolicy.ts';
 import { getFeedPubkeys } from '@/queries.ts';
 import { AdminStore } from '@/storages/AdminStore.ts';
 import { hydrateEvents } from '@/storages/hydrate.ts';

packages/ditto/controllers/nostr/relay.ts

@@ -1,5 +1,6 @@
 import { type DittoConf } from '@ditto/conf';
 import { relayConnectionsGauge, relayEventsCounter, relayMessagesCounter } from '@ditto/metrics';
+import { MemoryRateLimiter, MultiRateLimiter, type RateLimiter } from '@ditto/ratelimiter';
 import { logi } from '@soapbox/logi';
 import { JsonValue } from '@std/json';
 import {
@@ -19,9 +20,6 @@ import { relayInfoController } from '@/controllers/nostr/relay-info.ts';
 import { RelayError } from '@/RelayError.ts';
 import { errorJson } from '../../../utils/log.ts';
 import { purifyEvent } from '../../../utils/purify.ts';
-import { MemoryRateLimiter } from '../../../utils/ratelimiter/MemoryRateLimiter.ts';
-import { MultiRateLimiter } from '../../../utils/ratelimiter/MultiRateLimiter.ts';
-import { RateLimiter } from '../../../utils/ratelimiter/types.ts';
 import { Time } from '../../../utils/time.ts';
 import { DittoPipeline } from '@/DittoPipeline.ts';
 import { EventsDB } from '@/storages/EventsDB.ts';

packages/ditto/middleware/translatorMiddleware.ts

@@ -1,8 +1,7 @@
+import { DeepLTranslator, LibreTranslateTranslator } from '@ditto/translators';
 import { safeFetch } from '@soapbox/safe-fetch';
 
 import { AppMiddleware } from '@/app.ts';
-import { DeepLTranslator } from '@/translators/DeepLTranslator.ts';
-import { LibreTranslateTranslator } from '@/translators/LibreTranslateTranslator.ts';
 
 /** Set the translator used for translating posts. */
 export const translatorMiddleware: AppMiddleware = async (c, next) => {

packages/ditto/middleware/uploaderMiddleware.ts

@@ -1,10 +1,8 @@
+import { DenoUploader, IPFSUploader, S3Uploader } from '@ditto/uploaders';
 import { BlossomUploader, NostrBuildUploader } from '@nostrify/nostrify/uploaders';
 import { safeFetch } from '@soapbox/safe-fetch';
 
 import { AppMiddleware } from '@/app.ts';
-import { DenoUploader } from '../../uploaders/DenoUploader.ts';
-import { IPFSUploader } from '../../uploaders/IPFSUploader.ts';
-import { S3Uploader } from '../../uploaders/S3Uploader.ts';
 
 /** Set an uploader for the user. */
 export const uploaderMiddleware: AppMiddleware = async (c, next) => {

packages/ditto/test.ts

@@ -1,7 +1,5 @@
 import { DittoConf } from '@ditto/conf';
 import { DittoDB } from '@ditto/db';
-import ISO6391, { LanguageCode } from 'iso-639-1';
-import lande from 'lande';
 import { NostrEvent } from '@nostrify/nostrify';
 import { generateSecretKey, nip19 } from 'nostr-tools';
 
@@ -65,15 +63,3 @@ export function testConf(): DittoConf {
 export function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
-
-export function getLanguage(text: string): LanguageCode | undefined {
-  const [topResult] = lande(text);
-  if (topResult) {
-    const [iso6393] = topResult;
-    const locale = new Intl.Locale(iso6393);
-    if (ISO6391.validate(locale.language)) {
-      return locale.language;
-    }
-  }
-  return;
-}

packages/lang/deno.json

@@ -1,5 +1,6 @@
 {
   "name": "@ditto/lang",
+  "version": "1.1.0",
   "exports": {
     ".": "./language.ts"
   }

packages/lang/language.test.ts

@@ -1,6 +1,7 @@
-import { detectLanguage } from './language.ts';
 import { assertEquals } from '@std/assert';
 
+import { detectLanguage } from './language.ts';
+
 Deno.test('Detect English language', () => {
   assertEquals(detectLanguage(``, 0.90), undefined);
   assertEquals(detectLanguage(`Good morning my fellow friends`, 0.90), 'en');

packages/policies/MuteListPolicy.test.ts

@@ -16,14 +16,12 @@ Deno.test('block event: muted user cannot post', async () => {
   const blockEventCopy = structuredClone(blockEvent);
   const event1authorUserMeCopy = structuredClone(event1authorUserMe);
 
-  const db = new MockRelay();
+  const relay = new MockRelay();
+  const policy = new MuteListPolicy(userBlack.pubkey, relay);
 
-  const store = new UserStore(userBlackCopy.pubkey, db);
-  const policy = new MuteListPolicy(userBlack.pubkey, db);
-
-  await store.event(blockEventCopy);
-  await store.event(userBlackCopy);
-  await store.event(userMeCopy);
+  await relay.event(blockEventCopy);
+  await relay.event(userBlackCopy);
+  await relay.event(userMeCopy);
 
   const ok = await policy.call(event1authorUserMeCopy);
 
@@ -35,13 +33,11 @@ Deno.test('allow event: user is NOT muted because there is no muted event', asyn
   const userMeCopy = structuredClone(userMe);
   const event1authorUserMeCopy = structuredClone(event1authorUserMe);
 
-  const db = new MockRelay();
+  const relay = new MockRelay();
+  const policy = new MuteListPolicy(userBlack.pubkey, relay);
 
-  const store = new UserStore(userBlackCopy.pubkey, db);
-  const policy = new MuteListPolicy(userBlack.pubkey, db);
-
-  await store.event(userBlackCopy);
-  await store.event(userMeCopy);
+  await relay.event(userBlackCopy);
+  await relay.event(userMeCopy);
 
   const ok = await policy.call(event1authorUserMeCopy);
 
@@ -55,16 +51,15 @@ Deno.test('allow event: user is NOT muted because he is not in mute event', asyn
   const blockEventCopy = structuredClone(blockEvent);
   const event1copy = structuredClone(event1);
 
-  const db = new MockRelay();
+  const relay = new MockRelay();
 
-  const store = new UserStore(userBlackCopy.pubkey, db);
-  const policy = new MuteListPolicy(userBlack.pubkey, db);
+  const policy = new MuteListPolicy(userBlack.pubkey, relay);
 
-  await store.event(userBlackCopy);
-  await store.event(blockEventCopy);
-  await store.event(userMeCopy);
-  await store.event(event1copy);
-  await store.event(event1authorUserMeCopy);
+  await relay.event(userBlackCopy);
+  await relay.event(blockEventCopy);
+  await relay.event(userMeCopy);
+  await relay.event(event1copy);
+  await relay.event(event1authorUserMeCopy);
 
   const ok = await policy.call(event1copy);
 

packages/policies/MuteListPolicy.ts

@@ -1,13 +1,18 @@
-import { NostrEvent, NostrRelayOK, NPolicy, NStore } from '@nostrify/nostrify';
-
-import { getTagSet } from '@ditto/utils/tags';
+import type { NostrEvent, NostrRelayOK, NPolicy, NStore } from '@nostrify/nostrify';
 
 export class MuteListPolicy implements NPolicy {
   constructor(private pubkey: string, private store: NStore) {}
 
   async call(event: NostrEvent): Promise<NostrRelayOK> {
+    const pubkeys = new Set<string>();
+
     const [muteList] = await this.store.query([{ authors: [this.pubkey], kinds: [10000], limit: 1 }]);
-    const pubkeys = getTagSet(muteList?.tags ?? [], 'p');
+
+    for (const [name, value] of muteList?.tags ?? []) {
+      if (name === 'p') {
+        pubkeys.add(value);
+      }
+    }
 
     if (pubkeys.has(event.pubkey)) {
       return ['OK', event.id, false, 'blocked: Your account has been deactivated.'];

packages/policies/deno.json

@@ -1,5 +1,6 @@
 {
   "name": "@ditto/policies",
+  "version": "1.1.0",
   "exports": {
     ".": "./mod.ts"
   }

packages/ratelimiter/deno.json

@@ -1,5 +1,6 @@
 {
   "name": "@ditto/ratelimiter",
+  "version": "1.1.0",
   "exports": {
     ".": "./mod.ts"
   }

packages/translators/DeepLTranslator.test.ts

@@ -1,8 +1,8 @@
 import { DittoConf } from '@ditto/conf';
+import { detectLanguage } from '@ditto/lang';
 import { assert, assertEquals } from '@std/assert';
 
-import { DeepLTranslator } from '@/translators/DeepLTranslator.ts';
-import { getLanguage } from '@/test.ts';
+import { DeepLTranslator } from './DeepLTranslator.ts';
 
 const {
   deeplBaseUrl: baseUrl,
@@ -28,9 +28,9 @@ Deno.test('DeepL translation with source language omitted', {
   );
 
   assertEquals(data.source_lang, 'pt');
-  assertEquals(getLanguage(data.results[0]), 'en');
-  assertEquals(getLanguage(data.results[1]), 'en');
-  assertEquals(getLanguage(data.results[2]), 'en');
+  assertEquals(detectLanguage(data.results[0], 0), 'en');
+  assertEquals(detectLanguage(data.results[1], 0), 'en');
+  assertEquals(detectLanguage(data.results[2], 0), 'en');
 });
 
 Deno.test('DeepL translation with source language set', {
@@ -49,9 +49,9 @@ Deno.test('DeepL translation with source language set', {
   );
 
   assertEquals(data.source_lang, 'pt');
-  assertEquals(getLanguage(data.results[0]), 'en');
-  assertEquals(getLanguage(data.results[1]), 'en');
-  assertEquals(getLanguage(data.results[2]), 'en');
+  assertEquals(detectLanguage(data.results[0], 0), 'en');
+  assertEquals(detectLanguage(data.results[1], 0), 'en');
+  assertEquals(detectLanguage(data.results[2], 0), 'en');
 });
 
 Deno.test("DeepL translation doesn't alter Nostr URIs", {

packages/translators/DeepLTranslator.ts

@@ -1,8 +1,9 @@
-import { LanguageCode } from 'iso-639-1';
 import { z } from 'zod';
 
-import { DittoTranslator } from '@/interfaces/DittoTranslator.ts';
-import { languageSchema } from '@/schema.ts';
+import { languageSchema } from './schema.ts';
+
+import type { LanguageCode } from 'iso-639-1';
+import type { DittoTranslator } from './DittoTranslator.ts';
 
 interface DeepLTranslatorOpts {
   /** DeepL base URL to use. Default: 'https://api.deepl.com' */
@@ -31,7 +32,7 @@ export class DeepLTranslator implements DittoTranslator {
     source: LanguageCode | undefined,
     dest: LanguageCode,
     opts?: { signal?: AbortSignal },
-  ) {
+  ): Promise<{ results: string[]; source_lang: LanguageCode }> {
     const { translations } = await this.translateMany(texts, source, dest, opts);
 
     return {

packages/translators/LibreTranslateTranslator.test.ts

@@ -1,8 +1,8 @@
 import { DittoConf } from '@ditto/conf';
+import { detectLanguage } from '@ditto/lang';
 import { assertEquals } from '@std/assert';
 
-import { LibreTranslateTranslator } from '@/translators/LibreTranslateTranslator.ts';
-import { getLanguage } from '@/test.ts';
+import { LibreTranslateTranslator } from './LibreTranslateTranslator.ts';
 
 const {
   libretranslateBaseUrl: baseUrl,
@@ -28,9 +28,9 @@ Deno.test('LibreTranslate translation with source language omitted', {
   );
 
   assertEquals(data.source_lang, 'pt');
-  assertEquals(getLanguage(data.results[0]), 'ca');
-  assertEquals(getLanguage(data.results[1]), 'ca');
-  assertEquals(getLanguage(data.results[2]), 'ca');
+  assertEquals(detectLanguage(data.results[0], 0), 'ca');
+  assertEquals(detectLanguage(data.results[1], 0), 'ca');
+  assertEquals(detectLanguage(data.results[2], 0), 'ca');
 });
 
 Deno.test('LibreTranslate translation with source language set', {
@@ -49,7 +49,7 @@ Deno.test('LibreTranslate translation with source language set', {
   );
 
   assertEquals(data.source_lang, 'pt');
-  assertEquals(getLanguage(data.results[0]), 'ca');
-  assertEquals(getLanguage(data.results[1]), 'ca');
-  assertEquals(getLanguage(data.results[2]), 'ca');
+  assertEquals(detectLanguage(data.results[0], 0), 'ca');
+  assertEquals(detectLanguage(data.results[1], 0), 'ca');
+  assertEquals(detectLanguage(data.results[2], 0), 'ca');
 });

packages/translators/LibreTranslateTranslator.ts

@@ -1,8 +1,9 @@
-import { LanguageCode } from 'iso-639-1';
 import { z } from 'zod';
 
-import { DittoTranslator } from '@/interfaces/DittoTranslator.ts';
-import { languageSchema } from '@/schema.ts';
+import { languageSchema } from './schema.ts';
+
+import type { LanguageCode } from 'iso-639-1';
+import type { DittoTranslator } from './DittoTranslator.ts';
 
 interface LibreTranslateTranslatorOpts {
   /** Libretranslate endpoint to use. Default: 'https://libretranslate.com' */
@@ -31,7 +32,7 @@ export class LibreTranslateTranslator implements DittoTranslator {
     source: LanguageCode | undefined,
     dest: LanguageCode,
     opts?: { signal?: AbortSignal },
-  ) {
+  ): Promise<{ results: string[]; source_lang: LanguageCode }> {
     const translations = await Promise.all(
       texts.map((text) => this.translateOne(text, source, dest, 'html', { signal: opts?.signal })),
     );

packages/translators/mod.ts

@@ -1,5 +1,4 @@
 export { DeepLTranslator } from './DeepLTranslator.ts';
 export { LibreTranslateTranslator } from './LibreTranslateTranslator.ts';
 
-export type { LanguageCode } from 'iso-639-1';
 export type { DittoTranslator } from './DittoTranslator.ts';

packages/translators/schema.ts

@@ -0,0 +1,8 @@
+import ISO6391 from 'iso-639-1';
+import z from 'zod';
+
+/** Value is a ISO-639-1 language code. */
+export const languageSchema = z.string().refine(
+  (val) => ISO6391.validate(val),
+  { message: 'Not a valid language in ISO-639-1 format' },
+);

packages/uploaders/DenoUploader.ts

@@ -1,10 +1,11 @@
 import { join } from 'node:path';
 
-import { NUploader } from '@nostrify/nostrify';
 import { crypto } from '@std/crypto';
 import { encodeHex } from '@std/encoding/hex';
 import { extensionsByType } from '@std/media-types';
 
+import type { NUploader } from '@nostrify/nostrify';
+
 export interface DenoUploaderOpts {
   baseUrl: string;
   dir: string;

packages/uploaders/IPFSUploader.ts

@@ -1,6 +1,7 @@
-import { NUploader } from '@nostrify/nostrify';
 import { z } from 'zod';
 
+import type { NUploader } from '@nostrify/nostrify';
+
 export interface IPFSUploaderOpts {
   baseUrl: string;
   apiUrl?: string;

packages/uploaders/S3Uploader.ts

@@ -1,11 +1,12 @@
 import { join } from 'node:path';
 
 import { S3Client } from '@bradenmacdonald/s3-lite-client';
-import { NUploader } from '@nostrify/nostrify';
 import { crypto } from '@std/crypto';
 import { encodeHex } from '@std/encoding/hex';
 import { extensionsByType } from '@std/media-types';
 
+import type { NUploader } from '@nostrify/nostrify';
+
 export interface S3UploaderOpts {
   endPoint: string;
   region: string;

packages/uploaders/deno.json

@@ -1,5 +1,6 @@
 {
   "name": "@ditto/uploaders",
+  "version": "1.1.0",
   "exports": {
     ".": "./mod.ts"
   }