Merge remote-tracking branch 'origin/main' into rm-pubsub

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-image: denoland/deno:2.1.10
+image: denoland/deno:2.2.0
 
 default:
   interruptible: true

.tool-versions

@@ -1 +1 @@
-deno 2.1.10
+deno 2.2.0

Dockerfile

@@ -1,4 +1,4 @@
-FROM denoland/deno:2.1.10
+FROM denoland/deno:2.2.0
 ENV PORT 5000
 
 WORKDIR /app

packages/conf/DittoConf.ts

@@ -1,4 +1,3 @@
-import Module from 'node:module';
 import os from 'node:os';
 import path from 'node:path';
 
@@ -354,7 +353,7 @@ export class DittoConf {
 
   /** Absolute path to the data directory used by Ditto. */
   get dataDir(): string {
-    return this.env.get('DITTO_DATA_DIR') || path.join(cwd(), 'data');
+    return this.env.get('DITTO_DATA_DIR') || path.join(Deno.cwd(), 'data');
   }
 
   /** Absolute path of the Deno directory. */
@@ -465,12 +464,3 @@ export class DittoConf {
     return Number(this.env.get('STREAK_WINDOW') || 129600);
   }
 }
-
-/**
- * HACK: get cwd without read permissions.
- * https://github.com/denoland/deno/issues/27080#issuecomment-2504150155
- */
-function cwd() {
-  // @ts-ignore Internal method, but it does exist.
-  return Module._nodeModulePaths('a')[0].slice(0, -15);
-}

packages/ditto/controllers/api/accounts.ts

@@ -115,6 +115,7 @@ const accountSearchQuerySchema = z.object({
 });
 
 const accountSearchController: AppController = async (c) => {
+  const { store } = c.var;
   const { signal } = c.req.raw;
   const { limit } = c.get('pagination');
 
@@ -128,7 +129,6 @@ const accountSearchController: AppController = async (c) => {
   }
 
   const query = decodeURIComponent(result.data.q);
-  const store = await Storages.search();
 
   const lookup = extractIdentifier(query);
   const event = await lookupAccount(lookup ?? query);

packages/ditto/controllers/api/cashu.test.ts

@@ -1,12 +1,13 @@
 import { confMw } from '@ditto/api/middleware';
 import { Env as HonoEnv, Hono } from '@hono/hono';
 import { NostrSigner, NSecSigner, NStore } from '@nostrify/nostrify';
+import { genEvent } from '@nostrify/nostrify/test';
 import { generateSecretKey, getPublicKey } from 'nostr-tools';
 import { bytesToString, stringToBytes } from '@scure/base';
 import { stub } from '@std/testing/mock';
 import { assertEquals, assertExists, assertObjectMatch } from '@std/assert';
 
-import { createTestDB, genEvent } from '@/test.ts';
+import { createTestDB } from '@/test.ts';
 
 import cashuApp from '@/controllers/api/cashu.ts';
 import { walletSchema } from '@/schema.ts';

packages/ditto/controllers/api/search.ts

@@ -94,7 +94,7 @@ async function searchEvents(
     return Promise.resolve([]);
   }
 
-  const store = await Storages.search();
+  const store = await Storages.db();
 
   const filter: NostrFilter = {
     kinds: typeToKinds(type),
@@ -150,7 +150,7 @@ function typeToKinds(type: SearchQuery['type']): number[] {
 /** Resolve a searched value into an event, if applicable. */
 async function lookupEvent(query: SearchQuery, signal: AbortSignal): Promise<NostrEvent | undefined> {
   const filters = await getLookupFilters(query, signal);
-  const store = await Storages.search();
+  const store = await Storages.db();
 
   return store.query(filters, { limit: 1, signal })
     .then((events) => hydrateEvents({ events, store, signal }))

packages/ditto/filter.test.ts

@@ -1,46 +0,0 @@
-import { assertEquals } from '@std/assert';
-
-import event0 from '~/fixtures/events/event-0.json' with { type: 'json' };
-import event1 from '~/fixtures/events/event-1.json' with { type: 'json' };
-
-import { eventToMicroFilter, getFilterId, getFilterLimit, getMicroFilters, isMicrofilter } from './filter.ts';
-
-Deno.test('getMicroFilters', () => {
-  const event = event0;
-  const microfilters = getMicroFilters(event);
-  assertEquals(microfilters.length, 2);
-  assertEquals(microfilters[0], { authors: [event.pubkey], kinds: [0] });
-  assertEquals(microfilters[1], { ids: [event.id] });
-});
-
-Deno.test('eventToMicroFilter', () => {
-  assertEquals(eventToMicroFilter(event0), { authors: [event0.pubkey], kinds: [0] });
-  assertEquals(eventToMicroFilter(event1), { ids: [event1.id] });
-});
-
-Deno.test('isMicrofilter', () => {
-  assertEquals(isMicrofilter({ ids: [event0.id] }), true);
-  assertEquals(isMicrofilter({ authors: [event0.pubkey], kinds: [0] }), true);
-  assertEquals(isMicrofilter({ ids: [event0.id], authors: [event0.pubkey], kinds: [0] }), false);
-});
-
-Deno.test('getFilterId', () => {
-  assertEquals(
-    getFilterId({ ids: [event0.id] }),
-    '{"ids":["63d38c9b483d2d98a46382eadefd272e0e4bdb106a5b6eddb400c4e76f693d35"]}',
-  );
-  assertEquals(
-    getFilterId({ authors: [event0.pubkey], kinds: [0] }),
-    '{"authors":["79c2cae114ea28a981e7559b4fe7854a473521a8d22a66bbab9fa248eb820ff6"],"kinds":[0]}',
-  );
-});
-
-Deno.test('getFilterLimit', () => {
-  assertEquals(getFilterLimit({ ids: [event0.id] }), 1);
-  assertEquals(getFilterLimit({ ids: [event0.id], limit: 2 }), 1);
-  assertEquals(getFilterLimit({ ids: [event0.id], limit: 0 }), 0);
-  assertEquals(getFilterLimit({ ids: [event0.id], limit: -1 }), 0);
-  assertEquals(getFilterLimit({ kinds: [0], authors: [event0.pubkey] }), 1);
-  assertEquals(getFilterLimit({ kinds: [1], authors: [event0.pubkey] }), Infinity);
-  assertEquals(getFilterLimit({}), Infinity);
-});

packages/ditto/filter.ts

@@ -1,97 +0,0 @@
-import { NKinds, NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify';
-import stringifyStable from 'fast-stable-stringify';
-import { z } from 'zod';
-
-/** Microfilter to get one specific event by ID. */
-type IdMicrofilter = { ids: [NostrEvent['id']] };
-/** Microfilter to get an author. */
-type AuthorMicrofilter = { kinds: [0]; authors: [NostrEvent['pubkey']] };
-/** Filter to get one specific event. */
-type MicroFilter = IdMicrofilter | AuthorMicrofilter;
-
-/** Get deterministic ID for a microfilter. */
-function getFilterId(filter: MicroFilter): string {
-  if ('ids' in filter) {
-    return stringifyStable({ ids: [filter.ids[0]] });
-  } else {
-    return stringifyStable({
-      kinds: [filter.kinds[0]],
-      authors: [filter.authors[0]],
-    });
-  }
-}
-
-/** Get a microfilter from a Nostr event. */
-function eventToMicroFilter(event: NostrEvent): MicroFilter {
-  const [microfilter] = getMicroFilters(event);
-  return microfilter;
-}
-
-/** Get all the microfilters for an event, in order of priority. */
-function getMicroFilters(event: NostrEvent): MicroFilter[] {
-  const microfilters: MicroFilter[] = [];
-  if (event.kind === 0) {
-    microfilters.push({ kinds: [0], authors: [event.pubkey] });
-  }
-  microfilters.push({ ids: [event.id] });
-  return microfilters;
-}
-
-/** Microfilter schema. */
-const microFilterSchema = z.union([
-  z.object({ ids: z.tuple([n.id()]) }).strict(),
-  z.object({ kinds: z.tuple([z.literal(0)]), authors: z.tuple([n.id()]) }).strict(),
-]);
-
-/** Checks whether the filter is a microfilter. */
-function isMicrofilter(filter: NostrFilter): filter is MicroFilter {
-  return microFilterSchema.safeParse(filter).success;
-}
-
-/** Returns true if the filter could potentially return any stored events at all. */
-function canFilter(filter: NostrFilter): boolean {
-  return getFilterLimit(filter) > 0;
-}
-
-/** Normalize the `limit` of each filter, and remove filters that can't produce any events. */
-function normalizeFilters<F extends NostrFilter>(filters: F[]): F[] {
-  return filters.reduce<F[]>((acc, filter) => {
-    const limit = getFilterLimit(filter);
-    if (limit > 0) {
-      acc.push(limit === Infinity ? filter : { ...filter, limit });
-    }
-    return acc;
-  }, []);
-}
-
-/** Calculate the intrinsic limit of a filter. This function may return `Infinity`. */
-function getFilterLimit(filter: NostrFilter): number {
-  if (filter.ids && !filter.ids.length) return 0;
-  if (filter.kinds && !filter.kinds.length) return 0;
-  if (filter.authors && !filter.authors.length) return 0;
-
-  for (const [key, value] of Object.entries(filter)) {
-    if (key[0] === '#' && Array.isArray(value) && !value.length) return 0;
-  }
-
-  return Math.min(
-    Math.max(0, filter.limit ?? Infinity),
-    filter.ids?.length ?? Infinity,
-    filter.authors?.length && filter.kinds?.every((kind) => NKinds.replaceable(kind))
-      ? filter.authors.length * filter.kinds.length
-      : Infinity,
-  );
-}
-
-export {
-  type AuthorMicrofilter,
-  canFilter,
-  eventToMicroFilter,
-  getFilterId,
-  getFilterLimit,
-  getMicroFilters,
-  type IdMicrofilter,
-  isMicrofilter,
-  type MicroFilter,
-  normalizeFilters,
-};

packages/ditto/storages.ts

@@ -7,7 +7,6 @@ import { Conf } from '@/config.ts';
 import { wsUrlSchema } from '@/schema.ts';
 import { AdminStore } from '@/storages/AdminStore.ts';
 import { DittoPgStore } from '@/storages/DittoPgStore.ts';
-import { SearchStore } from '@/storages/search-store.ts';
 import { InternalRelay } from '@/storages/InternalRelay.ts';
 import { NPool, NRelay1 } from '@nostrify/nostrify';
 import { getRelays } from '@/utils/outbox.ts';
@@ -19,7 +18,6 @@ export class Storages {
   private static _admin: Promise<AdminStore> | undefined;
   private static _client: Promise<NPool<NRelay1>> | undefined;
   private static _pubsub: Promise<InternalRelay> | undefined;
-  private static _search: Promise<SearchStore> | undefined;
 
   public static async database(): Promise<DittoDatabase> {
     if (!this._database) {
@@ -124,17 +122,4 @@ export class Storages {
     }
     return this._client;
   }
-
-  /** Storage to use for remote search. */
-  public static async search(): Promise<SearchStore> {
-    if (!this._search) {
-      this._search = Promise.resolve(
-        new SearchStore({
-          relay: Conf.searchRelay,
-          fallback: await this.db(),
-        }),
-      );
-    }
-    return this._search;
-  }
 }

packages/ditto/storages/DittoAPIStore.ts

@@ -0,0 +1,38 @@
+import type {
+  NostrEvent,
+  NostrFilter,
+  NostrRelayCLOSED,
+  NostrRelayCOUNT,
+  NostrRelayEOSE,
+  NostrRelayEVENT,
+  NRelay,
+} from '@nostrify/nostrify';
+
+export class DittoAPIStore implements NRelay {
+  req(
+    filters: NostrFilter[],
+    opts?: { signal?: AbortSignal },
+  ): AsyncIterable<NostrRelayEVENT | NostrRelayEOSE | NostrRelayCLOSED> {
+    throw new Error('Method not implemented.');
+  }
+
+  close(): Promise<void> {
+    throw new Error('Method not implemented.');
+  }
+
+  event(event: NostrEvent, opts?: { signal?: AbortSignal }): Promise<void> {
+    throw new Error('Method not implemented.');
+  }
+
+  query(filters: NostrFilter[], opts?: { signal?: AbortSignal }): Promise<NostrEvent[]> {
+    throw new Error('Method not implemented.');
+  }
+
+  count(filters: NostrFilter[], opts?: { signal?: AbortSignal }): Promise<NostrRelayCOUNT[2]> {
+    throw new Error('Method not implemented.');
+  }
+
+  remove(filters: NostrFilter[], opts?: { signal?: AbortSignal }): Promise<void> {
+    throw new Error('Method not implemented.');
+  }
+}

packages/ditto/storages/DittoPgStore.test.ts

@@ -1,8 +1,9 @@
 import { assertEquals, assertRejects } from '@std/assert';
+import { genEvent } from '@nostrify/nostrify/test';
 import { generateSecretKey } from 'nostr-tools';
 
 import { RelayError } from '@/RelayError.ts';
-import { eventFixture, genEvent } from '@/test.ts';
+import { eventFixture } from '@/test.ts';
 import { Conf } from '@/config.ts';
 import { DittoPgStore } from '@/storages/DittoPgStore.ts';
 import { createTestDB } from '@/test.ts';

packages/ditto/storages/hydrate.bench.ts

@@ -1,5 +1,6 @@
+import { jsonlEvents } from '@nostrify/nostrify/test';
+
 import { assembleEvents } from '@/storages/hydrate.ts';
-import { jsonlEvents } from '@/test.ts';
 
 const testEvents = await jsonlEvents('fixtures/hydrated.jsonl');
 const testStats = JSON.parse(await Deno.readTextFile('fixtures/stats.json'));

packages/ditto/storages/search-store.ts

@@ -1,60 +0,0 @@
-import { NostrEvent, NostrFilter, NRelay1, NStore } from '@nostrify/nostrify';
-import { logi } from '@soapbox/logi';
-import { JsonValue } from '@std/json';
-
-import { normalizeFilters } from '@/filter.ts';
-import { type DittoEvent } from '@/interfaces/DittoEvent.ts';
-import { hydrateEvents } from '@/storages/hydrate.ts';
-import { abortError } from '@/utils/abort.ts';
-
-interface SearchStoreOpts {
-  relay: string | undefined;
-  fallback: NStore;
-  hydrator?: NStore;
-}
-
-class SearchStore implements NStore {
-  #fallback: NStore;
-  #hydrator: NStore;
-  #relay: NRelay1 | undefined;
-
-  constructor(opts: SearchStoreOpts) {
-    this.#fallback = opts.fallback;
-    this.#hydrator = opts.hydrator ?? this;
-
-    if (opts.relay) {
-      this.#relay = new NRelay1(opts.relay);
-    }
-  }
-
-  event(_event: NostrEvent, _opts?: { signal?: AbortSignal }): Promise<void> {
-    return Promise.reject(new Error('EVENT not implemented.'));
-  }
-
-  async query(filters: NostrFilter[], opts?: { signal?: AbortSignal; limit?: number }): Promise<DittoEvent[]> {
-    filters = normalizeFilters(filters);
-
-    if (opts?.signal?.aborted) return Promise.reject(abortError());
-    if (!filters.length) return Promise.resolve([]);
-
-    logi({ level: 'debug', ns: 'ditto.req', source: 'search', filters: filters as JsonValue });
-    const query = filters[0]?.search;
-
-    if (this.#relay && this.#relay.socket.readyState === WebSocket.OPEN) {
-      logi({ level: 'debug', ns: 'ditto.search', query, source: 'relay', relay: this.#relay.socket.url });
-
-      const events = await this.#relay.query(filters, opts);
-
-      return hydrateEvents({
-        events,
-        store: this.#hydrator,
-        signal: opts?.signal,
-      });
-    } else {
-      logi({ level: 'debug', ns: 'ditto.search', query, source: 'db' });
-      return this.#fallback.query(filters, opts);
-    }
-  }
-}
-
-export { SearchStore };

packages/ditto/test.ts

@@ -1,10 +1,8 @@
 import { DittoDB } from '@ditto/db';
 import { NostrEvent } from '@nostrify/nostrify';
-import { finalizeEvent, generateSecretKey } from 'nostr-tools';
 
 import { Conf } from '@/config.ts';
 import { DittoPgStore } from '@/storages/DittoPgStore.ts';
-import { purifyEvent } from '@/utils/purify.ts';
 import { sql } from 'kysely';
 
 /** Import an event fixture by name in tests. */
@@ -13,25 +11,6 @@ export async function eventFixture(name: string): Promise<NostrEvent> {
   return structuredClone(result.default);
 }
 
-/** Import a JSONL fixture by name in tests. */
-export async function jsonlEvents(path: string): Promise<NostrEvent[]> {
-  const data = await Deno.readTextFile(path);
-  return data.split('\n').map((line) => JSON.parse(line));
-}
-
-/** Generate an event for use in tests. */
-export function genEvent(t: Partial<NostrEvent> = {}, sk: Uint8Array = generateSecretKey()): NostrEvent {
-  const event = finalizeEvent({
-    kind: 255,
-    created_at: 0,
-    content: '',
-    tags: [],
-    ...t,
-  }, sk);
-
-  return purifyEvent(event);
-}
-
 /** Create a database for testing. It uses `DATABASE_URL`, or creates an in-memory database by default. */
 export async function createTestDB(opts?: { pure?: boolean }) {
   const db = DittoDB.create(Conf.databaseUrl, { poolSize: 1 });

packages/ditto/trends.test.ts

@@ -1,8 +1,9 @@
 import { assertEquals } from '@std/assert';
+import { genEvent } from '@nostrify/nostrify/test';
 import { generateSecretKey, NostrEvent } from 'nostr-tools';
 
 import { getTrendingTagValues } from '@/trends.ts';
-import { createTestDB, genEvent } from '@/test.ts';
+import { createTestDB } from '@/test.ts';
 
 Deno.test("getTrendingTagValues(): 'e' tag and WITHOUT language parameter", async () => {
   await using db = await createTestDB();

packages/ditto/utils/stats.test.ts

@@ -1,7 +1,8 @@
+import { genEvent } from '@nostrify/nostrify/test';
 import { assertEquals } from '@std/assert';
 import { generateSecretKey, getPublicKey } from 'nostr-tools';
 
-import { createTestDB, genEvent } from '@/test.ts';
+import { createTestDB } from '@/test.ts';
 import { countAuthorStats, getAuthorStats, getEventStats, getFollowDiff, updateStats } from '@/utils/stats.ts';
 
 Deno.test('updateStats with kind 1 increments notes count', async () => {