From aabe6350a765d4988bdc04d661f49d391405f045 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 18 Feb 2025 15:08:00 -0600 Subject: [PATCH 1/3] Remove SearchStore --- packages/ditto/controllers/api/accounts.ts | 2 +- packages/ditto/controllers/api/search.ts | 4 +- packages/ditto/filter.test.ts | 46 ---------- packages/ditto/filter.ts | 97 ---------------------- packages/ditto/storages.ts | 15 ---- packages/ditto/storages/search-store.ts | 60 ------------- 6 files changed, 3 insertions(+), 221 deletions(-) delete mode 100644 packages/ditto/filter.test.ts delete mode 100644 packages/ditto/filter.ts delete mode 100644 packages/ditto/storages/search-store.ts diff --git a/packages/ditto/controllers/api/accounts.ts b/packages/ditto/controllers/api/accounts.ts index 252ddad6..8a1b9e3d 100644 --- a/packages/ditto/controllers/api/accounts.ts +++ b/packages/ditto/controllers/api/accounts.ts @@ -115,6 +115,7 @@ const accountSearchQuerySchema = z.object({ }); const accountSearchController: AppController = async (c) => { + const { store } = c.var; const { signal } = c.req.raw; const { limit } = c.get('pagination'); @@ -128,7 +129,6 @@ const accountSearchController: AppController = async (c) => { } const query = decodeURIComponent(result.data.q); - const store = await Storages.search(); const lookup = extractIdentifier(query); const event = await lookupAccount(lookup ?? query); diff --git a/packages/ditto/controllers/api/search.ts b/packages/ditto/controllers/api/search.ts index e5761f32..e890f166 100644 --- a/packages/ditto/controllers/api/search.ts +++ b/packages/ditto/controllers/api/search.ts @@ -94,7 +94,7 @@ async function searchEvents( return Promise.resolve([]); } - const store = await Storages.search(); + const store = await Storages.db(); const filter: NostrFilter = { kinds: typeToKinds(type), @@ -150,7 +150,7 @@ function typeToKinds(type: SearchQuery['type']): number[] { /** Resolve a searched value into an event, if applicable. */ async function lookupEvent(query: SearchQuery, signal: AbortSignal): Promise { const filters = await getLookupFilters(query, signal); - const store = await Storages.search(); + const store = await Storages.db(); return store.query(filters, { limit: 1, signal }) .then((events) => hydrateEvents({ events, store, signal })) diff --git a/packages/ditto/filter.test.ts b/packages/ditto/filter.test.ts deleted file mode 100644 index 9379208e..00000000 --- a/packages/ditto/filter.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { assertEquals } from '@std/assert'; - -import event0 from '~/fixtures/events/event-0.json' with { type: 'json' }; -import event1 from '~/fixtures/events/event-1.json' with { type: 'json' }; - -import { eventToMicroFilter, getFilterId, getFilterLimit, getMicroFilters, isMicrofilter } from './filter.ts'; - -Deno.test('getMicroFilters', () => { - const event = event0; - const microfilters = getMicroFilters(event); - assertEquals(microfilters.length, 2); - assertEquals(microfilters[0], { authors: [event.pubkey], kinds: [0] }); - assertEquals(microfilters[1], { ids: [event.id] }); -}); - -Deno.test('eventToMicroFilter', () => { - assertEquals(eventToMicroFilter(event0), { authors: [event0.pubkey], kinds: [0] }); - assertEquals(eventToMicroFilter(event1), { ids: [event1.id] }); -}); - -Deno.test('isMicrofilter', () => { - assertEquals(isMicrofilter({ ids: [event0.id] }), true); - assertEquals(isMicrofilter({ authors: [event0.pubkey], kinds: [0] }), true); - assertEquals(isMicrofilter({ ids: [event0.id], authors: [event0.pubkey], kinds: [0] }), false); -}); - -Deno.test('getFilterId', () => { - assertEquals( - getFilterId({ ids: [event0.id] }), - '{"ids":["63d38c9b483d2d98a46382eadefd272e0e4bdb106a5b6eddb400c4e76f693d35"]}', - ); - assertEquals( - getFilterId({ authors: [event0.pubkey], kinds: [0] }), - '{"authors":["79c2cae114ea28a981e7559b4fe7854a473521a8d22a66bbab9fa248eb820ff6"],"kinds":[0]}', - ); -}); - -Deno.test('getFilterLimit', () => { - assertEquals(getFilterLimit({ ids: [event0.id] }), 1); - assertEquals(getFilterLimit({ ids: [event0.id], limit: 2 }), 1); - assertEquals(getFilterLimit({ ids: [event0.id], limit: 0 }), 0); - assertEquals(getFilterLimit({ ids: [event0.id], limit: -1 }), 0); - assertEquals(getFilterLimit({ kinds: [0], authors: [event0.pubkey] }), 1); - assertEquals(getFilterLimit({ kinds: [1], authors: [event0.pubkey] }), Infinity); - assertEquals(getFilterLimit({}), Infinity); -}); diff --git a/packages/ditto/filter.ts b/packages/ditto/filter.ts deleted file mode 100644 index f9288c8a..00000000 --- a/packages/ditto/filter.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { NKinds, NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; -import stringifyStable from 'fast-stable-stringify'; -import { z } from 'zod'; - -/** Microfilter to get one specific event by ID. */ -type IdMicrofilter = { ids: [NostrEvent['id']] }; -/** Microfilter to get an author. */ -type AuthorMicrofilter = { kinds: [0]; authors: [NostrEvent['pubkey']] }; -/** Filter to get one specific event. */ -type MicroFilter = IdMicrofilter | AuthorMicrofilter; - -/** Get deterministic ID for a microfilter. */ -function getFilterId(filter: MicroFilter): string { - if ('ids' in filter) { - return stringifyStable({ ids: [filter.ids[0]] }); - } else { - return stringifyStable({ - kinds: [filter.kinds[0]], - authors: [filter.authors[0]], - }); - } -} - -/** Get a microfilter from a Nostr event. */ -function eventToMicroFilter(event: NostrEvent): MicroFilter { - const [microfilter] = getMicroFilters(event); - return microfilter; -} - -/** Get all the microfilters for an event, in order of priority. */ -function getMicroFilters(event: NostrEvent): MicroFilter[] { - const microfilters: MicroFilter[] = []; - if (event.kind === 0) { - microfilters.push({ kinds: [0], authors: [event.pubkey] }); - } - microfilters.push({ ids: [event.id] }); - return microfilters; -} - -/** Microfilter schema. */ -const microFilterSchema = z.union([ - z.object({ ids: z.tuple([n.id()]) }).strict(), - z.object({ kinds: z.tuple([z.literal(0)]), authors: z.tuple([n.id()]) }).strict(), -]); - -/** Checks whether the filter is a microfilter. */ -function isMicrofilter(filter: NostrFilter): filter is MicroFilter { - return microFilterSchema.safeParse(filter).success; -} - -/** Returns true if the filter could potentially return any stored events at all. */ -function canFilter(filter: NostrFilter): boolean { - return getFilterLimit(filter) > 0; -} - -/** Normalize the `limit` of each filter, and remove filters that can't produce any events. */ -function normalizeFilters(filters: F[]): F[] { - return filters.reduce((acc, filter) => { - const limit = getFilterLimit(filter); - if (limit > 0) { - acc.push(limit === Infinity ? filter : { ...filter, limit }); - } - return acc; - }, []); -} - -/** Calculate the intrinsic limit of a filter. This function may return `Infinity`. */ -function getFilterLimit(filter: NostrFilter): number { - if (filter.ids && !filter.ids.length) return 0; - if (filter.kinds && !filter.kinds.length) return 0; - if (filter.authors && !filter.authors.length) return 0; - - for (const [key, value] of Object.entries(filter)) { - if (key[0] === '#' && Array.isArray(value) && !value.length) return 0; - } - - return Math.min( - Math.max(0, filter.limit ?? Infinity), - filter.ids?.length ?? Infinity, - filter.authors?.length && filter.kinds?.every((kind) => NKinds.replaceable(kind)) - ? filter.authors.length * filter.kinds.length - : Infinity, - ); -} - -export { - type AuthorMicrofilter, - canFilter, - eventToMicroFilter, - getFilterId, - getFilterLimit, - getMicroFilters, - type IdMicrofilter, - isMicrofilter, - type MicroFilter, - normalizeFilters, -}; diff --git a/packages/ditto/storages.ts b/packages/ditto/storages.ts index be61beb6..1494dc8c 100644 --- a/packages/ditto/storages.ts +++ b/packages/ditto/storages.ts @@ -7,7 +7,6 @@ import { Conf } from '@/config.ts'; import { wsUrlSchema } from '@/schema.ts'; import { AdminStore } from '@/storages/AdminStore.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; -import { SearchStore } from '@/storages/search-store.ts'; import { InternalRelay } from '@/storages/InternalRelay.ts'; import { NPool, NRelay1 } from '@nostrify/nostrify'; import { getRelays } from '@/utils/outbox.ts'; @@ -19,7 +18,6 @@ export class Storages { private static _admin: Promise | undefined; private static _client: Promise> | undefined; private static _pubsub: Promise | undefined; - private static _search: Promise | undefined; public static async database(): Promise { if (!this._database) { @@ -124,17 +122,4 @@ export class Storages { } return this._client; } - - /** Storage to use for remote search. */ - public static async search(): Promise { - if (!this._search) { - this._search = Promise.resolve( - new SearchStore({ - relay: Conf.searchRelay, - fallback: await this.db(), - }), - ); - } - return this._search; - } } diff --git a/packages/ditto/storages/search-store.ts b/packages/ditto/storages/search-store.ts deleted file mode 100644 index 44dc1519..00000000 --- a/packages/ditto/storages/search-store.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { NostrEvent, NostrFilter, NRelay1, NStore } from '@nostrify/nostrify'; -import { logi } from '@soapbox/logi'; -import { JsonValue } from '@std/json'; - -import { normalizeFilters } from '@/filter.ts'; -import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { hydrateEvents } from '@/storages/hydrate.ts'; -import { abortError } from '@/utils/abort.ts'; - -interface SearchStoreOpts { - relay: string | undefined; - fallback: NStore; - hydrator?: NStore; -} - -class SearchStore implements NStore { - #fallback: NStore; - #hydrator: NStore; - #relay: NRelay1 | undefined; - - constructor(opts: SearchStoreOpts) { - this.#fallback = opts.fallback; - this.#hydrator = opts.hydrator ?? this; - - if (opts.relay) { - this.#relay = new NRelay1(opts.relay); - } - } - - event(_event: NostrEvent, _opts?: { signal?: AbortSignal }): Promise { - return Promise.reject(new Error('EVENT not implemented.')); - } - - async query(filters: NostrFilter[], opts?: { signal?: AbortSignal; limit?: number }): Promise { - filters = normalizeFilters(filters); - - if (opts?.signal?.aborted) return Promise.reject(abortError()); - if (!filters.length) return Promise.resolve([]); - - logi({ level: 'debug', ns: 'ditto.req', source: 'search', filters: filters as JsonValue }); - const query = filters[0]?.search; - - if (this.#relay && this.#relay.socket.readyState === WebSocket.OPEN) { - logi({ level: 'debug', ns: 'ditto.search', query, source: 'relay', relay: this.#relay.socket.url }); - - const events = await this.#relay.query(filters, opts); - - return hydrateEvents({ - events, - store: this.#hydrator, - signal: opts?.signal, - }); - } else { - logi({ level: 'debug', ns: 'ditto.search', query, source: 'db' }); - return this.#fallback.query(filters, opts); - } - } -} - -export { SearchStore }; From c29fc57a8cc5bfa76315b3c9c11d12843347e5fa Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 18 Feb 2025 16:35:45 -0600 Subject: [PATCH 2/3] Switch to genEvent from Nostrify --- packages/ditto/controllers/api/cashu.test.ts | 3 ++- packages/ditto/storages/EventsDB.test.ts | 3 ++- packages/ditto/storages/hydrate.bench.ts | 3 ++- packages/ditto/test.ts | 21 -------------------- packages/ditto/trends.test.ts | 3 ++- packages/ditto/utils/stats.test.ts | 3 ++- 6 files changed, 10 insertions(+), 26 deletions(-) diff --git a/packages/ditto/controllers/api/cashu.test.ts b/packages/ditto/controllers/api/cashu.test.ts index 773e9800..ee73661b 100644 --- a/packages/ditto/controllers/api/cashu.test.ts +++ b/packages/ditto/controllers/api/cashu.test.ts @@ -1,12 +1,13 @@ import { confMw } from '@ditto/api/middleware'; import { Env as HonoEnv, Hono } from '@hono/hono'; import { NostrSigner, NSecSigner, NStore } from '@nostrify/nostrify'; +import { genEvent } from '@nostrify/nostrify/test'; import { generateSecretKey, getPublicKey } from 'nostr-tools'; import { bytesToString, stringToBytes } from '@scure/base'; import { stub } from '@std/testing/mock'; import { assertEquals, assertExists, assertObjectMatch } from '@std/assert'; -import { createTestDB, genEvent } from '@/test.ts'; +import { createTestDB } from '@/test.ts'; import cashuApp from '@/controllers/api/cashu.ts'; import { walletSchema } from '@/schema.ts'; diff --git a/packages/ditto/storages/EventsDB.test.ts b/packages/ditto/storages/EventsDB.test.ts index d0947075..03f31d35 100644 --- a/packages/ditto/storages/EventsDB.test.ts +++ b/packages/ditto/storages/EventsDB.test.ts @@ -1,8 +1,9 @@ import { assertEquals, assertRejects } from '@std/assert'; +import { genEvent } from '@nostrify/nostrify/test'; import { generateSecretKey } from 'nostr-tools'; import { RelayError } from '@/RelayError.ts'; -import { eventFixture, genEvent } from '@/test.ts'; +import { eventFixture } from '@/test.ts'; import { Conf } from '@/config.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; import { createTestDB } from '@/test.ts'; diff --git a/packages/ditto/storages/hydrate.bench.ts b/packages/ditto/storages/hydrate.bench.ts index eeacec50..026b1f81 100644 --- a/packages/ditto/storages/hydrate.bench.ts +++ b/packages/ditto/storages/hydrate.bench.ts @@ -1,5 +1,6 @@ +import { jsonlEvents } from '@nostrify/nostrify/test'; + import { assembleEvents } from '@/storages/hydrate.ts'; -import { jsonlEvents } from '@/test.ts'; const testEvents = await jsonlEvents('fixtures/hydrated.jsonl'); const testStats = JSON.parse(await Deno.readTextFile('fixtures/stats.json')); diff --git a/packages/ditto/test.ts b/packages/ditto/test.ts index 47052b8d..dcf428a6 100644 --- a/packages/ditto/test.ts +++ b/packages/ditto/test.ts @@ -1,10 +1,8 @@ import { DittoDB } from '@ditto/db'; import { NostrEvent } from '@nostrify/nostrify'; -import { finalizeEvent, generateSecretKey } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; -import { purifyEvent } from '@/utils/purify.ts'; import { sql } from 'kysely'; /** Import an event fixture by name in tests. */ @@ -13,25 +11,6 @@ export async function eventFixture(name: string): Promise { return structuredClone(result.default); } -/** Import a JSONL fixture by name in tests. */ -export async function jsonlEvents(path: string): Promise { - const data = await Deno.readTextFile(path); - return data.split('\n').map((line) => JSON.parse(line)); -} - -/** Generate an event for use in tests. */ -export function genEvent(t: Partial = {}, sk: Uint8Array = generateSecretKey()): NostrEvent { - const event = finalizeEvent({ - kind: 255, - created_at: 0, - content: '', - tags: [], - ...t, - }, sk); - - return purifyEvent(event); -} - /** Create a database for testing. It uses `DATABASE_URL`, or creates an in-memory database by default. */ export async function createTestDB(opts?: { pure?: boolean }) { const { kysely } = DittoDB.create(Conf.databaseUrl, { poolSize: 1 }); diff --git a/packages/ditto/trends.test.ts b/packages/ditto/trends.test.ts index 79eaf8e0..a99b4eb4 100644 --- a/packages/ditto/trends.test.ts +++ b/packages/ditto/trends.test.ts @@ -1,8 +1,9 @@ import { assertEquals } from '@std/assert'; +import { genEvent } from '@nostrify/nostrify/test'; import { generateSecretKey, NostrEvent } from 'nostr-tools'; import { getTrendingTagValues } from '@/trends.ts'; -import { createTestDB, genEvent } from '@/test.ts'; +import { createTestDB } from '@/test.ts'; Deno.test("getTrendingTagValues(): 'e' tag and WITHOUT language parameter", async () => { await using db = await createTestDB(); diff --git a/packages/ditto/utils/stats.test.ts b/packages/ditto/utils/stats.test.ts index 797f78da..762db37c 100644 --- a/packages/ditto/utils/stats.test.ts +++ b/packages/ditto/utils/stats.test.ts @@ -1,7 +1,8 @@ +import { genEvent } from '@nostrify/nostrify/test'; import { assertEquals } from '@std/assert'; import { generateSecretKey, getPublicKey } from 'nostr-tools'; -import { createTestDB, genEvent } from '@/test.ts'; +import { createTestDB } from '@/test.ts'; import { countAuthorStats, getAuthorStats, getEventStats, getFollowDiff, updateStats } from '@/utils/stats.ts'; Deno.test('updateStats with kind 1 increments notes count', async () => { From 7deec54a2ed4284634250d820bb0cc19c2506109 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 18 Feb 2025 20:03:03 -0600 Subject: [PATCH 3/3] Upgrade Deno to v2.2.0 --- .gitlab-ci.yml | 2 +- .tool-versions | 2 +- Dockerfile | 2 +- packages/conf/DittoConf.ts | 12 +----------- 4 files changed, 4 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 766a144d..b754ff1e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:2.1.10 +image: denoland/deno:2.2.0 default: interruptible: true diff --git a/.tool-versions b/.tool-versions index a3cfae3c..f9adf79b 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -deno 2.1.10 \ No newline at end of file +deno 2.2.0 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 78ae7fad..0b8724a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM denoland/deno:2.1.10 +FROM denoland/deno:2.2.0 ENV PORT 5000 WORKDIR /app diff --git a/packages/conf/DittoConf.ts b/packages/conf/DittoConf.ts index 6d4b45d7..456e9cd2 100644 --- a/packages/conf/DittoConf.ts +++ b/packages/conf/DittoConf.ts @@ -1,4 +1,3 @@ -import Module from 'node:module'; import os from 'node:os'; import path from 'node:path'; @@ -354,7 +353,7 @@ export class DittoConf { /** Absolute path to the data directory used by Ditto. */ get dataDir(): string { - return this.env.get('DITTO_DATA_DIR') || path.join(cwd(), 'data'); + return this.env.get('DITTO_DATA_DIR') || path.join(Deno.cwd(), 'data'); } /** Absolute path of the Deno directory. */ @@ -465,12 +464,3 @@ export class DittoConf { return Number(this.env.get('STREAK_WINDOW') || 129600); } } - -/** - * HACK: get cwd without read permissions. - * https://github.com/denoland/deno/issues/27080#issuecomment-2504150155 - */ -function cwd() { - // @ts-ignore Internal method, but it does exist. - return Module._nodeModulePaths('a')[0].slice(0, -15); -}