From d2fb3fd2534d3c722bc6533c54fad0261157e29d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Sep 2024 13:06:20 -0500 Subject: [PATCH] Make EventsDB not rely on Conf --- deno.json | 1 - scripts/admin-event.ts | 9 +++----- scripts/admin-role.ts | 10 ++++----- scripts/nostr-pull.ts | 8 +++---- src/controllers/api/statuses.ts | 3 ++- src/storages.ts | 2 +- src/storages/EventsDB.ts | 37 ++++++++++++++++++++------------- src/storages/InternalRelay.ts | 2 +- src/storages/hydrate.ts | 17 ++------------- src/test.ts | 9 ++++++-- src/utils/api.ts | 2 +- src/utils/purify.ts | 14 +++++++++++++ src/workers/policy.ts | 2 +- src/workers/policy.worker.ts | 15 +++++++++++-- 14 files changed, 75 insertions(+), 56 deletions(-) create mode 100644 src/utils/purify.ts diff --git a/deno.json b/deno.json index 4897cff4..699ab620 100644 --- a/deno.json +++ b/deno.json @@ -1,5 +1,4 @@ { - "$schema": "https://deno.land/x/deno@v1.41.0/cli/schemas/config-file.v1.json", "version": "1.1.0", "tasks": { "start": "deno run -A src/server.ts", diff --git a/scripts/admin-event.ts b/scripts/admin-event.ts index 313aa051..00711993 100644 --- a/scripts/admin-event.ts +++ b/scripts/admin-event.ts @@ -1,16 +1,13 @@ import { JsonParseStream } from '@std/json/json-parse-stream'; import { TextLineStream } from '@std/streams/text-line-stream'; -import { DittoDB } from '@/db/DittoDB.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; +import { Storages } from '@/storages.ts'; import { type EventStub } from '@/utils/api.ts'; import { nostrNow } from '@/utils.ts'; const signer = new AdminSigner(); - -const { kysely } = await DittoDB.getInstance(); -const eventsDB = new EventsDB(kysely); +const store = await Storages.db(); const readable = Deno.stdin.readable .pipeThrough(new TextDecoderStream()) @@ -25,7 +22,7 @@ for await (const t of readable) { ...t as EventStub, }); - await eventsDB.event(event); + await store.event(event); } Deno.exit(0); diff --git a/scripts/admin-role.ts b/scripts/admin-role.ts index 99986817..d275329f 100644 --- a/scripts/admin-role.ts +++ b/scripts/admin-role.ts @@ -1,13 +1,11 @@ import { NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { DittoDB } from '@/db/DittoDB.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; +import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; -const { kysely } = await DittoDB.getInstance(); -const eventsDB = new EventsDB(kysely); +const store = await Storages.db(); const [pubkeyOrNpub, role] = Deno.args; const pubkey = pubkeyOrNpub.startsWith('npub1') ? nip19.decode(pubkeyOrNpub as `npub1${string}`).data : pubkeyOrNpub; @@ -25,7 +23,7 @@ if (!['admin', 'user'].includes(role)) { const signer = new AdminSigner(); const admin = await signer.getPublicKey(); -const [existing] = await eventsDB.query([{ +const [existing] = await store.query([{ kinds: [30382], authors: [admin], '#d': [pubkey], @@ -59,6 +57,6 @@ const event = await signer.signEvent({ created_at: nostrNow(), }); -await eventsDB.event(event); +await store.event(event); Deno.exit(0); diff --git a/scripts/nostr-pull.ts b/scripts/nostr-pull.ts index 4b9d51db..f7a3840c 100644 --- a/scripts/nostr-pull.ts +++ b/scripts/nostr-pull.ts @@ -6,11 +6,9 @@ import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { DittoDB } from '@/db/DittoDB.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; +import { Storages } from '@/storages.ts'; -const { kysely } = await DittoDB.getInstance(); -const eventsDB = new EventsDB(kysely); +const store = await Storages.db(); interface ImportEventsOpts { profilesOnly: boolean; @@ -21,7 +19,7 @@ const importUsers = async ( authors: string[], relays: string[], opts?: Partial, - doEvent: DoEvent = async (event: NostrEvent) => await eventsDB.event(event), + doEvent: DoEvent = async (event: NostrEvent) => await store.event(event), ) => { // Kind 0s + follow lists. const profiles: Record> = {}; diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 05b5022c..bef98122 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -16,9 +16,10 @@ import { addTag, deleteTag } from '@/utils/tags.ts'; import { asyncReplaceAll } from '@/utils/text.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; import { Storages } from '@/storages.ts'; -import { hydrateEvents, purifyEvent } from '@/storages/hydrate.ts'; +import { hydrateEvents } from '@/storages/hydrate.ts'; import { createEvent, paginated, paginatedList, parseBody, updateListEvent } from '@/utils/api.ts'; import { getInvoice, getLnurl } from '@/utils/lnurl.ts'; +import { purifyEvent } from '@/utils/purify.ts'; import { getZapSplits } from '@/utils/zap-split.ts'; import { renderEventAccounts } from '@/views.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; diff --git a/src/storages.ts b/src/storages.ts index 7114a3d6..10510104 100644 --- a/src/storages.ts +++ b/src/storages.ts @@ -21,7 +21,7 @@ export class Storages { if (!this._db) { this._db = (async () => { const { kysely } = await DittoDB.getInstance(); - const store = new EventsDB(kysely); + const store = new EventsDB({ kysely, pubkey: Conf.pubkey, timeout: Conf.db.timeouts.default }); await seedZapSplits(store); return store; })(); diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 6c006dc2..72cd9bb3 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -1,6 +1,6 @@ // deno-lint-ignore-file require-await -import { NDatabase, NPostgres } from '@nostrify/db'; +import { NPostgres } from '@nostrify/db'; import { NIP50, NKinds, @@ -16,13 +16,12 @@ import { Stickynotes } from '@soapbox/stickynotes'; import { Kysely } from 'kysely'; import { nip27 } from 'nostr-tools'; -import { Conf } from '@/config.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { dbEventsCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; -import { purifyEvent } from '@/storages/hydrate.ts'; import { isNostrId, isURL } from '@/utils.ts'; import { abortError } from '@/utils/abort.ts'; +import { purifyEvent } from '@/utils/purify.ts'; /** Function to decide whether or not to index a tag. */ type TagCondition = ({ event, count, value }: { @@ -31,9 +30,19 @@ type TagCondition = ({ event, count, value }: { value: string; }) => boolean; +/** Options for the EventsDB store. */ +interface EventsDBOpts { + /** Kysely instance to use. */ + kysely: Kysely; + /** Pubkey of the admin account. */ + pubkey: string; + /** Timeout in milliseconds for database queries. */ + timeout: number; +} + /** SQL database storage adapter for Nostr events. */ class EventsDB implements NStore { - private store: NDatabase | NPostgres; + private store: NPostgres; private console = new Stickynotes('ditto:db:events'); /** Conditions for when to index certain tags. */ @@ -53,8 +62,8 @@ class EventsDB implements NStore { 't': ({ event, count, value }) => (event.kind === 1985 ? count < 20 : count < 5) && value.length < 50, }; - constructor(private kysely: Kysely) { - this.store = new NPostgres(kysely, { + constructor(private opts: EventsDBOpts) { + this.store = new NPostgres(opts.kysely, { indexTags: EventsDB.indexTags, indexSearch: EventsDB.searchText, }); @@ -73,7 +82,7 @@ class EventsDB implements NStore { await this.deleteEventsAdmin(event); try { - await this.store.event(event, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); + await this.store.event(event, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } catch (e) { if (e.message === 'Cannot add a deleted event') { throw new RelayError('blocked', 'event deleted by user'); @@ -88,7 +97,7 @@ class EventsDB implements NStore { /** Check if an event has been deleted by the admin. */ private async isDeletedAdmin(event: NostrEvent): Promise { const filters: NostrFilter[] = [ - { kinds: [5], authors: [Conf.pubkey], '#e': [event.id], limit: 1 }, + { kinds: [5], authors: [this.opts.pubkey], '#e': [event.id], limit: 1 }, ]; if (NKinds.replaceable(event.kind) || NKinds.parameterizedReplaceable(event.kind)) { @@ -96,7 +105,7 @@ class EventsDB implements NStore { filters.push({ kinds: [5], - authors: [Conf.pubkey], + authors: [this.opts.pubkey], '#a': [`${event.kind}:${event.pubkey}:${d}`], since: event.created_at, limit: 1, @@ -109,7 +118,7 @@ class EventsDB implements NStore { /** The DITTO_NSEC can delete any event from the database. NDatabase already handles user deletions. */ private async deleteEventsAdmin(event: NostrEvent): Promise { - if (event.kind === 5 && event.pubkey === Conf.pubkey) { + if (event.kind === 5 && event.pubkey === this.opts.pubkey) { const ids = new Set(event.tags.filter(([name]) => name === 'e').map(([_name, value]) => value)); const addrs = new Set(event.tags.filter(([name]) => name === 'a').map(([_name, value]) => value)); @@ -180,7 +189,7 @@ class EventsDB implements NStore { this.console.debug('REQ', JSON.stringify(filters)); - return this.store.query(filters, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); + return this.store.query(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } /** Delete events based on filters from the database. */ @@ -188,7 +197,7 @@ class EventsDB implements NStore { if (!filters.length) return Promise.resolve(); this.console.debug('DELETE', JSON.stringify(filters)); - return this.store.remove(filters, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); + return this.store.remove(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } /** Get number of events that would be returned by filters. */ @@ -201,7 +210,7 @@ class EventsDB implements NStore { this.console.debug('COUNT', JSON.stringify(filters)); - return this.store.count(filters, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); + return this.store.count(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } /** Return only the tags that should be indexed. */ @@ -277,7 +286,7 @@ class EventsDB implements NStore { ) as { key: 'domain'; value: string } | undefined)?.value; if (domain) { - const query = this.kysely + const query = this.opts.kysely .selectFrom('pubkey_domains') .select('pubkey') .where('domain', '=', domain); diff --git a/src/storages/InternalRelay.ts b/src/storages/InternalRelay.ts index 233a095c..93a480e1 100644 --- a/src/storages/InternalRelay.ts +++ b/src/storages/InternalRelay.ts @@ -12,7 +12,7 @@ import { Machina } from '@nostrify/nostrify/utils'; import { matchFilter } from 'nostr-tools'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { purifyEvent } from '@/storages/hydrate.ts'; +import { purifyEvent } from '@/utils/purify.ts'; /** * PubSub event store for streaming events within the application. diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 19ba0db4..c7a277c7 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -1,4 +1,4 @@ -import { NostrEvent, NStore } from '@nostrify/nostrify'; +import { NStore } from '@nostrify/nostrify'; import { matchFilter } from 'nostr-tools'; import { DittoDB } from '@/db/DittoDB.ts'; @@ -338,17 +338,4 @@ async function gatherEventStats( })); } -/** Return a normalized event without any non-standard keys. */ -function purifyEvent(event: NostrEvent): NostrEvent { - return { - id: event.id, - pubkey: event.pubkey, - kind: event.kind, - content: event.content, - tags: event.tags, - sig: event.sig, - created_at: event.created_at, - }; -} - -export { hydrateEvents, purifyEvent }; +export { hydrateEvents }; diff --git a/src/test.ts b/src/test.ts index 8b3dad80..45946f00 100644 --- a/src/test.ts +++ b/src/test.ts @@ -3,8 +3,8 @@ import { finalizeEvent, generateSecretKey } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; -import { purifyEvent } from '@/storages/hydrate.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; +import { purifyEvent } from '@/utils/purify.ts'; /** Import an event fixture by name in tests. */ export async function eventFixture(name: string): Promise { @@ -38,7 +38,12 @@ export async function createTestDB() { const { kysely } = DittoDB.create(testDatabaseUrl, { poolSize: 1 }); await DittoDB.migrate(kysely); - const store = new EventsDB(kysely); + + const store = new EventsDB({ + kysely, + timeout: Conf.db.timeouts.default, + pubkey: Conf.pubkey, + }); return { store, diff --git a/src/utils/api.ts b/src/utils/api.ts index b3b5a8b1..c6d3c6b6 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -13,7 +13,7 @@ import { RelayError } from '@/RelayError.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; -import { purifyEvent } from '@/storages/hydrate.ts'; +import { purifyEvent } from '@/utils/purify.ts'; const debug = Debug('ditto:api'); diff --git a/src/utils/purify.ts b/src/utils/purify.ts new file mode 100644 index 00000000..84c1e44b --- /dev/null +++ b/src/utils/purify.ts @@ -0,0 +1,14 @@ +import { NostrEvent } from '@nostrify/nostrify'; + +/** Return a normalized event without any non-standard keys. */ +export function purifyEvent(event: NostrEvent): NostrEvent { + return { + id: event.id, + pubkey: event.pubkey, + kind: event.kind, + content: event.content, + tags: event.tags, + sig: event.sig, + created_at: event.created_at, + }; +} diff --git a/src/workers/policy.ts b/src/workers/policy.ts index ef9aa2cd..08511ded 100644 --- a/src/workers/policy.ts +++ b/src/workers/policy.ts @@ -24,7 +24,7 @@ export const policyWorker = Comlink.wrap( ); try { - await policyWorker.import(Conf.policy); + await policyWorker.init(Conf.policy, Conf.databaseUrl, Conf.pubkey); console.debug(`Using custom policy: ${Conf.policy}`); } catch (e) { if (e.message.includes('Module not found')) { diff --git a/src/workers/policy.worker.ts b/src/workers/policy.worker.ts index 9f94a008..0036c4bd 100644 --- a/src/workers/policy.worker.ts +++ b/src/workers/policy.worker.ts @@ -3,6 +3,9 @@ import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify'; import { NoOpPolicy, ReadOnlyPolicy } from '@nostrify/nostrify/policies'; import * as Comlink from 'comlink'; +import { DittoDB } from '@/db/DittoDB.ts'; +import { EventsDB } from '@/storages/EventsDB.ts'; + export class CustomPolicy implements NPolicy { private policy: NPolicy = new ReadOnlyPolicy(); @@ -11,10 +14,18 @@ export class CustomPolicy implements NPolicy { return this.policy.call(event); } - async import(path: string): Promise { + async init(path: string, databaseUrl: string, adminPubkey: string): Promise { + const { kysely } = DittoDB.create(databaseUrl, { poolSize: 1 }); + + const store = new EventsDB({ + kysely, + pubkey: adminPubkey, + timeout: 1_000, + }); + try { const Policy = (await import(path)).default; - this.policy = new Policy(); + this.policy = new Policy({ store }); } catch (e) { if (e.message.includes('Module not found')) { this.policy = new NoOpPolicy();