From d2fb3fd2534d3c722bc6533c54fad0261157e29d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Sep 2024 13:06:20 -0500 Subject: [PATCH 1/3] Make EventsDB not rely on Conf --- deno.json | 1 - scripts/admin-event.ts | 9 +++----- scripts/admin-role.ts | 10 ++++----- scripts/nostr-pull.ts | 8 +++---- src/controllers/api/statuses.ts | 3 ++- src/storages.ts | 2 +- src/storages/EventsDB.ts | 37 ++++++++++++++++++++------------- src/storages/InternalRelay.ts | 2 +- src/storages/hydrate.ts | 17 ++------------- src/test.ts | 9 ++++++-- src/utils/api.ts | 2 +- src/utils/purify.ts | 14 +++++++++++++ src/workers/policy.ts | 2 +- src/workers/policy.worker.ts | 15 +++++++++++-- 14 files changed, 75 insertions(+), 56 deletions(-) create mode 100644 src/utils/purify.ts diff --git a/deno.json b/deno.json index 4897cff4..699ab620 100644 --- a/deno.json +++ b/deno.json @@ -1,5 +1,4 @@ { - "$schema": "https://deno.land/x/deno@v1.41.0/cli/schemas/config-file.v1.json", "version": "1.1.0", "tasks": { "start": "deno run -A src/server.ts", diff --git a/scripts/admin-event.ts b/scripts/admin-event.ts index 313aa051..00711993 100644 --- a/scripts/admin-event.ts +++ b/scripts/admin-event.ts @@ -1,16 +1,13 @@ import { JsonParseStream } from '@std/json/json-parse-stream'; import { TextLineStream } from '@std/streams/text-line-stream'; -import { DittoDB } from '@/db/DittoDB.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; +import { Storages } from '@/storages.ts'; import { type EventStub } from '@/utils/api.ts'; import { nostrNow } from '@/utils.ts'; const signer = new AdminSigner(); - -const { kysely } = await DittoDB.getInstance(); -const eventsDB = new EventsDB(kysely); +const store = await Storages.db(); const readable = Deno.stdin.readable .pipeThrough(new TextDecoderStream()) @@ -25,7 +22,7 @@ for await (const t of readable) { ...t as EventStub, }); - await eventsDB.event(event); + await store.event(event); } Deno.exit(0); diff --git a/scripts/admin-role.ts b/scripts/admin-role.ts index 99986817..d275329f 100644 --- a/scripts/admin-role.ts +++ b/scripts/admin-role.ts @@ -1,13 +1,11 @@ import { NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { DittoDB } from '@/db/DittoDB.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; +import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; -const { kysely } = await DittoDB.getInstance(); -const eventsDB = new EventsDB(kysely); +const store = await Storages.db(); const [pubkeyOrNpub, role] = Deno.args; const pubkey = pubkeyOrNpub.startsWith('npub1') ? nip19.decode(pubkeyOrNpub as `npub1${string}`).data : pubkeyOrNpub; @@ -25,7 +23,7 @@ if (!['admin', 'user'].includes(role)) { const signer = new AdminSigner(); const admin = await signer.getPublicKey(); -const [existing] = await eventsDB.query([{ +const [existing] = await store.query([{ kinds: [30382], authors: [admin], '#d': [pubkey], @@ -59,6 +57,6 @@ const event = await signer.signEvent({ created_at: nostrNow(), }); -await eventsDB.event(event); +await store.event(event); Deno.exit(0); diff --git a/scripts/nostr-pull.ts b/scripts/nostr-pull.ts index 4b9d51db..f7a3840c 100644 --- a/scripts/nostr-pull.ts +++ b/scripts/nostr-pull.ts @@ -6,11 +6,9 @@ import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { DittoDB } from '@/db/DittoDB.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; +import { Storages } from '@/storages.ts'; -const { kysely } = await DittoDB.getInstance(); -const eventsDB = new EventsDB(kysely); +const store = await Storages.db(); interface ImportEventsOpts { profilesOnly: boolean; @@ -21,7 +19,7 @@ const importUsers = async ( authors: string[], relays: string[], opts?: Partial, - doEvent: DoEvent = async (event: NostrEvent) => await eventsDB.event(event), + doEvent: DoEvent = async (event: NostrEvent) => await store.event(event), ) => { // Kind 0s + follow lists. const profiles: Record> = {}; diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 05b5022c..bef98122 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -16,9 +16,10 @@ import { addTag, deleteTag } from '@/utils/tags.ts'; import { asyncReplaceAll } from '@/utils/text.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; import { Storages } from '@/storages.ts'; -import { hydrateEvents, purifyEvent } from '@/storages/hydrate.ts'; +import { hydrateEvents } from '@/storages/hydrate.ts'; import { createEvent, paginated, paginatedList, parseBody, updateListEvent } from '@/utils/api.ts'; import { getInvoice, getLnurl } from '@/utils/lnurl.ts'; +import { purifyEvent } from '@/utils/purify.ts'; import { getZapSplits } from '@/utils/zap-split.ts'; import { renderEventAccounts } from '@/views.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; diff --git a/src/storages.ts b/src/storages.ts index 7114a3d6..10510104 100644 --- a/src/storages.ts +++ b/src/storages.ts @@ -21,7 +21,7 @@ export class Storages { if (!this._db) { this._db = (async () => { const { kysely } = await DittoDB.getInstance(); - const store = new EventsDB(kysely); + const store = new EventsDB({ kysely, pubkey: Conf.pubkey, timeout: Conf.db.timeouts.default }); await seedZapSplits(store); return store; })(); diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 6c006dc2..72cd9bb3 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -1,6 +1,6 @@ // deno-lint-ignore-file require-await -import { NDatabase, NPostgres } from '@nostrify/db'; +import { NPostgres } from '@nostrify/db'; import { NIP50, NKinds, @@ -16,13 +16,12 @@ import { Stickynotes } from '@soapbox/stickynotes'; import { Kysely } from 'kysely'; import { nip27 } from 'nostr-tools'; -import { Conf } from '@/config.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { dbEventsCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; -import { purifyEvent } from '@/storages/hydrate.ts'; import { isNostrId, isURL } from '@/utils.ts'; import { abortError } from '@/utils/abort.ts'; +import { purifyEvent } from '@/utils/purify.ts'; /** Function to decide whether or not to index a tag. */ type TagCondition = ({ event, count, value }: { @@ -31,9 +30,19 @@ type TagCondition = ({ event, count, value }: { value: string; }) => boolean; +/** Options for the EventsDB store. */ +interface EventsDBOpts { + /** Kysely instance to use. */ + kysely: Kysely; + /** Pubkey of the admin account. */ + pubkey: string; + /** Timeout in milliseconds for database queries. */ + timeout: number; +} + /** SQL database storage adapter for Nostr events. */ class EventsDB implements NStore { - private store: NDatabase | NPostgres; + private store: NPostgres; private console = new Stickynotes('ditto:db:events'); /** Conditions for when to index certain tags. */ @@ -53,8 +62,8 @@ class EventsDB implements NStore { 't': ({ event, count, value }) => (event.kind === 1985 ? count < 20 : count < 5) && value.length < 50, }; - constructor(private kysely: Kysely) { - this.store = new NPostgres(kysely, { + constructor(private opts: EventsDBOpts) { + this.store = new NPostgres(opts.kysely, { indexTags: EventsDB.indexTags, indexSearch: EventsDB.searchText, }); @@ -73,7 +82,7 @@ class EventsDB implements NStore { await this.deleteEventsAdmin(event); try { - await this.store.event(event, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); + await this.store.event(event, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } catch (e) { if (e.message === 'Cannot add a deleted event') { throw new RelayError('blocked', 'event deleted by user'); @@ -88,7 +97,7 @@ class EventsDB implements NStore { /** Check if an event has been deleted by the admin. */ private async isDeletedAdmin(event: NostrEvent): Promise { const filters: NostrFilter[] = [ - { kinds: [5], authors: [Conf.pubkey], '#e': [event.id], limit: 1 }, + { kinds: [5], authors: [this.opts.pubkey], '#e': [event.id], limit: 1 }, ]; if (NKinds.replaceable(event.kind) || NKinds.parameterizedReplaceable(event.kind)) { @@ -96,7 +105,7 @@ class EventsDB implements NStore { filters.push({ kinds: [5], - authors: [Conf.pubkey], + authors: [this.opts.pubkey], '#a': [`${event.kind}:${event.pubkey}:${d}`], since: event.created_at, limit: 1, @@ -109,7 +118,7 @@ class EventsDB implements NStore { /** The DITTO_NSEC can delete any event from the database. NDatabase already handles user deletions. */ private async deleteEventsAdmin(event: NostrEvent): Promise { - if (event.kind === 5 && event.pubkey === Conf.pubkey) { + if (event.kind === 5 && event.pubkey === this.opts.pubkey) { const ids = new Set(event.tags.filter(([name]) => name === 'e').map(([_name, value]) => value)); const addrs = new Set(event.tags.filter(([name]) => name === 'a').map(([_name, value]) => value)); @@ -180,7 +189,7 @@ class EventsDB implements NStore { this.console.debug('REQ', JSON.stringify(filters)); - return this.store.query(filters, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); + return this.store.query(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } /** Delete events based on filters from the database. */ @@ -188,7 +197,7 @@ class EventsDB implements NStore { if (!filters.length) return Promise.resolve(); this.console.debug('DELETE', JSON.stringify(filters)); - return this.store.remove(filters, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); + return this.store.remove(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } /** Get number of events that would be returned by filters. */ @@ -201,7 +210,7 @@ class EventsDB implements NStore { this.console.debug('COUNT', JSON.stringify(filters)); - return this.store.count(filters, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); + return this.store.count(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } /** Return only the tags that should be indexed. */ @@ -277,7 +286,7 @@ class EventsDB implements NStore { ) as { key: 'domain'; value: string } | undefined)?.value; if (domain) { - const query = this.kysely + const query = this.opts.kysely .selectFrom('pubkey_domains') .select('pubkey') .where('domain', '=', domain); diff --git a/src/storages/InternalRelay.ts b/src/storages/InternalRelay.ts index 233a095c..93a480e1 100644 --- a/src/storages/InternalRelay.ts +++ b/src/storages/InternalRelay.ts @@ -12,7 +12,7 @@ import { Machina } from '@nostrify/nostrify/utils'; import { matchFilter } from 'nostr-tools'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { purifyEvent } from '@/storages/hydrate.ts'; +import { purifyEvent } from '@/utils/purify.ts'; /** * PubSub event store for streaming events within the application. diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 19ba0db4..c7a277c7 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -1,4 +1,4 @@ -import { NostrEvent, NStore } from '@nostrify/nostrify'; +import { NStore } from '@nostrify/nostrify'; import { matchFilter } from 'nostr-tools'; import { DittoDB } from '@/db/DittoDB.ts'; @@ -338,17 +338,4 @@ async function gatherEventStats( })); } -/** Return a normalized event without any non-standard keys. */ -function purifyEvent(event: NostrEvent): NostrEvent { - return { - id: event.id, - pubkey: event.pubkey, - kind: event.kind, - content: event.content, - tags: event.tags, - sig: event.sig, - created_at: event.created_at, - }; -} - -export { hydrateEvents, purifyEvent }; +export { hydrateEvents }; diff --git a/src/test.ts b/src/test.ts index 8b3dad80..45946f00 100644 --- a/src/test.ts +++ b/src/test.ts @@ -3,8 +3,8 @@ import { finalizeEvent, generateSecretKey } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; -import { purifyEvent } from '@/storages/hydrate.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; +import { purifyEvent } from '@/utils/purify.ts'; /** Import an event fixture by name in tests. */ export async function eventFixture(name: string): Promise { @@ -38,7 +38,12 @@ export async function createTestDB() { const { kysely } = DittoDB.create(testDatabaseUrl, { poolSize: 1 }); await DittoDB.migrate(kysely); - const store = new EventsDB(kysely); + + const store = new EventsDB({ + kysely, + timeout: Conf.db.timeouts.default, + pubkey: Conf.pubkey, + }); return { store, diff --git a/src/utils/api.ts b/src/utils/api.ts index b3b5a8b1..c6d3c6b6 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -13,7 +13,7 @@ import { RelayError } from '@/RelayError.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; -import { purifyEvent } from '@/storages/hydrate.ts'; +import { purifyEvent } from '@/utils/purify.ts'; const debug = Debug('ditto:api'); diff --git a/src/utils/purify.ts b/src/utils/purify.ts new file mode 100644 index 00000000..84c1e44b --- /dev/null +++ b/src/utils/purify.ts @@ -0,0 +1,14 @@ +import { NostrEvent } from '@nostrify/nostrify'; + +/** Return a normalized event without any non-standard keys. */ +export function purifyEvent(event: NostrEvent): NostrEvent { + return { + id: event.id, + pubkey: event.pubkey, + kind: event.kind, + content: event.content, + tags: event.tags, + sig: event.sig, + created_at: event.created_at, + }; +} diff --git a/src/workers/policy.ts b/src/workers/policy.ts index ef9aa2cd..08511ded 100644 --- a/src/workers/policy.ts +++ b/src/workers/policy.ts @@ -24,7 +24,7 @@ export const policyWorker = Comlink.wrap( ); try { - await policyWorker.import(Conf.policy); + await policyWorker.init(Conf.policy, Conf.databaseUrl, Conf.pubkey); console.debug(`Using custom policy: ${Conf.policy}`); } catch (e) { if (e.message.includes('Module not found')) { diff --git a/src/workers/policy.worker.ts b/src/workers/policy.worker.ts index 9f94a008..0036c4bd 100644 --- a/src/workers/policy.worker.ts +++ b/src/workers/policy.worker.ts @@ -3,6 +3,9 @@ import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify'; import { NoOpPolicy, ReadOnlyPolicy } from '@nostrify/nostrify/policies'; import * as Comlink from 'comlink'; +import { DittoDB } from '@/db/DittoDB.ts'; +import { EventsDB } from '@/storages/EventsDB.ts'; + export class CustomPolicy implements NPolicy { private policy: NPolicy = new ReadOnlyPolicy(); @@ -11,10 +14,18 @@ export class CustomPolicy implements NPolicy { return this.policy.call(event); } - async import(path: string): Promise { + async init(path: string, databaseUrl: string, adminPubkey: string): Promise { + const { kysely } = DittoDB.create(databaseUrl, { poolSize: 1 }); + + const store = new EventsDB({ + kysely, + pubkey: adminPubkey, + timeout: 1_000, + }); + try { const Policy = (await import(path)).default; - this.policy = new Policy(); + this.policy = new Policy({ store }); } catch (e) { if (e.message.includes('Module not found')) { this.policy = new NoOpPolicy(); From ebc0250d81f74bd25e741e2d588f5b44a75af6f9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Sep 2024 13:23:06 -0500 Subject: [PATCH 2/3] DittoDB.getInstance() -> Storages.kysely() --- scripts/db-migrate.ts | 4 ++-- scripts/stats-recompute.ts | 3 +-- src/controllers/api/oauth.ts | 3 +-- src/controllers/api/statuses.ts | 3 +-- src/controllers/api/streaming.ts | 3 +-- src/controllers/metrics.ts | 4 ++-- src/db/DittoDB.ts | 12 ------------ src/middleware/signerMiddleware.ts | 4 ++-- src/pipeline.ts | 9 ++++----- src/storages.ts | 17 ++++++++++++++++- src/storages/hydrate.ts | 6 +++--- src/trends.ts | 4 ++-- 12 files changed, 35 insertions(+), 37 deletions(-) diff --git a/scripts/db-migrate.ts b/scripts/db-migrate.ts index 0e1d694d..d3e93783 100644 --- a/scripts/db-migrate.ts +++ b/scripts/db-migrate.ts @@ -1,7 +1,7 @@ -import { DittoDB } from '@/db/DittoDB.ts'; +import { Storages } from '@/storages.ts'; // This migrates kysely internally. -const { kysely } = await DittoDB.getInstance(); +const kysely = await Storages.kysely(); // Close the connection before exiting. await kysely.destroy(); diff --git a/scripts/stats-recompute.ts b/scripts/stats-recompute.ts index 7d6f721f..77be13fe 100644 --- a/scripts/stats-recompute.ts +++ b/scripts/stats-recompute.ts @@ -1,6 +1,5 @@ import { nip19 } from 'nostr-tools'; -import { DittoDB } from '@/db/DittoDB.ts'; import { Storages } from '@/storages.ts'; import { refreshAuthorStats } from '@/utils/stats.ts'; @@ -18,6 +17,6 @@ try { } const store = await Storages.db(); -const { kysely } = await DittoDB.getInstance(); +const kysely = await Storages.kysely(); await refreshAuthorStats({ pubkey, kysely, store }); diff --git a/src/controllers/api/oauth.ts b/src/controllers/api/oauth.ts index a736f5ca..94aaeecd 100644 --- a/src/controllers/api/oauth.ts +++ b/src/controllers/api/oauth.ts @@ -6,7 +6,6 @@ import { z } from 'zod'; import { AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { nostrNow } from '@/utils.ts'; import { parseBody } from '@/utils/api.ts'; import { Storages } from '@/storages.ts'; @@ -82,7 +81,7 @@ const createTokenController: AppController = async (c) => { async function getToken( { pubkey, secret, relays = [] }: { pubkey: string; secret?: string; relays?: string[] }, ): Promise<`token1${string}`> { - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); const token = generateToken(); const serverSeckey = generateSecretKey(); diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index bef98122..e0956e74 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -8,7 +8,6 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { DittoUpload, dittoUploads } from '@/DittoUploads.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { getAncestors, getAuthor, getDescendants, getEvent } from '@/queries.ts'; @@ -579,7 +578,7 @@ const zappedByController: AppController = async (c) => { const id = c.req.param('id'); const params = c.get('listPagination'); const store = await Storages.db(); - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); const zaps = await kysely.selectFrom('event_zaps') .selectAll() diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index 047aa573..cfa8c3c5 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -4,7 +4,6 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { streamingConnectionsGauge } from '@/metrics.ts'; import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; import { getFeedPubkeys } from '@/queries.ts'; @@ -222,7 +221,7 @@ async function topicToFilter( async function getTokenPubkey(token: string): Promise { if (token.startsWith('token1')) { - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); const { user_pubkey } = await kysely .selectFrom('nip46_tokens') diff --git a/src/controllers/metrics.ts b/src/controllers/metrics.ts index e25522ff..4ef378a0 100644 --- a/src/controllers/metrics.ts +++ b/src/controllers/metrics.ts @@ -1,12 +1,12 @@ import { register } from 'prom-client'; import { AppController } from '@/app.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { dbAvailableConnectionsGauge, dbPoolSizeGauge } from '@/metrics.ts'; +import { Storages } from '@/storages.ts'; /** Prometheus/OpenMetrics controller. */ export const metricsController: AppController = async (c) => { - const db = await DittoDB.getInstance(); + const db = await Storages.database(); // Update some metrics at request time. dbPoolSizeGauge.set(db.poolSize); diff --git a/src/db/DittoDB.ts b/src/db/DittoDB.ts index 63d2bfb1..445c3da2 100644 --- a/src/db/DittoDB.ts +++ b/src/db/DittoDB.ts @@ -3,24 +3,12 @@ import path from 'node:path'; import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; -import { Conf } from '@/config.ts'; import { DittoPglite } from '@/db/adapters/DittoPglite.ts'; import { DittoPostgres } from '@/db/adapters/DittoPostgres.ts'; import { DittoDatabase, DittoDatabaseOpts } from '@/db/DittoDatabase.ts'; import { DittoTables } from '@/db/DittoTables.ts'; export class DittoDB { - private static db: DittoDatabase | undefined; - - /** Create (and migrate) the database if it isn't been already, or return the existing connection. */ - static async getInstance(): Promise { - if (!this.db) { - this.db = this.create(Conf.databaseUrl, { poolSize: Conf.pg.poolSize }); - await this.migrate(this.db.kysely); - } - return this.db; - } - /** Open a new database connection. */ static create(databaseUrl: string, opts?: DittoDatabaseOpts): DittoDatabase { const { protocol } = new URL(databaseUrl); diff --git a/src/middleware/signerMiddleware.ts b/src/middleware/signerMiddleware.ts index 60826db9..344e14ef 100644 --- a/src/middleware/signerMiddleware.ts +++ b/src/middleware/signerMiddleware.ts @@ -5,7 +5,7 @@ import { nip19 } from 'nostr-tools'; import { AppMiddleware } from '@/app.ts'; import { ConnectSigner } from '@/signers/ConnectSigner.ts'; import { ReadOnlySigner } from '@/signers/ReadOnlySigner.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; +import { Storages } from '@/storages.ts'; /** We only accept "Bearer" type. */ const BEARER_REGEX = new RegExp(`^Bearer (${nip19.BECH32_REGEX.source})$`); @@ -20,7 +20,7 @@ export const signerMiddleware: AppMiddleware = async (c, next) => { if (bech32.startsWith('token1')) { try { - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); const { user_pubkey, server_seckey, relays } = await kysely .selectFrom('nip46_tokens') diff --git a/src/pipeline.ts b/src/pipeline.ts index dd59cb8d..88e5f29b 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -5,7 +5,6 @@ import { LRUCache } from 'lru-cache'; import { z } from 'zod'; import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { pipelineEventsCounter, policyEventsCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; @@ -53,7 +52,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { await hydrateEvents({ events: [event], store: await Storages.db(), signal }); - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); const domain = await kysely .selectFrom('pubkey_domains') .select('domain') @@ -118,7 +117,7 @@ async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { if (NKinds.ephemeral(event.kind)) return; const store = await Storages.db(); - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); await updateStats({ event, store, kysely }).catch(debug); await store.event(event, { signal }); @@ -146,7 +145,7 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise | undefined; + private static _database: DittoDatabase | undefined; private static _admin: Promise | undefined; private static _client: Promise | undefined; private static _pubsub: Promise | undefined; private static _search: Promise | undefined; + public static async database(): Promise { + if (!this._database) { + this._database = DittoDB.create(Conf.databaseUrl, { poolSize: Conf.pg.poolSize }); + await DittoDB.migrate(this._database.kysely); + } + return this._database; + } + + public static async kysely(): Promise { + const { kysely } = await this.database(); + return kysely; + } + /** SQL database to store events this Ditto server cares about. */ public static async db(): Promise { if (!this._db) { this._db = (async () => { - const { kysely } = await DittoDB.getInstance(); + const { kysely } = await this.database(); const store = new EventsDB({ kysely, pubkey: Conf.pubkey, timeout: Conf.db.timeouts.default }); await seedZapSplits(store); return store; diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index c7a277c7..7b11cfb8 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -1,13 +1,13 @@ import { NStore } from '@nostrify/nostrify'; +import { Kysely } from 'kysely'; import { matchFilter } from 'nostr-tools'; -import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Conf } from '@/config.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { findQuoteTag } from '@/utils/tags.ts'; import { findQuoteInContent } from '@/utils/note.ts'; -import { Kysely } from 'kysely'; +import { Storages } from '@/storages.ts'; interface HydrateOpts { events: DittoEvent[]; @@ -18,7 +18,7 @@ interface HydrateOpts { /** Hydrate events using the provided storage. */ async function hydrateEvents(opts: HydrateOpts): Promise { - const { events, store, signal, kysely = (await DittoDB.getInstance()).kysely } = opts; + const { events, store, signal, kysely = await Storages.kysely() } = opts; if (!events.length) { return events; diff --git a/src/trends.ts b/src/trends.ts index 337ee5c7..de91a33d 100644 --- a/src/trends.ts +++ b/src/trends.ts @@ -3,10 +3,10 @@ import { Stickynotes } from '@soapbox/stickynotes'; import { Kysely, sql } from 'kysely'; import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { handleEvent } from '@/pipeline.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; +import { Storages } from '@/storages.ts'; import { Time } from '@/utils/time.ts'; const console = new Stickynotes('ditto:trends'); @@ -70,7 +70,7 @@ export async function updateTrendingTags( aliases?: string[], ) { console.info(`Updating trending ${l}...`); - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); const signal = AbortSignal.timeout(1000); const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); From cae0f492f3dac0bb3b428312c99ffae3efb1307a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Sep 2024 14:04:11 -0500 Subject: [PATCH 3/3] Let PolicyWorker run in sandbox with store --- src/config.ts | 9 +++++++++ src/workers/policy.ts | 11 ++++++++--- src/workers/policy.worker.ts | 18 +++++++++++++++++- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/config.ts b/src/config.ts index 5fa7be9a..37b1d0f3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,3 +1,4 @@ +import os from 'node:os'; import * as dotenv from '@std/dotenv'; import { getPublicKey, nip19 } from 'nostr-tools'; import { z } from 'zod'; @@ -240,6 +241,14 @@ class Conf { static get policy(): string { return Deno.env.get('DITTO_POLICY') || new URL('../data/policy.ts', import.meta.url).pathname; } + /** Absolute path to the data directory used by Ditto. */ + static get dataDir(): string { + return Deno.env.get('DITTO_DATA_DIR') || new URL('../data', import.meta.url).pathname; + } + /** Absolute path of the Deno directory. */ + static get denoDir(): string { + return Deno.env.get('DENO_DIR') || `${os.userInfo().homedir}/.cache/deno`; + } /** Whether zap splits should be enabled. */ static get zapSplitsEnabled(): boolean { return optionalBooleanSchema.parse(Deno.env.get('ZAP_SPLITS_ENABLED')) ?? false; diff --git a/src/workers/policy.ts b/src/workers/policy.ts index 08511ded..f86f9d9b 100644 --- a/src/workers/policy.ts +++ b/src/workers/policy.ts @@ -13,8 +13,8 @@ export const policyWorker = Comlink.wrap( type: 'module', deno: { permissions: { - read: [Conf.policy], - write: false, + read: [Conf.denoDir, Conf.policy, Conf.dataDir], + write: [Conf.dataDir], net: 'inherit', env: false, }, @@ -24,7 +24,12 @@ export const policyWorker = Comlink.wrap( ); try { - await policyWorker.init(Conf.policy, Conf.databaseUrl, Conf.pubkey); + await policyWorker.init({ + path: Conf.policy, + cwd: Deno.cwd(), + databaseUrl: Conf.databaseUrl, + adminPubkey: Conf.pubkey, + }); console.debug(`Using custom policy: ${Conf.policy}`); } catch (e) { if (e.message.includes('Module not found')) { diff --git a/src/workers/policy.worker.ts b/src/workers/policy.worker.ts index 0036c4bd..3fb4ef3f 100644 --- a/src/workers/policy.worker.ts +++ b/src/workers/policy.worker.ts @@ -6,6 +6,18 @@ import * as Comlink from 'comlink'; import { DittoDB } from '@/db/DittoDB.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; +/** Serializable object the worker can use to set up the state. */ +interface PolicyInit { + /** Path to the policy module (https, jsr, file, etc) */ + path: string; + /** Current working directory. */ + cwd: string; + /** Database URL to connect to. */ + databaseUrl: string; + /** Admin pubkey to use for EventsDB checks. */ + adminPubkey: string; +} + export class CustomPolicy implements NPolicy { private policy: NPolicy = new ReadOnlyPolicy(); @@ -14,7 +26,11 @@ export class CustomPolicy implements NPolicy { return this.policy.call(event); } - async init(path: string, databaseUrl: string, adminPubkey: string): Promise { + async init({ path, cwd, databaseUrl, adminPubkey }: PolicyInit): Promise { + // HACK: PGlite uses `path.resolve`, which requires read permission on Deno (which we don't want to give). + // We can work around this getting the cwd from the caller and overwriting `Deno.cwd`. + Deno.cwd = () => cwd; + const { kysely } = DittoDB.create(databaseUrl, { poolSize: 1 }); const store = new EventsDB({