From ca5c8877053df83b442a3a0fb147d15a790416eb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Feb 2025 15:54:31 -0600 Subject: [PATCH] Remove storages.ts from scripts --- scripts/admin-event.ts | 14 +++++++++----- scripts/admin-role.ts | 16 ++++++++++------ scripts/db-export.ts | 12 ++++++++---- scripts/db-import.ts | 13 ++++++++----- scripts/db-migrate.ts | 10 +++++----- scripts/db-policy.ts | 16 +++++++++++----- scripts/db-populate-extensions.ts | 10 ++++++---- scripts/db-populate-nip05.ts | 19 ++++++++++++++----- scripts/db-populate-search.ts | 13 ++++++++----- scripts/db-streak-recompute.ts | 16 +++++++++------- scripts/nostr-pull.ts | 10 +++++++--- scripts/setup-kind0.ts | 18 +++++++++++------- scripts/stats-recompute.ts | 13 ++++++++----- 13 files changed, 114 insertions(+), 66 deletions(-) diff --git a/scripts/admin-event.ts b/scripts/admin-event.ts index aec9e145..bec49460 100644 --- a/scripts/admin-event.ts +++ b/scripts/admin-event.ts @@ -1,13 +1,17 @@ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { JsonParseStream } from '@std/json/json-parse-stream'; import { TextLineStream } from '@std/streams/text-line-stream'; -import { Conf } from '../packages/ditto/config.ts'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; import { type EventStub } from '../packages/ditto/utils/api.ts'; import { nostrNow } from '../packages/ditto/utils.ts'; -const signer = Conf.signer; -const store = await Storages.db(); +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); + +const { signer } = conf; const readable = Deno.stdin.readable .pipeThrough(new TextDecoderStream()) @@ -22,7 +26,7 @@ for await (const t of readable) { ...t as EventStub, }); - await store.event(event); + await relay.event(event); } Deno.exit(0); diff --git a/scripts/admin-role.ts b/scripts/admin-role.ts index 4da9610e..59b95878 100644 --- a/scripts/admin-role.ts +++ b/scripts/admin-role.ts @@ -1,15 +1,20 @@ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { Conf } from '../packages/ditto/config.ts'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; import { nostrNow } from '../packages/ditto/utils.ts'; -const store = await Storages.db(); +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); const [pubkeyOrNpub, role] = Deno.args; const pubkey = pubkeyOrNpub.startsWith('npub1') ? nip19.decode(pubkeyOrNpub as `npub1${string}`).data : pubkeyOrNpub; +const { signer } = conf; + if (!NSchema.id().safeParse(pubkey).success) { console.error('Invalid pubkey'); Deno.exit(1); @@ -20,10 +25,9 @@ if (!['admin', 'user'].includes(role)) { Deno.exit(1); } -const signer = Conf.signer; const admin = await signer.getPublicKey(); -const [existing] = await store.query([{ +const [existing] = await relay.query([{ kinds: [30382], authors: [admin], '#d': [pubkey], @@ -57,6 +61,6 @@ const event = await signer.signEvent({ created_at: nostrNow(), }); -await store.event(event); +await relay.event(event); Deno.exit(0); diff --git a/scripts/db-export.ts b/scripts/db-export.ts index d36d4f3f..d9295420 100644 --- a/scripts/db-export.ts +++ b/scripts/db-export.ts @@ -1,7 +1,13 @@ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { NostrFilter } from '@nostrify/nostrify'; import { Command, InvalidOptionArgumentError } from 'commander'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; + +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); interface ExportFilter { authors?: string[]; @@ -98,8 +104,6 @@ export function buildFilter(args: ExportFilter) { } async function exportEvents(args: ExportFilter) { - const store = await Storages.db(); - let filter: NostrFilter = {}; try { filter = buildFilter(args); @@ -108,7 +112,7 @@ async function exportEvents(args: ExportFilter) { } let count = 0; - for await (const msg of store.req([filter])) { + for await (const msg of relay.req([filter])) { if (msg[0] === 'EOSE') { break; } diff --git a/scripts/db-import.ts b/scripts/db-import.ts index 2f6c1595..4d27e54a 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -1,13 +1,16 @@ import { Semaphore } from '@core/asyncutil'; +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { NostrEvent } from '@nostrify/nostrify'; import { JsonParseStream } from '@std/json/json-parse-stream'; import { TextLineStream } from '@std/streams/text-line-stream'; -import { Conf } from '../packages/ditto/config.ts'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; -const store = await Storages.db(); -const sem = new Semaphore(Conf.pg.poolSize); +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); +const sem = new Semaphore(conf.pg.poolSize); console.warn('Importing events...'); @@ -27,7 +30,7 @@ for await (const line of readable) { sem.lock(async () => { try { - await store.event(event); + await relay.event(event); console.warn(`(${count}) Event<${event.kind}> ${event.id}`); } catch (error) { if (error instanceof Error && error.message.includes('violates unique constraint')) { diff --git a/scripts/db-migrate.ts b/scripts/db-migrate.ts index 21b8db22..23547eea 100644 --- a/scripts/db-migrate.ts +++ b/scripts/db-migrate.ts @@ -1,9 +1,9 @@ -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; -// This migrates kysely internally. -const kysely = await Storages.kysely(); +const conf = new DittoConf(Deno.env); +await using db = new DittoPolyPg(conf.databaseUrl); -// Close the connection before exiting. -await kysely.destroy(); +await db.migrate(); Deno.exit(); diff --git a/scripts/db-policy.ts b/scripts/db-policy.ts index caab55af..80e217c5 100644 --- a/scripts/db-policy.ts +++ b/scripts/db-policy.ts @@ -1,16 +1,22 @@ -import { policyWorker } from '../packages/ditto/workers/policy.ts'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; + +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; +import { policyWorker } from '../packages/ditto/workers/policy.ts'; + +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); -const db = await Storages.db(); let count = 0; -for await (const msg of db.req([{}])) { +for await (const msg of relay.req([{}])) { const [type, , event] = msg; if (type === 'EOSE') console.log('EOSE'); if (type !== 'EVENT') continue; const [, , ok] = await policyWorker.call(event, AbortSignal.timeout(5000)); if (!ok) { - await db.remove([{ ids: [event.id] }]); + await relay.remove([{ ids: [event.id] }]); count += 1; } } diff --git a/scripts/db-populate-extensions.ts b/scripts/db-populate-extensions.ts index 0cb3a49b..9af8be2a 100644 --- a/scripts/db-populate-extensions.ts +++ b/scripts/db-populate-extensions.ts @@ -1,11 +1,13 @@ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { NostrEvent } from '@nostrify/nostrify'; -import { Storages } from '../packages/ditto/storages.ts'; import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; -const kysely = await Storages.kysely(); +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); -const query = kysely +const query = db.kysely .selectFrom('nostr_events') .select(['id', 'kind', 'content', 'pubkey', 'tags', 'created_at', 'sig']); @@ -14,7 +16,7 @@ for await (const row of query.stream()) { const ext = DittoPgStore.indexExtensions(event); try { - await kysely + await db.kysely .updateTable('nostr_events') .set('search_ext', ext) .where('id', '=', event.id) diff --git a/scripts/db-populate-nip05.ts b/scripts/db-populate-nip05.ts index acfe70da..46e0686d 100644 --- a/scripts/db-populate-nip05.ts +++ b/scripts/db-populate-nip05.ts @@ -1,13 +1,22 @@ import { Semaphore } from '@core/asyncutil'; import { NostrEvent } from '@nostrify/nostrify'; +import { MockRelay } from '@nostrify/nostrify/test'; -import { updateAuthorData } from '../packages/ditto/pipeline.ts'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; + +import { DittoAPIStore } from '../packages/ditto/storages/DittoAPIStore.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; + +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); + +const pgstore = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); +const apistore = new DittoAPIStore({ conf, db, relay: pgstore, pool: new MockRelay() }); -const kysely = await Storages.kysely(); const sem = new Semaphore(5); -const query = kysely +const query = db.kysely .selectFrom('nostr_events') .select(['id', 'kind', 'content', 'pubkey', 'tags', 'created_at', 'sig']) .where('kind', '=', 0); @@ -19,7 +28,7 @@ for await (const row of query.stream(100)) { sem.lock(async () => { const event: NostrEvent = { ...row, created_at: Number(row.created_at) }; - await updateAuthorData(event, AbortSignal.timeout(3000)); + await apistore.updateAuthorData(event, AbortSignal.timeout(3000)); }); } diff --git a/scripts/db-populate-search.ts b/scripts/db-populate-search.ts index e73f79ac..7189b30c 100644 --- a/scripts/db-populate-search.ts +++ b/scripts/db-populate-search.ts @@ -1,11 +1,14 @@ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { NSchema as n } from '@nostrify/nostrify'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; -const store = await Storages.db(); -const kysely = await Storages.kysely(); +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); -for await (const msg of store.req([{ kinds: [0] }])) { +for await (const msg of relay.req([{ kinds: [0] }])) { if (msg[0] === 'EVENT') { const { pubkey, content } = msg[2]; @@ -13,7 +16,7 @@ for await (const msg of store.req([{ kinds: [0] }])) { const search = [name, nip05].filter(Boolean).join(' ').trim(); try { - await kysely.insertInto('author_stats').values({ + await db.kysely.insertInto('author_stats').values({ pubkey, search, followers_count: 0, diff --git a/scripts/db-streak-recompute.ts b/scripts/db-streak-recompute.ts index e45d4f64..6a0f313f 100644 --- a/scripts/db-streak-recompute.ts +++ b/scripts/db-streak-recompute.ts @@ -1,12 +1,14 @@ -import { Conf } from '../packages/ditto/config.ts'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; -const kysely = await Storages.kysely(); -const statsQuery = kysely.selectFrom('author_stats').select('pubkey'); -const { streakWindow } = Conf; +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); + +const statsQuery = db.kysely.selectFrom('author_stats').select('pubkey'); +const { streakWindow } = conf; for await (const { pubkey } of statsQuery.stream(10)) { - const eventsQuery = kysely + const eventsQuery = db.kysely .selectFrom('nostr_events') .select('created_at') .where('pubkey', '=', pubkey) @@ -38,7 +40,7 @@ for await (const { pubkey } of statsQuery.stream(10)) { } if (start && end) { - await kysely + await db.kysely .updateTable('author_stats') .set({ streak_end: end, diff --git a/scripts/nostr-pull.ts b/scripts/nostr-pull.ts index 7c21cb80..d8a4513a 100644 --- a/scripts/nostr-pull.ts +++ b/scripts/nostr-pull.ts @@ -3,12 +3,16 @@ * by looking them up on a list of relays. */ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; -const store = await Storages.db(); +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); interface ImportEventsOpts { profilesOnly: boolean; @@ -19,7 +23,7 @@ const importUsers = async ( authors: string[], relays: string[], opts?: Partial, - doEvent: DoEvent = async (event: NostrEvent) => await store.event(event), + doEvent: DoEvent = async (event: NostrEvent) => await relay.event(event), ) => { // Kind 0s + follow lists. const profiles: Record> = {}; diff --git a/scripts/setup-kind0.ts b/scripts/setup-kind0.ts index 85f7a6ca..b3dd0682 100644 --- a/scripts/setup-kind0.ts +++ b/scripts/setup-kind0.ts @@ -1,9 +1,13 @@ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { Command } from 'commander'; import { NostrEvent } from 'nostr-tools'; -import { nostrNow } from '../packages/ditto/utils.ts'; -import { Conf } from '../packages/ditto/config.ts'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; + +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); function die(code: number, ...args: unknown[]) { console.error(...args); @@ -33,19 +37,19 @@ if (import.meta.main) { content.lud16 = lightning; content.name = name; content.picture = image; - content.website = Conf.localDomain; + content.website = conf.localDomain; - const signer = Conf.signer; + const signer = conf.signer; const bare: Omit = { - created_at: nostrNow(), kind: 0, tags: [], content: JSON.stringify(content), + created_at: Math.floor(Date.now() / 1000), }; const signed = await signer.signEvent(bare); console.log({ content, signed }); - await Storages.db().then((store) => store.event(signed)); + await relay.event(signed); }); await kind0.parseAsync(); diff --git a/scripts/stats-recompute.ts b/scripts/stats-recompute.ts index 942d0012..c17e9047 100644 --- a/scripts/stats-recompute.ts +++ b/scripts/stats-recompute.ts @@ -1,8 +1,14 @@ +import { DittoConf } from '@ditto/conf'; +import { DittoPolyPg } from '@ditto/db'; import { nip19 } from 'nostr-tools'; -import { Storages } from '../packages/ditto/storages.ts'; +import { DittoPgStore } from '../packages/ditto/storages/DittoPgStore.ts'; import { refreshAuthorStats } from '../packages/ditto/utils/stats.ts'; +const conf = new DittoConf(Deno.env); +const db = new DittoPolyPg(conf.databaseUrl); +const relay = new DittoPgStore({ db, pubkey: await conf.signer.getPublicKey() }); + let pubkey: string; try { const result = nip19.decode(Deno.args[0]); @@ -16,7 +22,4 @@ try { Deno.exit(1); } -const store = await Storages.db(); -const kysely = await Storages.kysely(); - -await refreshAuthorStats({ pubkey, kysely, store }); +await refreshAuthorStats({ pubkey, kysely: db.kysely, store: relay });