diff --git a/deno.json b/deno.json index aa9729e1..bbd2f8f7 100644 --- a/deno.json +++ b/deno.json @@ -5,6 +5,7 @@ "start": "deno run -A src/server.ts", "dev": "deno run -A --watch src/server.ts", "hook": "deno run --allow-read --allow-run --allow-write https://deno.land/x/deno_hooks@0.1.1/mod.ts", + "db:export": "deno run -A scripts/db-export.ts", "db:migrate": "deno run -A scripts/db-migrate.ts", "nostr:pull": "deno run -A scripts/nostr-pull.ts", "debug": "deno run -A --inspect src/server.ts", @@ -28,7 +29,7 @@ "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.28.0", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.29.0", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", diff --git a/deno.lock b/deno.lock index 2f1f6090..1687f132 100644 --- a/deno.lock +++ b/deno.lock @@ -14,7 +14,7 @@ "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", - "jsr:@nostrify/nostrify@^0.28.0": "jsr:@nostrify/nostrify@0.28.0", + "jsr:@nostrify/nostrify@^0.29.0": "jsr:@nostrify/nostrify@0.29.0", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0": "jsr:@soapbox/kysely-deno-sqlite@2.2.0", "jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0", "jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0", @@ -24,6 +24,7 @@ "jsr:@std/bytes@^0.224.0": "jsr:@std/bytes@0.224.0", "jsr:@std/bytes@^1.0.0-rc.3": "jsr:@std/bytes@1.0.0", "jsr:@std/bytes@^1.0.1-rc.3": "jsr:@std/bytes@1.0.2", + "jsr:@std/bytes@^1.0.2-rc.3": "jsr:@std/bytes@1.0.2", "jsr:@std/crypto@^0.224.0": "jsr:@std/crypto@0.224.0", "jsr:@std/dotenv@^0.224.0": "jsr:@std/dotenv@0.224.2", "jsr:@std/encoding@^0.221.0": "jsr:@std/encoding@0.221.0", @@ -33,7 +34,7 @@ "jsr:@std/fs@^0.221.0": "jsr:@std/fs@0.221.0", "jsr:@std/fs@^0.229.3": "jsr:@std/fs@0.229.3", "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.1", - "jsr:@std/io@^0.224": "jsr:@std/io@0.224.3", + "jsr:@std/io@^0.224": "jsr:@std/io@0.224.4", "jsr:@std/media-types@^0.224.1": "jsr:@std/media-types@0.224.1", "jsr:@std/path@0.217": "jsr:@std/path@0.217.0", "jsr:@std/path@^0.221.0": "jsr:@std/path@0.221.0", @@ -159,8 +160,8 @@ "npm:zod@^3.23.8" ] }, - "@nostrify/nostrify@0.28.0": { - "integrity": "abaacd679e2a00a4394d60858d67e5b9a11605c785526d7f6ba354ccd8df087d", + "@nostrify/nostrify@0.29.0": { + "integrity": "d0489b62441c891324cce60c14bb398013259494b5ad9d21ec6dfbf0ca7368c9", "dependencies": [ "jsr:@std/crypto@^0.224.0", "jsr:@std/encoding@^0.224.1", @@ -263,6 +264,12 @@ "jsr:@std/bytes@^1.0.1-rc.3" ] }, + "@std/io@0.224.4": { + "integrity": "bce1151765e4e70e376039fd72c71672b4d4aae363878a5ee3e58361b81197ec", + "dependencies": [ + "jsr:@std/bytes@^1.0.2-rc.3" + ] + }, "@std/media-types@0.224.1": { "integrity": "9e69a5daed37c5b5c6d3ce4731dc191f80e67f79bed392b0957d1d03b87f11e1" }, @@ -1785,7 +1792,7 @@ "jsr:@db/sqlite@^0.11.1", "jsr:@hono/hono@^4.4.6", "jsr:@lambdalisue/async@^2.1.1", - "jsr:@nostrify/nostrify@^0.28.0", + "jsr:@nostrify/nostrify@^0.29.0", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", diff --git a/scripts/db-export.ts b/scripts/db-export.ts new file mode 100644 index 00000000..80b82162 --- /dev/null +++ b/scripts/db-export.ts @@ -0,0 +1,16 @@ +import { Storages } from '@/storages.ts'; + +const store = await Storages.db(); + +console.warn('Exporting events...'); + +for await (const msg of store.req([{}])) { + if (msg[0] === 'EVENT') console.log(JSON.stringify(msg[2])); + if (msg[0] === 'EOSE') break; + if (msg[0] === 'CLOSED') { + console.error('Database closed unexpectedly'); + break; + } +} + +console.warn('Done!'); diff --git a/src/db/KyselyLogger.ts b/src/db/KyselyLogger.ts index 5d285cc6..3b5b4398 100644 --- a/src/db/KyselyLogger.ts +++ b/src/db/KyselyLogger.ts @@ -1,6 +1,6 @@ import { Stickynotes } from '@soapbox/stickynotes'; import { Logger } from 'kysely'; -import { dbQueryTimeHistogram } from '@/metrics.ts'; +import { dbQueryCounter, dbQueryTimeHistogram } from '@/metrics.ts'; /** Log the SQL for queries. */ export const KyselyLogger: Logger = (event) => { @@ -9,6 +9,7 @@ export const KyselyLogger: Logger = (event) => { const { query, queryDurationMillis } = event; const { sql, parameters } = query; + dbQueryCounter.inc(); dbQueryTimeHistogram.observe(queryDurationMillis); console.debug( diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index abf076c7..011f5a02 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -1,13 +1,24 @@ // deno-lint-ignore-file require-await -import { NDatabase, NIP50, NKinds, NostrEvent, NostrFilter, NSchema as n, NStore } from '@nostrify/nostrify'; +import { + NDatabase, + NIP50, + NKinds, + NostrEvent, + NostrFilter, + NostrRelayCLOSED, + NostrRelayEOSE, + NostrRelayEVENT, + NSchema as n, + NStore, +} from '@nostrify/nostrify'; import { Stickynotes } from '@soapbox/stickynotes'; import { Kysely } from 'kysely'; import { nip27 } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { DittoTables } from '@/db/DittoTables.ts'; -import { dbEventCounter, dbQueryCounter } from '@/metrics.ts'; +import { dbEventCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; import { isNostrId, isURL } from '@/utils.ts'; @@ -137,13 +148,20 @@ class EventsDB implements NStore { } } + /** Stream events from the database. */ + req( + filters: NostrFilter[], + opts: { signal?: AbortSignal } = {}, + ): AsyncIterable { + return this.store.req(filters, opts); + } + /** Get events for filters from the database. */ async query( filters: NostrFilter[], opts: { signal?: AbortSignal; timeout?: number; limit?: number } = {}, ): Promise { filters = await this.expandFilters(filters); - dbQueryCounter.inc(); for (const filter of filters) { if (filter.since && filter.since >= 2_147_483_647) {