Add db:export command

deno.json

@@ -5,6 +5,7 @@
     "start": "deno run -A src/server.ts",
     "dev": "deno run -A --watch src/server.ts",
     "hook": "deno run --allow-read --allow-run --allow-write https://deno.land/x/deno_hooks@0.1.1/mod.ts",
+    "db:export": "deno run -A scripts/db-export.ts",
     "db:migrate": "deno run -A scripts/db-migrate.ts",
     "nostr:pull": "deno run -A scripts/nostr-pull.ts",
     "debug": "deno run -A --inspect src/server.ts",
@@ -28,7 +29,7 @@
     "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1",
     "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1",
     "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0",
-    "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.28.0",
+    "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.29.0",
     "@scure/base": "npm:@scure/base@^1.1.6",
     "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs",
     "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0",

deno.lock

@@ -14,7 +14,7 @@
       "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5",
       "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4",
       "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5",
-      "jsr:@nostrify/nostrify@^0.28.0": "jsr:@nostrify/nostrify@0.28.0",
+      "jsr:@nostrify/nostrify@^0.29.0": "jsr:@nostrify/nostrify@0.29.0",
       "jsr:@soapbox/kysely-deno-sqlite@^2.1.0": "jsr:@soapbox/kysely-deno-sqlite@2.2.0",
       "jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0",
       "jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0",
@@ -24,6 +24,7 @@
       "jsr:@std/bytes@^0.224.0": "jsr:@std/bytes@0.224.0",
       "jsr:@std/bytes@^1.0.0-rc.3": "jsr:@std/bytes@1.0.0",
       "jsr:@std/bytes@^1.0.1-rc.3": "jsr:@std/bytes@1.0.2",
+      "jsr:@std/bytes@^1.0.2-rc.3": "jsr:@std/bytes@1.0.2",
       "jsr:@std/crypto@^0.224.0": "jsr:@std/crypto@0.224.0",
       "jsr:@std/dotenv@^0.224.0": "jsr:@std/dotenv@0.224.2",
       "jsr:@std/encoding@^0.221.0": "jsr:@std/encoding@0.221.0",
@@ -33,7 +34,7 @@
       "jsr:@std/fs@^0.221.0": "jsr:@std/fs@0.221.0",
       "jsr:@std/fs@^0.229.3": "jsr:@std/fs@0.229.3",
       "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.1",
-      "jsr:@std/io@^0.224": "jsr:@std/io@0.224.3",
+      "jsr:@std/io@^0.224": "jsr:@std/io@0.224.4",
       "jsr:@std/media-types@^0.224.1": "jsr:@std/media-types@0.224.1",
       "jsr:@std/path@0.217": "jsr:@std/path@0.217.0",
       "jsr:@std/path@^0.221.0": "jsr:@std/path@0.221.0",
@@ -159,8 +160,8 @@
           "npm:zod@^3.23.8"
         ]
       },
-      "@nostrify/nostrify@0.28.0": {
-        "integrity": "abaacd679e2a00a4394d60858d67e5b9a11605c785526d7f6ba354ccd8df087d",
+      "@nostrify/nostrify@0.29.0": {
+        "integrity": "d0489b62441c891324cce60c14bb398013259494b5ad9d21ec6dfbf0ca7368c9",
         "dependencies": [
           "jsr:@std/crypto@^0.224.0",
           "jsr:@std/encoding@^0.224.1",
@@ -263,6 +264,12 @@
           "jsr:@std/bytes@^1.0.1-rc.3"
         ]
       },
+      "@std/io@0.224.4": {
+        "integrity": "bce1151765e4e70e376039fd72c71672b4d4aae363878a5ee3e58361b81197ec",
+        "dependencies": [
+          "jsr:@std/bytes@^1.0.2-rc.3"
+        ]
+      },
       "@std/media-types@0.224.1": {
         "integrity": "9e69a5daed37c5b5c6d3ce4731dc191f80e67f79bed392b0957d1d03b87f11e1"
       },
@@ -1785,7 +1792,7 @@
       "jsr:@db/sqlite@^0.11.1",
       "jsr:@hono/hono@^4.4.6",
       "jsr:@lambdalisue/async@^2.1.1",
-      "jsr:@nostrify/nostrify@^0.28.0",
+      "jsr:@nostrify/nostrify@^0.29.0",
       "jsr:@soapbox/kysely-deno-sqlite@^2.1.0",
       "jsr:@soapbox/stickynotes@^0.4.0",
       "jsr:@std/assert@^0.225.1",

scripts/db-export.ts

@@ -0,0 +1,16 @@
+import { Storages } from '@/storages.ts';
+
+const store = await Storages.db();
+
+console.warn('Exporting events...');
+
+for await (const msg of store.req([{}])) {
+  if (msg[0] === 'EVENT') console.log(JSON.stringify(msg[2]));
+  if (msg[0] === 'EOSE') break;
+  if (msg[0] === 'CLOSED') {
+    console.error('Database closed unexpectedly');
+    break;
+  }
+}
+
+console.warn('Done!');

src/db/KyselyLogger.ts

@@ -1,6 +1,6 @@
 import { Stickynotes } from '@soapbox/stickynotes';
 import { Logger } from 'kysely';
-import { dbQueryTimeHistogram } from '@/metrics.ts';
+import { dbQueryCounter, dbQueryTimeHistogram } from '@/metrics.ts';
 
 /** Log the SQL for queries. */
 export const KyselyLogger: Logger = (event) => {
@@ -9,6 +9,7 @@ export const KyselyLogger: Logger = (event) => {
   const { query, queryDurationMillis } = event;
   const { sql, parameters } = query;
 
+  dbQueryCounter.inc();
   dbQueryTimeHistogram.observe(queryDurationMillis);
 
   console.debug(

src/storages/EventsDB.ts

@@ -1,13 +1,24 @@
 // deno-lint-ignore-file require-await
 
-import { NDatabase, NIP50, NKinds, NostrEvent, NostrFilter, NSchema as n, NStore } from '@nostrify/nostrify';
+import {
+  NDatabase,
+  NIP50,
+  NKinds,
+  NostrEvent,
+  NostrFilter,
+  NostrRelayCLOSED,
+  NostrRelayEOSE,
+  NostrRelayEVENT,
+  NSchema as n,
+  NStore,
+} from '@nostrify/nostrify';
 import { Stickynotes } from '@soapbox/stickynotes';
 import { Kysely } from 'kysely';
 import { nip27 } from 'nostr-tools';
 
 import { Conf } from '@/config.ts';
 import { DittoTables } from '@/db/DittoTables.ts';
-import { dbEventCounter, dbQueryCounter } from '@/metrics.ts';
+import { dbEventCounter } from '@/metrics.ts';
 import { RelayError } from '@/RelayError.ts';
 import { purifyEvent } from '@/storages/hydrate.ts';
 import { isNostrId, isURL } from '@/utils.ts';
@@ -137,13 +148,20 @@ class EventsDB implements NStore {
     }
   }
 
+  /** Stream events from the database. */
+  req(
+    filters: NostrFilter[],
+    opts: { signal?: AbortSignal } = {},
+  ): AsyncIterable<NostrRelayEVENT | NostrRelayEOSE | NostrRelayCLOSED> {
+    return this.store.req(filters, opts);
+  }
+
   /** Get events for filters from the database. */
   async query(
     filters: NostrFilter[],
     opts: { signal?: AbortSignal; timeout?: number; limit?: number } = {},
   ): Promise<NostrEvent[]> {
     filters = await this.expandFilters(filters);
-    dbQueryCounter.inc();
 
     for (const filter of filters) {
       if (filter.since && filter.since >= 2_147_483_647) {