Merge branch 'script-import-user' into 'main'

deno task db:import

See merge request soapbox-pub/ditto!405

deno.json

@@ -6,6 +6,7 @@
     "dev": "deno run -A --watch src/server.ts",
     "hook": "deno run --allow-read --allow-run --allow-write https://deno.land/x/deno_hooks@0.1.1/mod.ts",
     "db:migrate": "deno run -A scripts/db-migrate.ts",
+    "nostr:pull": "deno run -A scripts/nostr-pull.ts",
     "debug": "deno run -A --inspect src/server.ts",
     "test": "DATABASE_URL=\"sqlite://:memory:\" deno test -A --junit-path=./deno-test.xml",
     "check": "deno check src/server.ts",

scripts/nostr-pull.ts

@@ -0,0 +1,135 @@
+/**
+ * Script to import a user/list of users into Ditto given their npub/pubkey by looking them up on a list of relays.
+ */
+
+import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify';
+import { nip19 } from 'nostr-tools';
+
+import { DittoDB } from '@/db/DittoDB.ts';
+import { EventsDB } from '@/storages/EventsDB.ts';
+
+const kysely = await DittoDB.getInstance();
+const eventsDB = new EventsDB(kysely);
+
+interface ImportEventsOpts {
+  profilesOnly: boolean;
+}
+
+type DoEvent = (event: NostrEvent) => void | Promise<void>;
+const importUsers = async (
+  authors: string[],
+  relays: string[],
+  opts?: Partial<ImportEventsOpts>,
+  doEvent: DoEvent = async (event: NostrEvent) => await eventsDB.event(event),
+) => {
+  // Kind 0s + follow lists.
+  const profiles: Record<string, Record<number, NostrEvent>> = {};
+  // Kind 1s.
+  const notes = new Set<string>();
+
+  const { profilesOnly = false } = opts || {};
+
+  await Promise.all(relays.map(async (relay) => {
+    if (!relay.startsWith('wss://')) console.error(`Invalid relay url ${relay}`);
+    const conn = new NRelay1(relay);
+    const kinds = [0, 3];
+    if (!profilesOnly) kinds.push(1);
+    const matched = await conn.query([{ kinds, authors, limit: 1000 }]);
+    await conn.close();
+    await Promise.all(
+      matched.map(async (event) => {
+        const { kind, pubkey } = event;
+        if (kind === 1 && !notes.has(event.id)) {
+          // add the event to eventsDB only if it has not been found already.
+          notes.add(event.id);
+          await doEvent(event);
+          return;
+        }
+
+        profiles[pubkey] ??= {};
+        const existing = profiles[pubkey][kind];
+        if (existing?.created_at > event.created_at) return;
+        else profiles[pubkey][kind] = event;
+      }),
+    );
+  }));
+
+  for (const user in profiles) {
+    const profile = profiles[user];
+    for (const kind in profile) {
+      await doEvent(profile[kind]);
+    }
+  }
+};
+
+if (import.meta.main) {
+  if (!Deno.args.length) {
+    showHelp();
+    Deno.exit(1);
+  }
+  const pubkeys: string[] = [];
+  const relays: string[] = [];
+
+  const opts: Partial<ImportEventsOpts> = {};
+
+  let optionsEnd = false;
+  let relaySectionBegun = false;
+  for (const arg of Deno.args) {
+    if (arg.startsWith('-')) {
+      if (optionsEnd) {
+        console.error('Option encountered after end of options section.');
+        showUsage();
+      }
+      switch (arg) {
+        case '-p':
+        case '--profile-only':
+          console.info('Only importing profiles.');
+          opts.profilesOnly = true;
+          break;
+      }
+    } else if (arg.startsWith('npub1')) {
+      optionsEnd = true;
+
+      if (relaySectionBegun) {
+        console.error('npub specified in relay section');
+        Deno.exit(1);
+      }
+      const decoded = nip19.decode(arg as `npub1${string}`).data;
+      if (!NSchema.id().safeParse(decoded).success) {
+        console.error(`invalid pubkey ${arg}, skipping...`);
+        continue;
+      }
+      pubkeys.push(decoded);
+    } else if (NSchema.id().safeParse(arg).success) {
+      pubkeys.push(arg);
+    } else {
+      relaySectionBegun = true;
+      if (!arg.startsWith('wss://')) {
+        console.error(`invalid relay url ${arg}, skipping...`);
+      }
+      relays.push(arg);
+    }
+  }
+
+  await importUsers(pubkeys, relays, opts);
+  Deno.exit(0);
+}
+
+function showHelp() {
+  console.info('ditto - db:import');
+  console.info("Import users' posts and kind 0s from a given set of relays.\n");
+  showUsage();
+  console.info(`
+OPTIONS:
+
+-p, --profile-only
+  Only import profiles and not posts. Default: off.
+`);
+}
+
+function showUsage() {
+  console.info(
+    'Usage: deno task db:import [options] npub1xxxxxx[ npub1yyyyyyy]...' +
+      ' wss://first.relay[ second.relay]...',
+  );
+}