From 80f6172a64fcf31148d04571863f974966c508b7 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 30 Jun 2024 15:35:46 +0530 Subject: [PATCH 1/8] create first version of import script --- deno.json | 1 + scripts/db-import.ts | 131 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 scripts/db-import.ts diff --git a/deno.json b/deno.json index 9c0cec5a..48c927f2 100644 --- a/deno.json +++ b/deno.json @@ -6,6 +6,7 @@ "dev": "deno run -A --watch src/server.ts", "hook": "deno run --allow-read --allow-run --allow-write https://deno.land/x/deno_hooks@0.1.1/mod.ts", "db:migrate": "deno run -A scripts/db-migrate.ts", + "db:import": "deno run -A scripts/db-import.ts", "debug": "deno run -A --inspect src/server.ts", "test": "DATABASE_URL=\"sqlite://:memory:\" deno test -A --junit-path=./deno-test.xml", "check": "deno check src/server.ts", diff --git a/scripts/db-import.ts b/scripts/db-import.ts new file mode 100644 index 00000000..33484507 --- /dev/null +++ b/scripts/db-import.ts @@ -0,0 +1,131 @@ +/** + * Script to import a user/list of users into Ditto given their npub/pubkey by looking them up on a list of relays. + */ + +import { nip19 } from 'npm:nostr-tools@^2.7.0'; +import { DittoDB } from '@/db/DittoDB.ts'; +import { EventsDB } from '@/storages/EventsDB.ts'; +import { NSchema, NRelay1, NostrEvent } from '@nostrify/nostrify'; + + +const kysely = await DittoDB.getInstance(); +const eventsDB = new EventsDB(kysely); + +interface ImportEventsOpts { + profilesOnly: boolean; +} + +type DoEvent = (evt: NostrEvent) => void | Promise; +const importUsers = async (authors: string[], relays: string[], doEvent: DoEvent = (evt: NostrEvent) => eventsDB.event(evt), opts?: Partial) => { + // Kind 0s + follow lists. + const profiles: Record> = {}; + // Kind 1s. + const notes = new Set(); + + await Promise.all(relays.map(async relay => { + if (!relay.startsWith('wss://')) console.error(`Invalid relay url ${relay}`); + const conn = new NRelay1(relay); + const kinds = [0, 3]; + if (!opts?.profilesOnly) { + kinds.push(1); + } + const matched = await conn.query([{ kinds, authors, limit: 1000 }]); + await conn.close(); + await Promise.all( + matched.map(async event => { + const { kind, pubkey } = event; + if (kind === 1 && !notes.has(event.id)) { + // add the event to eventsDB only if it has not been found already. + notes.add(event.id); + await doEvent(event); + return; + } + + profiles[pubkey] ??= {}; + const existing = profiles[pubkey][kind]; + if (existing.created_at > event.created_at) return; + else profiles[pubkey][kind] = event; + }) + ) + })) + + + for (const user in profiles) { + const profile = profiles[user]; + for (const kind in profile) { + await doEvent(profile[kind]); + } + } +} + +if (import.meta.main) { + if (!Deno.args.length) { + showHelp(); + Deno.exit(1); + } + const pubkeys: string[] = []; + const relays: string[] = []; + + const opts: Partial = {}; + + let optionsEnd = false; + let relaySectionBegun = false; + for (const arg of Deno.args) { + if (arg.startsWith('-')) { + if (optionsEnd) { + console.error("Option encountered after end of options section."); + showUsage(); + } + switch (arg) { + case '-p': + case '--profile-only': + console.log('Only importing profiles.'); + opts.profilesOnly = true; + break; + } + } + else if (arg.startsWith('npub1')) { + optionsEnd = true; + + if (relaySectionBegun) { + console.error('npub specified in relay section'); + Deno.exit(1); + } + const decoded = nip19.decode(arg as `npub1${string}`).data; + if (!NSchema.id().safeParse(decoded).success) { + console.error(`invalid pubkey ${arg}, skipping...`); + continue; + } + pubkeys.push(decoded); + } + else { + relaySectionBegun = true; + if (!arg.startsWith('wss://')) { + console.error(`invalid relay url ${arg}, skipping...`); + } + relays.push(arg); + } + } + + await importUsers(pubkeys, relays, console.log, opts); +} + +await kysely.destroy(); + +function showHelp() { + console.log('ditto - db:import'); + console.log('Import users\' posts and kind 0s from a given set of relays.\n'); + showUsage(); + console.log(` +OPTIONS: + +-p, --profile-only + Only import profiles and not posts. Default: off. +`); + +} + +function showUsage() { + console.log('Usage: deno task db:import [options] npub1xxxxxx[ npub1yyyyyyy]...' + + ' wss://first.relay[ second.relay]...'); +} From 9f3f6917d38fe995178c24ee931652544dfbd574 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 30 Jun 2024 15:50:25 +0530 Subject: [PATCH 2/8] import script works now --- scripts/db-import.ts | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/scripts/db-import.ts b/scripts/db-import.ts index 33484507..994728b0 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -16,19 +16,19 @@ interface ImportEventsOpts { } type DoEvent = (evt: NostrEvent) => void | Promise; -const importUsers = async (authors: string[], relays: string[], doEvent: DoEvent = (evt: NostrEvent) => eventsDB.event(evt), opts?: Partial) => { +const importUsers = async (authors: string[], relays: string[], opts?: Partial, doEvent: DoEvent = async (evt: NostrEvent) => await eventsDB.event(evt)) => { // Kind 0s + follow lists. const profiles: Record> = {}; // Kind 1s. const notes = new Set(); + const { profilesOnly = false } = opts || {}; + await Promise.all(relays.map(async relay => { if (!relay.startsWith('wss://')) console.error(`Invalid relay url ${relay}`); const conn = new NRelay1(relay); const kinds = [0, 3]; - if (!opts?.profilesOnly) { - kinds.push(1); - } + if (!profilesOnly) kinds.push(1); const matched = await conn.query([{ kinds, authors, limit: 1000 }]); await conn.close(); await Promise.all( @@ -43,7 +43,7 @@ const importUsers = async (authors: string[], relays: string[], doEvent: DoEvent profiles[pubkey] ??= {}; const existing = profiles[pubkey][kind]; - if (existing.created_at > event.created_at) return; + if (existing?.created_at > event.created_at) return; else profiles[pubkey][kind] = event; }) ) @@ -79,7 +79,7 @@ if (import.meta.main) { switch (arg) { case '-p': case '--profile-only': - console.log('Only importing profiles.'); + console.info('Only importing profiles.'); opts.profilesOnly = true; break; } @@ -107,25 +107,23 @@ if (import.meta.main) { } } - await importUsers(pubkeys, relays, console.log, opts); + await importUsers(pubkeys, relays, opts); + Deno.exit(0); } -await kysely.destroy(); - function showHelp() { - console.log('ditto - db:import'); - console.log('Import users\' posts and kind 0s from a given set of relays.\n'); + console.info('ditto - db:import'); + console.info('Import users\' posts and kind 0s from a given set of relays.\n'); showUsage(); - console.log(` + console.info(` OPTIONS: -p, --profile-only Only import profiles and not posts. Default: off. `); - } function showUsage() { - console.log('Usage: deno task db:import [options] npub1xxxxxx[ npub1yyyyyyy]...' + + console.info('Usage: deno task db:import [options] npub1xxxxxx[ npub1yyyyyyy]...' + ' wss://first.relay[ second.relay]...'); } From 480f4ed370736d06c48da0e2d52b9922fbc87119 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 30 Jun 2024 16:31:02 +0530 Subject: [PATCH 3/8] fmt --- scripts/db-import.ts | 189 ++++++++++++++++++++++--------------------- 1 file changed, 96 insertions(+), 93 deletions(-) diff --git a/scripts/db-import.ts b/scripts/db-import.ts index 994728b0..e1484c99 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -5,117 +5,118 @@ import { nip19 } from 'npm:nostr-tools@^2.7.0'; import { DittoDB } from '@/db/DittoDB.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; -import { NSchema, NRelay1, NostrEvent } from '@nostrify/nostrify'; - +import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify'; const kysely = await DittoDB.getInstance(); const eventsDB = new EventsDB(kysely); interface ImportEventsOpts { - profilesOnly: boolean; + profilesOnly: boolean; } type DoEvent = (evt: NostrEvent) => void | Promise; -const importUsers = async (authors: string[], relays: string[], opts?: Partial, doEvent: DoEvent = async (evt: NostrEvent) => await eventsDB.event(evt)) => { - // Kind 0s + follow lists. - const profiles: Record> = {}; - // Kind 1s. - const notes = new Set(); +const importUsers = async ( + authors: string[], + relays: string[], + opts?: Partial, + doEvent: DoEvent = async (evt: NostrEvent) => await eventsDB.event(evt), +) => { + // Kind 0s + follow lists. + const profiles: Record> = {}; + // Kind 1s. + const notes = new Set(); - const { profilesOnly = false } = opts || {}; + const { profilesOnly = false } = opts || {}; - await Promise.all(relays.map(async relay => { - if (!relay.startsWith('wss://')) console.error(`Invalid relay url ${relay}`); - const conn = new NRelay1(relay); - const kinds = [0, 3]; - if (!profilesOnly) kinds.push(1); - const matched = await conn.query([{ kinds, authors, limit: 1000 }]); - await conn.close(); - await Promise.all( - matched.map(async event => { - const { kind, pubkey } = event; - if (kind === 1 && !notes.has(event.id)) { - // add the event to eventsDB only if it has not been found already. - notes.add(event.id); - await doEvent(event); - return; - } - - profiles[pubkey] ??= {}; - const existing = profiles[pubkey][kind]; - if (existing?.created_at > event.created_at) return; - else profiles[pubkey][kind] = event; - }) - ) - })) - - - for (const user in profiles) { - const profile = profiles[user]; - for (const kind in profile) { - await doEvent(profile[kind]); + await Promise.all(relays.map(async (relay) => { + if (!relay.startsWith('wss://')) console.error(`Invalid relay url ${relay}`); + const conn = new NRelay1(relay); + const kinds = [0, 3]; + if (!profilesOnly) kinds.push(1); + const matched = await conn.query([{ kinds, authors, limit: 1000 }]); + await conn.close(); + await Promise.all( + matched.map(async (event) => { + const { kind, pubkey } = event; + if (kind === 1 && !notes.has(event.id)) { + // add the event to eventsDB only if it has not been found already. + notes.add(event.id); + await doEvent(event); + return; } + + profiles[pubkey] ??= {}; + const existing = profiles[pubkey][kind]; + if (existing?.created_at > event.created_at) return; + else profiles[pubkey][kind] = event; + }), + ); + })); + + for (const user in profiles) { + const profile = profiles[user]; + for (const kind in profile) { + await doEvent(profile[kind]); } -} + } +}; if (import.meta.main) { - if (!Deno.args.length) { - showHelp(); + if (!Deno.args.length) { + showHelp(); + Deno.exit(1); + } + const pubkeys: string[] = []; + const relays: string[] = []; + + const opts: Partial = {}; + + let optionsEnd = false; + let relaySectionBegun = false; + for (const arg of Deno.args) { + if (arg.startsWith('-')) { + if (optionsEnd) { + console.error('Option encountered after end of options section.'); + showUsage(); + } + switch (arg) { + case '-p': + case '--profile-only': + console.info('Only importing profiles.'); + opts.profilesOnly = true; + break; + } + } else if (arg.startsWith('npub1')) { + optionsEnd = true; + + if (relaySectionBegun) { + console.error('npub specified in relay section'); Deno.exit(1); + } + const decoded = nip19.decode(arg as `npub1${string}`).data; + if (!NSchema.id().safeParse(decoded).success) { + console.error(`invalid pubkey ${arg}, skipping...`); + continue; + } + pubkeys.push(decoded); + } else { + relaySectionBegun = true; + if (!arg.startsWith('wss://')) { + console.error(`invalid relay url ${arg}, skipping...`); + } + relays.push(arg); } - const pubkeys: string[] = []; - const relays: string[] = []; + } - const opts: Partial = {}; - - let optionsEnd = false; - let relaySectionBegun = false; - for (const arg of Deno.args) { - if (arg.startsWith('-')) { - if (optionsEnd) { - console.error("Option encountered after end of options section."); - showUsage(); - } - switch (arg) { - case '-p': - case '--profile-only': - console.info('Only importing profiles.'); - opts.profilesOnly = true; - break; - } - } - else if (arg.startsWith('npub1')) { - optionsEnd = true; - - if (relaySectionBegun) { - console.error('npub specified in relay section'); - Deno.exit(1); - } - const decoded = nip19.decode(arg as `npub1${string}`).data; - if (!NSchema.id().safeParse(decoded).success) { - console.error(`invalid pubkey ${arg}, skipping...`); - continue; - } - pubkeys.push(decoded); - } - else { - relaySectionBegun = true; - if (!arg.startsWith('wss://')) { - console.error(`invalid relay url ${arg}, skipping...`); - } - relays.push(arg); - } - } - - await importUsers(pubkeys, relays, opts); - Deno.exit(0); + await importUsers(pubkeys, relays, opts); + Deno.exit(0); } function showHelp() { - console.info('ditto - db:import'); - console.info('Import users\' posts and kind 0s from a given set of relays.\n'); - showUsage(); - console.info(` + console.info('ditto - db:import'); + console.info("Import users' posts and kind 0s from a given set of relays.\n"); + showUsage(); + console.info(` OPTIONS: -p, --profile-only @@ -124,6 +125,8 @@ OPTIONS: } function showUsage() { - console.info('Usage: deno task db:import [options] npub1xxxxxx[ npub1yyyyyyy]...' + - ' wss://first.relay[ second.relay]...'); + console.info( + 'Usage: deno task db:import [options] npub1xxxxxx[ npub1yyyyyyy]...' + + ' wss://first.relay[ second.relay]...', + ); } From ea987dfa14a9b7d7dc9aba5cb434985ed71c2c1f Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 1 Jul 2024 23:06:09 +0530 Subject: [PATCH 4/8] support raw pubkeys as well as npubs --- scripts/db-import.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/db-import.ts b/scripts/db-import.ts index e1484c99..954d83df 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -99,6 +99,8 @@ if (import.meta.main) { continue; } pubkeys.push(decoded); + } else if (NSchema.id().safeParse(arg).success) { + pubkeys.push(arg); } else { relaySectionBegun = true; if (!arg.startsWith('wss://')) { From 434056b83920bbd7312c35d3f893efabadaf148c Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 1 Jul 2024 23:06:41 +0530 Subject: [PATCH 5/8] rename evt to event --- scripts/db-import.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/db-import.ts b/scripts/db-import.ts index 954d83df..806ce326 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -14,12 +14,12 @@ interface ImportEventsOpts { profilesOnly: boolean; } -type DoEvent = (evt: NostrEvent) => void | Promise; +type DoEvent = (event: NostrEvent) => void | Promise; const importUsers = async ( authors: string[], relays: string[], opts?: Partial, - doEvent: DoEvent = async (evt: NostrEvent) => await eventsDB.event(evt), + doEvent: DoEvent = async (event: NostrEvent) => await eventsDB.event(event), ) => { // Kind 0s + follow lists. const profiles: Record> = {}; From 98e9ccfc46b0fdd4a1ed65bee53433e53d60c874 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 1 Jul 2024 23:08:15 +0530 Subject: [PATCH 6/8] fix import --- scripts/db-import.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/db-import.ts b/scripts/db-import.ts index 806ce326..2c8a60f4 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -2,7 +2,7 @@ * Script to import a user/list of users into Ditto given their npub/pubkey by looking them up on a list of relays. */ -import { nip19 } from 'npm:nostr-tools@^2.7.0'; +import { nip19 } from 'nostr-tools'; import { DittoDB } from '@/db/DittoDB.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify'; From e07be77de75f31387aa82b8f3f189e58ba8f166b Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 1 Jul 2024 23:13:20 +0530 Subject: [PATCH 7/8] fix import order --- scripts/db-import.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/db-import.ts b/scripts/db-import.ts index 2c8a60f4..03e389cb 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -2,10 +2,11 @@ * Script to import a user/list of users into Ditto given their npub/pubkey by looking them up on a list of relays. */ +import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; + import { DittoDB } from '@/db/DittoDB.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; -import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify'; const kysely = await DittoDB.getInstance(); const eventsDB = new EventsDB(kysely); From 6974b78952d9836134f3cc5cba2052da8bdba589 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 1 Jul 2024 23:21:29 +0530 Subject: [PATCH 8/8] db:import --> nostr:pull --- deno.json | 2 +- scripts/{db-import.ts => nostr-pull.ts} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename scripts/{db-import.ts => nostr-pull.ts} (100%) diff --git a/deno.json b/deno.json index 48c927f2..21ad4f2d 100644 --- a/deno.json +++ b/deno.json @@ -6,7 +6,7 @@ "dev": "deno run -A --watch src/server.ts", "hook": "deno run --allow-read --allow-run --allow-write https://deno.land/x/deno_hooks@0.1.1/mod.ts", "db:migrate": "deno run -A scripts/db-migrate.ts", - "db:import": "deno run -A scripts/db-import.ts", + "nostr:pull": "deno run -A scripts/nostr-pull.ts", "debug": "deno run -A --inspect src/server.ts", "test": "DATABASE_URL=\"sqlite://:memory:\" deno test -A --junit-path=./deno-test.xml", "check": "deno check src/server.ts", diff --git a/scripts/db-import.ts b/scripts/nostr-pull.ts similarity index 100% rename from scripts/db-import.ts rename to scripts/nostr-pull.ts