From 9298bf66081a8f0f037dd84005275c15ce2d90f9 Mon Sep 17 00:00:00 2001 From: Siddharth S Singh Date: Fri, 23 Aug 2024 16:25:34 +0530 Subject: [PATCH 1/3] first commit (nonfunctional) --- deno.json | 1 + deno.lock | 6 +++++ scripts/db-export.ts | 53 ++++++++++++++++++++++++++++++-------------- 3 files changed, 43 insertions(+), 17 deletions(-) diff --git a/deno.json b/deno.json index 96c86b19..75a10262 100644 --- a/deno.json +++ b/deno.json @@ -48,6 +48,7 @@ "@std/streams": "jsr:@std/streams@^0.223.0", "comlink": "npm:comlink@^4.4.1", "comlink-async-generator": "npm:comlink-async-generator@^0.0.1", + "commander": "npm:commander@12.1.0", "deno-safe-fetch/load": "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts", "deno.json": "./deno.json", "entities": "npm:entities@^4.5.0", diff --git a/deno.lock b/deno.lock index 68c03b53..0b4ce371 100644 --- a/deno.lock +++ b/deno.lock @@ -60,6 +60,7 @@ "npm:comlink-async-generator": "npm:comlink-async-generator@0.0.1", "npm:comlink-async-generator@^0.0.1": "npm:comlink-async-generator@0.0.1", "npm:comlink@^4.4.1": "npm:comlink@4.4.1", + "npm:commander@12.1.0": "npm:commander@12.1.0", "npm:entities@^4.5.0": "npm:entities@4.5.0", "npm:fast-stable-stringify@^1.0.0": "npm:fast-stable-stringify@1.0.0", "npm:formdata-helper@^0.3.0": "npm:formdata-helper@0.3.0", @@ -543,6 +544,10 @@ "integrity": "sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==", "dependencies": {} }, + "commander@12.1.0": { + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", + "dependencies": {} + }, "cross-spawn@7.0.3": { "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", "dependencies": { @@ -1904,6 +1909,7 @@ "npm:@scure/base@^1.1.6", "npm:comlink-async-generator@^0.0.1", "npm:comlink@^4.4.1", + "npm:commander@12.1.0", "npm:entities@^4.5.0", "npm:fast-stable-stringify@^1.0.0", "npm:formdata-helper@^0.3.0", diff --git a/scripts/db-export.ts b/scripts/db-export.ts index fbdac1b7..92166d96 100644 --- a/scripts/db-export.ts +++ b/scripts/db-export.ts @@ -1,24 +1,43 @@ import { Storages } from '@/storages.ts'; +import { Command } from 'commander'; const store = await Storages.db(); -console.warn('Exporting events...'); - -let count = 0; - -for await (const msg of store.req([{}])) { - if (msg[0] === 'EOSE') { - break; - } - if (msg[0] === 'EVENT') { - console.log(JSON.stringify(msg[2])); - count++; - } - if (msg[0] === 'CLOSED') { - console.error('Database closed unexpectedly'); - break; - } +interface ExportFilter { + authors: string[]; +} + +if (import.meta.main) { + const exporter = new Command() + .name('db:export') + .description('Export the specified set of events from the Ditto database.') + .version('0.1.0') + .showHelpAfterError(); + + exporter + // .option('') + .action(async (args: ExportFilter) => { + console.warn('Exporting events...'); + let count = 0; + + for await (const msg of store.req([{}])) { + if (msg[0] === 'EOSE') { + break; + } + if (msg[0] === 'EVENT') { + console.log(JSON.stringify(msg[2])); + count++; + } + if (msg[0] === 'CLOSED') { + console.error('Database closed unexpectedly'); + break; + } + } + + console.warn(`Exported ${count} events`); + }); + + exporter.parse(Deno.args, { from: 'user' }); } -console.warn(`Exported ${count} events`); Deno.exit(); From fb199123be9c34c090d76c03af7dfbc21e6c5c5b Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sat, 24 Aug 2024 22:56:35 +0530 Subject: [PATCH 2/3] first working version --- scripts/db-export.ts | 155 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 130 insertions(+), 25 deletions(-) diff --git a/scripts/db-export.ts b/scripts/db-export.ts index 92166d96..1120a190 100644 --- a/scripts/db-export.ts +++ b/scripts/db-export.ts @@ -1,43 +1,148 @@ import { Storages } from '@/storages.ts'; -import { Command } from 'commander'; +import { NostrFilter } from '@nostrify/nostrify'; +import { Command, InvalidOptionArgumentError } from 'commander'; const store = await Storages.db(); interface ExportFilter { - authors: string[]; + authors?: string[]; + ids?: string[]; + kinds?: number[]; + limit?: number; + search?: string; + /** + * Array of `key=value` pairs. + */ + tags?: string[]; + since?: number; + until?: number; + /** + * shortcut for `--tag d=` + */ + d?: string; + /** + * shortcut for `--tag e=` + */ + e?: string; + /** + * shortcut for `--tag p=` + */ + p?: string; +} + +function safeParseInt(s: string) { + const n = parseInt(s); + if (isNaN(n)) throw new InvalidOptionArgumentError('Not a number.'); + return n; +} + +function findInvalid(arr: string[], predicate = (v: string) => !/[a-f0-9]{64}/.test(v)) { + return arr.find(predicate); +} + +function die(code: number, ...args: any[]) { + console.error(...args); + Deno.exit(code); +} + +function tagFilterShortcut(name: 'd' | 'e' | 'p', value: string) { + const val = [value]; + if (findInvalid(val)) die(1, `ERROR: Invalid value supplied for ${name}-tag.`); + return val; +} + +async function exportEvents(args: ExportFilter) { + const filter: NostrFilter = {}; + const { authors, ids, kinds, d, e, limit, p, search, since, until, tags } = args; + if (since) { + filter.since = since; + } + if (until) { + filter.until = until; + } + if (authors && authors.length) { + const invalid = findInvalid(authors); + if (invalid) die(1, `ERROR: Invalid pubkey ${invalid} supplied.`); + filter.authors = authors; + } + if (ids) { + const invalid = findInvalid(ids); + if (invalid) die(1, `ERROR: Invalid event ID ${invalid} supplied.`); + } + if (kinds && kinds.length) { + filter.kinds = kinds; + } + if (d) { + filter['#d'] = tagFilterShortcut('d', d); + } + if (e) { + filter['#e'] = tagFilterShortcut('e', e); + } + if (p) { + filter['#p'] = tagFilterShortcut('e', p); + } + if (search) { + filter.search = search; + } + if (limit) { + filter.limit = limit; + } + if (tags) { + for (const val of tags) { + const [name, ...values] = val.split('='); + filter[`#${name}`] = [values.join('=')]; + } + } + + let count = 0; + for await (const msg of store.req([filter])) { + if (msg[0] === 'EOSE') { + break; + } + if (msg[0] === 'EVENT') { + console.log(JSON.stringify(msg[2])); + count++; + } + if (msg[0] === 'CLOSED') { + console.error('Database closed unexpectedly'); + break; + } + } + + console.warn(`Exported ${count} events`); } if (import.meta.main) { const exporter = new Command() .name('db:export') - .description('Export the specified set of events from the Ditto database.') + .description('Export the specified set of events from the Ditto database, in JSONL format.') .version('0.1.0') .showHelpAfterError(); exporter - // .option('') - .action(async (args: ExportFilter) => { - console.warn('Exporting events...'); - let count = 0; + .option('-a, --authors ', 'Pubkeys of authors whose events you want to export.', []) + .option('-i, --ids ', 'IDs of events you want to export.', []) + .option( + '-k --kinds ', + 'Event kinds you want to export.', + (v: string, arr: number[]) => arr.concat([safeParseInt(v)]), + [], + ) + .option( + '-t --tags ', + 'A list of key=value pairs of tags to search for events using. For tag values with spaces etc, simply quote the entire item, like `deno task db:export -t "name=A string with spaces in it"`.', + [], + ) + .option('--search ', 'A string to full-text search the db for.') + .option('-s --since ', 'The oldest time an exported event should be from.', safeParseInt) + .option('-u --until ', 'The newest time an exported event should be from.', safeParseInt) + .option('--limit ', 'Maximum number of events to export.', safeParseInt) + .option('-d ', 'Shortcut for `--tag d=`.') + .option('-e ', 'Shortcut for `--tag e=`.') + .option('-p ', 'Shortcut for `--tag p=`.') + .action(exportEvents); - for await (const msg of store.req([{}])) { - if (msg[0] === 'EOSE') { - break; - } - if (msg[0] === 'EVENT') { - console.log(JSON.stringify(msg[2])); - count++; - } - if (msg[0] === 'CLOSED') { - console.error('Database closed unexpectedly'); - break; - } - } - - console.warn(`Exported ${count} events`); - }); - - exporter.parse(Deno.args, { from: 'user' }); + await exporter.parseAsync(Deno.args, { from: 'user' }); } Deno.exit(); From 125dfb54f3ced16a5aebc09355b0ec949a0bba76 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 25 Aug 2024 19:41:35 +0530 Subject: [PATCH 3/3] Add tests for db:export filters --- scripts/db-export.test.ts | 94 +++++++++++++++++++++++++++++++++++++++ scripts/db-export.ts | 28 ++++++++---- 2 files changed, 113 insertions(+), 9 deletions(-) create mode 100644 scripts/db-export.test.ts diff --git a/scripts/db-export.test.ts b/scripts/db-export.test.ts new file mode 100644 index 00000000..939537d5 --- /dev/null +++ b/scripts/db-export.test.ts @@ -0,0 +1,94 @@ +import { assertEquals, assertThrows } from '@std/assert'; +import { buildFilter } from './db-export.ts'; + +Deno.test('buildFilter should return an empty filter when no arguments are provided', () => { + const filter = buildFilter({}); + assertEquals(Object.keys(filter).length, 0); +}); + +Deno.test('buildFilter should correctly handle valid authors', () => { + const filter = buildFilter({ + authors: ['a'.repeat(64)], + }); + + assertEquals(filter.authors, ['a'.repeat(64)]); +}); + +Deno.test('buildFilter throws on invalid author pubkey', () => { + assertThrows( + () => { + buildFilter({ + authors: ['invalid_pubkey'], + }); + }, + Error, + 'ERROR: Invalid pubkey invalid_pubkey supplied.', + ); +}); + +Deno.test('buildFilter should correctly handle valid ids', () => { + const filter = buildFilter({ + ids: ['b'.repeat(64)], + }); + + assertEquals(filter.ids, ['b'.repeat(64)]); +}); + +Deno.test('buildFilter should throw on invalid event IDs', () => { + assertThrows( + () => { + buildFilter({ + ids: ['invalid_id'], + }); + }, + Error, + 'ERROR: Invalid event ID invalid_id supplied.', + ); +}); + +Deno.test('buildFilter should correctly handle tag shortcuts', () => { + const filter = buildFilter({ + d: 'value1', + e: 'a'.repeat(64), + p: 'b'.repeat(64), + }); + + assertEquals(filter['#d'], ['value1']); + assertEquals(filter['#e'], ['a'.repeat(64)]); + assertEquals(filter['#p'], ['b'.repeat(64)]); +}); + +Deno.test('buildFilter should correctly handle since and until args', () => { + const filter = buildFilter({ + since: 1000, + until: 2000, + }); + + assertEquals(filter.since, 1000); + assertEquals(filter.until, 2000); +}); + +Deno.test('buildFilter should correctly handle search field', () => { + const filter = buildFilter({ + search: 'search_term', + }); + + assertEquals(filter.search, 'search_term'); +}); + +Deno.test('buildFilter should correctly handle tag k-v pairs', () => { + const filter = buildFilter({ + tags: ['tag1=value1', 'tag2=value2'], + }); + + assertEquals(filter['#tag1'], ['value1']); + assertEquals(filter['#tag2'], ['value2']); +}); + +Deno.test('buildFilter should correctly handle limit specifier', () => { + const filter = buildFilter({ + limit: 10, + }); + + assertEquals(filter.limit, 10); +}); diff --git a/scripts/db-export.ts b/scripts/db-export.ts index 1120a190..71939105 100644 --- a/scripts/db-export.ts +++ b/scripts/db-export.ts @@ -2,8 +2,6 @@ import { Storages } from '@/storages.ts'; import { NostrFilter } from '@nostrify/nostrify'; import { Command, InvalidOptionArgumentError } from 'commander'; -const store = await Storages.db(); - interface ExportFilter { authors?: string[]; ids?: string[]; @@ -47,11 +45,11 @@ function die(code: number, ...args: any[]) { function tagFilterShortcut(name: 'd' | 'e' | 'p', value: string) { const val = [value]; - if (findInvalid(val)) die(1, `ERROR: Invalid value supplied for ${name}-tag.`); + if (findInvalid(val)) throw new Error(`ERROR: Invalid value supplied for ${name}-tag.`); return val; } -async function exportEvents(args: ExportFilter) { +export function buildFilter(args: ExportFilter) { const filter: NostrFilter = {}; const { authors, ids, kinds, d, e, limit, p, search, since, until, tags } = args; if (since) { @@ -62,18 +60,19 @@ async function exportEvents(args: ExportFilter) { } if (authors && authors.length) { const invalid = findInvalid(authors); - if (invalid) die(1, `ERROR: Invalid pubkey ${invalid} supplied.`); + if (invalid) throw new Error(`ERROR: Invalid pubkey ${invalid} supplied.`); filter.authors = authors; } if (ids) { const invalid = findInvalid(ids); - if (invalid) die(1, `ERROR: Invalid event ID ${invalid} supplied.`); + if (invalid) throw new Error(`ERROR: Invalid event ID ${invalid} supplied.`); + filter.ids = ids; } if (kinds && kinds.length) { filter.kinds = kinds; } if (d) { - filter['#d'] = tagFilterShortcut('d', d); + filter['#d'] = [d]; } if (e) { filter['#e'] = tagFilterShortcut('e', e); @@ -94,6 +93,19 @@ async function exportEvents(args: ExportFilter) { } } + return filter; +} + +async function exportEvents(args: ExportFilter) { + const store = await Storages.db(); + + let filter: NostrFilter = {}; + try { + filter = buildFilter(args); + } catch (e) { + die(1, e.message || e.toString()); + } + let count = 0; for await (const msg of store.req([filter])) { if (msg[0] === 'EOSE') { @@ -144,5 +156,3 @@ if (import.meta.main) { await exporter.parseAsync(Deno.args, { from: 'user' }); } - -Deno.exit();