From b33a6cdfe0ac94b0f4df22dece8968478811bc1e Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 30 Sep 2024 13:53:30 -0300 Subject: [PATCH 01/26] feat: add TREND_LANGUAGES environment variable --- src/config.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/config.ts b/src/config.ts index 21fbbe01..0051d3d4 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,4 +1,5 @@ import os from 'node:os'; +import ISO6391, { LanguageCode } from 'iso-639-1'; import * as dotenv from '@std/dotenv'; import { getPublicKey, nip19 } from 'nostr-tools'; import { z } from 'zod'; @@ -247,6 +248,10 @@ class Conf { static get zapSplitsEnabled(): boolean { return optionalBooleanSchema.parse(Deno.env.get('ZAP_SPLITS_ENABLED')) ?? false; } + /** Filter trends by languages. */ + static get trendLanguages(): LanguageCode[] | undefined { + return Deno.env.get('TREND_LANGUAGES')?.split(',')?.filter(ISO6391.validate) as LanguageCode[]; + } /** Cache settings. */ static caches = { /** NIP-05 cache settings. */ From 61bc57c77851770636be2531c33fa0903734f347 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 30 Sep 2024 14:02:12 -0300 Subject: [PATCH 02/26] feat: support trendings by language --- src/trends.ts | 97 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 64 insertions(+), 33 deletions(-) diff --git a/src/trends.ts b/src/trends.ts index 23f7ea4d..0583fd60 100644 --- a/src/trends.ts +++ b/src/trends.ts @@ -1,3 +1,4 @@ +import ISO6391, { LanguageCode } from 'iso-639-1'; import { NostrFilter } from '@nostrify/nostrify'; import { Stickynotes } from '@soapbox/stickynotes'; import { Kysely, sql } from 'kysely'; @@ -19,34 +20,49 @@ export async function getTrendingTagValues( tagNames: string[], /** Filter of eligible events. */ filter: NostrFilter, + /** Only return trending events of 'language' */ + language?: LanguageCode, ): Promise<{ value: string; authors: number; uses: number }[]> { - let query = kysely - .selectFrom([ - 'nostr_events', - sql<{ key: string; value: string }>`jsonb_each_text(nostr_events.tags_index)`.as('kv'), - sql<{ key: string; value: string }>`jsonb_array_elements_text(kv.value::jsonb)`.as('element'), - ]) - .select(({ fn }) => [ - fn('lower', ['element.value']).as('value'), - fn.agg('count', ['nostr_events.pubkey']).distinct().as('authors'), - fn.countAll().as('uses'), - ]) - .where('kv.key', '=', (eb) => eb.fn.any(eb.val(tagNames))) - .groupBy((eb) => eb.fn('lower', ['element.value'])) - .orderBy((eb) => eb.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc'); + let query = kysely.with('trends', (db) => { + let query = db + .selectFrom([ + 'nostr_events', + sql<{ key: string; value: string }>`jsonb_each_text(nostr_events.tags_index)`.as('kv'), + sql<{ key: string; value: string }>`jsonb_array_elements_text(kv.value::jsonb)`.as('element'), + ]) + .select(({ fn }) => [ + fn('lower', ['element.value']).as('value'), + fn.agg('count', ['nostr_events.pubkey']).distinct().as('authors'), + fn.countAll().as('uses'), + ]) + .where('kv.key', '=', (eb) => eb.fn.any(eb.val(tagNames))) + .groupBy((eb) => eb.fn('lower', ['element.value'])) + .orderBy((eb) => eb.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc'); - if (filter.kinds) { - query = query.where('nostr_events.kind', '=', ({ fn, val }) => fn.any(val(filter.kinds))); - } - if (filter.authors) { - query = query.where('nostr_events.pubkey', '=', ({ fn, val }) => fn.any(val(filter.authors))); - } - if (typeof filter.since === 'number') { - query = query.where('nostr_events.created_at', '>=', filter.since); - } - if (typeof filter.until === 'number') { - query = query.where('nostr_events.created_at', '<=', filter.until); + if (filter.kinds) { + query = query.where('nostr_events.kind', '=', ({ fn, val }) => fn.any(val(filter.kinds))); + } + if (filter.authors) { + query = query.where('nostr_events.pubkey', '=', ({ fn, val }) => fn.any(val(filter.authors))); + } + if (typeof filter.since === 'number') { + query = query.where('nostr_events.created_at', '>=', filter.since); + } + if (typeof filter.until === 'number') { + query = query.where('nostr_events.created_at', '<=', filter.until); + } + return query; + }) + .selectFrom(['trends']) + .innerJoin('nostr_events', 'trends.value', 'nostr_events.id') + .select(['value', 'authors', 'uses']); + + if (language) { + query = query.where('nostr_events.language', '=', language); } + + query = query.orderBy('authors desc'); + if (typeof filter.limit === 'number') { query = query.limit(filter.limit); } @@ -68,6 +84,7 @@ export async function updateTrendingTags( limit: number, extra = '', aliases?: string[], + language?: LanguageCode, ) { console.info(`Updating trending ${l}...`); const kysely = await Storages.kysely(); @@ -84,7 +101,7 @@ export async function updateTrendingTags( since: yesterday, until: now, limit, - }); + }, language); if (!trends.length) { console.info(`No trending ${l} found. Skipping.`); @@ -93,14 +110,19 @@ export async function updateTrendingTags( const signer = new AdminSigner(); + const tags = [ + ['L', 'pub.ditto.trends'], + ['l', l, 'pub.ditto.trends'], + ...trends.map(({ value, authors, uses }) => [tagName, value, extra, authors.toString(), uses.toString()]), + ]; + if (language) { + tags.push(['lang', language]); + } + const label = await signer.signEvent({ kind: 1985, content: '', - tags: [ - ['L', 'pub.ditto.trends'], - ['l', l, 'pub.ditto.trends'], - ...trends.map(({ value, authors, uses }) => [tagName, value, extra, authors.toString(), uses.toString()]), - ], + tags, created_at: Math.floor(Date.now() / 1000), }); @@ -122,8 +144,17 @@ export function updateTrendingZappedEvents(): Promise { } /** Update trending events. */ -export function updateTrendingEvents(): Promise { - return updateTrendingTags('#e', 'e', [1, 6, 7, 9735], 40, Conf.relay, ['q']); +export async function updateTrendingEvents(): Promise { + const languages = Conf.trendLanguages; + if (!languages) return updateTrendingTags('#e', 'e', [1, 6, 7, 9735], 40, Conf.relay, ['q']); + + const promise: Promise[] = []; + + for (const language of languages) { + promise.push(updateTrendingTags('#e', 'e', [1, 6, 7, 9735], 40, Conf.relay, ['q'], language)); + } + + await Promise.allSettled(promise); } /** Update trending hashtags. */ From 5e23f4d6361dba28885e28ee346d9ae3d38c75f4 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 30 Sep 2024 14:03:22 -0300 Subject: [PATCH 03/26] test: trends without language and with language --- src/trends.test.ts | 103 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 src/trends.test.ts diff --git a/src/trends.test.ts b/src/trends.test.ts new file mode 100644 index 00000000..1788c496 --- /dev/null +++ b/src/trends.test.ts @@ -0,0 +1,103 @@ +import { assertEquals } from '@std/assert'; +import { generateSecretKey, NostrEvent } from 'nostr-tools'; + +import { getTrendingTagValues } from '@/trends.ts'; +import { createTestDB, genEvent } from '@/test.ts'; + +Deno.test("getTrendingTagValues(): 'e' tag and WITHOUT language parameter", async () => { + await using db = await createTestDB(); + + const events: NostrEvent[] = []; + + let sk = generateSecretKey(); + const post1 = genEvent({ kind: 1, content: 'SHOW ME THE MONEY' }, sk); + const numberOfAuthorsWhoLikedPost1 = 100; + const post1multiplier = 2; + const post1uses = numberOfAuthorsWhoLikedPost1 * post1multiplier; + for (let i = 0; i < numberOfAuthorsWhoLikedPost1; i++) { + const sk = generateSecretKey(); + events.push( + genEvent({ kind: 7, content: '+', tags: Array(post1multiplier).fill([...['e', post1.id]]) }, sk), + ); + } + events.push(post1); + + sk = generateSecretKey(); + const post2 = genEvent({ kind: 1, content: 'Ithaca' }, sk); + const numberOfAuthorsWhoLikedPost2 = 100; + const post2multiplier = 1; + const post2uses = numberOfAuthorsWhoLikedPost2 * post2multiplier; + for (let i = 0; i < numberOfAuthorsWhoLikedPost2; i++) { + const sk = generateSecretKey(); + events.push( + genEvent({ kind: 7, content: '+', tags: Array(post2multiplier).fill([...['e', post2.id]]) }, sk), + ); + } + events.push(post2); + + for (const event of events) { + await db.store.event(event); + } + + const trends = await getTrendingTagValues(db.kysely, ['e'], { kinds: [1, 7] }); + + const expected = [{ value: post1.id, authors: numberOfAuthorsWhoLikedPost1, uses: post1uses }, { + value: post2.id, + authors: numberOfAuthorsWhoLikedPost2, + uses: post2uses, + }]; + + assertEquals(trends, expected); +}); + +Deno.test("getTrendingTagValues(): 'e' tag and WITH language parameter", async () => { + await using db = await createTestDB(); + + const events: NostrEvent[] = []; + + let sk = generateSecretKey(); + const post1 = genEvent({ kind: 1, content: 'Irei cortar o cabelo.' }, sk); + const numberOfAuthorsWhoLikedPost1 = 100; + const post1multiplier = 2; + const post1uses = numberOfAuthorsWhoLikedPost1 * post1multiplier; + for (let i = 0; i < numberOfAuthorsWhoLikedPost1; i++) { + const sk = generateSecretKey(); + events.push( + genEvent({ kind: 7, content: '+', tags: Array(post1multiplier).fill([...['e', post1.id]]) }, sk), + ); + } + events.push(post1); + + sk = generateSecretKey(); + const post2 = genEvent({ kind: 1, content: 'Ithaca' }, sk); + const numberOfAuthorsWhoLikedPost2 = 100; + const post2multiplier = 1; + for (let i = 0; i < numberOfAuthorsWhoLikedPost2; i++) { + const sk = generateSecretKey(); + events.push( + genEvent({ kind: 7, content: '+', tags: Array(post2multiplier).fill([...['e', post2.id]]) }, sk), + ); + } + events.push(post2); + + for (const event of events) { + await db.store.event(event); + } + + await db.kysely.updateTable('nostr_events') + .set('language', 'pt') + .where('id', '=', post1.id) + .execute(); + + await db.kysely.updateTable('nostr_events') + .set('language', 'en') + .where('id', '=', post2.id) + .execute(); + + const trends = await getTrendingTagValues(db.kysely, ['e'], { kinds: [1, 7] }, 'pt'); + + // portuguese post + const expected = [{ value: post1.id, authors: numberOfAuthorsWhoLikedPost1, uses: post1uses }]; + + assertEquals(trends, expected); +}); From c0d9a90bfa6e1cfe829ac9b429ac14c169220203 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 30 Sep 2024 14:09:19 -0300 Subject: [PATCH 04/26] refactor: remove un-used variable --- src/trends.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/trends.ts b/src/trends.ts index 0583fd60..4efaf831 100644 --- a/src/trends.ts +++ b/src/trends.ts @@ -1,4 +1,4 @@ -import ISO6391, { LanguageCode } from 'iso-639-1'; +import { LanguageCode } from 'iso-639-1'; import { NostrFilter } from '@nostrify/nostrify'; import { Stickynotes } from '@soapbox/stickynotes'; import { Kysely, sql } from 'kysely'; From b549cdef536aa9e6ee5d11f5272def0c77d25f23 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 1 Oct 2024 13:52:30 -0300 Subject: [PATCH 05/26] refactor: rename TREND_LANGUAGES to DITTO_LANGUAGES --- src/config.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/config.ts b/src/config.ts index 0051d3d4..ae841997 100644 --- a/src/config.ts +++ b/src/config.ts @@ -248,9 +248,9 @@ class Conf { static get zapSplitsEnabled(): boolean { return optionalBooleanSchema.parse(Deno.env.get('ZAP_SPLITS_ENABLED')) ?? false; } - /** Filter trends by languages. */ - static get trendLanguages(): LanguageCode[] | undefined { - return Deno.env.get('TREND_LANGUAGES')?.split(',')?.filter(ISO6391.validate) as LanguageCode[]; + /** Languages this server wishes to highlight. Used when querying trends.*/ + static get preferredLanguages(): LanguageCode[] | undefined { + return Deno.env.get('DITTO_LANGUAGES')?.split(',')?.filter(ISO6391.validate) as LanguageCode[]; } /** Cache settings. */ static caches = { From d8b2c057b0f3f1bd1c30c131243fe1db0f9085b3 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 1 Oct 2024 13:58:08 -0300 Subject: [PATCH 06/26] feat: make trends fast again remove previous JOIN, now if a language is set, it will do '''query.where('trends.value', 'in', languagesIds);''', which is faster than a JOIN --- src/trends.ts | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/src/trends.ts b/src/trends.ts index 4efaf831..a60e28f3 100644 --- a/src/trends.ts +++ b/src/trends.ts @@ -1,5 +1,5 @@ import { LanguageCode } from 'iso-639-1'; -import { NostrFilter } from '@nostrify/nostrify'; +import { NostrEvent, NostrFilter } from '@nostrify/nostrify'; import { Stickynotes } from '@soapbox/stickynotes'; import { Kysely, sql } from 'kysely'; @@ -20,8 +20,8 @@ export async function getTrendingTagValues( tagNames: string[], /** Filter of eligible events. */ filter: NostrFilter, - /** Only return trending events of 'language' */ - language?: LanguageCode, + /** Results must be inside 'languagesIds' */ + languagesIds?: string[], ): Promise<{ value: string; authors: number; uses: number }[]> { let query = kysely.with('trends', (db) => { let query = db @@ -54,14 +54,13 @@ export async function getTrendingTagValues( return query; }) .selectFrom(['trends']) - .innerJoin('nostr_events', 'trends.value', 'nostr_events.id') .select(['value', 'authors', 'uses']); - if (language) { - query = query.where('nostr_events.language', '=', language); + if (languagesIds) { + query = query.where('trends.value', 'in', languagesIds); } - query = query.orderBy('authors desc'); + query = query.orderBy('authors desc').orderBy('uses desc'); if (typeof filter.limit === 'number') { query = query.limit(filter.limit); @@ -95,13 +94,24 @@ export async function updateTrendingTags( const tagNames = aliases ? [tagName, ...aliases] : [tagName]; + let languagesIds: NostrEvent['id'][] = []; + if (language) { + const result = (await kysely.selectFrom('nostr_events') + .select('id') + .where('language', '=', language) + .where('nostr_events.created_at', '>=', yesterday) + .where('nostr_events.created_at', '<=', now) + .execute()).map((event) => event.id); + languagesIds = result; + } + try { const trends = await getTrendingTagValues(kysely, tagNames, { kinds, since: yesterday, until: now, limit, - }, language); + }, languagesIds); if (!trends.length) { console.info(`No trending ${l} found. Skipping.`); @@ -110,19 +120,14 @@ export async function updateTrendingTags( const signer = new AdminSigner(); - const tags = [ - ['L', 'pub.ditto.trends'], - ['l', l, 'pub.ditto.trends'], - ...trends.map(({ value, authors, uses }) => [tagName, value, extra, authors.toString(), uses.toString()]), - ]; - if (language) { - tags.push(['lang', language]); - } - const label = await signer.signEvent({ kind: 1985, content: '', - tags, + tags: [ + ['L', 'pub.ditto.trends'], + ['l', languagesIds.length ? `${l}.${language}` : l, 'pub.ditto.trends'], + ...trends.map(({ value, authors, uses }) => [tagName, value, extra, authors.toString(), uses.toString()]), + ], created_at: Math.floor(Date.now() / 1000), }); @@ -145,7 +150,7 @@ export function updateTrendingZappedEvents(): Promise { /** Update trending events. */ export async function updateTrendingEvents(): Promise { - const languages = Conf.trendLanguages; + const languages = Conf.preferredLanguages; if (!languages) return updateTrendingTags('#e', 'e', [1, 6, 7, 9735], 40, Conf.relay, ['q']); const promise: Promise[] = []; From 7c29c81226ce0a9f932f640bec1d7d1e737b9b27 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 1 Oct 2024 13:58:51 -0300 Subject: [PATCH 07/26] test: pass languagesIds in getTrendingTagValues() function --- src/trends.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/trends.test.ts b/src/trends.test.ts index 1788c496..66cae23b 100644 --- a/src/trends.test.ts +++ b/src/trends.test.ts @@ -94,7 +94,9 @@ Deno.test("getTrendingTagValues(): 'e' tag and WITH language parameter", async ( .where('id', '=', post2.id) .execute(); - const trends = await getTrendingTagValues(db.kysely, ['e'], { kinds: [1, 7] }, 'pt'); + const languagesIds = (await db.store.query([{ search: 'language:pt' }])).map((event) => event.id); + + const trends = await getTrendingTagValues(db.kysely, ['e'], { kinds: [1, 7] }, languagesIds); // portuguese post const expected = [{ value: post1.id, authors: numberOfAuthorsWhoLikedPost1, uses: post1uses }]; From e7f5e563f58d892f256408604aca13b5a692bc6f Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 1 Oct 2024 13:59:21 -0300 Subject: [PATCH 08/26] feat: load dotenv in script/trends.ts --- scripts/trends.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/trends.ts b/scripts/trends.ts index 6600f7e2..c6ff63e0 100644 --- a/scripts/trends.ts +++ b/scripts/trends.ts @@ -1,3 +1,4 @@ +import * as dotenv from '@std/dotenv'; import { z } from 'zod'; import { @@ -8,6 +9,12 @@ import { updateTrendingZappedEvents, } from '@/trends.ts'; +await dotenv.load({ + export: true, + defaultsPath: null, + examplePath: null, +}); + const trendSchema = z.enum(['pubkeys', 'zapped_events', 'events', 'hashtags', 'links']); const trends = trendSchema.array().parse(Deno.args); From a5762628a546d7782b483134bf085e4aff6ad014 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Wed, 2 Oct 2024 20:47:46 +0530 Subject: [PATCH 09/26] add script for setting ditto kind 0 --- deno.json | 2 ++ deno.lock | 22 +++++++++++++ scripts/setup-kind0.ts | 72 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+) create mode 100644 scripts/setup-kind0.ts diff --git a/deno.json b/deno.json index ab2acc2a..00ab104f 100644 --- a/deno.json +++ b/deno.json @@ -15,6 +15,7 @@ "admin:event": "deno run -A scripts/admin-event.ts", "admin:role": "deno run -A scripts/admin-role.ts", "setup": "deno run -A scripts/setup.ts", + "setup:kind0": "deno run -A scripts/setup-kind0.ts", "stats:recompute": "deno run -A scripts/stats-recompute.ts", "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip -o soapbox.zip && rm soapbox.zip", "trends": "deno run -A scripts/trends.ts", @@ -71,6 +72,7 @@ "nostr-tools": "npm:nostr-tools@2.5.1", "nostr-wasm": "npm:nostr-wasm@^0.1.0", "path-to-regexp": "npm:path-to-regexp@^7.1.0", + "png-to-ico": "npm:png-to-ico@^2.1.8", "postgres": "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/mod.js", "prom-client": "npm:prom-client@^15.1.2", "question-deno": "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/mod.ts", diff --git a/deno.lock b/deno.lock index 7aa234d8..b64f2f95 100644 --- a/deno.lock +++ b/deno.lock @@ -103,6 +103,7 @@ "npm:nostr-tools@^2.7.0": "npm:nostr-tools@2.7.0", "npm:nostr-wasm@^0.1.0": "npm:nostr-wasm@0.1.0", "npm:path-to-regexp@^7.1.0": "npm:path-to-regexp@7.1.0", + "npm:png-to-ico@^2.1.8": "npm:png-to-ico@2.1.8", "npm:postgres@3.4.4": "npm:postgres@3.4.4", "npm:prom-client@^15.1.2": "npm:prom-client@15.1.2", "npm:tldts@^6.0.14": "npm:tldts@6.1.18", @@ -672,6 +673,10 @@ "@types/trusted-types": "@types/trusted-types@2.0.7" } }, + "@types/node@17.0.45": { + "integrity": "sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==", + "dependencies": {} + }, "@types/node@18.16.19": { "integrity": "sha512-IXl7o+R9iti9eBW4Wg2hx1xQDig183jj7YLn8F7udNceyfkbn1ZxmzZXuak20gR40D7pIkIY1kYGx5VIGbaHKA==", "dependencies": {} @@ -1131,6 +1136,10 @@ "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", "dependencies": {} }, + "minimist@1.2.8": { + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dependencies": {} + }, "ms@2.1.2": { "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dependencies": {} @@ -1240,6 +1249,18 @@ "integrity": "sha512-eG2dWTVw5bzqGRztnHExczNxt5VGsE6OwTeCG3fdUf9KBsZzO3R5OIIIzWR+iZA0NtZ+RDVdaoE2dK1cn6jH4g==", "dependencies": {} }, + "png-to-ico@2.1.8": { + "integrity": "sha512-Nf+IIn/cZ/DIZVdGveJp86NG5uNib1ZXMiDd/8x32HCTeKSvgpyg6D/6tUBn1QO/zybzoMK0/mc3QRgAyXdv9w==", + "dependencies": { + "@types/node": "@types/node@17.0.45", + "minimist": "minimist@1.2.8", + "pngjs": "pngjs@6.0.0" + } + }, + "pngjs@6.0.0": { + "integrity": "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==", + "dependencies": {} + }, "postgres@3.4.4": { "integrity": "sha512-IbyN+9KslkqcXa8AO9fxpk97PA4pzewvpi2B3Dwy9u4zpV32QicaEdgmF3eSQUzdRk7ttDHQejNgAEr4XoeH4A==", "dependencies": {} @@ -2166,6 +2187,7 @@ "npm:nostr-tools@2.5.1", "npm:nostr-wasm@^0.1.0", "npm:path-to-regexp@^7.1.0", + "npm:png-to-ico@^2.1.8", "npm:prom-client@^15.1.2", "npm:tldts@^6.0.14", "npm:tseep@^1.2.1", diff --git a/scripts/setup-kind0.ts b/scripts/setup-kind0.ts new file mode 100644 index 00000000..1ef06b9d --- /dev/null +++ b/scripts/setup-kind0.ts @@ -0,0 +1,72 @@ +import { AdminSigner } from '@/signers/AdminSigner.ts'; +import { Command } from 'commander'; +import { NostrEvent } from 'nostr-tools'; +import { nostrNow } from '@/utils.ts'; +import { Buffer } from 'node:buffer'; +import { Conf } from '@/config.ts'; +import pngToIco from 'png-to-ico'; +import { Storages } from '@/storages.ts'; + +function die(code: number, ...args: any[]) { + console.error(...args); + Deno.exit(code); +} + +if (import.meta.main) { + const kind0 = new Command() + .name('setup:kind0') + .description('Set up / change the kind 0 for a Ditto instance.') + .version('0.1.0') + .showHelpAfterError(); + + kind0 + .argument('', 'The name of the Ditto instance. Can just be your hostname.') + .option( + '-l --lightning ', + 'Lightning address for the server. Can just be your own lightning address.', + ) + .option('-a --about ', 'About text. This shows up whenever a description for your server is needed.') + .option('-i --image ', 'Image URL to use for OpenGraph previews and favicon.') + .action(async (name, args) => { + const { lightning, about, image } = args; + const content: Record = {}; + if (!name || !name.trim()) die(1, 'You must atleast supply a name!'); + content.bot = true; + content.about = about; + content.lud16 = lightning; + content.name = name; + content.picture = image; + content.website = Conf.localDomain; + + const signer = new AdminSigner(); + const bare: Omit = { + created_at: nostrNow(), + kind: 0, + tags: [], + content: JSON.stringify(content), + }; + const signed = await signer.signEvent(bare); + if (image) { + try { + await fetch(image) + .then((res) => { + if (!res.ok) throw new Error('Error attempting to fetch favicon.'); + if (res.headers.get('content-type') !== 'image/png') throw new Error('Non-png images are not supported!'); + return res.blob(); + }) + .then(async (blob) => + await pngToIco(Buffer.from(await blob.arrayBuffer())) + .then(async (buf) => { + await Deno.writeFile('./public/favicon.ico', buf); + }) + ); + } catch (e) { + die(1, `Error generating favicon from url ${image}: "${e}". Please check this or try again without --image.`); + } + } + console.log({ content, signed }); + await Storages.db().then((store) => store.event(signed)); + }); + + await kind0.parseAsync(); +} From 3df1fe4d3aad2898cd9f08d13401c3fd33bcc366 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Wed, 2 Oct 2024 20:53:57 +0530 Subject: [PATCH 10/26] neatness --- scripts/setup-kind0.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/setup-kind0.ts b/scripts/setup-kind0.ts index 1ef06b9d..c62cc706 100644 --- a/scripts/setup-kind0.ts +++ b/scripts/setup-kind0.ts @@ -56,9 +56,7 @@ if (import.meta.main) { }) .then(async (blob) => await pngToIco(Buffer.from(await blob.arrayBuffer())) - .then(async (buf) => { - await Deno.writeFile('./public/favicon.ico', buf); - }) + .then(async (buf) => await Deno.writeFile('./public/favicon.ico', buf)) ); } catch (e) { die(1, `Error generating favicon from url ${image}: "${e}". Please check this or try again without --image.`); From 23bedd82a068dbbfedab4f7dedd40b4d90ccc10a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 2 Oct 2024 13:35:34 -0500 Subject: [PATCH 11/26] utils: remove unused sha256 text function --- src/utils.ts | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index e361109d..ae257374 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -64,20 +64,6 @@ function findTag(tags: string[][], name: string): string[] | undefined { return tags.find((tag) => tag[0] === name); } -/** - * Get sha256 hash (hex) of some text. - * https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest#converting_a_digest_to_a_hex_string - */ -async function sha256(message: string): Promise { - const msgUint8 = new TextEncoder().encode(message); - const hashBuffer = await crypto.subtle.digest('SHA-256', msgUint8); - const hashArray = Array.from(new Uint8Array(hashBuffer)); - const hashHex = hashArray - .map((b) => b.toString(16).padStart(2, '0')) - .join(''); - return hashHex; -} - /** Test whether the value is a Nostr ID. */ function isNostrId(value: unknown): boolean { return n.id().safeParse(value).success; @@ -88,6 +74,6 @@ function isURL(value: unknown): boolean { return z.string().url().safeParse(value).success; } -export { bech32ToPubkey, eventAge, findTag, isNostrId, isURL, type Nip05, nostrDate, nostrNow, parseNip05, sha256 }; +export { bech32ToPubkey, eventAge, findTag, isNostrId, isURL, type Nip05, nostrDate, nostrNow, parseNip05 }; export { Time } from '@/utils/time.ts'; From 1d2bf07460290b587240415492f21a3b182e14da Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 2 Oct 2024 13:44:50 -0500 Subject: [PATCH 12/26] Remove unused nostr-relaypool library --- deno.json | 30 +++++++++++++++++++++++------- deno.lock | 35 ----------------------------------- 2 files changed, 23 insertions(+), 42 deletions(-) diff --git a/deno.json b/deno.json index 00ab104f..b081f323 100644 --- a/deno.json +++ b/deno.json @@ -22,8 +22,15 @@ "clean:deps": "deno cache --reload src/app.ts", "db:populate-search": "deno run -A scripts/db-populate-search.ts" }, - "unstable": ["cron", "ffi", "kv", "worker-options"], - "exclude": ["./public"], + "unstable": [ + "cron", + "ffi", + "kv", + "worker-options" + ], + "exclude": [ + "./public" + ], "imports": { "@/": "./src/", "@b-fuze/deno-dom": "jsr:@b-fuze/deno-dom@^0.1.47", @@ -68,7 +75,6 @@ "linkify-string": "npm:linkify-string@^4.1.1", "linkifyjs": "npm:linkifyjs@^4.1.1", "lru-cache": "npm:lru-cache@^10.2.2", - "nostr-relaypool": "npm:nostr-relaypool2@0.6.34", "nostr-tools": "npm:nostr-tools@2.5.1", "nostr-wasm": "npm:nostr-wasm@^0.1.0", "path-to-regexp": "npm:path-to-regexp@^7.1.0", @@ -84,14 +90,24 @@ "~/fixtures/": "./fixtures/" }, "lint": { - "include": ["src/", "scripts/"], + "include": [ + "src/", + "scripts/" + ], "rules": { - "tags": ["recommended"], - "exclude": ["no-explicit-any"] + "tags": [ + "recommended" + ], + "exclude": [ + "no-explicit-any" + ] } }, "fmt": { - "include": ["src/", "scripts/"], + "include": [ + "src/", + "scripts/" + ], "useTabs": false, "lineWidth": 120, "indentWidth": 2, diff --git a/deno.lock b/deno.lock index b64f2f95..7b7e7d1c 100644 --- a/deno.lock +++ b/deno.lock @@ -97,7 +97,6 @@ "npm:lint-staged": "npm:lint-staged@15.2.2", "npm:lru-cache@^10.2.0": "npm:lru-cache@10.2.2", "npm:lru-cache@^10.2.2": "npm:lru-cache@10.2.2", - "npm:nostr-relaypool2@0.6.34": "npm:nostr-relaypool2@0.6.34", "npm:nostr-tools@2.5.1": "npm:nostr-tools@2.5.1", "npm:nostr-tools@^2.5.0": "npm:nostr-tools@2.5.1", "npm:nostr-tools@^2.7.0": "npm:nostr-tools@2.7.0", @@ -583,10 +582,6 @@ "integrity": "sha512-RQgQ4uQ+pLbqXfOmieB91ejmLwvSgv9nLx6sT6sD83s7umBypgg+OIBOBbEUiJXrfpnp9j0mRhYYdzp9uqq3lA==", "dependencies": {} }, - "@noble/ciphers@0.2.0": { - "integrity": "sha512-6YBxJDAapHSdd3bLDv6x2wRPwq4QFMUaB3HvljNBUTThDd12eSm7/3F+2lnfzx2jvM+S6Nsy0jEt9QbPqSwqRw==", - "dependencies": {} - }, "@noble/ciphers@0.5.3": { "integrity": "sha512-B0+6IIHiqEs3BPMT0hcRmHvEj2QHOLu+uwt+tqDDeVd0oyVzh7BPrDcPjRnV1PV/5LaknXJJQvOuRGR0zQJz+w==", "dependencies": {} @@ -988,12 +983,6 @@ "jsdom": "jsdom@24.0.0" } }, - "isomorphic-ws@5.0.0_ws@8.17.0": { - "integrity": "sha512-muId7Zzn9ywDsyXgTIafTry2sV3nySZeUDe6YedVd1Hvuuep5AsIlqK+XefWpYTyJG5e503F2xIuT2lcU6rCSw==", - "dependencies": { - "ws": "ws@8.17.0" - } - }, "jsdom@24.0.0": { "integrity": "sha512-UDS2NayCvmXSXVP6mpTj+73JnNQadZlr9N68189xib2tx5Mls7swlTNao26IoHv46BZJFvXygyRtyXd1feAk1A==", "dependencies": { @@ -1154,25 +1143,6 @@ "whatwg-url": "whatwg-url@5.0.0" } }, - "nostr-relaypool2@0.6.34": { - "integrity": "sha512-e3FDh9w/wQkY513mvoJps1Hc/Y5wiWXeBM6MD+YKSyAg+px+/8uHSSHAuHhlavw7oOEOvEsIGlMDMc57DG3MOA==", - "dependencies": { - "isomorphic-ws": "isomorphic-ws@5.0.0_ws@8.17.0", - "nostr-tools": "nostr-tools@1.17.0", - "safe-stable-stringify": "safe-stable-stringify@2.4.3" - } - }, - "nostr-tools@1.17.0": { - "integrity": "sha512-LZmR8GEWKZeElbFV5Xte75dOeE9EFUW/QLI1Ncn3JKn0kFddDKEfBbFN8Mu4TMs+L4HR/WTPha2l+PPuRnJcMw==", - "dependencies": { - "@noble/ciphers": "@noble/ciphers@0.2.0", - "@noble/curves": "@noble/curves@1.1.0", - "@noble/hashes": "@noble/hashes@1.3.1", - "@scure/base": "@scure/base@1.1.1", - "@scure/bip32": "@scure/bip32@1.3.1", - "@scure/bip39": "@scure/bip39@1.2.1" - } - }, "nostr-tools@2.5.1": { "integrity": "sha512-bpkhGGAhdiCN0irfV+xoH3YP5CQeOXyXzUq7SYeM6D56xwTXZCPEmBlUGqFVfQidvRsoVeVxeAiOXW2c2HxoRQ==", "dependencies": { @@ -1303,10 +1273,6 @@ "integrity": "sha512-APM0Gt1KoXBz0iIkkdB/kfvGOwC4UuJFeG/c+yV7wSc7q96cG/kJ0HiYCnzivD9SB53cLV1MlHFNfOuPaadYSw==", "dependencies": {} }, - "safe-stable-stringify@2.4.3": { - "integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==", - "dependencies": {} - }, "safer-buffer@2.1.2": { "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "dependencies": {} @@ -2183,7 +2149,6 @@ "npm:linkify-string@^4.1.1", "npm:linkifyjs@^4.1.1", "npm:lru-cache@^10.2.2", - "npm:nostr-relaypool2@0.6.34", "npm:nostr-tools@2.5.1", "npm:nostr-wasm@^0.1.0", "npm:path-to-regexp@^7.1.0", From 70f56af281c40013283a05f693e03f936c4a27e5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 2 Oct 2024 15:05:37 -0500 Subject: [PATCH 13/26] Add auth utils for generating/hashing/encoding/decoding tokens --- src/utils/auth.bench.ts | 11 +++++++++++ src/utils/auth.test.ts | 18 ++++++++++++++++++ src/utils/auth.ts | 30 ++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 src/utils/auth.bench.ts create mode 100644 src/utils/auth.test.ts create mode 100644 src/utils/auth.ts diff --git a/src/utils/auth.bench.ts b/src/utils/auth.bench.ts new file mode 100644 index 00000000..fbffc857 --- /dev/null +++ b/src/utils/auth.bench.ts @@ -0,0 +1,11 @@ +import { generateToken, getTokenHash } from '@/utils/auth.ts'; + +Deno.bench('generateToken', async () => { + await generateToken(); +}); + +Deno.bench('getTokenHash', async (b) => { + const { token } = await generateToken(); + b.start(); + await getTokenHash(token); +}); diff --git a/src/utils/auth.test.ts b/src/utils/auth.test.ts new file mode 100644 index 00000000..a0256b5d --- /dev/null +++ b/src/utils/auth.test.ts @@ -0,0 +1,18 @@ +import { assertEquals } from '@std/assert'; +import { decodeHex } from '@std/encoding/hex'; + +import { generateToken, getTokenHash } from '@/utils/auth.ts'; + +Deno.test('generateToken', async () => { + const sk = decodeHex('a0968751df8fd42f362213f08751911672f2a037113b392403bbb7dd31b71c95'); + + const { token, hash } = await generateToken(sk); + + assertEquals(token, 'token15ztgw5wl3l2z7d3zz0cgw5v3zee09gphzyanjfqrhwma6vdhrj2sauwknd'); + assertEquals(hash, decodeHex('ab4c4ead4d1c72a38fffd45b999937b7e3f25f867b19aaf252df858e77b66a8a')); +}); + +Deno.test('getTokenHash', async () => { + const hash = await getTokenHash('token15ztgw5wl3l2z7d3zz0cgw5v3zee09gphzyanjfqrhwma6vdhrj2sauwknd'); + assertEquals(hash, decodeHex('ab4c4ead4d1c72a38fffd45b999937b7e3f25f867b19aaf252df858e77b66a8a')); +}); diff --git a/src/utils/auth.ts b/src/utils/auth.ts new file mode 100644 index 00000000..8d71ed6f --- /dev/null +++ b/src/utils/auth.ts @@ -0,0 +1,30 @@ +import { bech32 } from '@scure/base'; +import { generateSecretKey } from 'nostr-tools'; + +/** + * Generate an auth token for the API. + * + * Returns a bech32 encoded API token and the SHA-256 hash of the bytes. + * The token should be presented to the user, but only the hash should be stored in the database. + */ +export async function generateToken(sk = generateSecretKey()): Promise<{ token: `token1${string}`; hash: Uint8Array }> { + const words = bech32.toWords(sk); + const token = bech32.encode('token', words); + + const buffer = await crypto.subtle.digest('SHA-256', sk); + const hash = new Uint8Array(buffer); + + return { token, hash }; +} + +/** + * Get the SHA-256 hash of an API token. + * First decodes from bech32 then hashes the bytes. + * Used to identify the user in the database by the hash of their token. + */ +export async function getTokenHash(token: `token1${string}`): Promise { + const { bytes: sk } = bech32.decodeToBytes(token); + const buffer = await crypto.subtle.digest('SHA-256', sk); + + return new Uint8Array(buffer); +} From e73a8d71dc33cb7160b35fc9673e5d428c24dd58 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 2 Oct 2024 17:56:30 -0500 Subject: [PATCH 14/26] auth: add encryptSecretKey & decryptSecretKey functions --- src/utils/auth.bench.ts | 19 ++++++++++++++++++- src/utils/auth.test.ts | 19 +++++++++++++++---- src/utils/auth.ts | 24 ++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/src/utils/auth.bench.ts b/src/utils/auth.bench.ts index fbffc857..8c3da7cf 100644 --- a/src/utils/auth.bench.ts +++ b/src/utils/auth.bench.ts @@ -1,4 +1,6 @@ -import { generateToken, getTokenHash } from '@/utils/auth.ts'; +import { generateSecretKey } from 'nostr-tools'; + +import { decryptSecretKey, encryptSecretKey, generateToken, getTokenHash } from '@/utils/auth.ts'; Deno.bench('generateToken', async () => { await generateToken(); @@ -9,3 +11,18 @@ Deno.bench('getTokenHash', async (b) => { b.start(); await getTokenHash(token); }); + +Deno.bench('encryptSecretKey', async (b) => { + const sk = generateSecretKey(); + const decrypted = generateSecretKey(); + b.start(); + await encryptSecretKey(sk, decrypted); +}); + +Deno.bench('decryptSecretKey', async (b) => { + const sk = generateSecretKey(); + const decrypted = generateSecretKey(); + const encrypted = await encryptSecretKey(sk, decrypted); + b.start(); + await decryptSecretKey(sk, encrypted); +}); diff --git a/src/utils/auth.test.ts b/src/utils/auth.test.ts index a0256b5d..e9e610c1 100644 --- a/src/utils/auth.test.ts +++ b/src/utils/auth.test.ts @@ -1,7 +1,8 @@ import { assertEquals } from '@std/assert'; -import { decodeHex } from '@std/encoding/hex'; +import { decodeHex, encodeHex } from '@std/encoding/hex'; +import { generateSecretKey } from 'nostr-tools'; -import { generateToken, getTokenHash } from '@/utils/auth.ts'; +import { decryptSecretKey, encryptSecretKey, generateToken, getTokenHash } from '@/utils/auth.ts'; Deno.test('generateToken', async () => { const sk = decodeHex('a0968751df8fd42f362213f08751911672f2a037113b392403bbb7dd31b71c95'); @@ -9,10 +10,20 @@ Deno.test('generateToken', async () => { const { token, hash } = await generateToken(sk); assertEquals(token, 'token15ztgw5wl3l2z7d3zz0cgw5v3zee09gphzyanjfqrhwma6vdhrj2sauwknd'); - assertEquals(hash, decodeHex('ab4c4ead4d1c72a38fffd45b999937b7e3f25f867b19aaf252df858e77b66a8a')); + assertEquals(encodeHex(hash), 'ab4c4ead4d1c72a38fffd45b999937b7e3f25f867b19aaf252df858e77b66a8a'); }); Deno.test('getTokenHash', async () => { const hash = await getTokenHash('token15ztgw5wl3l2z7d3zz0cgw5v3zee09gphzyanjfqrhwma6vdhrj2sauwknd'); - assertEquals(hash, decodeHex('ab4c4ead4d1c72a38fffd45b999937b7e3f25f867b19aaf252df858e77b66a8a')); + assertEquals(encodeHex(hash), 'ab4c4ead4d1c72a38fffd45b999937b7e3f25f867b19aaf252df858e77b66a8a'); +}); + +Deno.test('encryptSecretKey & decryptSecretKey', async () => { + const sk = generateSecretKey(); + const data = generateSecretKey(); + + const encrypted = await encryptSecretKey(sk, data); + const decrypted = await decryptSecretKey(sk, encrypted); + + assertEquals(encodeHex(decrypted), encodeHex(data)); }); diff --git a/src/utils/auth.ts b/src/utils/auth.ts index 8d71ed6f..05e838a9 100644 --- a/src/utils/auth.ts +++ b/src/utils/auth.ts @@ -28,3 +28,27 @@ export async function getTokenHash(token: `token1${string}`): Promise { + const secretKey = await crypto.subtle.importKey('raw', sk, { name: 'AES-GCM' }, false, ['encrypt']); + const iv = crypto.getRandomValues(new Uint8Array(12)); + const buffer = await crypto.subtle.encrypt({ name: 'AES-GCM', iv }, secretKey, decrypted); + + return new Uint8Array([...iv, ...new Uint8Array(buffer)]); +} + +/** + * Decrypt a secret key with AES-GCM. + * This function is used to retrieve the secret key from the database. + */ +export async function decryptSecretKey(sk: Uint8Array, encrypted: Uint8Array): Promise { + const secretKey = await crypto.subtle.importKey('raw', sk, { name: 'AES-GCM' }, false, ['decrypt']); + const iv = encrypted.slice(0, 12); + const buffer = await crypto.subtle.decrypt({ name: 'AES-GCM', iv }, secretKey, encrypted.slice(12)); + + return new Uint8Array(buffer); +} From 432857c2ff593630cfec99f54eaf4223c75a833e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 2 Oct 2024 18:28:24 -0500 Subject: [PATCH 15/26] Rework auth tokens table to use hashed/encrypted data --- src/controllers/api/oauth.ts | 32 +++++++---------- src/controllers/api/streaming.ts | 12 ++++--- src/db/DittoTables.ts | 15 ++++---- src/db/migrations/037_auth_tokens.ts | 52 ++++++++++++++++++++++++++++ src/middleware/signerMiddleware.ts | 15 +++++--- src/utils/nip98.ts | 20 +++++++---- 6 files changed, 102 insertions(+), 44 deletions(-) create mode 100644 src/db/migrations/037_auth_tokens.ts diff --git a/src/controllers/api/oauth.ts b/src/controllers/api/oauth.ts index 94aaeecd..c4c3eca4 100644 --- a/src/controllers/api/oauth.ts +++ b/src/controllers/api/oauth.ts @@ -1,14 +1,14 @@ import { NConnectSigner, NSchema as n, NSecSigner } from '@nostrify/nostrify'; -import { bech32 } from '@scure/base'; import { escape } from 'entities'; -import { generateSecretKey, getPublicKey } from 'nostr-tools'; +import { generateSecretKey } from 'nostr-tools'; import { z } from 'zod'; import { AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; +import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; import { parseBody } from '@/utils/api.ts'; -import { Storages } from '@/storages.ts'; +import { encryptSecretKey, generateToken } from '@/utils/auth.ts'; const passwordGrantSchema = z.object({ grant_type: z.literal('password'), @@ -82,38 +82,30 @@ async function getToken( { pubkey, secret, relays = [] }: { pubkey: string; secret?: string; relays?: string[] }, ): Promise<`token1${string}`> { const kysely = await Storages.kysely(); - const token = generateToken(); + const { token, hash } = await generateToken(); - const serverSeckey = generateSecretKey(); - const serverPubkey = getPublicKey(serverSeckey); + const nip46Seckey = generateSecretKey(); const signer = new NConnectSigner({ pubkey, - signer: new NSecSigner(serverSeckey), + signer: new NSecSigner(nip46Seckey), relay: await Storages.pubsub(), // TODO: Use the relays from the request. timeout: 60_000, }); await signer.connect(secret); - await kysely.insertInto('nip46_tokens').values({ - api_token: token, - user_pubkey: pubkey, - server_seckey: serverSeckey, - server_pubkey: serverPubkey, - relays: JSON.stringify(relays), - connected_at: new Date(), + await kysely.insertInto('auth_tokens').values({ + token_hash: hash, + pubkey, + nip46_sk_enc: await encryptSecretKey(Conf.seckey, nip46Seckey), + nip46_relays: relays, + created_at: new Date(), }).execute(); return token; } -/** Generate a bech32 token for the API. */ -function generateToken(): `token1${string}` { - const words = bech32.toWords(generateSecretKey()); - return bech32.encode('token', words); -} - /** Display the OAuth form. */ const oauthController: AppController = (c) => { const encodedUri = c.req.query('redirect_uri'); diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index cee7c57e..9693a16c 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -14,6 +14,7 @@ import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; import { getFeedPubkeys } from '@/queries.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; +import { getTokenHash } from '@/utils/auth.ts'; import { bech32ToPubkey, Time } from '@/utils.ts'; import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts'; import { renderNotification } from '@/views/mastodon/notifications.ts'; @@ -233,14 +234,15 @@ async function topicToFilter( async function getTokenPubkey(token: string): Promise { if (token.startsWith('token1')) { const kysely = await Storages.kysely(); + const tokenHash = await getTokenHash(token as `token1${string}`); - const { user_pubkey } = await kysely - .selectFrom('nip46_tokens') - .select(['user_pubkey', 'server_seckey', 'relays']) - .where('api_token', '=', token) + const { pubkey } = await kysely + .selectFrom('auth_tokens') + .select('pubkey') + .where('token_hash', '=', tokenHash) .executeTakeFirstOrThrow(); - return user_pubkey; + return pubkey; } else { return bech32ToPubkey(token); } diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index c05ffe66..b6fa93f4 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -4,7 +4,7 @@ import { NPostgresSchema } from '@nostrify/db'; export interface DittoTables extends NPostgresSchema { nostr_events: NostrEventsRow; - nip46_tokens: NIP46TokenRow; + auth_tokens: AuthTokenRow; author_stats: AuthorStatsRow; event_stats: EventStatsRow; pubkey_domains: PubkeyDomainRow; @@ -33,13 +33,12 @@ interface EventStatsRow { zaps_amount: number; } -interface NIP46TokenRow { - api_token: string; - user_pubkey: string; - server_seckey: Uint8Array; - server_pubkey: string; - relays: string; - connected_at: Date; +interface AuthTokenRow { + token_hash: Uint8Array; + pubkey: string; + nip46_sk_enc: Uint8Array; + nip46_relays: string[]; + created_at: Date; } interface PubkeyDomainRow { diff --git a/src/db/migrations/037_auth_tokens.ts b/src/db/migrations/037_auth_tokens.ts new file mode 100644 index 00000000..9df133f5 --- /dev/null +++ b/src/db/migrations/037_auth_tokens.ts @@ -0,0 +1,52 @@ +import { Kysely, sql } from 'kysely'; + +import { encryptSecretKey, getTokenHash } from '@/utils/auth.ts'; +import { Conf } from '@/config.ts'; + +interface DB { + nip46_tokens: { + api_token: `token1${string}`; + user_pubkey: string; + server_seckey: Uint8Array; + server_pubkey: string; + relays: string; + connected_at: Date; + }; + auth_tokens: { + token_hash: Uint8Array; + pubkey: string; + nip46_sk_enc: Uint8Array; + nip46_relays: string[]; + created_at: Date; + }; +} + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('auth_tokens') + .addColumn('token_hash', 'bytea', (col) => col.primaryKey()) + .addColumn('pubkey', 'char(64)', (col) => col.notNull()) + .addColumn('nip46_sk_enc', 'bytea', (col) => col.notNull()) + .addColumn('nip46_relays', 'jsonb', (col) => col.defaultTo('[]')) + .addColumn('created_at', 'timestamp', (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`)) + .execute(); + + // There are probably not that many tokens in the database yet, so this should be fine. + const tokens = await db.selectFrom('nip46_tokens').selectAll().execute(); + + for (const token of tokens) { + await db.insertInto('auth_tokens').values({ + token_hash: await getTokenHash(token.api_token), + pubkey: token.user_pubkey, + nip46_sk_enc: await encryptSecretKey(Conf.seckey, token.server_seckey), + nip46_relays: JSON.parse(token.relays), + created_at: token.connected_at, + }).execute(); + } + + await db.schema.dropTable('nip46_tokens').execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable('auth_tokens').execute(); +} diff --git a/src/middleware/signerMiddleware.ts b/src/middleware/signerMiddleware.ts index 344e14ef..8200ae1d 100644 --- a/src/middleware/signerMiddleware.ts +++ b/src/middleware/signerMiddleware.ts @@ -3,9 +3,11 @@ import { NSecSigner } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; import { AppMiddleware } from '@/app.ts'; +import { Conf } from '@/config.ts'; import { ConnectSigner } from '@/signers/ConnectSigner.ts'; import { ReadOnlySigner } from '@/signers/ReadOnlySigner.ts'; import { Storages } from '@/storages.ts'; +import { decryptSecretKey, getTokenHash } from '@/utils/auth.ts'; /** We only accept "Bearer" type. */ const BEARER_REGEX = new RegExp(`^Bearer (${nip19.BECH32_REGEX.source})$`); @@ -21,14 +23,17 @@ export const signerMiddleware: AppMiddleware = async (c, next) => { if (bech32.startsWith('token1')) { try { const kysely = await Storages.kysely(); + const tokenHash = await getTokenHash(bech32 as `token1${string}`); - const { user_pubkey, server_seckey, relays } = await kysely - .selectFrom('nip46_tokens') - .select(['user_pubkey', 'server_seckey', 'relays']) - .where('api_token', '=', bech32) + const { pubkey, nip46_sk_enc, nip46_relays } = await kysely + .selectFrom('auth_tokens') + .select(['pubkey', 'nip46_sk_enc', 'nip46_relays']) + .where('token_hash', '=', tokenHash) .executeTakeFirstOrThrow(); - c.set('signer', new ConnectSigner(user_pubkey, new NSecSigner(server_seckey), JSON.parse(relays))); + const nep46Seckey = await decryptSecretKey(Conf.seckey, nip46_sk_enc); + + c.set('signer', new ConnectSigner(pubkey, new NSecSigner(nep46Seckey), nip46_relays)); } catch { throw new HTTPException(401); } diff --git a/src/utils/nip98.ts b/src/utils/nip98.ts index c33da877..f83fcddb 100644 --- a/src/utils/nip98.ts +++ b/src/utils/nip98.ts @@ -1,9 +1,10 @@ import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; +import { encodeHex } from '@std/encoding/hex'; import { EventTemplate, nip13 } from 'nostr-tools'; import { decode64Schema } from '@/schema.ts'; import { signedEventSchema } from '@/schemas/nostr.ts'; -import { eventAge, findTag, nostrNow, sha256 } from '@/utils.ts'; +import { eventAge, findTag, nostrNow } from '@/utils.ts'; import { Time } from '@/utils/time.ts'; /** Decode a Nostr event from a base64 encoded string. */ @@ -41,11 +42,10 @@ function validateAuthEvent(req: Request, event: NostrEvent, opts: ParseAuthReque .refine((event) => pow ? nip13.getPow(event.id) >= pow : true, 'Insufficient proof of work') .refine(validateBody, 'Event payload does not match request body'); - function validateBody(event: NostrEvent) { + async function validateBody(event: NostrEvent): Promise { if (!validatePayload) return true; - return req.clone().text() - .then(sha256) - .then((hash) => hash === tagValue(event, 'payload')); + const payload = await getPayload(req); + return payload === tagValue(event, 'payload'); } return schema.safeParseAsync(event); @@ -62,7 +62,7 @@ async function buildAuthEventTemplate(req: Request, opts: ParseAuthRequestOpts = ]; if (validatePayload) { - const payload = await req.clone().text().then(sha256); + const payload = await getPayload(req); tags.push(['payload', payload]); } @@ -74,6 +74,14 @@ async function buildAuthEventTemplate(req: Request, opts: ParseAuthRequestOpts = }; } +/** Get a SHA-256 hash of the request body encoded as a hex string. */ +async function getPayload(req: Request): Promise { + const text = await req.clone().text(); + const bytes = new TextEncoder().encode(text); + const buffer = await crypto.subtle.digest('SHA-256', bytes); + return encodeHex(buffer); +} + /** Get the value for the first matching tag name in the event. */ function tagValue(event: NostrEvent, tagName: string): string | undefined { return findTag(event.tags, tagName)?.[1]; From ff361a410662ae11993e915493d3a60a698d0378 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 2 Oct 2024 18:34:19 -0500 Subject: [PATCH 16/26] Recreate nip46_tokens in down migration --- src/db/migrations/037_auth_tokens.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/db/migrations/037_auth_tokens.ts b/src/db/migrations/037_auth_tokens.ts index 9df133f5..71e971d3 100644 --- a/src/db/migrations/037_auth_tokens.ts +++ b/src/db/migrations/037_auth_tokens.ts @@ -49,4 +49,14 @@ export async function up(db: Kysely): Promise { export async function down(db: Kysely): Promise { await db.schema.dropTable('auth_tokens').execute(); + + await db.schema + .createTable('nip46_tokens') + .addColumn('api_token', 'text', (col) => col.primaryKey().unique().notNull()) + .addColumn('user_pubkey', 'text', (col) => col.notNull()) + .addColumn('server_seckey', 'bytea', (col) => col.notNull()) + .addColumn('server_pubkey', 'text', (col) => col.notNull()) + .addColumn('relays', 'text', (col) => col.defaultTo('[]')) + .addColumn('connected_at', 'timestamp', (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`)) + .execute(); } From 031297f2533ec581e9b3a275ee6dfa3296b5fd4f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 2 Oct 2024 22:42:54 -0500 Subject: [PATCH 17/26] Improve relay/pubkey hints when creating a status --- src/controllers/api/statuses.ts | 55 ++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 486ea28b..383da870 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -88,14 +88,28 @@ const createStatusController: AppController = async (c) => { return c.json({ error: 'Original post not found.' }, 404); } - const root = ancestor.tags.find((tag) => tag[0] === 'e' && tag[3] === 'root')?.[1] ?? ancestor.id; + const rootId = ancestor.tags.find((tag) => tag[0] === 'e' && tag[3] === 'root')?.[1] ?? ancestor.id; + const root = rootId === ancestor.id ? ancestor : await getEvent(rootId); - tags.push(['e', root, Conf.relay, 'root']); - tags.push(['e', data.in_reply_to_id, Conf.relay, 'reply']); + if (root) { + tags.push(['e', root.id, Conf.relay, 'root', root.pubkey]); + } else { + tags.push(['e', rootId, Conf.relay, 'root']); + } + + tags.push(['e', ancestor.id, Conf.relay, 'reply', ancestor.pubkey]); } + let quoted: DittoEvent | undefined; + if (data.quote_id) { - tags.push(['q', data.quote_id]); + quoted = await getEvent(data.quote_id); + + if (!quoted) { + return c.json({ error: 'Quoted post not found.' }, 404); + } + + tags.push(['q', quoted.id, Conf.relay, '', quoted.pubkey]); } if (data.sensitive && data.spoiler_text) { @@ -143,7 +157,7 @@ const createStatusController: AppController = async (c) => { } try { - return `nostr:${nip19.npubEncode(pubkey)}`; + return `nostr:${nip19.nprofileEncode({ pubkey, relays: [Conf.relay] })}`; } catch { return match; } @@ -159,7 +173,7 @@ const createStatusController: AppController = async (c) => { } for (const pubkey of pubkeys) { - tags.push(['p', pubkey]); + tags.push(['p', pubkey, Conf.relay]); } for (const link of linkify.find(data.status ?? '')) { @@ -175,7 +189,12 @@ const createStatusController: AppController = async (c) => { .map(({ url }) => url) .filter((url): url is string => Boolean(url)); - const quoteCompat = data.quote_id ? `\n\nnostr:${nip19.noteEncode(data.quote_id)}` : ''; + const quoteCompat = quoted + ? `\n\nnostr:${ + nip19.neventEncode({ id: quoted.id, kind: quoted.kind, author: quoted.pubkey, relays: [Conf.relay] }) + }` + : ''; + const mediaCompat = mediaUrls.length ? `\n\n${mediaUrls.join('\n')}` : ''; const author = await getAuthor(await c.get('signer')?.getPublicKey()!); @@ -223,7 +242,7 @@ const deleteStatusController: AppController = async (c) => { if (event.pubkey === pubkey) { await createEvent({ kind: 5, - tags: [['e', id, Conf.relay]], + tags: [['e', id, Conf.relay, '', pubkey]], }, c); const author = await getAuthor(event.pubkey); @@ -281,7 +300,7 @@ const favouriteController: AppController = async (c) => { kind: 7, content: '+', tags: [ - ['e', target.id, Conf.relay], + ['e', target.id, Conf.relay, '', target.pubkey], ['p', target.pubkey, Conf.relay], ], }, c); @@ -324,7 +343,7 @@ const reblogStatusController: AppController = async (c) => { const reblogEvent = await createEvent({ kind: 6, tags: [ - ['e', event.id, Conf.relay], + ['e', event.id, Conf.relay, '', event.pubkey], ['p', event.pubkey, Conf.relay], ], }, c); @@ -361,7 +380,7 @@ const unreblogStatusController: AppController = async (c) => { await createEvent({ kind: 5, - tags: [['e', repostEvent.id, Conf.relay]], + tags: [['e', repostEvent.id, Conf.relay, '', repostEvent.pubkey]], }, c); return c.json(await renderStatus(event, { viewerPubkey: pubkey })); @@ -413,7 +432,7 @@ const bookmarkController: AppController = async (c) => { if (event) { await updateListEvent( { kinds: [10003], authors: [pubkey], limit: 1 }, - (tags) => addTag(tags, ['e', eventId, Conf.relay]), + (tags) => addTag(tags, ['e', event.id, Conf.relay, '', event.pubkey]), c, ); @@ -440,7 +459,7 @@ const unbookmarkController: AppController = async (c) => { if (event) { await updateListEvent( { kinds: [10003], authors: [pubkey], limit: 1 }, - (tags) => deleteTag(tags, ['e', eventId, Conf.relay]), + (tags) => deleteTag(tags, ['e', event.id, Conf.relay, '', event.pubkey]), c, ); @@ -467,7 +486,7 @@ const pinController: AppController = async (c) => { if (event) { await updateListEvent( { kinds: [10001], authors: [pubkey], limit: 1 }, - (tags) => addTag(tags, ['e', eventId, Conf.relay]), + (tags) => addTag(tags, ['e', event.id, Conf.relay, '', event.pubkey]), c, ); @@ -496,7 +515,7 @@ const unpinController: AppController = async (c) => { if (event) { await updateListEvent( { kinds: [10001], authors: [pubkey], limit: 1 }, - (tags) => deleteTag(tags, ['e', eventId, Conf.relay]), + (tags) => deleteTag(tags, ['e', event.id, Conf.relay, '', event.pubkey]), c, ); @@ -540,8 +559,8 @@ const zapController: AppController = async (c) => { lnurl = getLnurl(meta); if (target && lnurl) { tags.push( - ['e', target.id, Conf.relay], - ['p', target.pubkey], + ['e', target.id, Conf.relay, '', target.pubkey], + ['p', target.pubkey, Conf.relay], ['amount', amount.toString()], ['relays', Conf.relay], ['lnurl', lnurl], @@ -553,7 +572,7 @@ const zapController: AppController = async (c) => { lnurl = getLnurl(meta); if (target && lnurl) { tags.push( - ['p', target.pubkey], + ['p', target.pubkey, Conf.relay], ['amount', amount.toString()], ['relays', Conf.relay], ['lnurl', lnurl], From 7f8697f4f386a024ab78ac951d814f6a59f364ed Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 2 Oct 2024 22:50:34 -0500 Subject: [PATCH 18/26] Fix zap tag logic --- src/controllers/api/statuses.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 383da870..19907895 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -197,7 +197,8 @@ const createStatusController: AppController = async (c) => { const mediaCompat = mediaUrls.length ? `\n\n${mediaUrls.join('\n')}` : ''; - const author = await getAuthor(await c.get('signer')?.getPublicKey()!); + const pubkey = await c.get('signer')?.getPublicKey()!; + const author = pubkey ? await getAuthor(pubkey) : undefined; if (Conf.zapSplitsEnabled) { const meta = n.json().pipe(n.metadata()).catch({}).parse(author?.content); @@ -210,7 +211,7 @@ const createStatusController: AppController = async (c) => { tags.push(['zap', pubkey, Conf.relay, dittoZapSplit[pubkey].weight.toString(), dittoZapSplit[pubkey].message]); } if (totalSplit) { - tags.push(['zap', author?.pubkey as string, Conf.relay, Math.max(0, 100 - totalSplit).toString()]); + tags.push(['zap', pubkey, Conf.relay, Math.max(0, 100 - totalSplit).toString()]); } } } From e42c04736233ec1af02d9ecd9249f723eb51454c Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 3 Oct 2024 12:56:40 -0300 Subject: [PATCH 19/26] refactor: use Stickynotes instead of legacy Debug --- src/utils/lnurl.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/utils/lnurl.ts b/src/utils/lnurl.ts index 64e10fe3..1dd99769 100644 --- a/src/utils/lnurl.ts +++ b/src/utils/lnurl.ts @@ -1,5 +1,5 @@ import { LNURL, LNURLDetails } from '@nostrify/nostrify/ln'; -import Debug from '@soapbox/stickynotes/debug'; +import { Stickynotes } from '@soapbox/stickynotes'; import { cachedLnurlsSizeGauge } from '@/metrics.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; @@ -7,17 +7,17 @@ import { Time } from '@/utils/time.ts'; import { fetchWorker } from '@/workers/fetch.ts'; import { NostrEvent } from '@nostrify/nostrify'; -const debug = Debug('ditto:lnurl'); +const console = new Stickynotes('ditto:lnurl'); const lnurlCache = new SimpleLRU( async (lnurl, { signal }) => { - debug(`Lookup ${lnurl}`); + console.debug(`Lookup ${lnurl}`); try { const result = await LNURL.lookup(lnurl, { fetch: fetchWorker, signal }); - debug(`Found: ${lnurl}`); + console.debug(`Found: ${lnurl}`); return result; } catch (e) { - debug(`Not found: ${lnurl}`); + console.debug(`Not found: ${lnurl}`); throw e; } }, From bd3d7fda94e88d5d5181abf14d527cf0aecfe2a3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 3 Oct 2024 13:02:40 -0500 Subject: [PATCH 20/26] Treat .ts links in statuses as application/typescript Fixes https://gitlab.com/soapbox-pub/ditto/-/issues/241 --- src/utils/media.test.ts | 4 ++++ src/utils/media.ts | 12 +++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/utils/media.test.ts b/src/utils/media.test.ts index e88e97da..39abed23 100644 --- a/src/utils/media.test.ts +++ b/src/utils/media.test.ts @@ -7,6 +7,10 @@ Deno.test('getUrlMediaType', () => { assertEquals(getUrlMediaType('https://example.com/index.html'), 'text/html'); assertEquals(getUrlMediaType('https://example.com/yolo'), undefined); assertEquals(getUrlMediaType('https://example.com/'), undefined); + assertEquals( + getUrlMediaType('https://gitlab.com/soapbox-pub/nostrify/-/blob/main/packages/policies/WoTPolicy.ts'), + 'application/typescript', + ); }); Deno.test('isPermittedMediaType', () => { diff --git a/src/utils/media.ts b/src/utils/media.ts index 9c0ea9e3..82c9832d 100644 --- a/src/utils/media.ts +++ b/src/utils/media.ts @@ -1,4 +1,4 @@ -import { typeByExtension } from '@std/media-types'; +import { typeByExtension as _typeByExtension } from '@std/media-types'; /** Get media type of the filename in the URL by its extension, if any. */ export function getUrlMediaType(url: string): string | undefined { @@ -22,3 +22,13 @@ export function isPermittedMediaType(mediaType: string, permitted: string[]): bo const [baseType, _subType] = mediaType.split('/'); return permitted.includes(baseType); } + +/** Custom type-by-extension with overrides. */ +function typeByExtension(ext: string): string | undefined { + switch (ext) { + case 'ts': + return 'application/typescript'; + default: + return _typeByExtension(ext); + } +} From eac375b99da649c700adfdcf57056e0683e981de Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Fri, 4 Oct 2024 02:06:10 +0530 Subject: [PATCH 21/26] update dockerfile for tribes --- Dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f2dde5ec..d700de83 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,14 @@ FROM denoland/deno:1.44.2 EXPOSE 4036 + +ENV PORT 5000 + WORKDIR /app RUN mkdir -p data && chown -R deno data USER deno +RUN mkdir -p data COPY . . RUN deno cache src/server.ts -CMD deno task start +RUN apt-get update && apt-get install -y unzip curl +RUN deno task soapbox +CMD deno task start \ No newline at end of file From baae2974f31ef0bb1138da79a3ae68626e82c063 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Fri, 4 Oct 2024 02:14:21 +0530 Subject: [PATCH 22/26] typo fix --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d700de83..1500e69c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,6 @@ ENV PORT 5000 WORKDIR /app RUN mkdir -p data && chown -R deno data USER deno -RUN mkdir -p data COPY . . RUN deno cache src/server.ts RUN apt-get update && apt-get install -y unzip curl From 7107e389152291ef89c13b13c8d9a49601569c8f Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Fri, 4 Oct 2024 02:14:58 +0530 Subject: [PATCH 23/26] fix port nonsense --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1500e69c..f042820d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM denoland/deno:1.44.2 -EXPOSE 4036 +EXPOSE 5000 ENV PORT 5000 From 018600058a52151cda981332f4d180fb875bb8c3 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Fri, 4 Oct 2024 02:29:54 +0530 Subject: [PATCH 24/26] run Dockerfile as root --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f042820d..999c0efc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,9 +5,8 @@ ENV PORT 5000 WORKDIR /app RUN mkdir -p data && chown -R deno data -USER deno COPY . . RUN deno cache src/server.ts RUN apt-get update && apt-get install -y unzip curl RUN deno task soapbox -CMD deno task start \ No newline at end of file +CMD deno task start From a5def9fa6cedef0c67c3b1ce695f12ea462b7837 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 3 Oct 2024 18:16:23 -0300 Subject: [PATCH 25/26] refactor: just import config.ts directly instead of loading dotenv in trends.ts script --- scripts/trends.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/scripts/trends.ts b/scripts/trends.ts index c6ff63e0..627fb332 100644 --- a/scripts/trends.ts +++ b/scripts/trends.ts @@ -1,5 +1,5 @@ -import * as dotenv from '@std/dotenv'; import { z } from 'zod'; +import '@/config.ts'; import { updateTrendingEvents, @@ -9,12 +9,6 @@ import { updateTrendingZappedEvents, } from '@/trends.ts'; -await dotenv.load({ - export: true, - defaultsPath: null, - examplePath: null, -}); - const trendSchema = z.enum(['pubkeys', 'zapped_events', 'events', 'hashtags', 'links']); const trends = trendSchema.array().parse(Deno.args); From 67b0684a810399c62b2eb6964e7cf9c3860f099d Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 3 Oct 2024 19:40:29 -0300 Subject: [PATCH 26/26] refactor(trends.ts): move logic one level up, rename 'languagesIds' to 'values', remove WITH SQL statement --- src/trends.ts | 116 ++++++++++++++++++++++++-------------------------- 1 file changed, 55 insertions(+), 61 deletions(-) diff --git a/src/trends.ts b/src/trends.ts index a60e28f3..cf4f7c96 100644 --- a/src/trends.ts +++ b/src/trends.ts @@ -1,5 +1,4 @@ -import { LanguageCode } from 'iso-639-1'; -import { NostrEvent, NostrFilter } from '@nostrify/nostrify'; +import { NostrFilter } from '@nostrify/nostrify'; import { Stickynotes } from '@soapbox/stickynotes'; import { Kysely, sql } from 'kysely'; @@ -20,48 +19,39 @@ export async function getTrendingTagValues( tagNames: string[], /** Filter of eligible events. */ filter: NostrFilter, - /** Results must be inside 'languagesIds' */ - languagesIds?: string[], + /** If present, only tag values in this list are permitted to trend. */ + values?: string[], ): Promise<{ value: string; authors: number; uses: number }[]> { - let query = kysely.with('trends', (db) => { - let query = db - .selectFrom([ - 'nostr_events', - sql<{ key: string; value: string }>`jsonb_each_text(nostr_events.tags_index)`.as('kv'), - sql<{ key: string; value: string }>`jsonb_array_elements_text(kv.value::jsonb)`.as('element'), - ]) - .select(({ fn }) => [ - fn('lower', ['element.value']).as('value'), - fn.agg('count', ['nostr_events.pubkey']).distinct().as('authors'), - fn.countAll().as('uses'), - ]) - .where('kv.key', '=', (eb) => eb.fn.any(eb.val(tagNames))) - .groupBy((eb) => eb.fn('lower', ['element.value'])) - .orderBy((eb) => eb.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc'); + let query = kysely + .selectFrom([ + 'nostr_events', + sql<{ key: string; value: string }>`jsonb_each_text(nostr_events.tags_index)`.as('kv'), + sql<{ key: string; value: string }>`jsonb_array_elements_text(kv.value::jsonb)`.as('element'), + ]) + .select(({ fn }) => [ + fn('lower', ['element.value']).as('value'), + fn.agg('count', ['nostr_events.pubkey']).distinct().as('authors'), + fn.countAll().as('uses'), + ]) + .where('kv.key', '=', (eb) => eb.fn.any(eb.val(tagNames))) + .groupBy((eb) => eb.fn('lower', ['element.value'])) + .orderBy('authors desc').orderBy('uses desc'); - if (filter.kinds) { - query = query.where('nostr_events.kind', '=', ({ fn, val }) => fn.any(val(filter.kinds))); - } - if (filter.authors) { - query = query.where('nostr_events.pubkey', '=', ({ fn, val }) => fn.any(val(filter.authors))); - } - if (typeof filter.since === 'number') { - query = query.where('nostr_events.created_at', '>=', filter.since); - } - if (typeof filter.until === 'number') { - query = query.where('nostr_events.created_at', '<=', filter.until); - } - return query; - }) - .selectFrom(['trends']) - .select(['value', 'authors', 'uses']); - - if (languagesIds) { - query = query.where('trends.value', 'in', languagesIds); + if (filter.kinds) { + query = query.where('nostr_events.kind', '=', ({ fn, val }) => fn.any(val(filter.kinds))); + } + if (filter.authors) { + query = query.where('nostr_events.pubkey', '=', ({ fn, val }) => fn.any(val(filter.authors))); + } + if (typeof filter.since === 'number') { + query = query.where('nostr_events.created_at', '>=', filter.since); + } + if (typeof filter.until === 'number') { + query = query.where('nostr_events.created_at', '<=', filter.until); + } + if (values) { + query = query.where('element.value', 'in', values); } - - query = query.orderBy('authors desc').orderBy('uses desc'); - if (typeof filter.limit === 'number') { query = query.limit(filter.limit); } @@ -83,7 +73,7 @@ export async function updateTrendingTags( limit: number, extra = '', aliases?: string[], - language?: LanguageCode, + values?: string[], ) { console.info(`Updating trending ${l}...`); const kysely = await Storages.kysely(); @@ -94,25 +84,15 @@ export async function updateTrendingTags( const tagNames = aliases ? [tagName, ...aliases] : [tagName]; - let languagesIds: NostrEvent['id'][] = []; - if (language) { - const result = (await kysely.selectFrom('nostr_events') - .select('id') - .where('language', '=', language) - .where('nostr_events.created_at', '>=', yesterday) - .where('nostr_events.created_at', '<=', now) - .execute()).map((event) => event.id); - languagesIds = result; - } - try { const trends = await getTrendingTagValues(kysely, tagNames, { kinds, since: yesterday, until: now, limit, - }, languagesIds); + }, values); + console.log(trends); if (!trends.length) { console.info(`No trending ${l} found. Skipping.`); return; @@ -125,7 +105,7 @@ export async function updateTrendingTags( content: '', tags: [ ['L', 'pub.ditto.trends'], - ['l', languagesIds.length ? `${l}.${language}` : l, 'pub.ditto.trends'], + ['l', l, 'pub.ditto.trends'], ...trends.map(({ value, authors, uses }) => [tagName, value, extra, authors.toString(), uses.toString()]), ], created_at: Math.floor(Date.now() / 1000), @@ -150,16 +130,30 @@ export function updateTrendingZappedEvents(): Promise { /** Update trending events. */ export async function updateTrendingEvents(): Promise { - const languages = Conf.preferredLanguages; - if (!languages) return updateTrendingTags('#e', 'e', [1, 6, 7, 9735], 40, Conf.relay, ['q']); + const results: Promise[] = [ + updateTrendingTags('#e', 'e', [1, 6, 7, 9735], 40, Conf.relay, ['q']), + ]; - const promise: Promise[] = []; + const kysely = await Storages.kysely(); - for (const language of languages) { - promise.push(updateTrendingTags('#e', 'e', [1, 6, 7, 9735], 40, Conf.relay, ['q'], language)); + for (const language of Conf.preferredLanguages ?? []) { + const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); + const now = Math.floor(Date.now() / 1000); + + const rows = await kysely + .selectFrom('nostr_events') + .select('nostr_events.id') + .where('nostr_events.language', '=', language) + .where('nostr_events.created_at', '>=', yesterday) + .where('nostr_events.created_at', '<=', now) + .execute(); + + const ids = rows.map((row) => row.id); + + results.push(updateTrendingTags(`#e.${language}`, 'e', [1, 6, 7, 9735], 40, Conf.relay, ['q'], ids)); } - await Promise.allSettled(promise); + await Promise.allSettled(results); } /** Update trending hashtags. */