From e40064175766dac55b3e7bf86f8a7f57e3a1b125 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 23 Jul 2024 10:29:45 -0500 Subject: [PATCH 01/14] Improve performance of tag queries --- deno.json | 2 +- deno.lock | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/deno.json b/deno.json index 190ce017..272da841 100644 --- a/deno.json +++ b/deno.json @@ -27,7 +27,7 @@ "@hono/hono": "jsr:@hono/hono@^4.4.6", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.26.3", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@0.27.0-rc.1", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", diff --git a/deno.lock b/deno.lock index 5448018b..4548c23b 100644 --- a/deno.lock +++ b/deno.lock @@ -8,11 +8,11 @@ "jsr:@gleasonator/policy": "jsr:@gleasonator/policy@0.2.0", "jsr:@gleasonator/policy@0.2.0": "jsr:@gleasonator/policy@0.2.0", "jsr:@gleasonator/policy@0.4.0": "jsr:@gleasonator/policy@0.4.0", - "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.0", + "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.1", + "jsr:@nostrify/nostrify@0.27.0-rc.1": "jsr:@nostrify/nostrify@0.27.0-rc.1", "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", - "jsr:@nostrify/nostrify@^0.26.3": "jsr:@nostrify/nostrify@0.26.3", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0": "jsr:@soapbox/kysely-deno-sqlite@2.2.0", "jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0", "jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0", @@ -113,6 +113,9 @@ "@hono/hono@4.5.0": { "integrity": "4a410f7773ac4b5b0eb4520b26c7ab7795a271d57a9df7fa1953ded6b90ccaf7" }, + "@hono/hono@4.5.1": { + "integrity": "459748ed4d4146c6e4bdff0213ff1ac44749904066ae02e7550d6c7f28c9bc4c" + }, "@nostrify/nostrify@0.22.4": { "integrity": "1c8a7847e5773213044b491e85fd7cafae2ad194ce59da4d957d2b27c776b42d", "dependencies": [ @@ -142,8 +145,8 @@ "npm:zod@^3.23.8" ] }, - "@nostrify/nostrify@0.26.3": { - "integrity": "3e13e30f4fa3f76dcbcf9178630a9b2871186eb1d226d66234c0cdfd4841f548", + "@nostrify/nostrify@0.27.0-rc.1": { + "integrity": "13d5a001a7a422cbab3b0cef75f98586363e01a1440a8601bcb4c968d6e537af", "dependencies": [ "jsr:@std/crypto@^0.224.0", "jsr:@std/encoding@^0.224.1", @@ -1760,7 +1763,7 @@ "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "jsr:@db/sqlite@^0.11.1", "jsr:@hono/hono@^4.4.6", - "jsr:@nostrify/nostrify@^0.26.3", + "jsr:@nostrify/nostrify@0.27.0-rc.1", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", From ff353571210df620f6f796f10cf9a3ac7afa28a2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 23 Jul 2024 15:13:02 -0500 Subject: [PATCH 02/14] Upgrade Nostrify to v0.27.0-rc.2 --- deno.json | 2 +- deno.lock | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/deno.json b/deno.json index 272da841..de587ca1 100644 --- a/deno.json +++ b/deno.json @@ -27,7 +27,7 @@ "@hono/hono": "jsr:@hono/hono@^4.4.6", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@0.27.0-rc.1", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@0.27.0-rc.2", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", diff --git a/deno.lock b/deno.lock index 4548c23b..7f8559d8 100644 --- a/deno.lock +++ b/deno.lock @@ -9,7 +9,7 @@ "jsr:@gleasonator/policy@0.2.0": "jsr:@gleasonator/policy@0.2.0", "jsr:@gleasonator/policy@0.4.0": "jsr:@gleasonator/policy@0.4.0", "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.1", - "jsr:@nostrify/nostrify@0.27.0-rc.1": "jsr:@nostrify/nostrify@0.27.0-rc.1", + "jsr:@nostrify/nostrify@0.27.0-rc.2": "jsr:@nostrify/nostrify@0.27.0-rc.2", "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", @@ -30,7 +30,7 @@ "jsr:@std/fmt@^0.221.0": "jsr:@std/fmt@0.221.0", "jsr:@std/fs@^0.221.0": "jsr:@std/fs@0.221.0", "jsr:@std/fs@^0.229.3": "jsr:@std/fs@0.229.3", - "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.0", + "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.1", "jsr:@std/io@^0.224": "jsr:@std/io@0.224.3", "jsr:@std/media-types@^0.224.1": "jsr:@std/media-types@0.224.1", "jsr:@std/path@0.217": "jsr:@std/path@0.217.0", @@ -145,12 +145,10 @@ "npm:zod@^3.23.8" ] }, - "@nostrify/nostrify@0.27.0-rc.1": { - "integrity": "13d5a001a7a422cbab3b0cef75f98586363e01a1440a8601bcb4c968d6e537af", + "@nostrify/nostrify@0.27.0-rc.2": { + "integrity": "4f33dd4a61dbc6b47ce07f5a89f85c425474728655efd8f7845b5f371eb76dda", "dependencies": [ - "jsr:@std/crypto@^0.224.0", "jsr:@std/encoding@^0.224.1", - "npm:@scure/base@^1.1.6", "npm:@scure/bip32@^1.4.0", "npm:@scure/bip39@^1.3.0", "npm:kysely@^0.27.3", @@ -228,6 +226,9 @@ "@std/internal@1.0.0": { "integrity": "ac6a6dfebf838582c4b4f61a6907374e27e05bedb6ce276e0f1608fe84e7cd9a" }, + "@std/internal@1.0.1": { + "integrity": "6f8c7544d06a11dd256c8d6ba54b11ed870aac6c5aeafff499892662c57673e6" + }, "@std/io@0.224.0": { "integrity": "0aff885d21d829c050b8a08b1d71b54aed5841aecf227f8d77e99ec529a11e8e", "dependencies": [ @@ -1763,7 +1764,7 @@ "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "jsr:@db/sqlite@^0.11.1", "jsr:@hono/hono@^4.4.6", - "jsr:@nostrify/nostrify@0.27.0-rc.1", + "jsr:@nostrify/nostrify@0.27.0-rc.2", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", From 64eeee5ff7756c8586b7653a1f7355ecda52f6cc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 23 Jul 2024 22:09:51 -0500 Subject: [PATCH 03/14] Upgrade Nostrify to v0.27.0-rc.3 --- deno.json | 2 +- deno.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deno.json b/deno.json index de587ca1..905d9c3d 100644 --- a/deno.json +++ b/deno.json @@ -27,7 +27,7 @@ "@hono/hono": "jsr:@hono/hono@^4.4.6", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@0.27.0-rc.2", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@0.27.0-rc.3", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", diff --git a/deno.lock b/deno.lock index 7f8559d8..f94857fe 100644 --- a/deno.lock +++ b/deno.lock @@ -9,7 +9,7 @@ "jsr:@gleasonator/policy@0.2.0": "jsr:@gleasonator/policy@0.2.0", "jsr:@gleasonator/policy@0.4.0": "jsr:@gleasonator/policy@0.4.0", "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.1", - "jsr:@nostrify/nostrify@0.27.0-rc.2": "jsr:@nostrify/nostrify@0.27.0-rc.2", + "jsr:@nostrify/nostrify@0.27.0-rc.3": "jsr:@nostrify/nostrify@0.27.0-rc.3", "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", @@ -145,8 +145,8 @@ "npm:zod@^3.23.8" ] }, - "@nostrify/nostrify@0.27.0-rc.2": { - "integrity": "4f33dd4a61dbc6b47ce07f5a89f85c425474728655efd8f7845b5f371eb76dda", + "@nostrify/nostrify@0.27.0-rc.3": { + "integrity": "0cd3d26bb0d797e95e74b301efc85f4262bd913df12644ff42494a97a02a6c0d", "dependencies": [ "jsr:@std/encoding@^0.224.1", "npm:@scure/bip32@^1.4.0", @@ -1764,7 +1764,7 @@ "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "jsr:@db/sqlite@^0.11.1", "jsr:@hono/hono@^4.4.6", - "jsr:@nostrify/nostrify@0.27.0-rc.2", + "jsr:@nostrify/nostrify@0.27.0-rc.3", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", From 5ebde99320d6c330e42364e7372dd8e0faf50589 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 23 Jul 2024 22:19:10 -0500 Subject: [PATCH 04/14] trends: Deno.cron try/catch --- deno.lock | 2 ++ src/controllers/api/trends.ts | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/deno.lock b/deno.lock index f94857fe..a758bbb1 100644 --- a/deno.lock +++ b/deno.lock @@ -148,7 +148,9 @@ "@nostrify/nostrify@0.27.0-rc.3": { "integrity": "0cd3d26bb0d797e95e74b301efc85f4262bd913df12644ff42494a97a02a6c0d", "dependencies": [ + "jsr:@std/crypto@^0.224.0", "jsr:@std/encoding@^0.224.1", + "npm:@scure/base@^1.1.6", "npm:@scure/bip32@^1.4.0", "npm:@scure/bip39@^1.3.0", "npm:kysely@^0.27.3", diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts index 76c84f58..d581ddc8 100644 --- a/src/controllers/api/trends.ts +++ b/src/controllers/api/trends.ts @@ -12,8 +12,12 @@ import { renderStatus } from '@/views/mastodon/statuses.ts'; let trendingHashtagsCache = getTrendingHashtags(); Deno.cron('update trending hashtags cache', '35 * * * *', async () => { - const trends = await getTrendingHashtags(); - trendingHashtagsCache = Promise.resolve(trends); + try { + const trends = await getTrendingHashtags(); + trendingHashtagsCache = Promise.resolve(trends); + } catch (e) { + console.error(e); + } }); const trendingTagsQuerySchema = z.object({ @@ -51,8 +55,12 @@ async function getTrendingHashtags() { let trendingLinksCache = getTrendingLinks(); Deno.cron('update trending links cache', '50 * * * *', async () => { - const trends = await getTrendingLinks(); - trendingLinksCache = Promise.resolve(trends); + try { + const trends = await getTrendingLinks(); + trendingLinksCache = Promise.resolve(trends); + } catch (e) { + console.error(e); + } }); const trendingLinksController: AppController = async (c) => { From 7949c95f77d376ca3c11f4892d69ef798a347181 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 23 Jul 2024 22:32:16 -0500 Subject: [PATCH 05/14] try-catch trending tags cron --- src/trends.ts | 54 +++++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/src/trends.ts b/src/trends.ts index c346a492..3f0ab625 100644 --- a/src/trends.ts +++ b/src/trends.ts @@ -72,33 +72,37 @@ export async function updateTrendingTags( const tagNames = aliases ? [tagName, ...aliases] : [tagName]; - const trends = await getTrendingTagValues(kysely, tagNames, { - kinds, - since: yesterday, - until: now, - limit, - }); + try { + const trends = await getTrendingTagValues(kysely, tagNames, { + kinds, + since: yesterday, + until: now, + limit, + }); - if (!trends.length) { - console.info(`No trending ${l} found. Skipping.`); - return; + if (!trends.length) { + console.info(`No trending ${l} found. Skipping.`); + return; + } + + const signer = new AdminSigner(); + + const label = await signer.signEvent({ + kind: 1985, + content: '', + tags: [ + ['L', 'pub.ditto.trends'], + ['l', l, 'pub.ditto.trends'], + ...trends.map(({ value, authors, uses }) => [tagName, value, extra, authors.toString(), uses.toString()]), + ], + created_at: Math.floor(Date.now() / 1000), + }); + + await handleEvent(label, signal); + console.info(`Trending ${l} updated.`); + } catch (e) { + console.error(`Error updating trending ${l}: ${e.message}`); } - - const signer = new AdminSigner(); - - const label = await signer.signEvent({ - kind: 1985, - content: '', - tags: [ - ['L', 'pub.ditto.trends'], - ['l', l, 'pub.ditto.trends'], - ...trends.map(({ value, authors, uses }) => [tagName, value, extra, authors.toString(), uses.toString()]), - ], - created_at: Math.floor(Date.now() / 1000), - }); - - await handleEvent(label, signal); - console.info(`Trending ${l} updated.`); } /** Update trending pubkeys. */ From 2b8009d8405f168d586aac51286b42f854ce7158 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 23 Jul 2024 23:06:00 -0500 Subject: [PATCH 06/14] trends: catch initial queries --- src/controllers/api/trends.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts index d581ddc8..181d8dc9 100644 --- a/src/controllers/api/trends.ts +++ b/src/controllers/api/trends.ts @@ -9,7 +9,10 @@ import { generateDateRange, Time } from '@/utils/time.ts'; import { unfurlCardCached } from '@/utils/unfurl.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; -let trendingHashtagsCache = getTrendingHashtags(); +let trendingHashtagsCache = getTrendingHashtags().catch((e) => { + console.error(`Failed to get trending hashtags: ${e}`); + return Promise.resolve([]); +}); Deno.cron('update trending hashtags cache', '35 * * * *', async () => { try { @@ -52,7 +55,10 @@ async function getTrendingHashtags() { }); } -let trendingLinksCache = getTrendingLinks(); +let trendingLinksCache = getTrendingLinks().catch((e) => { + console.error(`Failed to get trending links: ${e}`); + return Promise.resolve([]); +}); Deno.cron('update trending links cache', '50 * * * *', async () => { try { From 75e9837c09f9cc939d559e5d02058958cdb46804 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 24 Jul 2024 13:59:21 -0500 Subject: [PATCH 07/14] Upgrade Nostrify to v0.27.0-rc.4 --- deno.json | 2 +- deno.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deno.json b/deno.json index 905d9c3d..a4f8fba9 100644 --- a/deno.json +++ b/deno.json @@ -27,7 +27,7 @@ "@hono/hono": "jsr:@hono/hono@^4.4.6", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@0.27.0-rc.3", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@0.27.0-rc.4", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", diff --git a/deno.lock b/deno.lock index a758bbb1..a9a42d5a 100644 --- a/deno.lock +++ b/deno.lock @@ -9,7 +9,7 @@ "jsr:@gleasonator/policy@0.2.0": "jsr:@gleasonator/policy@0.2.0", "jsr:@gleasonator/policy@0.4.0": "jsr:@gleasonator/policy@0.4.0", "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.1", - "jsr:@nostrify/nostrify@0.27.0-rc.3": "jsr:@nostrify/nostrify@0.27.0-rc.3", + "jsr:@nostrify/nostrify@0.27.0-rc.4": "jsr:@nostrify/nostrify@0.27.0-rc.4", "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", @@ -145,8 +145,8 @@ "npm:zod@^3.23.8" ] }, - "@nostrify/nostrify@0.27.0-rc.3": { - "integrity": "0cd3d26bb0d797e95e74b301efc85f4262bd913df12644ff42494a97a02a6c0d", + "@nostrify/nostrify@0.27.0-rc.4": { + "integrity": "e49de7bc8fe8091f482d7b2ce83f02daca126abd358719633f85a90d21476e43", "dependencies": [ "jsr:@std/crypto@^0.224.0", "jsr:@std/encoding@^0.224.1", @@ -1766,7 +1766,7 @@ "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "jsr:@db/sqlite@^0.11.1", "jsr:@hono/hono@^4.4.6", - "jsr:@nostrify/nostrify@0.27.0-rc.3", + "jsr:@nostrify/nostrify@0.27.0-rc.4", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", From df9ae26a3a6c6087e8a1d83a63e6bb3aeaf779b6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 29 Jul 2024 14:18:46 -0500 Subject: [PATCH 08/14] Upgrade Nostrify to v0.27.0 --- deno.json | 2 +- deno.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deno.json b/deno.json index db3da221..ff91487a 100644 --- a/deno.json +++ b/deno.json @@ -27,7 +27,7 @@ "@hono/hono": "jsr:@hono/hono@^4.4.6", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@0.27.0-rc.4", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@0.27.0", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", diff --git a/deno.lock b/deno.lock index a9a42d5a..4749d2b3 100644 --- a/deno.lock +++ b/deno.lock @@ -9,7 +9,7 @@ "jsr:@gleasonator/policy@0.2.0": "jsr:@gleasonator/policy@0.2.0", "jsr:@gleasonator/policy@0.4.0": "jsr:@gleasonator/policy@0.4.0", "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.1", - "jsr:@nostrify/nostrify@0.27.0-rc.4": "jsr:@nostrify/nostrify@0.27.0-rc.4", + "jsr:@nostrify/nostrify@0.27.0": "jsr:@nostrify/nostrify@0.27.0", "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", @@ -145,8 +145,8 @@ "npm:zod@^3.23.8" ] }, - "@nostrify/nostrify@0.27.0-rc.4": { - "integrity": "e49de7bc8fe8091f482d7b2ce83f02daca126abd358719633f85a90d21476e43", + "@nostrify/nostrify@0.27.0": { + "integrity": "c4461dd93ed78c7bd0f3a4fbc0d77ba68acafa05ffcf6b82f3f3962a3a7e9698", "dependencies": [ "jsr:@std/crypto@^0.224.0", "jsr:@std/encoding@^0.224.1", @@ -1766,7 +1766,7 @@ "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "jsr:@db/sqlite@^0.11.1", "jsr:@hono/hono@^4.4.6", - "jsr:@nostrify/nostrify@0.27.0-rc.4", + "jsr:@nostrify/nostrify@0.27.0", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", From ece18c9bd88219a511965e61b9375b440641d57d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 29 Jul 2024 15:09:36 -0500 Subject: [PATCH 09/14] Add tags table migration --- src/db/DittoTables.ts | 3 + src/db/migrations/029_tag_queries.ts | 85 ++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 src/db/migrations/029_tag_queries.ts diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index 863ca61e..73f0cdad 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -46,6 +46,9 @@ interface TagRow { event_id: string; name: string; value: string; + kind: number; + pubkey: string; + created_at: number; } interface NIP46TokenRow { diff --git a/src/db/migrations/029_tag_queries.ts b/src/db/migrations/029_tag_queries.ts new file mode 100644 index 00000000..3211f9d1 --- /dev/null +++ b/src/db/migrations/029_tag_queries.ts @@ -0,0 +1,85 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('nostr_tags_new') + .addColumn('event_id', 'text', (col) => col.notNull().references('nostr_events.id').onDelete('cascade')) + .addColumn('name', 'text', (col) => col.notNull()) + .addColumn('value', 'text', (col) => col.notNull()) + .addColumn('kind', 'integer', (col) => col.notNull()) + .addColumn('pubkey', 'text', (col) => col.notNull()) + .addColumn('created_at', 'integer', (col) => col.notNull()) + .execute(); + + setTimeout(() => { + console.warn( + 'Recreating the tags table to boost performance. Depending on the size of your database, this could take a very long time, even as long as 2 days!', + ); + const emojis = ['⚡', '🐛', '🔎', '😂', '😅', '😬', '😭', '🙃', '🤔', '🧐', '🧐', '🫠']; + setInterval(() => { + const emoji = emojis[Math.floor(Math.random() * emojis.length)]; + console.info(`Recreating tags table... ${emoji}`); + }, 60_000); + }, 10_000); + + // Copy data to the new table. + await sql` + INSERT INTO + nostr_tags_new (name, value, event_id, kind, pubkey, created_at) + SELECT + t.name, t.value, t.event_id, e.kind, e.pubkey, e.created_at + FROM + nostr_tags as t LEFT JOIN nostr_events e on t.event_id = e.id; + `.execute(db); + + // Drop the old table and rename it. + await db.schema.dropTable('nostr_tags').execute(); + await db.schema.alterTable('nostr_tags_new').renameTo('nostr_tags').execute(); + + await db.schema + .createIndex('nostr_tags_created_at') + .on('nostr_tags') + .ifNotExists() + .columns(['value', 'name', 'created_at desc', 'event_id asc']) + .execute(); + await db.schema + .createIndex('nostr_tags_kind_created_at') + .on('nostr_tags') + .ifNotExists() + .columns(['value', 'name', 'kind', 'created_at desc', 'event_id asc']) + .execute(); + await db.schema + .createIndex('nostr_tags_kind_pubkey_created_at') + .on('nostr_tags') + .ifNotExists() + .columns(['value', 'name', 'kind', 'pubkey', 'created_at desc', 'event_id asc']) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema + .createTable('nostr_tags_old') + .addColumn('event_id', 'text', (col) => col.references('nostr_events.id').onDelete('cascade')) + .addColumn('name', 'text', (col) => col.notNull()) + .addColumn('value', 'text', (col) => col.notNull()) + .addColumn('kind', 'integer', (col) => col.notNull()) + .addColumn('pubkey', 'text', (col) => col.notNull()) + .addColumn('created_at', 'integer', (col) => col.notNull()) + .execute(); + + await sql` + INSERT INTO + nostr_tags_old (name, value, event_id) + SELECT + name, value, event_id + FROM + nostr_tags; + `.execute(db); + + await db.schema.dropTable('nostr_tags').execute(); + await db.schema.alterTable('nostr_tags_old').renameTo('nostr_tags').execute(); + + await db.schema.createIndex('idx_tags_event_id').on('nostr_tags').ifNotExists().column('event_id').execute(); + await db.schema.createIndex('idx_tags_name').on('nostr_tags').ifNotExists().column('name').execute(); + await db.schema.createIndex('idx_tags_tag_value').on('nostr_tags').ifNotExists().columns(['name', 'value']).execute(); +} From 48a4e30e3811fb722e42802566da0c28f6a8ce1a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 29 Jul 2024 15:33:16 -0500 Subject: [PATCH 10/14] trends: read directly from the tags table instead of doing an inner join on events --- src/db/migrations/029_tag_queries.ts | 6 ++++++ src/trends.ts | 11 +++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/db/migrations/029_tag_queries.ts b/src/db/migrations/029_tag_queries.ts index 3211f9d1..932b9599 100644 --- a/src/db/migrations/029_tag_queries.ts +++ b/src/db/migrations/029_tag_queries.ts @@ -54,6 +54,12 @@ export async function up(db: Kysely): Promise { .ifNotExists() .columns(['value', 'name', 'kind', 'pubkey', 'created_at desc', 'event_id asc']) .execute(); + await db.schema + .createIndex('nostr_tags_trends') + .on('nostr_tags') + .ifNotExists() + .columns(['created_at', 'name', 'kind']) + .execute(); } export async function down(db: Kysely): Promise { diff --git a/src/trends.ts b/src/trends.ts index 3f0ab625..c5165f33 100644 --- a/src/trends.ts +++ b/src/trends.ts @@ -22,24 +22,23 @@ export async function getTrendingTagValues( ): Promise<{ value: string; authors: number; uses: number }[]> { let query = kysely .selectFrom('nostr_tags') - .innerJoin('nostr_events', 'nostr_events.id', 'nostr_tags.event_id') .select(({ fn }) => [ 'nostr_tags.value', - fn.agg('count', ['nostr_events.pubkey']).distinct().as('authors'), + fn.agg('count', ['nostr_tags.pubkey']).distinct().as('authors'), fn.countAll().as('uses'), ]) .where('nostr_tags.name', 'in', tagNames) .groupBy('nostr_tags.value') - .orderBy((c) => c.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc'); + .orderBy((c) => c.fn.agg('count', ['nostr_tags.pubkey']).distinct(), 'desc'); if (filter.kinds) { - query = query.where('nostr_events.kind', 'in', filter.kinds); + query = query.where('nostr_tags.kind', 'in', filter.kinds); } if (typeof filter.since === 'number') { - query = query.where('nostr_events.created_at', '>=', filter.since); + query = query.where('nostr_tags.created_at', '>=', filter.since); } if (typeof filter.until === 'number') { - query = query.where('nostr_events.created_at', '<=', filter.until); + query = query.where('nostr_tags.created_at', '<=', filter.until); } if (typeof filter.limit === 'number') { query = query.limit(filter.limit); From 31f5254fb3d7ac8530946cdf557e0833f8072bd2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 29 Jul 2024 15:37:45 -0500 Subject: [PATCH 11/14] Clear timeouts in tag queries migration --- src/db/migrations/029_tag_queries.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/db/migrations/029_tag_queries.ts b/src/db/migrations/029_tag_queries.ts index 932b9599..5a27d720 100644 --- a/src/db/migrations/029_tag_queries.ts +++ b/src/db/migrations/029_tag_queries.ts @@ -11,12 +11,13 @@ export async function up(db: Kysely): Promise { .addColumn('created_at', 'integer', (col) => col.notNull()) .execute(); - setTimeout(() => { + let iid: number | undefined; + const tid = setTimeout(() => { console.warn( 'Recreating the tags table to boost performance. Depending on the size of your database, this could take a very long time, even as long as 2 days!', ); const emojis = ['⚡', '🐛', '🔎', '😂', '😅', '😬', '😭', '🙃', '🤔', '🧐', '🧐', '🫠']; - setInterval(() => { + iid = setInterval(() => { const emoji = emojis[Math.floor(Math.random() * emojis.length)]; console.info(`Recreating tags table... ${emoji}`); }, 60_000); @@ -32,6 +33,9 @@ export async function up(db: Kysely): Promise { nostr_tags as t LEFT JOIN nostr_events e on t.event_id = e.id; `.execute(db); + clearTimeout(tid); + if (iid) clearInterval(iid); + // Drop the old table and rename it. await db.schema.dropTable('nostr_tags').execute(); await db.schema.alterTable('nostr_tags_new').renameTo('nostr_tags').execute(); From 37f229408cf02ad2d1e6c6482259b36fefd5c4d0 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 29 Jul 2024 15:59:29 -0500 Subject: [PATCH 12/14] Seed zap splits when Storages.db is first accessed --- src/startup.ts | 3 --- src/storages.ts | 5 ++++- src/utils/zap-split.ts | 12 +++++------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/startup.ts b/src/startup.ts index 21df4d50..f3282d73 100644 --- a/src/startup.ts +++ b/src/startup.ts @@ -1,7 +1,6 @@ // Starts up applications required to run before the HTTP server is on. import { Conf } from '@/config.ts'; -import { seedZapSplits } from '@/utils/zap-split.ts'; import { cron } from '@/cron.ts'; import { startFirehose } from '@/firehose.ts'; @@ -12,5 +11,3 @@ if (Conf.firehoseEnabled) { if (Conf.cronEnabled) { cron(); } - -await seedZapSplits(); diff --git a/src/storages.ts b/src/storages.ts index 303a6eb7..c8229880 100644 --- a/src/storages.ts +++ b/src/storages.ts @@ -7,6 +7,7 @@ import { SearchStore } from '@/storages/search-store.ts'; import { InternalRelay } from '@/storages/InternalRelay.ts'; import { NPool, NRelay1 } from '@nostrify/nostrify'; import { getRelays } from '@/utils/outbox.ts'; +import { seedZapSplits } from '@/utils/zap-split.ts'; export class Storages { private static _db: Promise | undefined; @@ -20,7 +21,9 @@ export class Storages { if (!this._db) { this._db = (async () => { const kysely = await DittoDB.getInstance(); - return new EventsDB(kysely); + const store = new EventsDB(kysely); + await seedZapSplits(store); + return store; })(); } return this._db; diff --git a/src/utils/zap-split.ts b/src/utils/zap-split.ts index 553d9c8c..80d6cb2e 100644 --- a/src/utils/zap-split.ts +++ b/src/utils/zap-split.ts @@ -1,10 +1,8 @@ import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Conf } from '@/config.ts'; -import { handleEvent } from '@/pipeline.ts'; import { NSchema as n, NStore } from '@nostrify/nostrify'; import { nostrNow } from '@/utils.ts'; import { percentageSchema } from '@/schema.ts'; -import { Storages } from '@/storages.ts'; type Pubkey = string; type ExtraMessage = string; @@ -39,11 +37,10 @@ export async function getZapSplits(store: NStore, pubkey: string): Promise Date: Mon, 29 Jul 2024 16:42:18 -0500 Subject: [PATCH 13/14] zap-split.test: use using --- src/utils/zap-split.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/utils/zap-split.test.ts b/src/utils/zap-split.test.ts index 08454160..33a8023e 100644 --- a/src/utils/zap-split.test.ts +++ b/src/utils/zap-split.test.ts @@ -6,7 +6,8 @@ import { getZapSplits } from '@/utils/zap-split.ts'; import { getTestDB } from '@/test.ts'; Deno.test('Get zap splits in DittoZapSplits format', async () => { - const { store } = await getTestDB(); + await using db = await getTestDB(); + const store = db.store; const sk = generateSecretKey(); const pubkey = getPublicKey(sk); @@ -19,6 +20,7 @@ Deno.test('Get zap splits in DittoZapSplits format', async () => { ['p', '0461fcbecc4c3374439932d6b8f11269ccdb7cc973ad7a50ae362db135a474dd', '3', 'Alex creator of Ditto'], ], }, sk); + await store.event(event); const eventFromDb = await store.query([{ kinds: [30078], authors: [pubkey] }]); @@ -36,7 +38,8 @@ Deno.test('Get zap splits in DittoZapSplits format', async () => { }); Deno.test('Zap split is empty', async () => { - const { store } = await getTestDB(); + await using db = await getTestDB(); + const store = db.store; const sk = generateSecretKey(); const pubkey = getPublicKey(sk); From 18e446bd056b051135da0ad086ae0bd944e827b9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 29 Jul 2024 16:46:27 -0500 Subject: [PATCH 14/14] Delete zap split tests --- src/utils/zap-split.test.ts | 63 ------------------------------------- 1 file changed, 63 deletions(-) delete mode 100644 src/utils/zap-split.test.ts diff --git a/src/utils/zap-split.test.ts b/src/utils/zap-split.test.ts deleted file mode 100644 index 33a8023e..00000000 --- a/src/utils/zap-split.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { assertEquals } from '@std/assert'; -import { generateSecretKey, getPublicKey } from 'nostr-tools'; - -import { genEvent } from '@/test.ts'; -import { getZapSplits } from '@/utils/zap-split.ts'; -import { getTestDB } from '@/test.ts'; - -Deno.test('Get zap splits in DittoZapSplits format', async () => { - await using db = await getTestDB(); - const store = db.store; - - const sk = generateSecretKey(); - const pubkey = getPublicKey(sk); - - const event = genEvent({ - kind: 30078, - tags: [ - ['d', 'pub.ditto.zapSplits'], - ['p', '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', '2', 'Patrick developer'], - ['p', '0461fcbecc4c3374439932d6b8f11269ccdb7cc973ad7a50ae362db135a474dd', '3', 'Alex creator of Ditto'], - ], - }, sk); - - await store.event(event); - - const eventFromDb = await store.query([{ kinds: [30078], authors: [pubkey] }]); - - assertEquals(eventFromDb.length, 1); - - const zapSplits = await getZapSplits(store, pubkey); - - assertEquals(zapSplits, { - '0461fcbecc4c3374439932d6b8f11269ccdb7cc973ad7a50ae362db135a474dd': { amount: 3, message: 'Alex creator of Ditto' }, - '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4': { amount: 2, message: 'Patrick developer' }, - }); - - assertEquals(await getZapSplits(store, 'garbage'), undefined); -}); - -Deno.test('Zap split is empty', async () => { - await using db = await getTestDB(); - const store = db.store; - - const sk = generateSecretKey(); - const pubkey = getPublicKey(sk); - - const event = genEvent({ - kind: 30078, - tags: [ - ['d', 'pub.ditto.zapSplits'], - ['p', 'baka'], - ], - }, sk); - await store.event(event); - - const eventFromDb = await store.query([{ kinds: [30078], authors: [pubkey] }]); - - assertEquals(eventFromDb.length, 1); - - const zapSplits = await getZapSplits(store, pubkey); - - assertEquals(zapSplits, {}); -});