From a39910fa984e5f4896aa860143af07c1044e0271 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 13:12:40 -0500 Subject: [PATCH 1/6] Add a function to recalculate author stats --- scripts/stats-recompute.ts | 25 ++----------------------- src/stats.ts | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/scripts/stats-recompute.ts b/scripts/stats-recompute.ts index dcb0bc07..4037a85b 100644 --- a/scripts/stats-recompute.ts +++ b/scripts/stats-recompute.ts @@ -1,8 +1,6 @@ import { nip19 } from 'nostr-tools'; -import { DittoDB } from '@/db/DittoDB.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; -import { Storages } from '@/storages.ts'; +import { refreshAuthorStats } from '@/stats.ts'; let pubkey: string; try { @@ -17,23 +15,4 @@ try { Deno.exit(1); } -const store = await Storages.db(); -const kysely = await DittoDB.getInstance(); - -const [followList] = await store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]); - -const authorStats: DittoTables['author_stats'] = { - pubkey, - followers_count: (await store.count([{ kinds: [3], '#p': [pubkey] }])).count, - following_count: followList?.tags.filter(([name]) => name === 'p')?.length ?? 0, - notes_count: (await store.count([{ kinds: [1], authors: [pubkey] }])).count, -}; - -await kysely.insertInto('author_stats') - .values(authorStats) - .onConflict((oc) => - oc - .column('pubkey') - .doUpdateSet(authorStats) - ) - .execute(); +await refreshAuthorStats(pubkey); diff --git a/src/stats.ts b/src/stats.ts index 92040710..74242f7a 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -1,11 +1,12 @@ -import { NKinds, NostrEvent } from '@nostrify/nostrify'; +import { NKinds, NostrEvent, NStore } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; import { InsertQueryBuilder, Kysely } from 'kysely'; +import { SetRequired } from 'type-fest'; import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Storages } from '@/storages.ts'; -import { findReplyTag } from '@/tags.ts'; +import { findReplyTag, getTagSet } from '@/tags.ts'; type AuthorStat = keyof Omit; type EventStat = keyof Omit; @@ -216,4 +217,35 @@ function getFollowDiff(event: NostrEvent, prev?: NostrEvent): AuthorStatDiff[] { ]; } -export { updateStats }; +/** Refresh the author's stats in the database. */ +async function refreshAuthorStats(pubkey: string): Promise { + const store = await Storages.db(); + const stats = await countAuthorStats(store, pubkey); + + const kysely = await DittoDB.getInstance(); + await kysely.insertInto('author_stats') + .values(stats) + .onConflict((oc) => oc.column('pubkey').doUpdateSet(stats)) + .execute(); +} + +/** Calculate author stats from the database. */ +async function countAuthorStats( + store: SetRequired, + pubkey: string, +): Promise { + const [{ count: followers_count }, { count: notes_count }, [followList]] = await Promise.all([ + store.count([{ kinds: [3], '#p': [pubkey] }]), + store.count([{ kinds: [1], authors: [pubkey] }]), + store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]), + ]); + + return { + pubkey, + followers_count, + following_count: getTagSet(followList?.tags ?? [], 'p').size, + notes_count, + }; +} + +export { refreshAuthorStats, updateStats }; From 6995bd2b292810913c6c8d5877ea8f76b91e6b42 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 16:23:23 -0500 Subject: [PATCH 2/6] Upgrade Deno to the latest version --- .gitlab-ci.yml | 2 +- .tool-versions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a9dee457..8e728884 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:1.41.3 +image: denoland/deno:1.43.4 default: interruptible: true diff --git a/.tool-versions b/.tool-versions index a13fd5ff..9bbaf96d 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -deno 1.41.3 \ No newline at end of file +deno 1.43.4 \ No newline at end of file From ae9516b445332df74aa817a26e5be522e243a923 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 16:23:38 -0500 Subject: [PATCH 3/6] refreshAuthorStats: return the stats --- src/stats.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/stats.ts b/src/stats.ts index 74242f7a..08d4bb99 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -218,7 +218,7 @@ function getFollowDiff(event: NostrEvent, prev?: NostrEvent): AuthorStatDiff[] { } /** Refresh the author's stats in the database. */ -async function refreshAuthorStats(pubkey: string): Promise { +async function refreshAuthorStats(pubkey: string): Promise { const store = await Storages.db(); const stats = await countAuthorStats(store, pubkey); @@ -227,6 +227,8 @@ async function refreshAuthorStats(pubkey: string): Promise { .values(stats) .onConflict((oc) => oc.column('pubkey').doUpdateSet(stats)) .execute(); + + return stats; } /** Calculate author stats from the database. */ From 17b633019339506d2414bc27056531088544499a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 16:59:45 -0500 Subject: [PATCH 4/6] Downgrade Deno to v1.43.3 due to TypeScript issues --- .gitlab-ci.yml | 2 +- .tool-versions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8e728884..b2140dbf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:1.43.4 +image: denoland/deno:1.43.3 default: interruptible: true diff --git a/.tool-versions b/.tool-versions index 9bbaf96d..b3e19cd6 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -deno 1.43.4 \ No newline at end of file +deno 1.43.3 \ No newline at end of file From 5c2e3450a9ae2b06645f570ef2059109120a648f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 17:50:30 -0500 Subject: [PATCH 5/6] Refresh author stats: less naive way --- src/storages/hydrate.ts | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 9b958416..7b964d77 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -1,10 +1,12 @@ import { NostrEvent, NStore } from '@nostrify/nostrify'; +import { LRUCache } from 'lru-cache'; import { matchFilter } from 'nostr-tools'; import { DittoDB } from '@/db/DittoDB.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Conf } from '@/config.ts'; +import { refreshAuthorStats } from '@/stats.ts'; interface HydrateOpts { events: DittoEvent[]; @@ -55,6 +57,8 @@ async function hydrateEvents(opts: HydrateOpts): Promise { events: await gatherEventStats(cache), }; + requestMissingAuthorStats(events, stats.authors); + // Dedupe events. const results = [...new Map(cache.map((event) => [event.id, event])).values()]; @@ -266,6 +270,31 @@ async function gatherAuthorStats(events: DittoEvent[]): Promise( + events + .filter((event) => event.kind === 0) + .map((event) => event.pubkey), + ); + + const missing = pubkeys.difference( + new Set(stats.map((stat) => stat.pubkey)), + ); + + for (const pubkey of missing) { + refreshAuthorStatsDebounced(pubkey); + } +} + +const lru = new LRUCache({ max: 1000 }); + +/** Calls `refreshAuthorStats` only once per author. */ +function refreshAuthorStatsDebounced(pubkey: string): void { + if (lru.get(pubkey)) return; + lru.set(pubkey, true); + refreshAuthorStats(pubkey).catch(() => {}); +} + /** Collect event stats from the events. */ async function gatherEventStats(events: DittoEvent[]): Promise { const ids = new Set( From bf479d01625497eac9190286eaf383f381ca606b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 18:26:55 -0500 Subject: [PATCH 6/6] Move refreshAuthorStatsDebounced to stats.ts --- src/stats.ts | 12 +++++++++++- src/storages/hydrate.ts | 16 +++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/stats.ts b/src/stats.ts index 08d4bb99..43647818 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -1,6 +1,7 @@ import { NKinds, NostrEvent, NStore } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; import { InsertQueryBuilder, Kysely } from 'kysely'; +import { LRUCache } from 'lru-cache'; import { SetRequired } from 'type-fest'; import { DittoDB } from '@/db/DittoDB.ts'; @@ -250,4 +251,13 @@ async function countAuthorStats( }; } -export { refreshAuthorStats, updateStats }; +const lru = new LRUCache({ max: 1000 }); + +/** Calls `refreshAuthorStats` only once per author. */ +function refreshAuthorStatsDebounced(pubkey: string): void { + if (lru.get(pubkey)) return; + lru.set(pubkey, true); + refreshAuthorStats(pubkey).catch(() => {}); +} + +export { refreshAuthorStats, refreshAuthorStatsDebounced, updateStats }; diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 7b964d77..e5c488e3 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -1,12 +1,11 @@ import { NostrEvent, NStore } from '@nostrify/nostrify'; -import { LRUCache } from 'lru-cache'; import { matchFilter } from 'nostr-tools'; import { DittoDB } from '@/db/DittoDB.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Conf } from '@/config.ts'; -import { refreshAuthorStats } from '@/stats.ts'; +import { refreshAuthorStatsDebounced } from '@/stats.ts'; interface HydrateOpts { events: DittoEvent[]; @@ -57,7 +56,7 @@ async function hydrateEvents(opts: HydrateOpts): Promise { events: await gatherEventStats(cache), }; - requestMissingAuthorStats(events, stats.authors); + refreshMissingAuthorStats(events, stats.authors); // Dedupe events. const results = [...new Map(cache.map((event) => [event.id, event])).values()]; @@ -270,7 +269,7 @@ async function gatherAuthorStats(events: DittoEvent[]): Promise( events .filter((event) => event.kind === 0) @@ -286,15 +285,6 @@ function requestMissingAuthorStats(events: NostrEvent[], stats: DittoTables['aut } } -const lru = new LRUCache({ max: 1000 }); - -/** Calls `refreshAuthorStats` only once per author. */ -function refreshAuthorStatsDebounced(pubkey: string): void { - if (lru.get(pubkey)) return; - lru.set(pubkey, true); - refreshAuthorStats(pubkey).catch(() => {}); -} - /** Collect event stats from the events. */ async function gatherEventStats(events: DittoEvent[]): Promise { const ids = new Set(