From de9fecaf65ffcc4a752758b202cb456e8f89956f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 7 Dec 2023 18:43:24 -0600 Subject: [PATCH 01/20] Add a stats module (draft) --- src/stats.ts | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 src/stats.ts diff --git a/src/stats.ts b/src/stats.ts new file mode 100644 index 00000000..741c418c --- /dev/null +++ b/src/stats.ts @@ -0,0 +1,32 @@ +import { open as lmdb } from 'npm:lmdb'; +import { Event } from '@/deps.ts'; + +const db = lmdb({ path: 'data/ditto.lmdb' }); + +/** Store stats for the event in LMDB. */ +async function saveStats(event: Event): Promise { + switch (event.kind) { + case 6: + return await incrementMentionedEvent(event, 'reposts'); + case 7: + return await incrementMentionedEvent(event, 'reactions'); + } +} + +/** Increment the subkey for the first mentioned event. */ +async function incrementMentionedEvent(event: Event, subkey: string): Promise { + const eventId = event.tags.find(([name]) => name === 'e')?.[1]; + if (eventId) { + return await incrementKey([eventId, subkey]); + } +} + +/** Increase the counter by 1, or set the key if it doesn't exist. */ +function incrementKey(key: string[]): Promise { + return db.transaction(() => { + const value = db.get(key) || 0; + db.put(key, value + 1); + }); +} + +export { saveStats }; From 2ab76167952f2c8ab591c63eaf3162a0056f49ad Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 10:47:13 -0600 Subject: [PATCH 02/20] Upgrade Deno to v1.38.5 --- .gitlab-ci.yml | 2 +- .tool-versions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bcf4615f..aa164bb9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:1.38.4 +image: denoland/deno:1.38.5 default: interruptible: true diff --git a/.tool-versions b/.tool-versions index 12523898..fd4bb2f3 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -deno 1.38.4 +deno 1.38.5 \ No newline at end of file From bababe56f30f72b7a57e5c52d2b1108955f721d0 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 11:10:11 -0600 Subject: [PATCH 03/20] stats: update note count --- src/db.ts | 20 ++++++++++++++++++-- src/pipeline.ts | 6 +++++- src/stats.ts | 47 +++++++++++++++++++++++++++++++---------------- 3 files changed, 54 insertions(+), 19 deletions(-) diff --git a/src/db.ts b/src/db.ts index a722d4e2..6f0397f9 100644 --- a/src/db.ts +++ b/src/db.ts @@ -13,6 +13,22 @@ interface DittoDB { users: UserRow; relays: RelayRow; unattached_media: UnattachedMediaRow; + pubkey_stats: PubkeyStatsRow; + event_stats: EventStatsRow; +} + +interface PubkeyStatsRow { + pubkey: string; + followers_count: number; + following_count: number; + notes_count: number; +} + +interface EventStatsRow { + event_id: string; + replies_count: number; + reposts_count: number; + reactions_count: number; } interface EventRow { @@ -101,7 +117,7 @@ async function migrate() { console.log('Everything up-to-date.'); } else { console.log('Migrations finished!'); - for (const { migrationName, status } of results.results) { + for (const { migrationName, status } of results.results!) { console.log(` - ${migrationName}: ${status}`); } } @@ -110,4 +126,4 @@ async function migrate() { await migrate(); -export { db, type DittoDB, type EventRow, type TagRow, type UserRow }; +export { db, type DittoDB, type EventRow, type EventStatsRow, type PubkeyStatsRow, type TagRow, type UserRow }; diff --git a/src/pipeline.ts b/src/pipeline.ts index 9749a9cb..d4811e40 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -8,6 +8,7 @@ import { isEphemeralKind } from '@/kinds.ts'; import * as mixer from '@/mixer.ts'; import { publish } from '@/pool.ts'; import { isLocallyFollowed } from '@/queries.ts'; +import { updateStats } from '@/stats.ts'; import { Sub } from '@/subs.ts'; import { getTagSet } from '@/tags.ts'; import { eventAge, isRelay, nostrDate, Time } from '@/utils.ts'; @@ -68,7 +69,10 @@ async function storeEvent(event: Event, data: EventData): Promise { if (deletion) { return Promise.reject(new RelayError('blocked', 'event was deleted')); } else { - await eventsDB.insertEvent(event, data).catch(console.warn); + await Promise.all([ + eventsDB.insertEvent(event, data).catch(console.warn), + updateStats(event), + ]); } } else { return Promise.reject(new RelayError('blocked', 'only registered users can post')); diff --git a/src/stats.ts b/src/stats.ts index 741c418c..d0d14c06 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -1,11 +1,17 @@ -import { open as lmdb } from 'npm:lmdb'; +import { db, type PubkeyStatsRow } from '@/db.ts'; import { Event } from '@/deps.ts'; -const db = lmdb({ path: 'data/ditto.lmdb' }); +type PubkeyStat = keyof Omit; /** Store stats for the event in LMDB. */ -async function saveStats(event: Event): Promise { +function updateStats(event: Event) { + return updateStatsQuery(event).execute(); +} + +async function updateStatsQuery(event: Event) { switch (event.kind) { + case 1: + return incrementPubkeyStatQuery(event.pubkey, 'notes_count', 1); case 6: return await incrementMentionedEvent(event, 'reposts'); case 7: @@ -13,20 +19,29 @@ async function saveStats(event: Event): Promise { } } -/** Increment the subkey for the first mentioned event. */ -async function incrementMentionedEvent(event: Event, subkey: string): Promise { - const eventId = event.tags.find(([name]) => name === 'e')?.[1]; - if (eventId) { - return await incrementKey([eventId, subkey]); - } +function incrementPubkeyStatQuery(pubkey: string, stat: PubkeyStat, diff: number) { + const row: PubkeyStatsRow = { + pubkey, + followers_count: 0, + following_count: 0, + notes_count: 0, + }; + + row[stat] = diff; + + return db.insertInto('pubkey_stats') + .values(row) + .onConflict((oc) => + oc + .column('pubkey') + .doUpdateSet((eb) => ({ + [stat]: eb(stat, '+', diff), + })) + ); } -/** Increase the counter by 1, or set the key if it doesn't exist. */ -function incrementKey(key: string[]): Promise { - return db.transaction(() => { - const value = db.get(key) || 0; - db.put(key, value + 1); - }); +function findFirstTag({ tags }: Event, name: string): string | undefined { + return tags.find(([n]) => n === name)?.[1]; } -export { saveStats }; +export { updateStats }; From eca923d7c8f2c6a6205d1ebf9c58cbf5ebf6b39d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 11:43:41 -0600 Subject: [PATCH 04/20] stats: make the logic kind of make sense --- src/stats.ts | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src/stats.ts b/src/stats.ts index d0d14c06..15cf5e26 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -1,21 +1,24 @@ -import { db, type PubkeyStatsRow } from '@/db.ts'; +import { db, type EventStatsRow, type PubkeyStatsRow } from '@/db.ts'; import { Event } from '@/deps.ts'; type PubkeyStat = keyof Omit; +type EventStat = keyof Omit; /** Store stats for the event in LMDB. */ function updateStats(event: Event) { - return updateStatsQuery(event).execute(); + return updateStatsQuery(event)?.execute(); } -async function updateStatsQuery(event: Event) { +function updateStatsQuery(event: Event) { + const firstE = findFirstTag(event, 'e'); + switch (event.kind) { case 1: return incrementPubkeyStatQuery(event.pubkey, 'notes_count', 1); case 6: - return await incrementMentionedEvent(event, 'reposts'); + return firstE ? incrementEventStatQuery(firstE, 'reposts_count', 1) : undefined; case 7: - return await incrementMentionedEvent(event, 'reactions'); + return firstE ? incrementEventStatQuery(firstE, 'reactions_count', 1) : undefined; } } @@ -40,6 +43,27 @@ function incrementPubkeyStatQuery(pubkey: string, stat: PubkeyStat, diff: number ); } +function incrementEventStatQuery(eventId: string, stat: EventStat, diff: number) { + const row: EventStatsRow = { + event_id: eventId, + replies_count: 0, + reposts_count: 0, + reactions_count: 0, + }; + + row[stat] = diff; + + return db.insertInto('event_stats') + .values(row) + .onConflict((oc) => + oc + .column('event_id') + .doUpdateSet((eb) => ({ + [stat]: eb(stat, '+', diff), + })) + ); +} + function findFirstTag({ tags }: Event, name: string): string | undefined { return tags.find(([n]) => n === name)?.[1]; } From a8944dd7ea5928edf821275cbd700013eefa86b7 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 13:12:35 -0600 Subject: [PATCH 05/20] stats: support multiple values --- src/stats.ts | 50 +++++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/src/stats.ts b/src/stats.ts index 15cf5e26..edd74793 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -14,26 +14,28 @@ function updateStatsQuery(event: Event) { switch (event.kind) { case 1: - return incrementPubkeyStatQuery(event.pubkey, 'notes_count', 1); + return incrementPubkeyStatsQuery([event.pubkey], 'notes_count', 1); case 6: - return firstE ? incrementEventStatQuery(firstE, 'reposts_count', 1) : undefined; + return firstE ? incrementEventStatsQuery([firstE], 'reposts_count', 1) : undefined; case 7: - return firstE ? incrementEventStatQuery(firstE, 'reactions_count', 1) : undefined; + return firstE ? incrementEventStatsQuery([firstE], 'reactions_count', 1) : undefined; } } -function incrementPubkeyStatQuery(pubkey: string, stat: PubkeyStat, diff: number) { - const row: PubkeyStatsRow = { - pubkey, - followers_count: 0, - following_count: 0, - notes_count: 0, - }; - - row[stat] = diff; +function incrementPubkeyStatsQuery(pubkeys: string[], stat: PubkeyStat, diff: number) { + const values: PubkeyStatsRow[] = pubkeys.map((pubkey) => { + const row: PubkeyStatsRow = { + pubkey, + followers_count: 0, + following_count: 0, + notes_count: 0, + }; + row[stat] = diff; + return row; + }); return db.insertInto('pubkey_stats') - .values(row) + .values(values) .onConflict((oc) => oc .column('pubkey') @@ -43,18 +45,20 @@ function incrementPubkeyStatQuery(pubkey: string, stat: PubkeyStat, diff: number ); } -function incrementEventStatQuery(eventId: string, stat: EventStat, diff: number) { - const row: EventStatsRow = { - event_id: eventId, - replies_count: 0, - reposts_count: 0, - reactions_count: 0, - }; - - row[stat] = diff; +function incrementEventStatsQuery(eventIds: string[], stat: EventStat, diff: number) { + const values: EventStatsRow[] = eventIds.map((event_id) => { + const row: EventStatsRow = { + event_id, + replies_count: 0, + reposts_count: 0, + reactions_count: 0, + }; + row[stat] = diff; + return row; + }); return db.insertInto('event_stats') - .values(row) + .values(values) .onConflict((oc) => oc .column('event_id') From 7167553afeffae2d504f324574dc333450d097e9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 13:53:51 -0600 Subject: [PATCH 06/20] stats: switch to a system based on diff tuples --- src/stats.ts | 64 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/src/stats.ts b/src/stats.ts index edd74793..2496ed67 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -1,29 +1,59 @@ import { db, type EventStatsRow, type PubkeyStatsRow } from '@/db.ts'; -import { Event } from '@/deps.ts'; +import { Event, findReplyTag } from '@/deps.ts'; type PubkeyStat = keyof Omit; type EventStat = keyof Omit; +type PubkeyStatDiff = ['pubkey_stats', pubkey: string, stat: PubkeyStat, diff: number]; +type EventStatDiff = ['event_stats', eventId: string, stat: EventStat, diff: number]; +type StatDiff = PubkeyStatDiff | EventStatDiff; + /** Store stats for the event in LMDB. */ function updateStats(event: Event) { - return updateStatsQuery(event)?.execute(); + const statDiffs = getStatsDiff(event); + if (!statDiffs.length) return; + + const pubkeyDiffs = statDiffs.filter(([table]) => table === 'pubkey_stats') as PubkeyStatDiff[]; + const eventDiffs = statDiffs.filter(([table]) => table === 'event_stats') as EventStatDiff[]; + + return db.transaction().execute(() => { + return Promise.all([ + pubkeyStatsQuery(pubkeyDiffs).execute(), + eventStatsQuery(eventDiffs).execute(), + ]); + }); } -function updateStatsQuery(event: Event) { - const firstE = findFirstTag(event, 'e'); +/** Calculate stats changes ahead of time so we can build an efficient query. */ +function getStatsDiff(event: Event): StatDiff[] { + const statDiffs: StatDiff[] = []; + + const firstE = event.tags.find(([name]) => name === 'e')?.[1]; + const replyTag = findReplyTag(event as Event<1>); switch (event.kind) { case 1: - return incrementPubkeyStatsQuery([event.pubkey], 'notes_count', 1); + statDiffs.push(['pubkey_stats', event.pubkey, 'notes_count', 1]); + if (replyTag && replyTag[1]) { + statDiffs.push(['event_stats', replyTag[1], 'replies_count', 1]); + } + break; case 6: - return firstE ? incrementEventStatsQuery([firstE], 'reposts_count', 1) : undefined; + if (firstE) { + statDiffs.push(['event_stats', firstE, 'reposts_count', 1]); + } + break; case 7: - return firstE ? incrementEventStatsQuery([firstE], 'reactions_count', 1) : undefined; + if (firstE) { + statDiffs.push(['event_stats', firstE, 'reactions_count', 1]); + } } + + return statDiffs; } -function incrementPubkeyStatsQuery(pubkeys: string[], stat: PubkeyStat, diff: number) { - const values: PubkeyStatsRow[] = pubkeys.map((pubkey) => { +function pubkeyStatsQuery(diffs: PubkeyStatDiff[]) { + const values: PubkeyStatsRow[] = diffs.map(([_, pubkey, stat, diff]) => { const row: PubkeyStatsRow = { pubkey, followers_count: 0, @@ -40,13 +70,15 @@ function incrementPubkeyStatsQuery(pubkeys: string[], stat: PubkeyStat, diff: nu oc .column('pubkey') .doUpdateSet((eb) => ({ - [stat]: eb(stat, '+', diff), + followers_count: eb('followers_count', '+', eb.ref('excluded.followers_count')), + following_count: eb('following_count', '+', eb.ref('excluded.following_count')), + notes_count: eb('notes_count', '+', eb.ref('excluded.notes_count')), })) ); } -function incrementEventStatsQuery(eventIds: string[], stat: EventStat, diff: number) { - const values: EventStatsRow[] = eventIds.map((event_id) => { +function eventStatsQuery(diffs: EventStatDiff[]) { + const values: EventStatsRow[] = diffs.map(([_, event_id, stat, diff]) => { const row: EventStatsRow = { event_id, replies_count: 0, @@ -63,13 +95,11 @@ function incrementEventStatsQuery(eventIds: string[], stat: EventStat, diff: num oc .column('event_id') .doUpdateSet((eb) => ({ - [stat]: eb(stat, '+', diff), + replies_count: eb('replies_count', '+', eb.ref('excluded.replies_count')), + reposts_count: eb('reposts_count', '+', eb.ref('excluded.reposts_count')), + reactions_count: eb('reactions_count', '+', eb.ref('excluded.reactions_count')), })) ); } -function findFirstTag({ tags }: Event, name: string): string | undefined { - return tags.find(([n]) => n === name)?.[1]; -} - export { updateStats }; From 0f10a7c3a2c11f25365b7dc18bbaa1aa4ad00289 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 13:58:35 -0600 Subject: [PATCH 07/20] stats: refactor inReplyToId --- src/stats.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/stats.ts b/src/stats.ts index 2496ed67..f54b7dea 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -29,13 +29,13 @@ function getStatsDiff(event: Event): StatDiff[] { const statDiffs: StatDiff[] = []; const firstE = event.tags.find(([name]) => name === 'e')?.[1]; - const replyTag = findReplyTag(event as Event<1>); + const inReplyToId = findReplyTag(event as Event<1>)?.[1]; switch (event.kind) { case 1: statDiffs.push(['pubkey_stats', event.pubkey, 'notes_count', 1]); - if (replyTag && replyTag[1]) { - statDiffs.push(['event_stats', replyTag[1], 'replies_count', 1]); + if (inReplyToId) { + statDiffs.push(['event_stats', inReplyToId, 'replies_count', 1]); } break; case 6: From 8b03d492a10c41e2730cc55102e85dc0b54a4ff5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 14:03:01 -0600 Subject: [PATCH 08/20] stats: add migration for stats tables --- src/db/migrations/009_add_stats.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 src/db/migrations/009_add_stats.ts diff --git a/src/db/migrations/009_add_stats.ts b/src/db/migrations/009_add_stats.ts new file mode 100644 index 00000000..9655c051 --- /dev/null +++ b/src/db/migrations/009_add_stats.ts @@ -0,0 +1,24 @@ +import { Kysely } from '@/deps.ts'; + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('pubkey_stats') + .addColumn('pubkey', 'text', (col) => col.primaryKey()) + .addColumn('followers_count', 'integer', (col) => col.notNull().defaultTo(0)) + .addColumn('following_count', 'integer', (col) => col.notNull().defaultTo(0)) + .addColumn('notes_count', 'integer', (col) => col.notNull().defaultTo(0)) + .execute(); + + await db.schema + .createTable('event_stats') + .addColumn('event_id', 'text', (col) => col.primaryKey().references('events.id').onDelete('cascade')) + .addColumn('replies_count', 'integer', (col) => col.notNull().defaultTo(0)) + .addColumn('reposts_count', 'integer', (col) => col.notNull().defaultTo(0)) + .addColumn('reactions_count', 'integer', (col) => col.notNull().defaultTo(0)) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable('pubkey_stats').execute(); + await db.schema.dropTable('event_stats').execute(); +} From 21b6a02ff32909376954d16a3d036f653be741e1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 14:27:15 -0600 Subject: [PATCH 09/20] views: avoid counting directly in the view, take from the event object if it has it --- src/db/events.ts | 28 +++++++++++++++++++++++++++- src/views/mastodon/accounts.ts | 21 ++++++++++----------- src/views/mastodon/statuses.ts | 11 ++++------- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/src/db/events.ts b/src/db/events.ts index b8db69a4..54ed10b9 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -80,6 +80,9 @@ type EventQuery = SelectQueryBuilder; /** Build the query for a filter. */ @@ -184,8 +190,12 @@ function getFiltersQuery(filters: DittoFilter[]) { .reduce((result, query) => result.unionAll(query)); } +type AuthorStats = Omit; +type EventStats = Omit; + interface DittoEvent extends Event { - author?: Event<0>; + author?: Event<0> & { stats?: AuthorStats }; + stats?: EventStats; } /** Get events for filters from the database. */ @@ -221,6 +231,22 @@ async function getFilters( tags: JSON.parse(row.author_tags!), sig: row.author_sig!, }; + + if (typeof row.author_stats_followers_count === 'number') { + event.author.stats = { + followers_count: row.author_stats_followers_count, + following_count: row.author_stats_following_count!, + notes_count: row.author_stats_notes_count!, + }; + } + } + + if (typeof row.stats_replies_count === 'number') { + event.stats = { + replies_count: row.stats_replies_count, + reposts_count: row.stats_reposts_count!, + reactions_count: row.stats_reactions_count!, + }; } return event; diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index 3a55fb75..f2d2f586 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -1,8 +1,7 @@ import { Conf } from '@/config.ts'; -import * as eventsDB from '@/db/events.ts'; +import { type DittoEvent } from '@/db/events.ts'; import { findUser } from '@/db/users.ts'; import { lodash, nip19, type UnsignedEvent } from '@/deps.ts'; -import { getFollowedPubkeys } from '@/queries.ts'; import { jsonMetaContentSchema } from '@/schemas/nostr.ts'; import { verifyNip05Cached } from '@/utils/nip05.ts'; import { Nip05, nostrDate, nostrNow, parseNip05 } from '@/utils.ts'; @@ -12,7 +11,10 @@ interface ToAccountOpts { withSource?: boolean; } -async function renderAccount(event: UnsignedEvent<0>, opts: ToAccountOpts = {}) { +async function renderAccount( + event: Omit, 'id' | 'sig'>, + opts: ToAccountOpts = {}, +) { const { withSource = false } = opts; const { pubkey } = event; @@ -26,12 +28,9 @@ async function renderAccount(event: UnsignedEvent<0>, opts: ToAccountOpts = {}) const npub = nip19.npubEncode(pubkey); - const [user, parsed05, followersCount, followingCount, statusesCount] = await Promise.all([ + const [user, parsed05] = await Promise.all([ findUser({ pubkey }), parseAndVerifyNip05(nip05, pubkey), - eventsDB.countFilters([{ kinds: [3], '#p': [pubkey] }]), - getFollowedPubkeys(pubkey).then((pubkeys) => pubkeys.length), - eventsDB.countFilters([{ kinds: [1], authors: [pubkey] }]), ]); return { @@ -40,14 +39,14 @@ async function renderAccount(event: UnsignedEvent<0>, opts: ToAccountOpts = {}) avatar: picture, avatar_static: picture, bot: false, - created_at: event ? nostrDate(event.created_at).toISOString() : new Date().toISOString(), + created_at: nostrDate(event.created_at).toISOString(), discoverable: true, display_name: name, emojis: renderEmojis(event), fields: [], follow_requests_count: 0, - followers_count: followersCount, - following_count: followingCount, + followers_count: event.stats?.followers_count ?? 0, + following_count: event.stats?.following_count ?? 0, fqn: parsed05?.handle || npub, header: banner, header_static: banner, @@ -65,7 +64,7 @@ async function renderAccount(event: UnsignedEvent<0>, opts: ToAccountOpts = {}) follow_requests_count: 0, } : undefined, - statuses_count: statusesCount, + statuses_count: event.stats?.notes_count ?? 0, url: Conf.local(`/users/${pubkey}`), username: parsed05?.nickname || npub.substring(0, 8), pleroma: { diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 86bdd13e..88596d29 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -26,13 +26,10 @@ async function renderStatus(event: eventsDB.DittoEvent<1>, viewerPubkey?: string const { html, links, firstUrl } = parseNoteContent(event.content); - const [mentions, card, repliesCount, reblogsCount, favouritesCount, [repostEvent], [reactionEvent]] = await Promise + const [mentions, card, [repostEvent], [reactionEvent]] = await Promise .all([ Promise.all(mentionedPubkeys.map(toMention)), firstUrl ? unfurlCardCached(firstUrl) : null, - eventsDB.countFilters([{ kinds: [1], '#e': [event.id] }]), - eventsDB.countFilters([{ kinds: [6], '#e': [event.id] }]), - eventsDB.countFilters([{ kinds: [7], '#e': [event.id] }]), viewerPubkey ? eventsDB.getFilters([{ kinds: [6], '#e': [event.id], authors: [viewerPubkey] }], { limit: 1 }) : [], @@ -66,9 +63,9 @@ async function renderStatus(event: eventsDB.DittoEvent<1>, viewerPubkey?: string spoiler_text: (cw ? cw[1] : subject?.[1]) || '', visibility: 'public', language: event.tags.find((tag) => tag[0] === 'lang')?.[1] || null, - replies_count: repliesCount, - reblogs_count: reblogsCount, - favourites_count: favouritesCount, + replies_count: event.stats?.replies_count ?? 0, + reblogs_count: event.stats?.reposts_count ?? 0, + favourites_count: event.stats?.reactions_count ?? 0, favourited: reactionEvent?.content === '+', reblogged: Boolean(repostEvent), muted: false, From 6a92c5135da5b9d452445c5a9c64d3d28028e6db Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 15:02:19 -0600 Subject: [PATCH 10/20] db/events: support 'stats' relation --- src/db/events.ts | 38 ++++++++++++++++++++++++++++++++------ src/filter.ts | 2 +- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/db/events.ts b/src/db/events.ts index 54ed10b9..adb3abc8 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -140,7 +140,7 @@ function getFilterQuery(filter: DittoFilter): EventQuery { query = query .leftJoin('tags', 'tags.event_id', 'events.id') .where('tags.tag', '=', tag) - .where('tags.value', 'in', value) as typeof query; + .where('tags.value', 'in', value); } } @@ -153,14 +153,27 @@ function getFilterQuery(filter: DittoFilter): EventQuery { if (filter.relations?.includes('author')) { query = query .leftJoin( - (eb) => - eb + (eb) => { + let exp: EventQuery = eb .selectFrom('events') .selectAll() - .where('kind', '=', 0) + .where('kind', '=', 0); + + if (filter.relations?.includes('stats')) { + exp = exp + .leftJoin('pubkey_stats', 'pubkey_stats.pubkey', 'events.pubkey') + .select((eb) => [ + eb.fn.coalesce('pubkey_stats.followers_count', eb.val(0)).as('author_stats_followers_count'), + eb.fn.coalesce('pubkey_stats.following_count', eb.val(0)).as('author_stats_following_count'), + eb.fn.coalesce('pubkey_stats.notes_count', eb.val(0)).as('author_stats_notes_count'), + ]) as typeof exp; + } + + return exp .orderBy('created_at', 'desc') .groupBy('pubkey') - .as('authors'), + .as('authors'); + }, (join) => join.onRef('authors.pubkey', '=', 'events.pubkey'), ) .select([ @@ -171,7 +184,20 @@ function getFilterQuery(filter: DittoFilter): EventQuery { 'authors.tags as author_tags', 'authors.created_at as author_created_at', 'authors.sig as author_sig', - ]) as typeof query; + 'authors.author_stats_followers_count', + 'authors.author_stats_following_count', + 'authors.author_stats_notes_count', + ]); + } + + if (filter.relations?.includes('stats')) { + query = query + .leftJoin('event_stats', 'event_stats.event_id', 'events.id') + .select((eb) => [ + eb.fn.coalesce('event_stats.replies_count', eb.val(0)).as('stats_replies_count'), + eb.fn.coalesce('event_stats.reposts_count', eb.val(0)).as('stats_reposts_count'), + eb.fn.coalesce('event_stats.reactions_count', eb.val(0)).as('stats_reactions_count'), + ]); } if (filter.search) { diff --git a/src/filter.ts b/src/filter.ts index e4e336fd..7f41b63c 100644 --- a/src/filter.ts +++ b/src/filter.ts @@ -4,7 +4,7 @@ import { type Event, type Filter, matchFilters } from '@/deps.ts'; import type { EventData } from '@/types.ts'; /** Additional properties that may be added by Ditto to events. */ -type Relation = 'author'; +type Relation = 'author' | 'stats'; /** Custom filter interface that extends Nostr filters with extra options for Ditto. */ interface DittoFilter extends Filter { From a48c1e51e180ae0ccd8a4dafb1c0eea3aeeabbad Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 15:33:01 -0600 Subject: [PATCH 11/20] stats: fix queries getting stuck --- src/db/events.ts | 12 ++++++++---- src/stats.ts | 12 +++++------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/db/events.ts b/src/db/events.ts index adb3abc8..b88d38d1 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -171,7 +171,7 @@ function getFilterQuery(filter: DittoFilter): EventQuery { return exp .orderBy('created_at', 'desc') - .groupBy('pubkey') + .groupBy('events.pubkey') .as('authors'); }, (join) => join.onRef('authors.pubkey', '=', 'events.pubkey'), @@ -184,9 +184,13 @@ function getFilterQuery(filter: DittoFilter): EventQuery { 'authors.tags as author_tags', 'authors.created_at as author_created_at', 'authors.sig as author_sig', - 'authors.author_stats_followers_count', - 'authors.author_stats_following_count', - 'authors.author_stats_notes_count', + ...(filter.relations?.includes('stats') + ? [ + 'authors.author_stats_followers_count', + 'authors.author_stats_following_count', + 'authors.author_stats_notes_count', + ] as const + : []), ]); } diff --git a/src/stats.ts b/src/stats.ts index f54b7dea..05e6be35 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -9,19 +9,17 @@ type EventStatDiff = ['event_stats', eventId: string, stat: EventStat, diff: num type StatDiff = PubkeyStatDiff | EventStatDiff; /** Store stats for the event in LMDB. */ -function updateStats(event: Event) { +async function updateStats(event: Event) { const statDiffs = getStatsDiff(event); if (!statDiffs.length) return; const pubkeyDiffs = statDiffs.filter(([table]) => table === 'pubkey_stats') as PubkeyStatDiff[]; const eventDiffs = statDiffs.filter(([table]) => table === 'event_stats') as EventStatDiff[]; - return db.transaction().execute(() => { - return Promise.all([ - pubkeyStatsQuery(pubkeyDiffs).execute(), - eventStatsQuery(eventDiffs).execute(), - ]); - }); + await Promise.all([ + pubkeyDiffs.length ? pubkeyStatsQuery(pubkeyDiffs).execute() : undefined, + eventDiffs.length ? eventStatsQuery(eventDiffs).execute() : undefined, + ]); } /** Calculate stats changes ahead of time so we can build an efficient query. */ From 5415656b4d9dfdd46169e28a1ffcf5459c84a03f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 15:40:05 -0600 Subject: [PATCH 12/20] Make author_stats and event_stats two separate keys on an event --- src/db/events.ts | 9 +++++---- src/views/mastodon/accounts.ts | 6 +++--- src/views/mastodon/statuses.ts | 6 +++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/db/events.ts b/src/db/events.ts index b88d38d1..4e3c020b 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -224,8 +224,9 @@ type AuthorStats = Omit; type EventStats = Omit; interface DittoEvent extends Event { - author?: Event<0> & { stats?: AuthorStats }; - stats?: EventStats; + author?: DittoEvent<0>; + author_stats?: AuthorStats; + event_stats?: EventStats; } /** Get events for filters from the database. */ @@ -263,7 +264,7 @@ async function getFilters( }; if (typeof row.author_stats_followers_count === 'number') { - event.author.stats = { + event.author.author_stats = { followers_count: row.author_stats_followers_count, following_count: row.author_stats_following_count!, notes_count: row.author_stats_notes_count!, @@ -272,7 +273,7 @@ async function getFilters( } if (typeof row.stats_replies_count === 'number') { - event.stats = { + event.event_stats = { replies_count: row.stats_replies_count, reposts_count: row.stats_reposts_count!, reactions_count: row.stats_reactions_count!, diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index f2d2f586..bfcd7efa 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -45,8 +45,8 @@ async function renderAccount( emojis: renderEmojis(event), fields: [], follow_requests_count: 0, - followers_count: event.stats?.followers_count ?? 0, - following_count: event.stats?.following_count ?? 0, + followers_count: event.author_stats?.followers_count ?? 0, + following_count: event.author_stats?.following_count ?? 0, fqn: parsed05?.handle || npub, header: banner, header_static: banner, @@ -64,7 +64,7 @@ async function renderAccount( follow_requests_count: 0, } : undefined, - statuses_count: event.stats?.notes_count ?? 0, + statuses_count: event.author_stats?.notes_count ?? 0, url: Conf.local(`/users/${pubkey}`), username: parsed05?.nickname || npub.substring(0, 8), pleroma: { diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 88596d29..380682de 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -63,9 +63,9 @@ async function renderStatus(event: eventsDB.DittoEvent<1>, viewerPubkey?: string spoiler_text: (cw ? cw[1] : subject?.[1]) || '', visibility: 'public', language: event.tags.find((tag) => tag[0] === 'lang')?.[1] || null, - replies_count: event.stats?.replies_count ?? 0, - reblogs_count: event.stats?.reposts_count ?? 0, - favourites_count: event.stats?.reactions_count ?? 0, + replies_count: event.event_stats?.replies_count ?? 0, + reblogs_count: event.event_stats?.reposts_count ?? 0, + favourites_count: event.event_stats?.reactions_count ?? 0, favourited: reactionEvent?.content === '+', reblogged: Boolean(repostEvent), muted: false, From 07dc07ab713ae260d863e9cb2d3db4edd36be846 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 15:54:31 -0600 Subject: [PATCH 13/20] Simplify author_stats relation --- src/db/events.ts | 56 ++++++++++++++-------------------- src/filter.ts | 2 +- src/views/mastodon/statuses.ts | 5 ++- 3 files changed, 28 insertions(+), 35 deletions(-) diff --git a/src/db/events.ts b/src/db/events.ts index 4e3c020b..efefbba7 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -153,27 +153,14 @@ function getFilterQuery(filter: DittoFilter): EventQuery { if (filter.relations?.includes('author')) { query = query .leftJoin( - (eb) => { - let exp: EventQuery = eb + (eb) => + eb .selectFrom('events') .selectAll() - .where('kind', '=', 0); - - if (filter.relations?.includes('stats')) { - exp = exp - .leftJoin('pubkey_stats', 'pubkey_stats.pubkey', 'events.pubkey') - .select((eb) => [ - eb.fn.coalesce('pubkey_stats.followers_count', eb.val(0)).as('author_stats_followers_count'), - eb.fn.coalesce('pubkey_stats.following_count', eb.val(0)).as('author_stats_following_count'), - eb.fn.coalesce('pubkey_stats.notes_count', eb.val(0)).as('author_stats_notes_count'), - ]) as typeof exp; - } - - return exp + .where('kind', '=', 0) .orderBy('created_at', 'desc') - .groupBy('events.pubkey') - .as('authors'); - }, + .groupBy('pubkey') + .as('authors'), (join) => join.onRef('authors.pubkey', '=', 'events.pubkey'), ) .select([ @@ -184,17 +171,20 @@ function getFilterQuery(filter: DittoFilter): EventQuery { 'authors.tags as author_tags', 'authors.created_at as author_created_at', 'authors.sig as author_sig', - ...(filter.relations?.includes('stats') - ? [ - 'authors.author_stats_followers_count', - 'authors.author_stats_following_count', - 'authors.author_stats_notes_count', - ] as const - : []), ]); } - if (filter.relations?.includes('stats')) { + if (filter.relations?.includes('author_stats')) { + query = query + .leftJoin('pubkey_stats', 'pubkey_stats.pubkey', 'events.pubkey') + .select((eb) => [ + eb.fn.coalesce('pubkey_stats.followers_count', eb.val(0)).as('author_stats_followers_count'), + eb.fn.coalesce('pubkey_stats.following_count', eb.val(0)).as('author_stats_following_count'), + eb.fn.coalesce('pubkey_stats.notes_count', eb.val(0)).as('author_stats_notes_count'), + ]); + } + + if (filter.relations?.includes('event_stats')) { query = query .leftJoin('event_stats', 'event_stats.event_id', 'events.id') .select((eb) => [ @@ -262,14 +252,14 @@ async function getFilters( tags: JSON.parse(row.author_tags!), sig: row.author_sig!, }; + } - if (typeof row.author_stats_followers_count === 'number') { - event.author.author_stats = { - followers_count: row.author_stats_followers_count, - following_count: row.author_stats_following_count!, - notes_count: row.author_stats_notes_count!, - }; - } + if (typeof row.author_stats_followers_count === 'number') { + event.author_stats = { + followers_count: row.author_stats_followers_count, + following_count: row.author_stats_following_count!, + notes_count: row.author_stats_notes_count!, + }; } if (typeof row.stats_replies_count === 'number') { diff --git a/src/filter.ts b/src/filter.ts index 7f41b63c..76a0fcde 100644 --- a/src/filter.ts +++ b/src/filter.ts @@ -4,7 +4,7 @@ import { type Event, type Filter, matchFilters } from '@/deps.ts'; import type { EventData } from '@/types.ts'; /** Additional properties that may be added by Ditto to events. */ -type Relation = 'author' | 'stats'; +type Relation = 'author' | 'author_stats' | 'event_stats'; /** Custom filter interface that extends Nostr filters with extra options for Ditto. */ interface DittoFilter extends Filter { diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 380682de..120ba93a 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -13,7 +13,10 @@ import { DittoAttachment, renderAttachment } from '@/views/mastodon/attachments. import { renderEmojis } from '@/views/mastodon/emojis.ts'; async function renderStatus(event: eventsDB.DittoEvent<1>, viewerPubkey?: string) { - const account = event.author ? await renderAccount(event.author) : await accountFromPubkey(event.pubkey); + const account = event.author + ? await renderAccount({ ...event.author, author_stats: event.author_stats }) + : await accountFromPubkey(event.pubkey); + const replyTag = findReplyTag(event); const mentionedPubkeys = [ From 733b8ba9c549bb547d529837429043a10569aba1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 16:04:52 -0600 Subject: [PATCH 14/20] pubkey_stats --> author_stats --- src/db.ts | 6 +++--- src/db/events.ts | 10 +++++----- src/db/migrations/009_add_stats.ts | 4 ++-- src/stats.ts | 22 +++++++++++----------- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/db.ts b/src/db.ts index 6f0397f9..4f1a8d91 100644 --- a/src/db.ts +++ b/src/db.ts @@ -13,11 +13,11 @@ interface DittoDB { users: UserRow; relays: RelayRow; unattached_media: UnattachedMediaRow; - pubkey_stats: PubkeyStatsRow; + author_stats: AuthorStatsRow; event_stats: EventStatsRow; } -interface PubkeyStatsRow { +interface AuthorStatsRow { pubkey: string; followers_count: number; following_count: number; @@ -126,4 +126,4 @@ async function migrate() { await migrate(); -export { db, type DittoDB, type EventRow, type EventStatsRow, type PubkeyStatsRow, type TagRow, type UserRow }; +export { type AuthorStatsRow, db, type DittoDB, type EventRow, type EventStatsRow, type TagRow, type UserRow }; diff --git a/src/db/events.ts b/src/db/events.ts index efefbba7..2a5f2fd7 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -176,11 +176,11 @@ function getFilterQuery(filter: DittoFilter): EventQuery { if (filter.relations?.includes('author_stats')) { query = query - .leftJoin('pubkey_stats', 'pubkey_stats.pubkey', 'events.pubkey') + .leftJoin('author_stats', 'author_stats.pubkey', 'events.pubkey') .select((eb) => [ - eb.fn.coalesce('pubkey_stats.followers_count', eb.val(0)).as('author_stats_followers_count'), - eb.fn.coalesce('pubkey_stats.following_count', eb.val(0)).as('author_stats_following_count'), - eb.fn.coalesce('pubkey_stats.notes_count', eb.val(0)).as('author_stats_notes_count'), + eb.fn.coalesce('author_stats.followers_count', eb.val(0)).as('author_stats_followers_count'), + eb.fn.coalesce('author_stats.following_count', eb.val(0)).as('author_stats_following_count'), + eb.fn.coalesce('author_stats.notes_count', eb.val(0)).as('author_stats_notes_count'), ]); } @@ -210,7 +210,7 @@ function getFiltersQuery(filters: DittoFilter[]) { .reduce((result, query) => result.unionAll(query)); } -type AuthorStats = Omit; +type AuthorStats = Omit; type EventStats = Omit; interface DittoEvent extends Event { diff --git a/src/db/migrations/009_add_stats.ts b/src/db/migrations/009_add_stats.ts index 9655c051..60d9447b 100644 --- a/src/db/migrations/009_add_stats.ts +++ b/src/db/migrations/009_add_stats.ts @@ -2,7 +2,7 @@ import { Kysely } from '@/deps.ts'; export async function up(db: Kysely): Promise { await db.schema - .createTable('pubkey_stats') + .createTable('author_stats') .addColumn('pubkey', 'text', (col) => col.primaryKey()) .addColumn('followers_count', 'integer', (col) => col.notNull().defaultTo(0)) .addColumn('following_count', 'integer', (col) => col.notNull().defaultTo(0)) @@ -19,6 +19,6 @@ export async function up(db: Kysely): Promise { } export async function down(db: Kysely): Promise { - await db.schema.dropTable('pubkey_stats').execute(); + await db.schema.dropTable('author_stats').execute(); await db.schema.dropTable('event_stats').execute(); } diff --git a/src/stats.ts b/src/stats.ts index 05e6be35..e076e148 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -1,23 +1,23 @@ -import { db, type EventStatsRow, type PubkeyStatsRow } from '@/db.ts'; +import { type AuthorStatsRow, db, type EventStatsRow } from '@/db.ts'; import { Event, findReplyTag } from '@/deps.ts'; -type PubkeyStat = keyof Omit; +type AuthorStat = keyof Omit; type EventStat = keyof Omit; -type PubkeyStatDiff = ['pubkey_stats', pubkey: string, stat: PubkeyStat, diff: number]; +type AuthorStatDiff = ['author_stats', pubkey: string, stat: AuthorStat, diff: number]; type EventStatDiff = ['event_stats', eventId: string, stat: EventStat, diff: number]; -type StatDiff = PubkeyStatDiff | EventStatDiff; +type StatDiff = AuthorStatDiff | EventStatDiff; /** Store stats for the event in LMDB. */ async function updateStats(event: Event) { const statDiffs = getStatsDiff(event); if (!statDiffs.length) return; - const pubkeyDiffs = statDiffs.filter(([table]) => table === 'pubkey_stats') as PubkeyStatDiff[]; + const pubkeyDiffs = statDiffs.filter(([table]) => table === 'author_stats') as AuthorStatDiff[]; const eventDiffs = statDiffs.filter(([table]) => table === 'event_stats') as EventStatDiff[]; await Promise.all([ - pubkeyDiffs.length ? pubkeyStatsQuery(pubkeyDiffs).execute() : undefined, + pubkeyDiffs.length ? authorStatsQuery(pubkeyDiffs).execute() : undefined, eventDiffs.length ? eventStatsQuery(eventDiffs).execute() : undefined, ]); } @@ -31,7 +31,7 @@ function getStatsDiff(event: Event): StatDiff[] { switch (event.kind) { case 1: - statDiffs.push(['pubkey_stats', event.pubkey, 'notes_count', 1]); + statDiffs.push(['author_stats', event.pubkey, 'notes_count', 1]); if (inReplyToId) { statDiffs.push(['event_stats', inReplyToId, 'replies_count', 1]); } @@ -50,9 +50,9 @@ function getStatsDiff(event: Event): StatDiff[] { return statDiffs; } -function pubkeyStatsQuery(diffs: PubkeyStatDiff[]) { - const values: PubkeyStatsRow[] = diffs.map(([_, pubkey, stat, diff]) => { - const row: PubkeyStatsRow = { +function authorStatsQuery(diffs: AuthorStatDiff[]) { + const values: AuthorStatsRow[] = diffs.map(([_, pubkey, stat, diff]) => { + const row: AuthorStatsRow = { pubkey, followers_count: 0, following_count: 0, @@ -62,7 +62,7 @@ function pubkeyStatsQuery(diffs: PubkeyStatDiff[]) { return row; }); - return db.insertInto('pubkey_stats') + return db.insertInto('author_stats') .values(values) .onConflict((oc) => oc From a5369d982685e2435d5b7c18b22a99e860ee2ec2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 16:21:18 -0600 Subject: [PATCH 15/20] Ensure relations are loaded throughout the API where needed --- src/controllers/api/accounts.ts | 16 +++++++++++++--- src/controllers/api/search.ts | 18 +++++++++++------- src/controllers/api/statuses.ts | 6 +++--- src/controllers/api/timelines.ts | 2 +- src/queries.ts | 17 +++++++++++++---- src/views.ts | 2 +- src/views/mastodon/statuses.ts | 4 ++-- 7 files changed, 44 insertions(+), 21 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 2896b96b..22c9a63f 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -59,7 +59,7 @@ const createAccountController: AppController = async (c) => { const verifyCredentialsController: AppController = async (c) => { const pubkey = c.get('pubkey')!; - const event = await getAuthor(pubkey); + const event = await getAuthor(pubkey, { relations: ['author_stats'] }); if (event) { return c.json(await renderAccount(event, { withSource: true })); } else { @@ -138,7 +138,15 @@ const accountStatusesController: AppController = async (c) => { return c.json([]); } - const filter: DittoFilter<1> = { authors: [pubkey], kinds: [1], relations: ['author'], since, until, limit }; + const filter: DittoFilter<1> = { + authors: [pubkey], + kinds: [1], + relations: ['author', 'event_stats', 'author_stats'], + since, + until, + limit, + }; + if (tagged) { filter['#t'] = [tagged]; } @@ -257,7 +265,9 @@ const favouritesController: AppController = async (c) => { .map((event) => event.tags.find((tag) => tag[0] === 'e')?.[1]) .filter((id): id is string => !!id); - const events1 = await mixer.getFilters([{ kinds: [1], ids, relations: ['author'] }], { timeout: Time.seconds(1) }); + const events1 = await mixer.getFilters([{ kinds: [1], ids, relations: ['author', 'event_stats', 'author_stats'] }], { + timeout: Time.seconds(1), + }); const statuses = await Promise.all(events1.map((event) => renderStatus(event, c.get('pubkey')))); return paginated(c, events1, statuses); diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index b0f71cef..e5f1778c 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -69,7 +69,7 @@ function searchEvents({ q, type, limit, account_id }: SearchQuery): Promise ({ ...filter, relations: ['author'] })); + return filters; } export { searchController }; diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 114923da..cf0446a3 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -29,7 +29,7 @@ const createStatusSchema = z.object({ const statusController: AppController = async (c) => { const id = c.req.param('id'); - const event = await getEvent(id, { kind: 1, relations: ['author'] }); + const event = await getEvent(id, { kind: 1, relations: ['author', 'event_stats', 'author_stats'] }); if (event) { return c.json(await renderStatus(event, c.get('pubkey'))); } @@ -89,7 +89,7 @@ const createStatusController: AppController = async (c) => { const contextController: AppController = async (c) => { const id = c.req.param('id'); - const event = await getEvent(id, { kind: 1, relations: ['author'] }); + const event = await getEvent(id, { kind: 1, relations: ['author', 'event_stats', 'author_stats'] }); async function renderStatuses(events: Event<1>[]) { const statuses = await Promise.all(events.map((event) => renderStatus(event, c.get('pubkey')))); @@ -110,7 +110,7 @@ const contextController: AppController = async (c) => { const favouriteController: AppController = async (c) => { const id = c.req.param('id'); - const target = await getEvent(id, { kind: 1, relations: ['author'] }); + const target = await getEvent(id, { kind: 1, relations: ['author', 'event_stats', 'author_stats'] }); if (target) { await createEvent({ diff --git a/src/controllers/api/timelines.ts b/src/controllers/api/timelines.ts index d3ffcdc4..a29cdc89 100644 --- a/src/controllers/api/timelines.ts +++ b/src/controllers/api/timelines.ts @@ -35,7 +35,7 @@ const hashtagTimelineController: AppController = (c) => { /** Render statuses for timelines. */ async function renderStatuses(c: AppContext, filters: DittoFilter<1>[]) { const events = await mixer.getFilters( - filters.map((filter) => ({ ...filter, relations: ['author'] })), + filters.map((filter) => ({ ...filter, relations: ['author', 'event_stats', 'author_stats'] })), { timeout: Time.seconds(1) }, ); diff --git a/src/queries.ts b/src/queries.ts index de9bd33e..edd5f4aa 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -27,8 +27,14 @@ const getEvent = async ( }; /** Get a Nostr `set_medatadata` event for a user's pubkey. */ -const getAuthor = async (pubkey: string, timeout = 1000): Promise | undefined> => { - const [event] = await mixer.getFilters([{ authors: [pubkey], kinds: [0], limit: 1 }], { limit: 1, timeout }); +const getAuthor = async (pubkey: string, opts: GetEventOpts<0> = {}): Promise | undefined> => { + const { relations, timeout = 1000 } = opts; + + const [event] = await mixer.getFilters( + [{ authors: [pubkey], relations, kinds: [0], limit: 1 }], + { limit: 1, timeout }, + ); + return event; }; @@ -60,7 +66,7 @@ async function getAncestors(event: Event<1>, result = [] as Event<1>[]): Promise const inReplyTo = replyTag ? replyTag[1] : undefined; if (inReplyTo) { - const parentEvent = await getEvent(inReplyTo, { kind: 1, relations: ['author'] }); + const parentEvent = await getEvent(inReplyTo, { kind: 1, relations: ['author', 'event_stats', 'author_stats'] }); if (parentEvent) { result.push(parentEvent); @@ -73,7 +79,10 @@ async function getAncestors(event: Event<1>, result = [] as Event<1>[]): Promise } function getDescendants(eventId: string): Promise[]> { - return mixer.getFilters([{ kinds: [1], '#e': [eventId], relations: ['author'] }], { limit: 200, timeout: 2000 }); + return mixer.getFilters( + [{ kinds: [1], '#e': [eventId], relations: ['author', 'event_stats', 'author_stats'] }], + { limit: 200, timeout: 2000 }, + ); } /** Returns whether the pubkey is followed by a local user. */ diff --git a/src/views.ts b/src/views.ts index 30497108..2314a844 100644 --- a/src/views.ts +++ b/src/views.ts @@ -15,7 +15,7 @@ async function renderEventAccounts(c: AppContext, filters: Filter[]) { } const accounts = await Promise.all([...pubkeys].map(async (pubkey) => { - const author = await getAuthor(pubkey); + const author = await getAuthor(pubkey, { relations: ['author_stats'] }); if (author) { return renderAccount(author); } diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 120ba93a..17b16770 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -86,8 +86,8 @@ async function renderStatus(event: eventsDB.DittoEvent<1>, viewerPubkey?: string } async function toMention(pubkey: string) { - const profile = await getAuthor(pubkey); - const account = profile ? await renderAccount(profile) : undefined; + const author = await getAuthor(pubkey); + const account = author ? await renderAccount(author) : undefined; if (account) { return { From 2d3f12dc7288f173472533a70ec53b4197ed5c07 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 16:32:43 -0600 Subject: [PATCH 16/20] stats: firstE -> firstTaggedId --- src/stats.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/stats.ts b/src/stats.ts index e076e148..a0d32d70 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -26,7 +26,7 @@ async function updateStats(event: Event) { function getStatsDiff(event: Event): StatDiff[] { const statDiffs: StatDiff[] = []; - const firstE = event.tags.find(([name]) => name === 'e')?.[1]; + const firstTaggedId = event.tags.find(([name]) => name === 'e')?.[1]; const inReplyToId = findReplyTag(event as Event<1>)?.[1]; switch (event.kind) { @@ -37,13 +37,13 @@ function getStatsDiff(event: Event): StatDiff[] { } break; case 6: - if (firstE) { - statDiffs.push(['event_stats', firstE, 'reposts_count', 1]); + if (firstTaggedId) { + statDiffs.push(['event_stats', firstTaggedId, 'reposts_count', 1]); } break; case 7: - if (firstE) { - statDiffs.push(['event_stats', firstE, 'reactions_count', 1]); + if (firstTaggedId) { + statDiffs.push(['event_stats', firstTaggedId, 'reactions_count', 1]); } } From 4f79b7ec29d75a56062c1bbff6517481a893df8c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 17:42:44 -0600 Subject: [PATCH 17/20] stats: handle follow/following counts --- src/deps.ts | 1 + src/pipeline.ts | 2 +- src/stats.ts | 94 +++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 86 insertions(+), 11 deletions(-) diff --git a/src/deps.ts b/src/deps.ts index f1cedc56..a6b893cd 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -63,6 +63,7 @@ export { type CompiledQuery, FileMigrationProvider, type Insertable, + type InsertQueryBuilder, Kysely, Migrator, type NullableInsertKeys, diff --git a/src/pipeline.ts b/src/pipeline.ts index d4811e40..4f0c51ab 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -71,7 +71,7 @@ async function storeEvent(event: Event, data: EventData): Promise { } else { await Promise.all([ eventsDB.insertEvent(event, data).catch(console.warn), - updateStats(event), + updateStats(event).catch(console.warn), ]); } } else { diff --git a/src/stats.ts b/src/stats.ts index a0d32d70..578e2a35 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -1,5 +1,6 @@ -import { type AuthorStatsRow, db, type EventStatsRow } from '@/db.ts'; -import { Event, findReplyTag } from '@/deps.ts'; +import { type AuthorStatsRow, db, type DittoDB, type EventStatsRow } from '@/db.ts'; +import * as eventsDB from '@/db/events.ts'; +import { type Event, findReplyTag, type InsertQueryBuilder } from '@/deps.ts'; type AuthorStat = keyof Omit; type EventStat = keyof Omit; @@ -9,21 +10,29 @@ type EventStatDiff = ['event_stats', eventId: string, stat: EventStat, diff: num type StatDiff = AuthorStatDiff | EventStatDiff; /** Store stats for the event in LMDB. */ -async function updateStats(event: Event) { - const statDiffs = getStatsDiff(event); - if (!statDiffs.length) return; +async function updateStats(event: Event & { prev?: Event }) { + const queries: InsertQueryBuilder[] = []; + // Kind 3 is a special case - replace the count with the new list. + if (event.kind === 3) { + await maybeSetPrev(event); + queries.push(updateFollowingCountQuery(event as Event<3>)); + } + + const statDiffs = getStatsDiff(event); const pubkeyDiffs = statDiffs.filter(([table]) => table === 'author_stats') as AuthorStatDiff[]; const eventDiffs = statDiffs.filter(([table]) => table === 'event_stats') as EventStatDiff[]; - await Promise.all([ - pubkeyDiffs.length ? authorStatsQuery(pubkeyDiffs).execute() : undefined, - eventDiffs.length ? eventStatsQuery(eventDiffs).execute() : undefined, - ]); + if (pubkeyDiffs.length) queries.push(authorStatsQuery(pubkeyDiffs)); + if (eventDiffs.length) queries.push(eventStatsQuery(eventDiffs)); + + if (queries.length) { + await Promise.all(queries.map((query) => query.execute())); + } } /** Calculate stats changes ahead of time so we can build an efficient query. */ -function getStatsDiff(event: Event): StatDiff[] { +function getStatsDiff(event: Event & { prev?: Event }): StatDiff[] { const statDiffs: StatDiff[] = []; const firstTaggedId = event.tags.find(([name]) => name === 'e')?.[1]; @@ -36,6 +45,9 @@ function getStatsDiff(event: Event): StatDiff[] { statDiffs.push(['event_stats', inReplyToId, 'replies_count', 1]); } break; + case 3: + statDiffs.push(...getFollowDiff(event as Event<3>, event.prev as Event<3> | undefined)); + break; case 6: if (firstTaggedId) { statDiffs.push(['event_stats', firstTaggedId, 'reposts_count', 1]); @@ -50,6 +62,7 @@ function getStatsDiff(event: Event): StatDiff[] { return statDiffs; } +/** Create an author stats query from the list of diffs. */ function authorStatsQuery(diffs: AuthorStatDiff[]) { const values: AuthorStatsRow[] = diffs.map(([_, pubkey, stat, diff]) => { const row: AuthorStatsRow = { @@ -75,6 +88,7 @@ function authorStatsQuery(diffs: AuthorStatDiff[]) { ); } +/** Create an event stats query from the list of diffs. */ function eventStatsQuery(diffs: EventStatDiff[]) { const values: EventStatsRow[] = diffs.map(([_, event_id, stat, diff]) => { const row: EventStatsRow = { @@ -100,4 +114,64 @@ function eventStatsQuery(diffs: EventStatDiff[]) { ); } +/** Set the `prev` value on the event to the last version of the event, if any. */ +async function maybeSetPrev(event: Event & { prev?: Event }): Promise { + if (event.prev?.kind === event.kind) return; + + const [prev] = await eventsDB.getFilters([ + { kinds: [event.kind], authors: [event.pubkey], limit: 1 }, + ]); + + if (prev.created_at < event.created_at) { + event.prev = prev; + } +} + +/** Set the following count to the total number of unique "p" tags in the follow list. */ +function updateFollowingCountQuery({ pubkey, tags }: Event<3>) { + const following_count = new Set( + tags + .filter(([name]) => name === 'p') + .map(([_, value]) => value), + ).size; + + return db.insertInto('author_stats') + .values({ + pubkey, + following_count, + followers_count: 0, + notes_count: 0, + }) + .onConflict((oc) => + oc + .column('pubkey') + .doUpdateSet({ following_count }) + ); +} + +/** Compare the old and new follow events (if any), and return a diff array. */ +function getFollowDiff(event: Event<3>, prev?: Event<3>): AuthorStatDiff[] { + const prevTags = prev?.tags ?? []; + + const prevPubkeys = new Set( + prevTags + .filter(([name]) => name === 'p') + .map(([_, value]) => value), + ); + + const pubkeys = new Set( + event.tags + .filter(([name]) => name === 'p') + .map(([_, value]) => value), + ); + + const added = [...pubkeys].filter((pubkey) => !prevPubkeys.has(pubkey)); + const removed = [...prevPubkeys].filter((pubkey) => !pubkeys.has(pubkey)); + + return [ + ...added.map((pubkey): AuthorStatDiff => ['author_stats', pubkey, 'followers_count', 1]), + ...removed.map((pubkey): AuthorStatDiff => ['author_stats', pubkey, 'followers_count', -1]), + ]; +} + export { updateStats }; From a32b0e7066d0212bf2e3ea15d10be5f50b67813d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 17:48:21 -0600 Subject: [PATCH 18/20] stats: clean up prev usage --- src/stats.ts | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/stats.ts b/src/stats.ts index 578e2a35..32208df8 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -10,16 +10,19 @@ type EventStatDiff = ['event_stats', eventId: string, stat: EventStat, diff: num type StatDiff = AuthorStatDiff | EventStatDiff; /** Store stats for the event in LMDB. */ -async function updateStats(event: Event & { prev?: Event }) { +async function updateStats(event: Event) { + let prev: Event | undefined; const queries: InsertQueryBuilder[] = []; // Kind 3 is a special case - replace the count with the new list. if (event.kind === 3) { - await maybeSetPrev(event); - queries.push(updateFollowingCountQuery(event as Event<3>)); + prev = await maybeGetPrev(event); + if (!prev || event.created_at >= prev.created_at) { + queries.push(updateFollowingCountQuery(event as Event<3>)); + } } - const statDiffs = getStatsDiff(event); + const statDiffs = getStatsDiff(event, prev); const pubkeyDiffs = statDiffs.filter(([table]) => table === 'author_stats') as AuthorStatDiff[]; const eventDiffs = statDiffs.filter(([table]) => table === 'event_stats') as EventStatDiff[]; @@ -32,7 +35,7 @@ async function updateStats(event: Event & { prev?: Event } /** Calculate stats changes ahead of time so we can build an efficient query. */ -function getStatsDiff(event: Event & { prev?: Event }): StatDiff[] { +function getStatsDiff(event: Event, prev: Event | undefined): StatDiff[] { const statDiffs: StatDiff[] = []; const firstTaggedId = event.tags.find(([name]) => name === 'e')?.[1]; @@ -46,7 +49,7 @@ function getStatsDiff(event: Event & { prev?: Event }): } break; case 3: - statDiffs.push(...getFollowDiff(event as Event<3>, event.prev as Event<3> | undefined)); + statDiffs.push(...getFollowDiff(event as Event<3>, prev as Event<3> | undefined)); break; case 6: if (firstTaggedId) { @@ -114,17 +117,13 @@ function eventStatsQuery(diffs: EventStatDiff[]) { ); } -/** Set the `prev` value on the event to the last version of the event, if any. */ -async function maybeSetPrev(event: Event & { prev?: Event }): Promise { - if (event.prev?.kind === event.kind) return; - +/** Get the last version of the event, if any. */ +async function maybeGetPrev(event: Event): Promise> { const [prev] = await eventsDB.getFilters([ { kinds: [event.kind], authors: [event.pubkey], limit: 1 }, ]); - if (prev.created_at < event.created_at) { - event.prev = prev; - } + return prev; } /** Set the following count to the total number of unique "p" tags in the follow list. */ From ff278487e8d48a80eefec60bad8c1354535eb746 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 17:55:07 -0600 Subject: [PATCH 19/20] Sentry: decrease tracesSampleRate to 0.2 --- src/sentry.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentry.ts b/src/sentry.ts index eefe9c59..8a30e3bf 100644 --- a/src/sentry.ts +++ b/src/sentry.ts @@ -6,6 +6,6 @@ if (Conf.sentryDsn) { console.log('Sentry enabled'); Sentry.init({ dsn: Conf.sentryDsn, - tracesSampleRate: 1.0, + tracesSampleRate: .2, }); } From 862ff74d7b382af984604fa270e84b0de5641436 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 10 Dec 2023 17:56:51 -0600 Subject: [PATCH 20/20] relays: don't automatically add crawled relays --- scripts/relays.ts | 2 +- src/db/relays.ts | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/relays.ts b/scripts/relays.ts index 94d0afd8..84f8a7e6 100644 --- a/scripts/relays.ts +++ b/scripts/relays.ts @@ -18,6 +18,6 @@ async function sync([url]: string[]) { const response = await fetch(url); const data = await response.json(); const values = filteredArray(relaySchema).parse(data) as `wss://${string}`[]; - await addRelays(values); + await addRelays(values, { active: true }); console.log(`Done: added ${values.length} relays.`); } diff --git a/src/db/relays.ts b/src/db/relays.ts index d41948f2..836f520e 100644 --- a/src/db/relays.ts +++ b/src/db/relays.ts @@ -1,14 +1,19 @@ import { tldts } from '@/deps.ts'; import { db } from '@/db.ts'; +interface AddRelaysOpts { + active?: boolean; +} + /** Inserts relays into the database, skipping duplicates. */ -function addRelays(relays: `wss://${string}`[]) { +function addRelays(relays: `wss://${string}`[], opts: AddRelaysOpts = {}) { if (!relays.length) return Promise.resolve(); + const { active = false } = opts; const values = relays.map((url) => ({ url: new URL(url).toString(), domain: tldts.getDomain(url)!, - active: true, + active, })); return db.insertInto('relays')