From 5da5848ca6a3901d156c60490ed9aa2f6de02eba Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 19:48:26 -0500 Subject: [PATCH 1/2] Trending statuses first iteration --- src/app.ts | 3 +- src/controllers/api/trends.ts | 64 +++++++++++++++++++++++++++++++---- 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/src/app.ts b/src/app.ts index 9b82bee2..86faa242 100644 --- a/src/app.ts +++ b/src/app.ts @@ -74,7 +74,7 @@ import { homeTimelineController, publicTimelineController, } from '@/controllers/api/timelines.ts'; -import { trendingTagsController } from '@/controllers/api/trends.ts'; +import { trendingStatusesController, trendingTagsController } from '@/controllers/api/trends.ts'; import { indexController } from '@/controllers/site.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; import { nostrController } from '@/controllers/well-known/nostr.ts'; @@ -194,6 +194,7 @@ app.get('/api/v2/search', searchController); app.get('/api/pleroma/frontend_configurations', frontendConfigController); +app.get('/api/v1/trends/statuses', trendingStatusesController); app.get('/api/v1/trends/tags', trendingTagsController); app.get('/api/v1/trends', trendingTagsController); diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts index 64b3019a..d531ffcb 100644 --- a/src/controllers/api/trends.ts +++ b/src/controllers/api/trends.ts @@ -1,29 +1,35 @@ +import { NostrEvent } from '@nostrify/nostrify'; +import { sql } from 'kysely'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; +import { DittoDB } from '@/db/DittoDB.ts'; +import { hydrateEvents } from '@/storages/hydrate.ts'; +import { Storages } from '@/storages.ts'; import { Time } from '@/utils.ts'; import { stripTime } from '@/utils/time.ts'; +import { renderStatus } from '@/views/mastodon/statuses.ts'; import { TrendsWorker } from '@/workers/trends.ts'; await TrendsWorker.open('data/trends.sqlite3'); const limitSchema = z.coerce.number().catch(10).transform((value) => Math.min(Math.max(value, 0), 20)); -let cache = getTrends(); +let trendingHashtagsCache = getTrendingHashtags(); Deno.cron('update trends cache', { minute: { every: 15 } }, async () => { - const trends = await getTrends(); - cache = Promise.resolve(trends); + const trends = await getTrendingHashtags(); + trendingHashtagsCache = Promise.resolve(trends); }); const trendingTagsController: AppController = async (c) => { const limit = limitSchema.parse(c.req.query('limit')); - const trends = await cache; + const trends = await trendingHashtagsCache; return c.json(trends.slice(0, limit)); }; -async function getTrends() { +async function getTrendingHashtags() { const now = new Date(); const yesterday = new Date(now.getTime() - Time.days(1)); const lastWeek = new Date(now.getTime() - Time.days(7)); @@ -62,4 +68,50 @@ async function getTrends() { }))); } -export { trendingTagsController }; +let trendingNotesCache = getTrendingNotes(); + +Deno.cron('update trending notes cache', { minute: { every: 15 } }, async () => { + const events = await getTrendingNotes(); + trendingNotesCache = Promise.resolve(events); +}); + +const trendingStatusesController: AppController = async (c) => { + const store = await Storages.db(); + const limit = limitSchema.parse(c.req.query('limit')); + + const events = await trendingNotesCache + .then((events) => events.slice(0, limit)) + .then((events) => hydrateEvents({ events, store })); + + const statuses = await Promise.all( + events.map((event) => renderStatus(event, {})), + ); + + return c.json(statuses.filter(Boolean)); +}; + +async function getTrendingNotes(): Promise { + const store = await Storages.db(); + const kysely = await DittoDB.getInstance(); + const since = Math.floor((Date.now() - Time.days(1)) / 1000); + + const tags = await kysely + .selectFrom('nostr_tags') + .select('nostr_tags.value') + .leftJoin('nostr_events', 'nostr_events.id', 'nostr_tags.event_id') + .where('nostr_events.kind', 'in', [1, 6, 7]) + .where('nostr_events.created_at', '>', since) + .where('nostr_tags.name', '=', 'e') + .groupBy('nostr_tags.value') + .orderBy( + sql`SUM(case when nostr_events.kind = 6 then 2 else 0 end) + SUM(case when nostr_events.kind = 1 or nostr_events.kind = 7 then 1 else 0 end)`, + 'desc', + ) + .limit(20) + .execute(); + + const ids = tags.map(({ value }) => value); + return store.query([{ kinds: [1], ids, limit: ids.length }]); +} + +export { trendingStatusesController, trendingTagsController }; From 80e886bfff7cc384479b28ecc985a404e2af6cff Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 20:38:57 -0500 Subject: [PATCH 2/2] Rework trending posts (it's much faster) --- src/controllers/api/trends.ts | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts index d531ffcb..b5a56a43 100644 --- a/src/controllers/api/trends.ts +++ b/src/controllers/api/trends.ts @@ -91,27 +91,26 @@ const trendingStatusesController: AppController = async (c) => { }; async function getTrendingNotes(): Promise { - const store = await Storages.db(); const kysely = await DittoDB.getInstance(); const since = Math.floor((Date.now() - Time.days(1)) / 1000); - const tags = await kysely - .selectFrom('nostr_tags') - .select('nostr_tags.value') - .leftJoin('nostr_events', 'nostr_events.id', 'nostr_tags.event_id') - .where('nostr_events.kind', 'in', [1, 6, 7]) + const rows = await kysely + .selectFrom('nostr_events') + .selectAll('nostr_events') + .innerJoin('event_stats', 'event_stats.event_id', 'nostr_events.id') + .where('nostr_events.kind', '=', 1) .where('nostr_events.created_at', '>', since) - .where('nostr_tags.name', '=', 'e') - .groupBy('nostr_tags.value') .orderBy( - sql`SUM(case when nostr_events.kind = 6 then 2 else 0 end) + SUM(case when nostr_events.kind = 1 or nostr_events.kind = 7 then 1 else 0 end)`, + sql`(event_stats.reposts_count * 2) + (event_stats.replies_count) + (event_stats.reactions_count)`, 'desc', ) .limit(20) .execute(); - const ids = tags.map(({ value }) => value); - return store.query([{ kinds: [1], ids, limit: ids.length }]); + return rows.map((row) => ({ + ...row, + tags: JSON.parse(row.tags), + })); } export { trendingStatusesController, trendingTagsController };