From 8d8e46eae80e099067228fa4ba2328fc23433a96 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Jun 2024 11:51:55 -0500 Subject: [PATCH] Add a script to update trends --- deno.json | 3 +- scripts/trends.ts | 44 +++++++++++ src/cron.ts | 88 ++------------------- src/trends.ts | 127 ++++++++++++++++++++++++++++++ src/trends/trending-tag-values.ts | 47 ----------- 5 files changed, 181 insertions(+), 128 deletions(-) create mode 100644 scripts/trends.ts create mode 100644 src/trends.ts delete mode 100644 src/trends/trending-tag-values.ts diff --git a/deno.json b/deno.json index 1a7f5e8b..18c88749 100644 --- a/deno.json +++ b/deno.json @@ -13,7 +13,8 @@ "admin:role": "deno run -A scripts/admin-role.ts", "setup": "deno run -A scripts/setup.ts", "stats:recompute": "deno run -A scripts/stats-recompute.ts", - "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip soapbox.zip && rm soapbox.zip" + "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip soapbox.zip && rm soapbox.zip", + "trends": "deno run -A scripts/trends.ts" }, "unstable": ["cron", "ffi", "kv", "worker-options"], "exclude": ["./public"], diff --git a/scripts/trends.ts b/scripts/trends.ts new file mode 100644 index 00000000..6600f7e2 --- /dev/null +++ b/scripts/trends.ts @@ -0,0 +1,44 @@ +import { z } from 'zod'; + +import { + updateTrendingEvents, + updateTrendingHashtags, + updateTrendingLinks, + updateTrendingPubkeys, + updateTrendingZappedEvents, +} from '@/trends.ts'; + +const trendSchema = z.enum(['pubkeys', 'zapped_events', 'events', 'hashtags', 'links']); +const trends = trendSchema.array().parse(Deno.args); + +if (!trends.length) { + trends.push('pubkeys', 'zapped_events', 'events', 'hashtags', 'links'); +} + +for (const trend of trends) { + switch (trend) { + case 'pubkeys': + console.log('Updating trending pubkeys...'); + await updateTrendingPubkeys(); + break; + case 'zapped_events': + console.log('Updating trending zapped events...'); + await updateTrendingZappedEvents(); + break; + case 'events': + console.log('Updating trending events...'); + await updateTrendingEvents(); + break; + case 'hashtags': + console.log('Updating trending hashtags...'); + await updateTrendingHashtags(); + break; + case 'links': + console.log('Updating trending links...'); + await updateTrendingLinks(); + break; + } +} + +console.log('Trends updated.'); +Deno.exit(0); diff --git a/src/cron.ts b/src/cron.ts index 166e42ee..6994561e 100644 --- a/src/cron.ts +++ b/src/cron.ts @@ -1,84 +1,12 @@ -import { Stickynotes } from '@soapbox/stickynotes'; - -import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; -import { handleEvent } from '@/pipeline.ts'; -import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { getTrendingTagValues } from '@/trends/trending-tag-values.ts'; -import { Time } from '@/utils/time.ts'; - -const console = new Stickynotes('ditto:trends'); - -async function updateTrendingTags( - l: string, - tagName: string, - kinds: number[], - limit: number, - extra = '', - aliases?: string[], -) { - console.info(`Updating trending ${l}...`); - const kysely = await DittoDB.getInstance(); - const signal = AbortSignal.timeout(1000); - - const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); - const now = Math.floor(Date.now() / 1000); - - const tagNames = aliases ? [tagName, ...aliases] : [tagName]; - - const trends = await getTrendingTagValues(kysely, tagNames, { - kinds, - since: yesterday, - until: now, - limit, - }); - - if (!trends.length) { - return; - } - - const signer = new AdminSigner(); - - const label = await signer.signEvent({ - kind: 1985, - content: '', - tags: [ - ['L', 'pub.ditto.trends'], - ['l', l, 'pub.ditto.trends'], - ...trends.map(({ value, authors, uses }) => [tagName, value, extra, authors.toString(), uses.toString()]), - ], - created_at: Math.floor(Date.now() / 1000), - }); - - await handleEvent(label, signal); - console.info(`Trending ${l} updated.`); -} +import { updateTrendingLinks } from '@/trends.ts'; +import { updateTrendingHashtags } from '@/trends.ts'; +import { updateTrendingEvents, updateTrendingPubkeys, updateTrendingZappedEvents } from '@/trends.ts'; /** Start cron jobs for the application. */ export function cron() { - Deno.cron( - 'update trending pubkeys', - '0 * * * *', - () => updateTrendingTags('#p', 'p', [1, 3, 6, 7, 9735], 40, Conf.relay), - ); - Deno.cron( - 'update trending zapped events', - '7 * * * *', - () => updateTrendingTags('zapped', 'e', [9735], 40, Conf.relay, ['q']), - ); - Deno.cron( - 'update trending events', - '15 * * * *', - () => updateTrendingTags('#e', 'e', [1, 6, 7, 9735], 40, Conf.relay, ['q']), - ); - Deno.cron( - 'update trending hashtags', - '30 * * * *', - () => updateTrendingTags('#t', 't', [1], 20), - ); - Deno.cron( - 'update trending links', - '45 * * * *', - () => updateTrendingTags('#r', 'r', [1], 20), - ); + Deno.cron('update trending pubkeys', '0 * * * *', updateTrendingPubkeys); + Deno.cron('update trending zapped events', '7 * * * *', updateTrendingZappedEvents); + Deno.cron('update trending events', '15 * * * *', updateTrendingEvents); + Deno.cron('update trending hashtags', '30 * * * *', updateTrendingHashtags); + Deno.cron('update trending links', '45 * * * *', updateTrendingLinks); } diff --git a/src/trends.ts b/src/trends.ts new file mode 100644 index 00000000..c346a492 --- /dev/null +++ b/src/trends.ts @@ -0,0 +1,127 @@ +import { NostrFilter } from '@nostrify/nostrify'; +import { Stickynotes } from '@soapbox/stickynotes'; +import { Kysely } from 'kysely'; + +import { Conf } from '@/config.ts'; +import { DittoDB } from '@/db/DittoDB.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; +import { handleEvent } from '@/pipeline.ts'; +import { AdminSigner } from '@/signers/AdminSigner.ts'; +import { Time } from '@/utils/time.ts'; + +const console = new Stickynotes('ditto:trends'); + +/** Get trending tag values for a given tag in the given time frame. */ +export async function getTrendingTagValues( + /** Kysely instance to execute queries on. */ + kysely: Kysely, + /** Tag name to filter by, eg `t` or `r`. */ + tagNames: string[], + /** Filter of eligible events. */ + filter: NostrFilter, +): Promise<{ value: string; authors: number; uses: number }[]> { + let query = kysely + .selectFrom('nostr_tags') + .innerJoin('nostr_events', 'nostr_events.id', 'nostr_tags.event_id') + .select(({ fn }) => [ + 'nostr_tags.value', + fn.agg('count', ['nostr_events.pubkey']).distinct().as('authors'), + fn.countAll().as('uses'), + ]) + .where('nostr_tags.name', 'in', tagNames) + .groupBy('nostr_tags.value') + .orderBy((c) => c.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc'); + + if (filter.kinds) { + query = query.where('nostr_events.kind', 'in', filter.kinds); + } + if (typeof filter.since === 'number') { + query = query.where('nostr_events.created_at', '>=', filter.since); + } + if (typeof filter.until === 'number') { + query = query.where('nostr_events.created_at', '<=', filter.until); + } + if (typeof filter.limit === 'number') { + query = query.limit(filter.limit); + } + + const rows = await query.execute(); + + return rows.map((row) => ({ + value: row.value, + authors: Number(row.authors), + uses: Number(row.uses), + })); +} + +/** Get trending tags and publish an event with them. */ +export async function updateTrendingTags( + l: string, + tagName: string, + kinds: number[], + limit: number, + extra = '', + aliases?: string[], +) { + console.info(`Updating trending ${l}...`); + const kysely = await DittoDB.getInstance(); + const signal = AbortSignal.timeout(1000); + + const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); + const now = Math.floor(Date.now() / 1000); + + const tagNames = aliases ? [tagName, ...aliases] : [tagName]; + + const trends = await getTrendingTagValues(kysely, tagNames, { + kinds, + since: yesterday, + until: now, + limit, + }); + + if (!trends.length) { + console.info(`No trending ${l} found. Skipping.`); + return; + } + + const signer = new AdminSigner(); + + const label = await signer.signEvent({ + kind: 1985, + content: '', + tags: [ + ['L', 'pub.ditto.trends'], + ['l', l, 'pub.ditto.trends'], + ...trends.map(({ value, authors, uses }) => [tagName, value, extra, authors.toString(), uses.toString()]), + ], + created_at: Math.floor(Date.now() / 1000), + }); + + await handleEvent(label, signal); + console.info(`Trending ${l} updated.`); +} + +/** Update trending pubkeys. */ +export function updateTrendingPubkeys(): Promise { + return updateTrendingTags('#p', 'p', [1, 3, 6, 7, 9735], 40, Conf.relay); +} + +/** Update trending zapped events. */ +export function updateTrendingZappedEvents(): Promise { + return updateTrendingTags('zapped', 'e', [9735], 40, Conf.relay, ['q']); +} + +/** Update trending events. */ +export function updateTrendingEvents(): Promise { + return updateTrendingTags('#e', 'e', [1, 6, 7, 9735], 40, Conf.relay, ['q']); +} + +/** Update trending hashtags. */ +export function updateTrendingHashtags(): Promise { + return updateTrendingTags('#t', 't', [1], 20); +} + +/** Update trending links. */ +export function updateTrendingLinks(): Promise { + return updateTrendingTags('#r', 'r', [1], 20); +} diff --git a/src/trends/trending-tag-values.ts b/src/trends/trending-tag-values.ts deleted file mode 100644 index 17ec53d2..00000000 --- a/src/trends/trending-tag-values.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { NostrFilter } from '@nostrify/nostrify'; -import { Kysely } from 'kysely'; - -import { DittoTables } from '@/db/DittoTables.ts'; - -/** Get trending tag values for a given tag in the given time frame. */ -export async function getTrendingTagValues( - /** Kysely instance to execute queries on. */ - kysely: Kysely, - /** Tag name to filter by, eg `t` or `r`. */ - tagNames: string[], - /** Filter of eligible events. */ - filter: NostrFilter, -): Promise<{ value: string; authors: number; uses: number }[]> { - let query = kysely - .selectFrom('nostr_tags') - .innerJoin('nostr_events', 'nostr_events.id', 'nostr_tags.event_id') - .select(({ fn }) => [ - 'nostr_tags.value', - fn.agg('count', ['nostr_events.pubkey']).distinct().as('authors'), - fn.countAll().as('uses'), - ]) - .where('nostr_tags.name', 'in', tagNames) - .groupBy('nostr_tags.value') - .orderBy((c) => c.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc'); - - if (filter.kinds) { - query = query.where('nostr_events.kind', 'in', filter.kinds); - } - if (typeof filter.since === 'number') { - query = query.where('nostr_events.created_at', '>=', filter.since); - } - if (typeof filter.until === 'number') { - query = query.where('nostr_events.created_at', '<=', filter.until); - } - if (typeof filter.limit === 'number') { - query = query.limit(filter.limit); - } - - const rows = await query.execute(); - - return rows.map((row) => ({ - value: row.value, - authors: Number(row.authors), - uses: Number(row.uses), - })); -}