Merge branch 'trends-any-language' into 'main'

Trends in any language

Closes #222

See merge request soapbox-pub/ditto!523

scripts/trends.ts

@@ -1,4 +1,5 @@
 import { z } from 'zod';
+import '@/config.ts';
 
 import {
   updateTrendingEvents,

src/config.ts

@@ -1,4 +1,5 @@
 import os from 'node:os';
+import ISO6391, { LanguageCode } from 'iso-639-1';
 import * as dotenv from '@std/dotenv';
 import { getPublicKey, nip19 } from 'nostr-tools';
 import { z } from 'zod';
@@ -247,6 +248,10 @@ class Conf {
   static get zapSplitsEnabled(): boolean {
     return optionalBooleanSchema.parse(Deno.env.get('ZAP_SPLITS_ENABLED')) ?? false;
   }
+  /** Languages this server wishes to highlight. Used when querying trends.*/
+  static get preferredLanguages(): LanguageCode[] | undefined {
+    return Deno.env.get('DITTO_LANGUAGES')?.split(',')?.filter(ISO6391.validate) as LanguageCode[];
+  }
   /** Cache settings. */
   static caches = {
     /** NIP-05 cache settings. */

src/trends.test.ts

@@ -0,0 +1,105 @@
+import { assertEquals } from '@std/assert';
+import { generateSecretKey, NostrEvent } from 'nostr-tools';
+
+import { getTrendingTagValues } from '@/trends.ts';
+import { createTestDB, genEvent } from '@/test.ts';
+
+Deno.test("getTrendingTagValues(): 'e' tag and WITHOUT language parameter", async () => {
+  await using db = await createTestDB();
+
+  const events: NostrEvent[] = [];
+
+  let sk = generateSecretKey();
+  const post1 = genEvent({ kind: 1, content: 'SHOW ME THE MONEY' }, sk);
+  const numberOfAuthorsWhoLikedPost1 = 100;
+  const post1multiplier = 2;
+  const post1uses = numberOfAuthorsWhoLikedPost1 * post1multiplier;
+  for (let i = 0; i < numberOfAuthorsWhoLikedPost1; i++) {
+    const sk = generateSecretKey();
+    events.push(
+      genEvent({ kind: 7, content: '+', tags: Array(post1multiplier).fill([...['e', post1.id]]) }, sk),
+    );
+  }
+  events.push(post1);
+
+  sk = generateSecretKey();
+  const post2 = genEvent({ kind: 1, content: 'Ithaca' }, sk);
+  const numberOfAuthorsWhoLikedPost2 = 100;
+  const post2multiplier = 1;
+  const post2uses = numberOfAuthorsWhoLikedPost2 * post2multiplier;
+  for (let i = 0; i < numberOfAuthorsWhoLikedPost2; i++) {
+    const sk = generateSecretKey();
+    events.push(
+      genEvent({ kind: 7, content: '+', tags: Array(post2multiplier).fill([...['e', post2.id]]) }, sk),
+    );
+  }
+  events.push(post2);
+
+  for (const event of events) {
+    await db.store.event(event);
+  }
+
+  const trends = await getTrendingTagValues(db.kysely, ['e'], { kinds: [1, 7] });
+
+  const expected = [{ value: post1.id, authors: numberOfAuthorsWhoLikedPost1, uses: post1uses }, {
+    value: post2.id,
+    authors: numberOfAuthorsWhoLikedPost2,
+    uses: post2uses,
+  }];
+
+  assertEquals(trends, expected);
+});
+
+Deno.test("getTrendingTagValues(): 'e' tag and WITH language parameter", async () => {
+  await using db = await createTestDB();
+
+  const events: NostrEvent[] = [];
+
+  let sk = generateSecretKey();
+  const post1 = genEvent({ kind: 1, content: 'Irei cortar o cabelo.' }, sk);
+  const numberOfAuthorsWhoLikedPost1 = 100;
+  const post1multiplier = 2;
+  const post1uses = numberOfAuthorsWhoLikedPost1 * post1multiplier;
+  for (let i = 0; i < numberOfAuthorsWhoLikedPost1; i++) {
+    const sk = generateSecretKey();
+    events.push(
+      genEvent({ kind: 7, content: '+', tags: Array(post1multiplier).fill([...['e', post1.id]]) }, sk),
+    );
+  }
+  events.push(post1);
+
+  sk = generateSecretKey();
+  const post2 = genEvent({ kind: 1, content: 'Ithaca' }, sk);
+  const numberOfAuthorsWhoLikedPost2 = 100;
+  const post2multiplier = 1;
+  for (let i = 0; i < numberOfAuthorsWhoLikedPost2; i++) {
+    const sk = generateSecretKey();
+    events.push(
+      genEvent({ kind: 7, content: '+', tags: Array(post2multiplier).fill([...['e', post2.id]]) }, sk),
+    );
+  }
+  events.push(post2);
+
+  for (const event of events) {
+    await db.store.event(event);
+  }
+
+  await db.kysely.updateTable('nostr_events')
+    .set('language', 'pt')
+    .where('id', '=', post1.id)
+    .execute();
+
+  await db.kysely.updateTable('nostr_events')
+    .set('language', 'en')
+    .where('id', '=', post2.id)
+    .execute();
+
+  const languagesIds = (await db.store.query([{ search: 'language:pt' }])).map((event) => event.id);
+
+  const trends = await getTrendingTagValues(db.kysely, ['e'], { kinds: [1, 7] }, languagesIds);
+
+  // portuguese post
+  const expected = [{ value: post1.id, authors: numberOfAuthorsWhoLikedPost1, uses: post1uses }];
+
+  assertEquals(trends, expected);
+});

src/trends.ts

@@ -19,6 +19,8 @@ export async function getTrendingTagValues(
   tagNames: string[],
   /** Filter of eligible events. */
   filter: NostrFilter,
+  /** If present, only tag values in this list are permitted to trend. */
+  values?: string[],
 ): Promise<{ value: string; authors: number; uses: number }[]> {
   let query = kysely
     .selectFrom([
@@ -33,7 +35,7 @@ export async function getTrendingTagValues(
     ])
     .where('kv.key', '=', (eb) => eb.fn.any(eb.val(tagNames)))
     .groupBy((eb) => eb.fn<string>('lower', ['element.value']))
-    .orderBy((eb) => eb.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc');
+    .orderBy('authors desc').orderBy('uses desc');
 
   if (filter.kinds) {
     query = query.where('nostr_events.kind', '=', ({ fn, val }) => fn.any(val(filter.kinds)));
@@ -47,6 +49,9 @@ export async function getTrendingTagValues(
   if (typeof filter.until === 'number') {
     query = query.where('nostr_events.created_at', '<=', filter.until);
   }
+  if (values) {
+    query = query.where('element.value', 'in', values);
+  }
   if (typeof filter.limit === 'number') {
     query = query.limit(filter.limit);
   }
@@ -68,6 +73,7 @@ export async function updateTrendingTags(
   limit: number,
   extra = '',
   aliases?: string[],
+  values?: string[],
 ) {
   console.info(`Updating trending ${l}...`);
   const kysely = await Storages.kysely();
@@ -84,8 +90,9 @@ export async function updateTrendingTags(
       since: yesterday,
       until: now,
       limit,
-    });
+    }, values);
 
+    console.log(trends);
     if (!trends.length) {
       console.info(`No trending ${l} found. Skipping.`);
       return;
@@ -122,8 +129,31 @@ export function updateTrendingZappedEvents(): Promise<void> {
 }
 
 /** Update trending events. */
-export function updateTrendingEvents(): Promise<void> {
+export async function updateTrendingEvents(): Promise<void> {
-  return updateTrendingTags('#e', 'e', [1, 6, 7, 9735], 40, Conf.relay, ['q']);
+  const results: Promise<void>[] = [
+    updateTrendingTags('#e', 'e', [1, 6, 7, 9735], 40, Conf.relay, ['q']),
+  ];
+
+  const kysely = await Storages.kysely();
+
+  for (const language of Conf.preferredLanguages ?? []) {
+    const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000);
+    const now = Math.floor(Date.now() / 1000);
+
+    const rows = await kysely
+      .selectFrom('nostr_events')
+      .select('nostr_events.id')
+      .where('nostr_events.language', '=', language)
+      .where('nostr_events.created_at', '>=', yesterday)
+      .where('nostr_events.created_at', '<=', now)
+      .execute();
+
+    const ids = rows.map((row) => row.id);
+
+    results.push(updateTrendingTags(`#e.${language}`, 'e', [1, 6, 7, 9735], 40, Conf.relay, ['q'], ids));
+  }
+
+  await Promise.allSettled(results);
 }
 
 /** Update trending hashtags. */