Merge branch 'main' into cashu

deno.json

@@ -23,6 +23,7 @@
     "clean:deps": "deno cache --reload src/app.ts",
     "db:populate-search": "deno run -A --env-file --deny-read=.env scripts/db-populate-search.ts",
     "db:populate-extensions": "deno run -A --env-file --deny-read=.env scripts/db-populate-extensions.ts",
+    "db:streak:recompute": "deno run -A --env-file --deny-read=.env scripts/db-streak-recompute.ts",
     "vapid": "deno run scripts/vapid.ts"
   },
   "unstable": [

scripts/db-streak-recompute.ts

@@ -0,0 +1,52 @@
+import { Conf } from '@/config.ts';
+import { Storages } from '@/storages.ts';
+
+const kysely = await Storages.kysely();
+const statsQuery = kysely.selectFrom('author_stats').select('pubkey');
+const { streakWindow } = Conf;
+
+for await (const { pubkey } of statsQuery.stream(10)) {
+  const eventsQuery = kysely
+    .selectFrom('nostr_events')
+    .select('created_at')
+    .where('pubkey', '=', pubkey)
+    .where('kind', 'in', [1, 20, 1111, 30023])
+    .orderBy('nostr_events.created_at', 'desc')
+    .orderBy('nostr_events.id', 'asc');
+
+  let end: number | null = null;
+  let start: number | null = null;
+
+  for await (const { created_at } of eventsQuery.stream(20)) {
+    const createdAt = Number(created_at);
+
+    if (!end) {
+      const now = Math.floor(Date.now() / 1000);
+
+      if (now - createdAt > streakWindow) {
+        break; // streak broken
+      }
+
+      end = createdAt;
+    }
+
+    if (start && (start - createdAt > streakWindow)) {
+      break; // streak broken
+    }
+
+    start = createdAt;
+  }
+
+  if (start && end) {
+    await kysely
+      .updateTable('author_stats')
+      .set({
+        streak_end: end,
+        streak_start: start,
+      })
+      .where('pubkey', '=', pubkey)
+      .execute();
+  }
+}
+
+Deno.exit();

src/config.ts

@@ -357,6 +357,10 @@ class Conf {
       return Number(Deno.env.get('PROFILE_FIELDS_VALUE_LENGTH') || 2047);
     },
   };
+  /** Maximum time between events before a streak is broken, *in seconds*. */
+  static get streakWindow(): number {
+    return Number(Deno.env.get('STREAK_WINDOW') || 129600);
+  }
 }
 
 const optionalBooleanSchema = z

src/db/DittoTables.ts

@@ -17,6 +17,8 @@ interface AuthorStatsRow {
   following_count: number;
   notes_count: number;
   search: string;
+  streak_start: number | null;
+  streak_end: number | null;
 }
 
 interface EventStatsRow {

src/db/migrations/045_streaks.ts

@@ -0,0 +1,17 @@
+import { Kysely } from 'kysely';
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .alterTable('author_stats')
+    .addColumn('streak_start', 'integer')
+    .addColumn('streak_end', 'integer')
+    .execute();
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .alterTable('author_stats')
+    .dropColumn('streak_start')
+    .dropColumn('streak_end')
+    .execute();
+}

src/entities/MastodonAccount.ts

@@ -45,6 +45,11 @@ export interface MastodonAccount {
   ditto: {
     accepts_zaps: boolean;
     external_url: string;
+    streak: {
+      days: number;
+      start: string | null;
+      end: string | null;
+    };
   };
   domain?: string;
   pleroma: {

src/interfaces/DittoEvent.ts

@@ -6,6 +6,8 @@ export interface AuthorStats {
   followers_count: number;
   following_count: number;
   notes_count: number;
+  streak_start?: number;
+  streak_end?: number;
 }
 
 /** Ditto internal stats for the event. */

src/storages/EventsDB.ts

@@ -68,7 +68,13 @@ class EventsDB extends NPostgres {
 
     if (event.kind === 1) {
       ext.reply = event.tags.some(([name]) => name === 'e').toString();
+    } else if (event.kind === 1111) {
+      ext.reply = event.tags.some(([name]) => ['e', 'E'].includes(name)).toString();
+    } else if (event.kind === 6) {
+      ext.reply = 'false';
+    }
 
+    if ([1, 20, 30023].includes(event.kind)) {
       const language = detectLanguage(event.content, 0.90);
 
       if (language) {

src/storages/hydrate.ts

@@ -89,10 +89,22 @@ export function assembleEvents(
 ): DittoEvent[] {
   const admin = Conf.pubkey;
 
-  const eventStats = stats.events.map((stat) => ({
+  const authorStats = stats.authors.reduce((result, { pubkey, ...stat }) => {
+    result[pubkey] = {
+      ...stat,
+      streak_start: stat.streak_start ?? undefined,
+      streak_end: stat.streak_end ?? undefined,
+    };
+    return result;
+  }, {} as Record<string, DittoEvent['author_stats']>);
+
+  const eventStats = stats.events.reduce((result, { event_id, ...stat }) => {
+    result[event_id] = {
       ...stat,
       reactions: JSON.parse(stat.reactions),
-  }));
+    };
+    return result;
+  }, {} as Record<string, DittoEvent['event_stats']>);
 
   for (const event of a) {
     event.author = b.find((e) => matchFilter({ kinds: [0], authors: [event.pubkey] }, e));
@@ -161,8 +173,8 @@ export function assembleEvents(
       event.zap_message = zapRequest?.content ?? '';
     }
 
-    event.author_stats = stats.authors.find((stats) => stats.pubkey === event.pubkey);
-    event.event_stats = eventStats.find((stats) => stats.event_id === event.id);
+    event.author_stats = authorStats[event.pubkey];
+    event.event_stats = eventStats[event.id];
   }
 
   return a;
@@ -383,6 +395,8 @@ async function gatherAuthorStats(
     following_count: Math.max(0, row.following_count),
     notes_count: Math.max(0, row.notes_count),
     search: row.search,
+    streak_start: row.streak_start,
+    streak_end: row.streak_end,
   }));
 }
 

src/utils/language.test.ts

@@ -35,9 +35,17 @@ Deno.test('Detects definitive texts', () => {
   assertEquals(detectLanguage('Γειά σου!', 1), 'el');
   assertEquals(detectLanguage('שלום!', 1), 'he');
   assertEquals(detectLanguage('こんにちは。', 1), 'ja');
+  assertEquals(
+    detectLanguage(
+      '最近、長女から「中学生男子全員クソ」という話を良く聞き中学生女子側の視点が分かってよかった。父からは「中学生男子は自分がクソだということを3年間かかって学習するんだよ」と言っておいた',
+      1,
+    ),
+    'ja',
+  );
 
   // ambiguous
   assertEquals(detectLanguage('你好', 1), undefined);
+  assertEquals(detectLanguage('東京', 1), undefined);
   assertEquals(detectLanguage('Привет', 1), undefined);
   assertEquals(detectLanguage('Hello', 1), undefined);
 });

src/utils/language.ts

@@ -12,9 +12,10 @@ export function detectLanguage(text: string, minConfidence: number): LanguageCod
   // It's better to remove the emojis first
   const sanitizedText = linkify.tokenize(
     text
-      .replaceAll(/\p{Extended_Pictographic}/gu, '')
-      .replaceAll(/[\s\uFEFF\u00A0\u200B-\u200D\u{0FE0E}]+/gu, ' '),
-  ).reduce((acc, { t, v }) => t === 'text' ? acc + v : acc, '').trim();
+      .replaceAll(/\p{Extended_Pictographic}/gu, '') // strip emojis
+      .replaceAll(/[\s\uFEFF\u00A0\u200B-\u200D\u{0FE0E}]+/gu, ' '), // strip invisible characters
+  )
+    .reduce((acc, { t, v }) => t === 'text' ? acc + v : acc, '').trim();
 
   // Definite patterns for some languages.
   // Text which matches MUST unambiguously be in the given language.
@@ -30,7 +31,11 @@ export function detectLanguage(text: string, minConfidence: number): LanguageCod
 
   // If any pattern matches, the language is known.
   for (const [lang, pattern] of Object.entries(languagePatterns) as [LanguageCode, RegExp][]) {
-    if (pattern.test(text.replace(/[\p{P}\p{S}]/gu, ''))) { // strip punctuation and symbols before checking
+    const text = sanitizedText
+      .replaceAll(/[\p{P}\p{S}]/gu, '') // strip punctuation and symbols
+      .replaceAll(/\p{N}/gu, ''); // strip numbers
+
+    if (pattern.test(text)) {
       return lang;
     }
   }

src/utils/stats.ts

@@ -1,8 +1,9 @@
 import { NostrEvent, NSchema as n, NStore } from '@nostrify/nostrify';
-import { Kysely, UpdateObject } from 'kysely';
+import { Insertable, Kysely, UpdateObject } from 'kysely';
 import { SetRequired } from 'type-fest';
 import { z } from 'zod';
 
+import { Conf } from '@/config.ts';
 import { DittoTables } from '@/db/DittoTables.ts';
 import { findQuoteTag, findReplyTag, getTagSet } from '@/utils/tags.ts';
 
@@ -18,6 +19,9 @@ interface UpdateStatsOpts {
 export async function updateStats({ event, kysely, store, x = 1 }: UpdateStatsOpts): Promise<void> {
   switch (event.kind) {
     case 1:
+    case 20:
+    case 1111:
+    case 30023:
       return handleEvent1(kysely, event, x);
     case 3:
       return handleEvent3(kysely, event, x, store);
@@ -34,7 +38,34 @@ export async function updateStats({ event, kysely, store, x = 1 }: UpdateStatsOp
 
 /** Update stats for kind 1 event. */
 async function handleEvent1(kysely: Kysely<DittoTables>, event: NostrEvent, x: number): Promise<void> {
-  await updateAuthorStats(kysely, event.pubkey, ({ notes_count }) => ({ notes_count: Math.max(0, notes_count + x) }));
+  await updateAuthorStats(kysely, event.pubkey, (prev) => {
+    const now = event.created_at;
+
+    let start = prev.streak_start;
+    let end = prev.streak_end;
+
+    if (start && end) { // Streak exists.
+      if (now <= end) {
+        // Streak cannot go backwards in time. Skip it.
+      } else if (now - end > Conf.streakWindow) {
+        // Streak is broken. Start a new streak.
+        start = now;
+        end = now;
+      } else {
+        // Extend the streak.
+        end = now;
+      }
+    } else { // New streak.
+      start = now;
+      end = now;
+    }
+
+    return {
+      notes_count: Math.max(0, prev.notes_count + x),
+      streak_start: start || null,
+      streak_end: end || null,
+    };
+  });
 
   const replyId = findReplyTag(event.tags)?.[1];
   const quoteId = findQuoteTag(event.tags)?.[1];
@@ -187,9 +218,9 @@ export function getAuthorStats(
 export async function updateAuthorStats(
   kysely: Kysely<DittoTables>,
   pubkey: string,
-  fn: (prev: DittoTables['author_stats']) => UpdateObject<DittoTables, 'author_stats'>,
+  fn: (prev: Insertable<DittoTables['author_stats']>) => UpdateObject<DittoTables, 'author_stats'>,
 ): Promise<void> {
-  const empty: DittoTables['author_stats'] = {
+  const empty: Insertable<DittoTables['author_stats']> = {
     pubkey,
     followers_count: 0,
     following_count: 0,
@@ -290,6 +321,8 @@ export async function countAuthorStats(
     following_count: getTagSet(followList?.tags ?? [], 'p').size,
     notes_count,
     search,
+    streak_start: null,
+    streak_end: null,
   };
 }
 

src/views/mastodon/accounts.ts

@@ -69,6 +69,22 @@ async function renderAccount(
       verified_at: null,
     })) ?? [];
 
+  let streakDays = 0;
+  let streakStart = event.author_stats?.streak_start ?? null;
+  let streakEnd = event.author_stats?.streak_end ?? null;
+  const { streakWindow } = Conf;
+
+  if (streakStart && streakEnd) {
+    const broken = nostrNow() - streakEnd > streakWindow;
+    if (broken) {
+      streakStart = null;
+      streakEnd = null;
+    } else {
+      const delta = streakEnd - streakStart;
+      streakDays = Math.max(Math.ceil(delta / 86400), 1);
+    }
+  }
+
   return {
     id: pubkey,
     acct,
@@ -113,6 +129,11 @@ async function renderAccount(
     ditto: {
       accepts_zaps: Boolean(getLnurl({ lud06, lud16 })),
       external_url: Conf.external(nprofile),
+      streak: {
+        days: streakDays,
+        start: streakStart ? nostrDate(streakStart).toISOString() : null,
+        end: streakEnd ? nostrDate(streakEnd).toISOString() : null,
+      },
     },
     domain: parsed05?.domain,
     pleroma: {