Merge branch 'last-kind7' into 'main'

EventsDB: index only the final `e` and `p` tag of kind 7 events

Closes #220

See merge request soapbox-pub/ditto!628

src/storages/EventsDB.test.ts

@@ -4,6 +4,7 @@ import { generateSecretKey } from 'nostr-tools';
 import { RelayError } from '@/RelayError.ts';
 import { eventFixture, genEvent } from '@/test.ts';
 import { Conf } from '@/config.ts';
+import { EventsDB } from '@/storages/EventsDB.ts';
 import { createTestDB } from '@/test.ts';
 
 Deno.test('count filters', async () => {
@@ -244,3 +245,42 @@ Deno.test('NPostgres.query with search', async (t) => {
     assertEquals(await store.query([{ search: "this shouldn't match" }]), []);
   });
 });
+
+Deno.test('EventsDB.indexTags indexes only the final `e` and `p` tag of kind 7 events', () => {
+  const event = {
+    kind: 7,
+    id: 'a92549a442d306b32273aa9456ba48e3851a4e6203af3f567543298ab964b35b',
+    pubkey: 'f288a224a61b7361aa9dc41a90aba8a2dff4544db0bc386728e638b21da1792c',
+    created_at: 1737908284,
+    tags: [
+      ['e', '2503cea56931fb25914866e12ffc739741539db4d6815220b9974ef0967fe3f9', '', 'root'],
+      ['p', 'fad5c18326fb26d9019f1b2aa503802f0253494701bf311d7588a1e65cb8046b'],
+      ['p', '26d6a946675e603f8de4bf6f9cef442037b70c7eee170ff06ed7673fc34c98f1'],
+      ['p', '04c960497af618ae18f5147b3e5c309ef3d8a6251768a1c0820e02c93768cc3b'],
+      ['p', '0114bb11dd8eb89bfb40669509b2a5a473d27126e27acae58257f2fd7cd95776'],
+      ['p', '9fce3aea32b35637838fb45b75be32595742e16bb3e4742cc82bb3d50f9087e6'],
+      ['p', '26bd32c67232bdf16d05e763ec67d883015eb99fd1269025224c20c6cfdb0158'],
+      ['p', 'eab0e756d32b80bcd464f3d844b8040303075a13eabc3599a762c9ac7ab91f4f'],
+      ['p', 'edcd20558f17d99327d841e4582f9b006331ac4010806efa020ef0d40078e6da'],
+      ['p', 'bd1e19980e2c91e6dc657e92c25762ca882eb9272d2579e221f037f93788de91'],
+      ['p', 'bf2376e17ba4ec269d10fcc996a4746b451152be9031fa48e74553dde5526bce'],
+      ['p', '3878d95db7b854c3a0d3b2d6b7bf9bf28b36162be64326f5521ba71cf3b45a69'],
+      ['p', 'ede3866ddfc40aa4e458952c11c67e827e3cbb8a6a4f0a934c009aa2ed2fb477'],
+      ['p', 'f288a224a61b7361aa9dc41a90aba8a2dff4544db0bc386728e638b21da1792c'],
+      ['p', '9ce71f1506ccf4b99f234af49bd6202be883a80f95a155c6e9a1c36fd7e780c7', '', 'mention'],
+      ['p', '932614571afcbad4d17a191ee281e39eebbb41b93fac8fd87829622aeb112f4d', '', 'mention'],
+      ['e', 'e3653ae41ffb510e5fc071555ecfbc94d2fc31e355d61d941e39a97ac6acb15b'],
+      ['p', '4e088f3087f6a7e7097ce5fe7fd884ec04ddc69ed6cdd37c55e200f7744b1792'],
+    ],
+    content: '🤙',
+    sig:
+      '44639d039a7f7fb8772fcfa13d134d3cda684ec34b6a777ead589676f9e8d81b08a24234066dcde1aacfbe193224940fba7586e7197c159757d3caf8f2b57e1b',
+  };
+
+  const tags = EventsDB.indexTags(event);
+
+  assertEquals(tags, [
+    ['e', 'e3653ae41ffb510e5fc071555ecfbc94d2fc31e355d61d941e39a97ac6acb15b'],
+    ['p', '4e088f3087f6a7e7097ce5fe7fd884ec04ddc69ed6cdd37c55e200f7744b1792'],
+  ]);
+});

src/storages/EventsDB.ts

@@ -17,11 +17,19 @@ import { purifyEvent } from '@/utils/purify.ts';
 import { DittoEvent } from '@/interfaces/DittoEvent.ts';
 
 /** Function to decide whether or not to index a tag. */
-type TagCondition = ({ event, count, value }: {
+type TagCondition = (opts: TagConditionOpts) => boolean;
+
+/** Options for the tag condition function. */
+interface TagConditionOpts {
+  /** Nostr event whose tags are being indexed. */
   event: NostrEvent;
+  /** Count of the current tag name so far. Each tag name has a separate counter starting at 0. */
   count: number;
+  /** Overall tag index. */
+  index: number;
+  /** Current vag value. */
   value: string;
-}) => boolean;
+}
 
 /** Options for the EventsDB store. */
 interface EventsDBOpts {
@@ -41,13 +49,13 @@ class EventsDB extends NPostgres {
   static tagConditions: Record<string, TagCondition> = {
     'a': ({ count }) => count < 15,
     'd': ({ event, count }) => count === 0 && NKinds.parameterizedReplaceable(event.kind),
-    'e': ({ event, count, value }) => ((event.kind === 10003) || count < 15) && isNostrId(value),
+    'e': EventsDB.eTagCondition,
     'k': ({ count, value }) => count === 0 && Number.isInteger(Number(value)),
     'L': ({ event, count }) => event.kind === 1985 || count === 0,
     'l': ({ event, count }) => event.kind === 1985 || count === 0,
     'n': ({ count, value }) => count < 50 && value.length < 50,
     'P': ({ count, value }) => count === 0 && isNostrId(value),
-    'p': ({ event, count, value }) => (count < 15 || event.kind === 3) && isNostrId(value),
+    'p': EventsDB.pTagCondition,
     'proxy': ({ count, value }) => count === 0 && value.length < 256,
     'q': ({ event, count, value }) => count === 0 && event.kind === 1 && isNostrId(value),
     'r': ({ event, count }) => (event.kind === 1985 ? count < 20 : count < 3),
@@ -243,6 +251,28 @@ class EventsDB extends NPostgres {
     return super.count(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout });
   }
 
+  /** Rule for indexing `e` tags. */
+  private static eTagCondition({ event, count, value, index }: TagConditionOpts): boolean {
+    if (!isNostrId(value)) return false;
+
+    if (event.kind === 7) {
+      return index === event.tags.findLastIndex(([name]) => name === 'e');
+    }
+
+    return event.kind === 10003 || count < 15;
+  }
+
+  /** Rule for indexing `p` tags. */
+  private static pTagCondition({ event, count, value, index }: TagConditionOpts): boolean {
+    if (!isNostrId(value)) return false;
+
+    if (event.kind === 7) {
+      return index === event.tags.findLastIndex(([name]) => name === 'p');
+    }
+
+    return count < 15 || event.kind === 3;
+  }
+
   /** Return only the tags that should be indexed. */
   static override indexTags(event: NostrEvent): string[][] {
     const tagCounts: Record<string, number> = {};
@@ -255,19 +285,20 @@ class EventsDB extends NPostgres {
       tagCounts[name] = getCount(name) + 1;
     }
 
-    function checkCondition(name: string, value: string, condition: TagCondition) {
+    function checkCondition(name: string, value: string, condition: TagCondition, index: number): boolean {
       return condition({
         event,
         count: getCount(name),
         value,
+        index,
       });
     }
 
-    return event.tags.reduce<string[][]>((results, tag) => {
+    return event.tags.reduce<string[][]>((results, tag, index) => {
       const [name, value] = tag;
       const condition = EventsDB.tagConditions[name] as TagCondition | undefined;
 
-      if (value && condition && value.length < 200 && checkCondition(name, value, condition)) {
+      if (value && condition && value.length < 200 && checkCondition(name, value, condition, index)) {
         results.push(tag);
       }
 

src/trends.test.ts

@@ -16,10 +16,12 @@ Deno.test("getTrendingTagValues(): 'e' tag and WITHOUT language parameter", asyn
   const post1uses = numberOfAuthorsWhoLikedPost1 * post1multiplier;
   for (let i = 0; i < numberOfAuthorsWhoLikedPost1; i++) {
     const sk = generateSecretKey();
+    for (let j = 0; j < post1multiplier; j++) {
       events.push(
-      genEvent({ kind: 7, content: '+', tags: Array(post1multiplier).fill([...['e', post1.id]]) }, sk),
+        genEvent({ kind: 7, content: '+', tags: [['e', post1.id, `${j}`]] }, sk),
       );
     }
+  }
   events.push(post1);
 
   sk = generateSecretKey();
@@ -29,10 +31,12 @@ Deno.test("getTrendingTagValues(): 'e' tag and WITHOUT language parameter", asyn
   const post2uses = numberOfAuthorsWhoLikedPost2 * post2multiplier;
   for (let i = 0; i < numberOfAuthorsWhoLikedPost2; i++) {
     const sk = generateSecretKey();
+    for (let j = 0; j < post2multiplier; j++) {
       events.push(
-      genEvent({ kind: 7, content: '+', tags: Array(post2multiplier).fill([...['e', post2.id]]) }, sk),
+        genEvent({ kind: 7, content: '+', tags: [['e', post2.id, `${j}`]] }, sk),
       );
     }
+  }
   events.push(post2);
 
   for (const event of events) {
@@ -62,10 +66,12 @@ Deno.test("getTrendingTagValues(): 'e' tag and WITH language parameter", async (
   const post1uses = numberOfAuthorsWhoLikedPost1 * post1multiplier;
   for (let i = 0; i < numberOfAuthorsWhoLikedPost1; i++) {
     const sk = generateSecretKey();
+    for (let j = 0; j < post1multiplier; j++) {
       events.push(
-      genEvent({ kind: 7, content: '+', tags: Array(post1multiplier).fill([...['e', post1.id]]) }, sk),
+        genEvent({ kind: 7, content: '+', tags: [['e', post1.id, `${j}`]] }, sk),
       );
     }
+  }
   events.push(post1);
 
   sk = generateSecretKey();
@@ -74,10 +80,12 @@ Deno.test("getTrendingTagValues(): 'e' tag and WITH language parameter", async (
   const post2multiplier = 1;
   for (let i = 0; i < numberOfAuthorsWhoLikedPost2; i++) {
     const sk = generateSecretKey();
+    for (let j = 0; j < post2multiplier; j++) {
       events.push(
-      genEvent({ kind: 7, content: '+', tags: Array(post2multiplier).fill([...['e', post2.id]]) }, sk),
+        genEvent({ kind: 7, content: '+', tags: [['e', post2.id, `${j}`]] }, sk),
       );
     }
+  }
   events.push(post2);
 
   for (const event of events) {