From 8267c170f0988e8fe95cfece97577722d965c0e2 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Mon, 10 Feb 2025 16:15:13 -0600
Subject: [PATCH 1/2] hydrate: batch queries together

---
 src/storages/hydrate.ts | 196 ++++++++++------------------------------
 1 file changed, 47 insertions(+), 149 deletions(-)

diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts
index a162571a..a656d590 100644
--- a/src/storages/hydrate.ts
+++ b/src/storages/hydrate.ts
@@ -30,23 +30,11 @@ async function hydrateEvents(opts: HydrateOpts): Promise<DittoEvent[]> {
 
   const cache = [...events];
 
-  for (const event of await gatherReposts({ events: cache, store, signal })) {
+  for (const event of await gatherRelatedEvents({ events: cache, store, signal })) {
     cache.push(event);
   }
 
-  for (const event of await gatherReacted({ events: cache, store, signal })) {
-    cache.push(event);
-  }
-
-  for (const event of await gatherQuotes({ events: cache, store, signal })) {
-    cache.push(event);
-  }
-
-  for (const event of await gatherMentions({ events: cache, store, signal })) {
-    cache.push(event);
-  }
-
-  for (const event of await gatherAuthors({ events: cache, store, signal })) {
+  for (const event of await gatherProfiles({ events: cache, store, signal })) {
     cache.push(event);
   }
 
@@ -58,18 +46,6 @@ async function hydrateEvents(opts: HydrateOpts): Promise<DittoEvent[]> {
     cache.push(event);
   }
 
-  for (const event of await gatherReportedProfiles({ events: cache, store, signal })) {
-    cache.push(event);
-  }
-
-  for (const event of await gatherReportedNotes({ events: cache, store, signal })) {
-    cache.push(event);
-  }
-
-  for (const event of await gatherZapped({ events: cache, store, signal })) {
-    cache.push(event);
-  }
-
   const authorStats = await gatherAuthorStats(cache, kysely as Kysely<DittoTables>);
   const eventStats = await gatherEventStats(cache, kysely as Kysely<DittoTables>);
 
@@ -217,17 +193,47 @@ export function assembleEvents(
   return a;
 }
 
-/** Collect reposts from the events. */
-function gatherReposts({ events, store, signal }: HydrateOpts): Promise<DittoEvent[]> {
+/** Collect event targets (eg reposts, quote posts, reacted posts, etc.) */
+function gatherRelatedEvents({ events, store, signal }: HydrateOpts): Promise<DittoEvent[]> {
   const ids = new Set<string>();
 
   for (const event of events) {
+    // Quoted events
+    if (event.kind === 1) {
+      const id = findQuoteTag(event.tags)?.[1] || findQuoteInContent(event.content);
+      if (id) {
+        ids.add(id);
+      }
+    }
+    // Reposted events
     if (event.kind === 6) {
       const id = event.tags.find(([name]) => name === 'e')?.[1];
       if (id) {
         ids.add(id);
       }
     }
+    // Reacted events
+    if (event.kind === 7) {
+      const id = event.tags.findLast(([name]) => name === 'e')?.[1];
+      if (id) {
+        ids.add(id);
+      }
+    }
+    // Reported events
+    if (event.kind === 1984) {
+      for (const [name, value] of event.tags) {
+        if (name === 'e') {
+          ids.add(value);
+        }
+      }
+    }
+    // Zapped events
+    if (event.kind === 9735) {
+      const id = event.tags.find(([name]) => name === 'e')?.[1];
+      if (id) {
+        ids.add(id);
+      }
+    }
   }
 
   return store.query(
@@ -236,49 +242,15 @@ function gatherReposts({ events, store, signal }: HydrateOpts): Promise<DittoEve
   );
 }
 
-/** Collect events being reacted to by the events. */
-function gatherReacted({ events, store, signal }: HydrateOpts): Promise<DittoEvent[]> {
-  const ids = new Set<string>();
-
-  for (const event of events) {
-    if (event.kind === 7) {
-      const id = event.tags.findLast(([name]) => name === 'e')?.[1];
-      if (id) {
-        ids.add(id);
-      }
-    }
-  }
-
-  return store.query(
-    [{ ids: [...ids], limit: ids.size }],
-    { signal },
-  );
-}
-
-/** Collect quotes from the events. */
-function gatherQuotes({ events, store, signal }: HydrateOpts): Promise<DittoEvent[]> {
-  const ids = new Set<string>();
-
-  for (const event of events) {
-    if (event.kind === 1) {
-      const id = findQuoteTag(event.tags)?.[1] || findQuoteInContent(event.content);
-      if (id) {
-        ids.add(id);
-      }
-    }
-  }
-
-  return store.query(
-    [{ ids: [...ids], limit: ids.size }],
-    { signal },
-  );
-}
-
-/** Collect mentioned profiles from notes. */
-async function gatherMentions({ events, store, signal }: HydrateOpts): Promise<DittoEvent[]> {
+/** Collect profiles from the events. */
+async function gatherProfiles({ events, store, signal }: HydrateOpts): Promise<DittoEvent[]> {
   const pubkeys = new Set<string>();
 
   for (const event of events) {
+    // Authors
+    pubkeys.add(event.pubkey);
+
+    // Mentions
     if (event.kind === 1) {
       for (const [name, value] of event.tags) {
         if (name === 'p') {
@@ -286,29 +258,14 @@ async function gatherMentions({ events, store, signal }: HydrateOpts): Promise<D
         }
       }
     }
-  }
-
-  const authors = await store.query(
-    [{ kinds: [0], authors: [...pubkeys], limit: pubkeys.size }],
-    { signal },
-  );
-
-  for (const pubkey of pubkeys) {
-    const author = authors.find((e) => matchFilter({ kinds: [0], authors: [pubkey] }, e));
-    if (!author) {
-      const fallback = fallbackAuthor(pubkey);
-      authors.push(fallback);
+    // Reported profiles
+    if (event.kind === 1984) {
+      const pubkey = event.tags.find(([name]) => name === 'p')?.[1];
+      if (pubkey) {
+        pubkeys.add(pubkey);
+      }
     }
-  }
-
-  return authors;
-}
-
-/** Collect authors from the events. */
-async function gatherAuthors({ events, store, signal }: HydrateOpts): Promise<DittoEvent[]> {
-  const pubkeys = new Set<string>();
-
-  for (const event of events) {
+    // Zap recipients
     if (event.kind === 9735) {
       const zapReceiver = event.tags.find(([name]) => name === 'p')?.[1];
       if (zapReceiver) {
@@ -324,7 +281,6 @@ async function gatherAuthors({ events, store, signal }: HydrateOpts): Promise<Di
         pubkeys.add(zapSender);
       }
     }
-    pubkeys.add(event.pubkey);
   }
 
   const authors = await store.query(
@@ -377,64 +333,6 @@ function gatherInfo({ events, store, signal }: HydrateOpts): Promise<DittoEvent[
   );
 }
 
-/** Collect reported notes from the events. */
-function gatherReportedNotes({ events, store, signal }: HydrateOpts): Promise<DittoEvent[]> {
-  const ids = new Set<string>();
-  for (const event of events) {
-    if (event.kind === 1984) {
-      const status_ids = event.tags.filter(([name]) => name === 'e').map((tag) => tag[1]);
-      if (status_ids.length > 0) {
-        for (const id of status_ids) {
-          ids.add(id);
-        }
-      }
-    }
-  }
-
-  return store.query(
-    [{ kinds: [1, 20], ids: [...ids], limit: ids.size }],
-    { signal },
-  );
-}
-
-/** Collect reported profiles from the events. */
-function gatherReportedProfiles({ events, store, signal }: HydrateOpts): Promise<DittoEvent[]> {
-  const pubkeys = new Set<string>();
-
-  for (const event of events) {
-    if (event.kind === 1984) {
-      const pubkey = event.tags.find(([name]) => name === 'p')?.[1];
-      if (pubkey) {
-        pubkeys.add(pubkey);
-      }
-    }
-  }
-
-  return store.query(
-    [{ kinds: [0], authors: [...pubkeys], limit: pubkeys.size }],
-    { signal },
-  );
-}
-
-/** Collect events being zapped. */
-function gatherZapped({ events, store, signal }: HydrateOpts): Promise<DittoEvent[]> {
-  const ids = new Set<string>();
-
-  for (const event of events) {
-    if (event.kind === 9735) {
-      const id = event.tags.find(([name]) => name === 'e')?.[1];
-      if (id) {
-        ids.add(id);
-      }
-    }
-  }
-
-  return store.query(
-    [{ ids: [...ids], limit: ids.size }],
-    { signal },
-  );
-}
-
 /** Collect author stats from the events. */
 async function gatherAuthorStats(
   events: DittoEvent[],

From 4e86b6bf3f987c9a2ffe1a502a1ab43e00cf99de Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Mon, 10 Feb 2025 16:21:01 -0600
Subject: [PATCH 2/2] hydrate: quotes must be gathered in a separate step

---
 src/storages/hydrate.ts | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts
index a656d590..36df74f6 100644
--- a/src/storages/hydrate.ts
+++ b/src/storages/hydrate.ts
@@ -34,6 +34,10 @@ async function hydrateEvents(opts: HydrateOpts): Promise<DittoEvent[]> {
     cache.push(event);
   }
 
+  for (const event of await gatherQuotes({ events: cache, store, signal })) {
+    cache.push(event);
+  }
+
   for (const event of await gatherProfiles({ events: cache, store, signal })) {
     cache.push(event);
   }
@@ -198,13 +202,6 @@ function gatherRelatedEvents({ events, store, signal }: HydrateOpts): Promise<Di
   const ids = new Set<string>();
 
   for (const event of events) {
-    // Quoted events
-    if (event.kind === 1) {
-      const id = findQuoteTag(event.tags)?.[1] || findQuoteInContent(event.content);
-      if (id) {
-        ids.add(id);
-      }
-    }
     // Reposted events
     if (event.kind === 6) {
       const id = event.tags.find(([name]) => name === 'e')?.[1];
@@ -242,6 +239,25 @@ function gatherRelatedEvents({ events, store, signal }: HydrateOpts): Promise<Di
   );
 }
 
+/** Collect quotes from the events. */
+function gatherQuotes({ events, store, signal }: HydrateOpts): Promise<DittoEvent[]> {
+  const ids = new Set<string>();
+
+  for (const event of events) {
+    if (event.kind === 1) {
+      const id = findQuoteTag(event.tags)?.[1] || findQuoteInContent(event.content);
+      if (id) {
+        ids.add(id);
+      }
+    }
+  }
+
+  return store.query(
+    [{ ids: [...ids], limit: ids.size }],
+    { signal },
+  );
+}
+
 /** Collect profiles from the events. */
 async function gatherProfiles({ events, store, signal }: HydrateOpts): Promise<DittoEvent[]> {
   const pubkeys = new Set<string>();