From b77c8a00cd4b624764658f10804070df61582d98 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 22 Apr 2024 19:51:29 -0300 Subject: [PATCH 1/9] perf: make up to 5 calls to database in hydrateEvents & remove old hydrate functions --- src/controllers/api/accounts.ts | 9 +- src/controllers/api/search.ts | 8 +- src/controllers/api/statuses.ts | 3 - src/controllers/api/streaming.ts | 1 - src/controllers/api/timelines.ts | 1 - src/interfaces/DittoEvent.ts | 4 +- src/pipeline.ts | 2 +- src/queries.ts | 14 ++- src/storages/hydrate.ts | 176 ++++++++++--------------------- src/storages/search-store.ts | 1 - src/views.ts | 8 +- 11 files changed, 71 insertions(+), 156 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 472c9e50..b6cd7353 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -95,7 +95,6 @@ const accountSearchController: AppController = async (c) => { const results = await hydrateEvents({ events: event ? [event, ...events] : events, - relations: ['author_stats'], storage: eventsDB, signal: c.req.raw.signal, }); @@ -164,9 +163,7 @@ const accountStatusesController: AppController = async (c) => { } const events = await eventsDB.query([filter], { signal }) - .then((events) => - hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal }) - ) + .then((events) => hydrateEvents({ events, storage: eventsDB, signal })) .then((events) => { if (exclude_replies) { return events.filter((event) => !findReplyTag(event.tags)); @@ -317,9 +314,7 @@ const favouritesController: AppController = async (c) => { .filter((id): id is string => !!id); const events1 = await eventsDB.query([{ kinds: [1], ids }], { signal }) - .then((events) => - hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal }) - ); + .then((events) => hydrateEvents({ events, storage: eventsDB, signal })); const statuses = await Promise.all(events1.map((event) => renderStatus(event, { viewerPubkey: c.get('pubkey') }))); return paginated(c, events1, statuses); diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 628bb3ae..6c1c972b 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -90,9 +90,7 @@ function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: Abort } return searchStore.query([filter], { signal }) - .then((events) => - hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: searchStore, signal }) - ); + .then((events) => hydrateEvents({ events, storage: searchStore, signal })); } /** Get event kinds to search from `type` query param. */ @@ -112,9 +110,7 @@ async function lookupEvent(query: SearchQuery, signal: AbortSignal): Promise - hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: searchStore, signal }) - ) + .then((events) => hydrateEvents({ events, storage: searchStore, signal })) .then(([event]) => event); } diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index b063abd9..a26ce08a 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -41,7 +41,6 @@ const statusController: AppController = async (c) => { const event = await getEvent(id, { kind: 1, - relations: ['author', 'event_stats', 'author_stats', 'quote_repost'], signal: AbortSignal.timeout(1500), }); @@ -135,7 +134,6 @@ const createStatusController: AppController = async (c) => { if (data.quote_id) { await hydrateEvents({ events: [event], - relations: ['quote_repost'], storage: eventsDB, signal: c.req.raw.signal, }); @@ -241,7 +239,6 @@ const reblogStatusController: AppController = async (c) => { await hydrateEvents({ events: [reblogEvent], - relations: ['repost', 'author'], storage: eventsDB, signal: signal, }); diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index 4ad878e9..3855b9bc 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -68,7 +68,6 @@ const streamingController: AppController = (c) => { if (event.kind === 6) { await hydrateEvents({ events: [event], - relations: ['repost', 'author'], storage: eventsDB, signal: AbortSignal.timeout(1000), }); diff --git a/src/controllers/api/timelines.ts b/src/controllers/api/timelines.ts index ebf19123..8bd4c52f 100644 --- a/src/controllers/api/timelines.ts +++ b/src/controllers/api/timelines.ts @@ -51,7 +51,6 @@ async function renderStatuses(c: AppContext, filters: NostrFilter[]) { .then((events) => hydrateEvents({ events, - relations: ['author', 'author_stats', 'event_stats', 'repost', 'quote_repost'], storage: eventsDB, signal, }) diff --git a/src/interfaces/DittoEvent.ts b/src/interfaces/DittoEvent.ts index 84c8a3ec..08879f81 100644 --- a/src/interfaces/DittoEvent.ts +++ b/src/interfaces/DittoEvent.ts @@ -22,6 +22,6 @@ export interface DittoEvent extends NostrEvent { event_stats?: EventStats; d_author?: DittoEvent; user?: DittoEvent; - repost?: NostrEvent; - quote_repost?: NostrEvent; + repost?: DittoEvent; + quote_repost?: DittoEvent; } diff --git a/src/pipeline.ts b/src/pipeline.ts index 56d7bb23..000a105c 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -56,7 +56,7 @@ async function encounterEvent(event: NostrEvent, signal: AbortSignal): Promise { - await hydrateEvents({ events: [event], relations: ['author', 'user'], storage: eventsDB, signal }); + await hydrateEvents({ events: [event], storage: eventsDB, signal }); const domain = await db .selectFrom('pubkey_domains') diff --git a/src/queries.ts b/src/queries.ts index 61887b25..cf61b84f 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -24,7 +24,7 @@ const getEvent = async ( opts: GetEventOpts = {}, ): Promise => { debug(`getEvent: ${id}`); - const { kind, relations = [], signal = AbortSignal.timeout(1000) } = opts; + const { kind, signal = AbortSignal.timeout(1000) } = opts; const filter: NostrFilter = { ids: [id], limit: 1 }; if (kind) { @@ -32,16 +32,16 @@ const getEvent = async ( } return await optimizer.query([filter], { limit: 1, signal }) - .then((events) => hydrateEvents({ events, relations, storage: optimizer, signal })) + .then((events) => hydrateEvents({ events, storage: optimizer, signal })) .then(([event]) => event); }; /** Get a Nostr `set_medatadata` event for a user's pubkey. */ const getAuthor = async (pubkey: string, opts: GetEventOpts = {}): Promise => { - const { relations = [], signal = AbortSignal.timeout(1000) } = opts; + const { signal = AbortSignal.timeout(1000) } = opts; return await optimizer.query([{ authors: [pubkey], kinds: [0], limit: 1 }], { limit: 1, signal }) - .then((events) => hydrateEvents({ events, relations, storage: optimizer, signal })) + .then((events) => hydrateEvents({ events, storage: optimizer, signal })) .then(([event]) => event); }; @@ -70,7 +70,7 @@ async function getAncestors(event: NostrEvent, result: NostrEvent[] = []): Promi const inReplyTo = replyTag ? replyTag[1] : undefined; if (inReplyTo) { - const parentEvent = await getEvent(inReplyTo, { kind: 1, relations: ['author', 'event_stats', 'author_stats'] }); + const parentEvent = await getEvent(inReplyTo, { kind: 1 }); if (parentEvent) { result.push(parentEvent); @@ -84,9 +84,7 @@ async function getAncestors(event: NostrEvent, result: NostrEvent[] = []): Promi function getDescendants(eventId: string, signal = AbortSignal.timeout(2000)): Promise { return eventsDB.query([{ kinds: [1], '#e': [eventId] }], { limit: 200, signal }) - .then((events) => - hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal }) - ); + .then((events) => hydrateEvents({ events, storage: eventsDB, signal })); } /** Returns whether the pubkey is followed by a local user. */ diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 56518e24..8785a9a6 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -1,46 +1,75 @@ import { NostrEvent, NStore } from '@nostrify/nostrify'; -import { Conf } from '@/config.ts'; import { db } from '@/db.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { type DittoRelation } from '@/interfaces/DittoFilter.ts'; interface HydrateEventOpts { events: DittoEvent[]; - relations: DittoRelation[]; storage: NStore; signal?: AbortSignal; } -/** Hydrate event relationships using the provided storage. */ +/** Hydrate events using the provided storage. */ async function hydrateEvents(opts: HydrateEventOpts): Promise { - const { events, relations, storage, signal } = opts; + const { events, storage, signal } = opts; - if (!events.length || !relations.length) { + if (!events.length) { return events; } - for (const relation of relations) { - switch (relation) { - case 'author': - await hydrateAuthors({ events, storage, signal }); - break; - case 'author_stats': - await hydrateAuthorStats(events); - break; - case 'event_stats': - await hydrateEventStats(events); - break; - case 'user': - await hydrateUsers({ events, storage, signal }); - break; - case 'repost': - await hydrateRepostEvents({ events, storage, signal }); - break; - case 'quote_repost': - await hydrateQuoteRepostEvents({ events, storage, signal }); - break; + const allEvents: DittoEvent[] = structuredClone(events); + + const childrenEventsIds = (events.map((event) => { + if (event.kind === 1) return event.tags.find(([name]) => name === 'q')?.[1]; // possible quote repost + if (event.kind === 6) return event.tags.find(([name]) => name === 'e')?.[1]; // possible repost + return; + }).filter(Boolean)) as string[]; + + if (childrenEventsIds.length > 0) { + const childrenEvents = await storage.query([{ ids: childrenEventsIds }], { signal }); + allEvents.push(...childrenEvents); + + if (childrenEvents.length > 0) { + const grandChildrenEventsIds = (childrenEvents.map((event) => { + if (event.kind === 1) return event.tags.find(([name]) => name === 'q')?.[1]; // possible quote repost + return; + }).filter(Boolean)) as string[]; + if (grandChildrenEventsIds.length > 0) { + const grandChildrenEvents = await storage.query([{ ids: grandChildrenEventsIds }], { signal }); + allEvents.push(...grandChildrenEvents); + } } } + await hydrateAuthors({ events: allEvents, storage, signal }); + await hydrateAuthorStats(allEvents); + await hydrateEventStats(allEvents); + + events.forEach((event) => { + const correspondingEvent = allEvents.find((element) => element.id === event.id); + if (correspondingEvent?.author) event.author = correspondingEvent.author; + if (correspondingEvent?.author_stats) event.author_stats = correspondingEvent.author_stats; + if (correspondingEvent?.event_stats) event.event_stats = correspondingEvent.event_stats; + + if (event.kind === 1) { + const quoteId = event.tags.find(([name]) => name === 'q')?.[1]; + if (quoteId) { + event.quote_repost = allEvents.find((element) => element.id === quoteId); + } + } else if (event.kind === 6) { + const repostedId = event.tags.find(([name]) => name === 'e')?.[1]; + if (repostedId) { + const repostedEvent = allEvents.find((element) => element.id === repostedId); + if (repostedEvent && repostedEvent.tags.find(([name]) => name === 'q')?.[1]) { // The repost is a repost of a quote repost + const postBeingQuoteRepostedId = repostedEvent.tags.find(([name]) => name === 'q')?.[1]; + event.repost = { + quote_repost: allEvents.find((element) => element.id === postBeingQuoteRepostedId), + ...allEvents.find((element) => element.id === repostedId) as DittoEvent, + }; + } else { // The repost is a repost of a normal post + event.repost = allEvents.find((element) => element.id === repostedId); + } + } + } + }); return events; } @@ -58,23 +87,6 @@ async function hydrateAuthors(opts: Omit): Promis return events; } -async function hydrateUsers(opts: Omit): Promise { - const { events, storage, signal } = opts; - - const pubkeys = new Set([...events].map((event) => event.pubkey)); - - const users = await storage.query( - [{ kinds: [30361], authors: [Conf.pubkey], '#d': [...pubkeys], limit: pubkeys.size }], - { signal }, - ); - - for (const event of events) { - event.user = users.find((user) => user.tags.find(([name]) => name === 'd')?.[1] === event.pubkey); - } - - return events; -} - async function hydrateAuthorStats(events: DittoEvent[]): Promise { const results = await db .selectFrom('author_stats') @@ -117,84 +129,6 @@ async function hydrateEventStats(events: DittoEvent[]): Promise { return events; } -async function hydrateRepostEvents(opts: Omit): Promise { - const { events, storage, signal } = opts; - const results = await storage.query([{ - kinds: [1], - ids: events.map((event) => { - if (event.kind === 6) { - const originalPostId = event.tags.find(([name]) => name === 'e')?.[1]; - if (!originalPostId) return event.id; - else return originalPostId; - } - return event.id; - }), - }], { signal }); - - for (const event of events) { - if (event.kind === 6) { - const originalPostId = event.tags.find(([name]) => name === 'e')?.[1]; - if (!originalPostId) continue; - - const originalPostEvent = results.find((event) => event.id === originalPostId); - if (!originalPostEvent) continue; - - await hydrateEvents({ - events: [originalPostEvent], - storage: storage, - signal: signal, - relations: ['author', 'event_stats'], - }); - event.repost = originalPostEvent; - } - } - - return events; -} - -async function hydrateQuoteRepostEvents(opts: Omit): Promise { - const { events, storage, signal } = opts; - - const results = await storage.query([{ - kinds: [1], - ids: events.map((event) => { - if (event.kind === 1) { - const originalPostId = event.tags.find(([name]) => name === 'q')?.[1]; - if (!originalPostId) return event.id; - else return originalPostId; - } - return event.id; - }), - }], { signal }); - - for (const event of events) { - if (event.kind === 1) { - const originalPostId = event.tags.find(([name]) => name === 'q')?.[1]; - if (!originalPostId) continue; - - const originalPostEvent = events.find((event) => event.id === originalPostId); - if (!originalPostEvent) { - const originalPostEvent = results.find((event) => event.id === originalPostId); - if (!originalPostEvent) continue; - - await hydrateEvents({ events: [originalPostEvent], storage: storage, signal: signal, relations: ['author'] }); - - event.quote_repost = originalPostEvent; - continue; - } - if (!originalPostEvent.author) { - await hydrateEvents({ events: [originalPostEvent], storage: storage, signal: signal, relations: ['author'] }); - - event.quote_repost = originalPostEvent; - continue; - } - event.quote_repost = originalPostEvent; - } - } - - return events; -} - /** Return a normalized event without any non-standard keys. */ function purifyEvent(event: NostrEvent): NostrEvent { return { diff --git a/src/storages/search-store.ts b/src/storages/search-store.ts index 51641087..61508966 100644 --- a/src/storages/search-store.ts +++ b/src/storages/search-store.ts @@ -47,7 +47,6 @@ class SearchStore implements NStore { return hydrateEvents({ events, - relations: ['author', 'event_stats', 'author_stats'], storage: this.#hydrator, signal: opts?.signal, }); diff --git a/src/views.ts b/src/views.ts index 618bd9fc..94bb8df3 100644 --- a/src/views.ts +++ b/src/views.ts @@ -20,7 +20,7 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal } const authors = await eventsDB.query([{ kinds: [0], authors: [...pubkeys] }], { signal }) - .then((events) => hydrateEvents({ events, relations: ['author_stats'], storage: eventsDB, signal })); + .then((events) => hydrateEvents({ events, storage: eventsDB, signal })); const accounts = await Promise.all( authors.map((event) => renderAccount(event)), @@ -33,7 +33,7 @@ async function renderAccounts(c: AppContext, authors: string[], signal = AbortSi const { since, until, limit } = paginationSchema.parse(c.req.query()); const events = await eventsDB.query([{ kinds: [0], authors, since, until, limit }], { signal }) - .then((events) => hydrateEvents({ events, relations: ['author_stats'], storage: eventsDB, signal })); + .then((events) => hydrateEvents({ events, storage: eventsDB, signal })); const accounts = await Promise.all( events.map((event) => renderAccount(event)), @@ -51,9 +51,7 @@ async function renderStatuses(c: AppContext, ids: string[], signal = AbortSignal const { limit } = paginationSchema.parse(c.req.query()); const events = await eventsDB.query([{ kinds: [1], ids, limit }], { signal }) - .then((events) => - hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal }) - ); + .then((events) => hydrateEvents({ events, storage: eventsDB, signal })); if (!events.length) { return c.json([]); From 062e21e8a89d4dd7e48e2d2b80a7cd3fb52e2a76 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 22 Apr 2024 19:52:11 -0300 Subject: [PATCH 2/9] test: remove 'hydrate quote repost WITHOUT hydrate author' --- src/storages/hydrate.test.ts | 37 ------------------------------------ 1 file changed, 37 deletions(-) diff --git a/src/storages/hydrate.test.ts b/src/storages/hydrate.test.ts index 0c36717b..78863851 100644 --- a/src/storages/hydrate.test.ts +++ b/src/storages/hydrate.test.ts @@ -161,40 +161,3 @@ Deno.test('hydrate quote repost and original post with hydrate author ', async ( clearTimeout(timeoutId); }); - -Deno.test('hydrate quote repost WITHOUT hydrate author', async () => { - const db = new NCache({ max: 100 }); - - const event0madeQuoteRepostCopy = structuredClone(event0madeQuoteRepost); - const event0copy = structuredClone(event0); - const event1quoteRepostCopy = structuredClone(event1quoteRepost); - const event1willBeQuoteRepostedCopy = structuredClone(event1willBeQuoteReposted); - - // Save events to database - await db.event(event0madeQuoteRepostCopy); - await db.event(event0copy); - await db.event(event1quoteRepostCopy); - await db.event(event1willBeQuoteRepostedCopy); - - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 1000); - - await hydrateEvents({ - events: [event1quoteRepostCopy, event1willBeQuoteRepostedCopy], - relations: ['quote_repost'], - storage: db, - signal: controller.signal, - }); - - const expectedEvent1quoteRepost = { - ...event1quoteRepost, - quote_repost: { ...event1willBeQuoteRepostedCopy, author: event0copy }, - }; - - assertEquals(event1quoteRepostCopy, expectedEvent1quoteRepost); - - await db.remove([{ kinds: [0, 1] }]); - assertEquals(await db.query([{ kinds: [0, 1] }]), []); - - clearTimeout(timeoutId); -}); From 90eb6ede2b63bd9f43678a080f3afff9da17ebfd Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 22 Apr 2024 20:56:41 -0300 Subject: [PATCH 3/9] test: kind 0 (user 'me') fixture --- .../events/event-0-makes-repost-with-quote-repost.json | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 fixtures/events/event-0-makes-repost-with-quote-repost.json diff --git a/fixtures/events/event-0-makes-repost-with-quote-repost.json b/fixtures/events/event-0-makes-repost-with-quote-repost.json new file mode 100644 index 00000000..b024d5fa --- /dev/null +++ b/fixtures/events/event-0-makes-repost-with-quote-repost.json @@ -0,0 +1,9 @@ +{ + "id": "4acbf01269a2b09aaa4559b6d950ceffe37985dc3eb56c3d1bb3200ca93fae3d", + "pubkey": "2894578b2c8570825f48dc647e746e690f83f92d19446b3051f59cd0196a7991", + "created_at": 1713452168, + "kind": 0, + "tags": [], + "content": "{\"name\":\"me\",\"about\":\"\",\"nip05\":\"\"}", + "sig": "373ca965fc3772804cf448db8da3add6f59653cb1ba8ba89b8d8fc88e4ed326b446e2641ed675dcaab886eb2678cca5293c6312e03ed9e73ccebca14ef47eaaa" +} From 351d81f2c847669b976f7a9443ed288c7a25dd13 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 22 Apr 2024 20:58:35 -0300 Subject: [PATCH 4/9] test: kind 1 (quote repost) fixture --- .../event-1-quote-repost-will-be-reposted.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 fixtures/events/event-1-quote-repost-will-be-reposted.json diff --git a/fixtures/events/event-1-quote-repost-will-be-reposted.json b/fixtures/events/event-1-quote-repost-will-be-reposted.json new file mode 100644 index 00000000..11531368 --- /dev/null +++ b/fixtures/events/event-1-quote-repost-will-be-reposted.json @@ -0,0 +1,14 @@ +{ + "id": "00b325bba6de17030091558f439d41d4c08b82e60fbcaee3e0331c4c690ef205", + "pubkey": "2894578b2c8570825f48dc647e746e690f83f92d19446b3051f59cd0196a7991", + "created_at": 1713735562, + "kind": 1, + "tags": [ + [ + "q", + "f7b82508931bbeeadb901931e3dea8c4f96f0f9e353ba6cf1ec3a93eb8ad7e05" + ] + ], + "content": "Deus futurus est deus aquae deiectus!", + "sig": "72d8365f3c6b6de89fdfd005798c242629145fdc97bfc25e57bb78a4444c2a297bf41a47d7d0e2ee819d77f73fa3fcfcc4b455928ede7fca715e261c567b0b3b" +} From 51f24fed87f0d7840bd441c4c9604fb5c53ab8e3 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 22 Apr 2024 20:59:23 -0300 Subject: [PATCH 5/9] test: kind 1 (normal post) fixture --- .../event-1-will-be-reposted-with-quote-repost.json | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 fixtures/events/event-1-will-be-reposted-with-quote-repost.json diff --git a/fixtures/events/event-1-will-be-reposted-with-quote-repost.json b/fixtures/events/event-1-will-be-reposted-with-quote-repost.json new file mode 100644 index 00000000..dc661549 --- /dev/null +++ b/fixtures/events/event-1-will-be-reposted-with-quote-repost.json @@ -0,0 +1,9 @@ +{ + "id": "f7b82508931bbeeadb901931e3dea8c4f96f0f9e353ba6cf1ec3a93eb8ad7e05", + "pubkey": "2894578b2c8570825f48dc647e746e690f83f92d19446b3051f59cd0196a7991", + "created_at": 1713735505, + "kind": 1, + "tags": [], + "content": "The present is theirs, the future, for which I really worked, is mine.", + "sig": "b27fff3ec821e529e74ceede28ecf368682677de1aa2cc2cc65083b8f4a789f53e6a5da899cb0f03e4e6a3555a0fe4421971c427c5c9dd50758127c4da3e9405" +} From c7b84e5438281e6ab74f31e654e71d071e9f1ebf Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 22 Apr 2024 20:59:50 -0300 Subject: [PATCH 6/9] test: kind 6 fixture --- fixtures/events/event-6-of-quote-repost.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 fixtures/events/event-6-of-quote-repost.json diff --git a/fixtures/events/event-6-of-quote-repost.json b/fixtures/events/event-6-of-quote-repost.json new file mode 100644 index 00000000..37d03743 --- /dev/null +++ b/fixtures/events/event-6-of-quote-repost.json @@ -0,0 +1,18 @@ +{ + "id": "04ee8a34c398ef20bdb56064979aff879f81b6b746232811845eca872e0ebe8d", + "pubkey": "2894578b2c8570825f48dc647e746e690f83f92d19446b3051f59cd0196a7991", + "created_at": 1713735600, + "kind": 6, + "tags": [ + [ + "e", + "00b325bba6de17030091558f439d41d4c08b82e60fbcaee3e0331c4c690ef205" + ], + [ + "p", + "2894578b2c8570825f48dc647e746e690f83f92d19446b3051f59cd0196a7991" + ] + ], + "content": "", + "sig": "061b741a8d399db4c1151ed003a76afcf04cac25b98f2df4d4b6467ea9e0dcb54de9d5a6f959ef86b82e8c6e547a87596aecb904cf5fa99e7f8b67fefd43c0f6" +} From 7a12e5ec7bcdbc07d5644701b6deaf5527d9b72d Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 22 Apr 2024 22:00:28 -0300 Subject: [PATCH 7/9] test: rough adapt tests for new performance hydratation --- src/storages/hydrate.test.ts | 54 ++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/src/storages/hydrate.test.ts b/src/storages/hydrate.test.ts index 78863851..d9acfa1b 100644 --- a/src/storages/hydrate.test.ts +++ b/src/storages/hydrate.test.ts @@ -6,16 +6,24 @@ import event0 from '~/fixtures/events/event-0.json' with { type: 'json' }; import event0madePost from '~/fixtures/events/event-0-the-one-who-post-and-users-repost.json' with { type: 'json' }; import event0madeRepost from '~/fixtures/events/event-0-the-one-who-repost.json' with { type: 'json' }; import event0madeQuoteRepost from '~/fixtures/events/event-0-the-one-who-quote-repost.json' with { type: 'json' }; +import event0madeRepostWithQuoteRepost from '~/fixtures/events/event-0-makes-repost-with-quote-repost.json' with { + type: 'json', +}; import event1 from '~/fixtures/events/event-1.json' with { type: 'json' }; import event1quoteRepost from '~/fixtures/events/event-1-quote-repost.json' with { type: 'json' }; +import event1futureIsMine from '~/fixtures/events/event-1-will-be-reposted-with-quote-repost.json' with { + type: 'json', +}; +import event1quoteRepostLatin from '~/fixtures/events/event-1-quote-repost-will-be-reposted.json' with { type: 'json' }; import event1willBeQuoteReposted from '~/fixtures/events/event-1-that-will-be-quote-reposted.json' with { type: 'json', }; import event1reposted from '~/fixtures/events/event-1-reposted.json' with { type: 'json' }; import event6 from '~/fixtures/events/event-6.json' with { type: 'json' }; +import event6ofQuoteRepost from '~/fixtures/events/event-6-of-quote-repost.json' with { type: 'json' }; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -Deno.test('hydrate author', async () => { +Deno.test('hydrateEvents(): author --- WITHOUT stats', async () => { const db = new NCache({ max: 100 }); const event0copy = structuredClone(event0); @@ -32,7 +40,6 @@ Deno.test('hydrate author', async () => { await hydrateEvents({ events: [event1copy], - relations: ['author'], storage: db, signal: controller.signal, }); @@ -46,7 +53,7 @@ Deno.test('hydrate author', async () => { clearTimeout(timeoutId); }); -Deno.test('hydrate repost', async () => { +Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { const db = new NCache({ max: 100 }); const event0madePostCopy = structuredClone(event0madePost); @@ -68,7 +75,6 @@ Deno.test('hydrate repost', async () => { await hydrateEvents({ events: [event6copy], - relations: ['repost', 'author'], storage: db, signal: controller.signal, }); @@ -86,7 +92,7 @@ Deno.test('hydrate repost', async () => { clearTimeout(timeoutId); }); -Deno.test('hydrate quote repost with hydrate author', async () => { +Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { const db = new NCache({ max: 100 }); const event0madeQuoteRepostCopy = structuredClone(event0madeQuoteRepost); @@ -105,7 +111,6 @@ Deno.test('hydrate quote repost with hydrate author', async () => { await hydrateEvents({ events: [event1quoteRepostCopy], - relations: ['author', 'quote_repost'], // if author is called first the performance will be better storage: db, signal: controller.signal, }); @@ -124,40 +129,41 @@ Deno.test('hydrate quote repost with hydrate author', async () => { clearTimeout(timeoutId); }); -Deno.test('hydrate quote repost and original post with hydrate author ', async () => { +Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () => { const db = new NCache({ max: 100 }); - const event0madeQuoteRepostCopy = structuredClone(event0madeQuoteRepost); - const event0copy = structuredClone(event0); - const event1quoteRepostCopy = structuredClone(event1quoteRepost); - const event1willBeQuoteRepostedCopy = structuredClone(event1willBeQuoteReposted); + const event0copy = structuredClone(event0madeRepostWithQuoteRepost); + const event1copy = structuredClone(event1futureIsMine); + const event1quoteCopy = structuredClone(event1quoteRepostLatin); + const event6copy = structuredClone(event6ofQuoteRepost); // Save events to database - await db.event(event0madeQuoteRepostCopy); await db.event(event0copy); - await db.event(event1quoteRepostCopy); - await db.event(event1willBeQuoteRepostedCopy); + await db.event(event1copy); + await db.event(event1quoteCopy); + await db.event(event6copy); + + assertEquals((event6copy as DittoEvent).author, undefined, "Event hasn't been hydrated author yet"); + assertEquals((event6copy as DittoEvent).repost, undefined, "Event hasn't been hydrated repost yet"); const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), 1000); await hydrateEvents({ - events: [event1quoteRepostCopy, event1willBeQuoteRepostedCopy], - relations: ['author', 'quote_repost'], // if author is called first the performance will be better + events: [event6copy], storage: db, signal: controller.signal, }); - const expectedEvent1quoteRepost = { - ...event1quoteRepostCopy, - author: event0madeQuoteRepostCopy, - quote_repost: { ...event1willBeQuoteRepostedCopy, author: event0copy }, + const expectedEvent6 = { + ...event6copy, + author: event0copy, + repost: { ...event1quoteCopy, author: event0copy, quote_repost: { author: event0copy, ...event1copy } }, }; + assertEquals(event6copy, expectedEvent6); - assertEquals(event1quoteRepostCopy, expectedEvent1quoteRepost); - - await db.remove([{ kinds: [0, 1] }]); - assertEquals(await db.query([{ kinds: [0, 1] }]), []); + await db.remove([{ kinds: [0, 1, 6] }]); + assertEquals(await db.query([{ kinds: [0, 1, 6] }]), []); clearTimeout(timeoutId); }); From 5fca482e5c61a78cb194fb2171725129585b8384 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 23 Apr 2024 15:56:35 -0300 Subject: [PATCH 8/9] refactor(hydrate events): change array to Map --- src/storages/hydrate.ts | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 8785a9a6..63d7a11b 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -16,7 +16,9 @@ async function hydrateEvents(opts: HydrateEventOpts): Promise { return events; } - const allEvents: DittoEvent[] = structuredClone(events); + const allEventsMap: Map = new Map(events.map((event) => { + return [event.id, structuredClone(event)]; + })); const childrenEventsIds = (events.map((event) => { if (event.kind === 1) return event.tags.find(([name]) => name === 'q')?.[1]; // possible quote repost @@ -26,7 +28,9 @@ async function hydrateEvents(opts: HydrateEventOpts): Promise { if (childrenEventsIds.length > 0) { const childrenEvents = await storage.query([{ ids: childrenEventsIds }], { signal }); - allEvents.push(...childrenEvents); + childrenEvents.forEach((event) => { + allEventsMap.set(event.id, structuredClone(event)); + }); if (childrenEvents.length > 0) { const grandChildrenEventsIds = (childrenEvents.map((event) => { @@ -35,16 +39,18 @@ async function hydrateEvents(opts: HydrateEventOpts): Promise { }).filter(Boolean)) as string[]; if (grandChildrenEventsIds.length > 0) { const grandChildrenEvents = await storage.query([{ ids: grandChildrenEventsIds }], { signal }); - allEvents.push(...grandChildrenEvents); + grandChildrenEvents.forEach((event) => { + allEventsMap.set(event.id, structuredClone(event)); + }); } } } - await hydrateAuthors({ events: allEvents, storage, signal }); - await hydrateAuthorStats(allEvents); - await hydrateEventStats(allEvents); + await hydrateAuthors({ events: [...allEventsMap.values()], storage, signal }); + await hydrateAuthorStats([...allEventsMap.values()]); + await hydrateEventStats([...allEventsMap.values()]); events.forEach((event) => { - const correspondingEvent = allEvents.find((element) => element.id === event.id); + const correspondingEvent = allEventsMap.get(event.id); if (correspondingEvent?.author) event.author = correspondingEvent.author; if (correspondingEvent?.author_stats) event.author_stats = correspondingEvent.author_stats; if (correspondingEvent?.event_stats) event.event_stats = correspondingEvent.event_stats; @@ -52,25 +58,24 @@ async function hydrateEvents(opts: HydrateEventOpts): Promise { if (event.kind === 1) { const quoteId = event.tags.find(([name]) => name === 'q')?.[1]; if (quoteId) { - event.quote_repost = allEvents.find((element) => element.id === quoteId); + event.quote_repost = allEventsMap.get(quoteId); } } else if (event.kind === 6) { const repostedId = event.tags.find(([name]) => name === 'e')?.[1]; if (repostedId) { - const repostedEvent = allEvents.find((element) => element.id === repostedId); + const repostedEvent = allEventsMap.get(repostedId); if (repostedEvent && repostedEvent.tags.find(([name]) => name === 'q')?.[1]) { // The repost is a repost of a quote repost const postBeingQuoteRepostedId = repostedEvent.tags.find(([name]) => name === 'q')?.[1]; event.repost = { - quote_repost: allEvents.find((element) => element.id === postBeingQuoteRepostedId), - ...allEvents.find((element) => element.id === repostedId) as DittoEvent, + quote_repost: allEventsMap.get(postBeingQuoteRepostedId!), + ...allEventsMap.get(repostedId)!, }; } else { // The repost is a repost of a normal post - event.repost = allEvents.find((element) => element.id === repostedId); + event.repost = allEventsMap.get(repostedId); } } } }); - return events; } From 58d75d19397a7fe833ae9437e0f62e5e0781a816 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 23 Apr 2024 17:16:15 -0300 Subject: [PATCH 9/9] fix: hydrate events stats and author stats with filter by kind --- src/storages/hydrate.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 63d7a11b..f65641fd 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -46,8 +46,8 @@ async function hydrateEvents(opts: HydrateEventOpts): Promise { } } await hydrateAuthors({ events: [...allEventsMap.values()], storage, signal }); - await hydrateAuthorStats([...allEventsMap.values()]); - await hydrateEventStats([...allEventsMap.values()]); + await hydrateAuthorStats([...allEventsMap.values()].filter((e) => e.kind === 0)); + await hydrateEventStats([...allEventsMap.values()].filter((e) => e.kind === 1)); events.forEach((event) => { const correspondingEvent = allEventsMap.get(event.id);