From 017c17c8a273842a2a92b908f5ece2bd702cee34 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 19 Sep 2024 19:35:05 -0300 Subject: [PATCH 1/7] refactor: remove author_search table, put search in author_stats --- .../034_move_author_search_to_author_stats.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 src/db/migrations/034_move_author_search_to_author_stats.ts diff --git a/src/db/migrations/034_move_author_search_to_author_stats.ts b/src/db/migrations/034_move_author_search_to_author_stats.ts new file mode 100644 index 00000000..6d21ca39 --- /dev/null +++ b/src/db/migrations/034_move_author_search_to_author_stats.ts @@ -0,0 +1,32 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .alterTable('author_stats') + .addColumn('search', 'text', (col) => col.notNull().defaultTo('')) + .execute(); + + await sql`CREATE INDEX author_stats_search_idx ON author_stats USING GIN (search gin_trgm_ops)`.execute(db); + + await db.insertInto('author_stats') + .columns(['pubkey', 'search']) + .expression( + db.selectFrom('author_search') + .select(['pubkey', 'search']), + ) + .onConflict((oc) => + oc.column('pubkey') + .doUpdateSet((eb) => ({ + search: eb.ref('excluded.search'), + })) + ) + .execute(); + + await db.schema.dropIndex('author_search_search_idx').ifExists().execute(); + await db.schema.dropTable('author_search').execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropIndex('author_stats_search_idx').ifExists().execute(); + await db.schema.alterTable('author_stats').dropColumn('search').execute(); +} From f063da1b86204702eb33721a8849932f60203ce1 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 19 Sep 2024 19:35:38 -0300 Subject: [PATCH 2/7] refactor: add search field to AuthorStatsRow, remove AuthorSearch interface --- src/db/DittoTables.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index 642db484..c05ffe66 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -9,7 +9,6 @@ export interface DittoTables extends NPostgresSchema { event_stats: EventStatsRow; pubkey_domains: PubkeyDomainRow; event_zaps: EventZapRow; - author_search: AuthorSearch; } type NostrEventsRow = NPostgresSchema['nostr_events'] & { @@ -21,6 +20,7 @@ interface AuthorStatsRow { followers_count: number; following_count: number; notes_count: number; + search: string; } interface EventStatsRow { @@ -55,8 +55,3 @@ interface EventZapRow { amount_millisats: number; comment: string; } - -interface AuthorSearch { - pubkey: string; - search: string; -} From 2727523540c3df6a6abc0a1911b43899de2fe1a5 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 19 Sep 2024 19:36:13 -0300 Subject: [PATCH 3/7] feat: order search by followers count also --- src/utils/search.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/search.ts b/src/utils/search.ts index 5ca43417..e17be135 100644 --- a/src/utils/search.ts +++ b/src/utils/search.ts @@ -10,13 +10,14 @@ export async function getPubkeysBySearch( const { q, limit, followedPubkeys } = opts; let query = kysely - .selectFrom('author_search') + .selectFrom('author_stats') .select((eb) => [ 'pubkey', 'search', eb.fn('word_similarity', [sql`${q}`, 'search']).as('sml'), ]) .where(() => sql`${q} <% search`) + .orderBy(['followers_count desc']) .orderBy(['sml desc', 'search']) .limit(limit); From 1b6e9160ec7db3df9a01e30a0bc4e7d2ca8ec0da Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 19 Sep 2024 19:36:44 -0300 Subject: [PATCH 4/7] test: update to use author_stats table --- src/utils/search.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/utils/search.test.ts b/src/utils/search.test.ts index 7dc37750..1acd2f60 100644 --- a/src/utils/search.test.ts +++ b/src/utils/search.test.ts @@ -6,9 +6,12 @@ import { getPubkeysBySearch } from '@/utils/search.ts'; Deno.test('fuzzy search works', async () => { await using db = await createTestDB(); - await db.kysely.insertInto('author_search').values({ + await db.kysely.insertInto('author_stats').values({ pubkey: '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', search: 'patrickReiis patrickdosreis.com', + notes_count: 0, + followers_count: 0, + following_count: 0, }).execute(); assertEquals(await getPubkeysBySearch(db.kysely, { q: 'pat rick', limit: 1, followedPubkeys: new Set() }), new Set()); From ebeb150463d5ff211fd7c9e68c1f440ff327843a Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 19 Sep 2024 19:37:13 -0300 Subject: [PATCH 5/7] refactor: use search in author_stats --- src/pipeline.ts | 4 ++-- src/storages/hydrate.ts | 1 + src/utils/stats.test.ts | 11 ++++++++++- src/utils/stats.ts | 13 ++++++++++--- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index aaa6ca07..d56653c4 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -157,8 +157,8 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise oc.column('pubkey').doUpdateSet({ search })) .execute(); } diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 7b11cfb8..5948018b 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -303,6 +303,7 @@ async function gatherAuthorStats( followers_count: Math.max(0, row.followers_count), following_count: Math.max(0, row.following_count), notes_count: Math.max(0, row.notes_count), + search: row.search, })); } diff --git a/src/utils/stats.test.ts b/src/utils/stats.test.ts index 69633ae3..797f78da 100644 --- a/src/utils/stats.test.ts +++ b/src/utils/stats.test.ts @@ -171,7 +171,16 @@ Deno.test('countAuthorStats counts author stats from the database', async () => await db.store.event(genEvent({ kind: 1, content: 'yolo' }, sk)); await db.store.event(genEvent({ kind: 3, tags: [['p', pubkey]] })); - const stats = await countAuthorStats(db.store, pubkey); + await db.kysely.insertInto('author_stats').values({ + pubkey, + search: 'Yolo Lolo', + notes_count: 0, + followers_count: 0, + following_count: 0, + }).onConflict((oc) => oc.column('pubkey').doUpdateSet({ 'search': 'baka' })) + .execute(); + + const stats = await countAuthorStats({ store: db.store, pubkey, kysely: db.kysely }); assertEquals(stats!.notes_count, 2); assertEquals(stats!.followers_count, 1); diff --git a/src/utils/stats.ts b/src/utils/stats.ts index e4d4d3f2..4946be3a 100644 --- a/src/utils/stats.ts +++ b/src/utils/stats.ts @@ -194,6 +194,7 @@ export async function updateAuthorStats( followers_count: 0, following_count: 0, notes_count: 0, + search: '', }; const prev = await kysely @@ -268,8 +269,7 @@ export async function updateEventStats( /** Calculate author stats from the database. */ export async function countAuthorStats( - store: SetRequired, - pubkey: string, + { pubkey, kysely, store }: RefreshAuthorStatsOpts, ): Promise { const [{ count: followers_count }, { count: notes_count }, [followList]] = await Promise.all([ store.count([{ kinds: [3], '#p': [pubkey] }]), @@ -277,11 +277,18 @@ export async function countAuthorStats( store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]), ]); + const [{ search }] = await kysely + .selectFrom('author_stats') + .select('search') + .where('pubkey', '=', [pubkey]) + .execute(); + return { pubkey, followers_count, following_count: getTagSet(followList?.tags ?? [], 'p').size, notes_count, + search, }; } @@ -295,7 +302,7 @@ export interface RefreshAuthorStatsOpts { export async function refreshAuthorStats( { pubkey, kysely, store }: RefreshAuthorStatsOpts, ): Promise { - const stats = await countAuthorStats(store, pubkey); + const stats = await countAuthorStats({ store, pubkey, kysely }); await kysely.insertInto('author_stats') .values(stats) From 578f269a65df62e3a744c70506f0613b7abfffcf Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 20 Sep 2024 09:53:30 -0300 Subject: [PATCH 6/7] refactor: countAuthorStats() function does not return search --- src/utils/stats.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/utils/stats.ts b/src/utils/stats.ts index 4946be3a..a44f8882 100644 --- a/src/utils/stats.ts +++ b/src/utils/stats.ts @@ -270,25 +270,18 @@ export async function updateEventStats( /** Calculate author stats from the database. */ export async function countAuthorStats( { pubkey, kysely, store }: RefreshAuthorStatsOpts, -): Promise { +): Promise> { const [{ count: followers_count }, { count: notes_count }, [followList]] = await Promise.all([ store.count([{ kinds: [3], '#p': [pubkey] }]), store.count([{ kinds: [1], authors: [pubkey] }]), store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]), ]); - const [{ search }] = await kysely - .selectFrom('author_stats') - .select('search') - .where('pubkey', '=', [pubkey]) - .execute(); - return { pubkey, followers_count, following_count: getTagSet(followList?.tags ?? [], 'p').size, notes_count, - search, }; } From b13e92400105881705ed66147166721f8731fd5c Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 20 Sep 2024 10:24:41 -0300 Subject: [PATCH 7/7] fix: build search and also return it in countAuthorStats() function --- src/utils/stats.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/utils/stats.ts b/src/utils/stats.ts index a44f8882..4573bb60 100644 --- a/src/utils/stats.ts +++ b/src/utils/stats.ts @@ -269,19 +269,27 @@ export async function updateEventStats( /** Calculate author stats from the database. */ export async function countAuthorStats( - { pubkey, kysely, store }: RefreshAuthorStatsOpts, -): Promise> { - const [{ count: followers_count }, { count: notes_count }, [followList]] = await Promise.all([ + { pubkey, store }: RefreshAuthorStatsOpts, +): Promise { + const [{ count: followers_count }, { count: notes_count }, [followList], [kind0]] = await Promise.all([ store.count([{ kinds: [3], '#p': [pubkey] }]), store.count([{ kinds: [1], authors: [pubkey] }]), store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]), + store.query([{ kinds: [0], authors: [pubkey], limit: 1 }]), ]); + let search: string = ''; + const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(kind0?.content); + if (metadata.success) { + const { name, nip05 } = metadata.data; + search = [name, nip05].filter(Boolean).join(' ').trim(); + } return { pubkey, followers_count, following_count: getTagSet(followList?.tags ?? [], 'p').size, notes_count, + search, }; }