diff --git a/scripts/db.ts b/scripts/db.ts index cdc5174e..cbaca91f 100644 --- a/scripts/db.ts +++ b/scripts/db.ts @@ -1,8 +1,8 @@ import { Conf } from '@/config.ts'; import { db } from '@/db.ts'; -import { eventsDB } from '@/db/events.ts'; import { type Kysely } from '@/deps.ts'; import { signAdminEvent } from '@/sign.ts'; +import { eventsDB } from '@/storages.ts'; interface DB { users: { diff --git a/src/client.ts b/src/client.ts index 511cbc75..34d45441 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1,18 +1,23 @@ import { Debug, type Event, type Filter, matchFilters } from '@/deps.ts'; +import { normalizeFilters } from '@/filter.ts'; import * as pipeline from '@/pipeline.ts'; import { activeRelays, pool } from '@/pool.ts'; -import { type EventStore, type GetEventsOpts, type StoreEventOpts } from '@/store.ts'; +import { type EventStore, type GetEventsOpts, type StoreEventOpts } from '@/storages/types.ts'; +import { EventSet } from '@/utils/event-set.ts'; const debug = Debug('ditto:client'); /** Get events from a NIP-01 filter. */ function getEvents(filters: Filter[], opts: GetEventsOpts = {}): Promise[]> { + filters = normalizeFilters(filters); + if (opts.signal?.aborted) return Promise.resolve([]); if (!filters.length) return Promise.resolve([]); + debug('REQ', JSON.stringify(filters)); return new Promise((resolve) => { - const results: Event[] = []; + const results = new EventSet>(); const unsub = pool.subscribe( filters, @@ -20,9 +25,9 @@ function getEvents(filters: Filter[], opts: GetEventsOpts = (event: Event | null) => { if (event && matchFilters(filters, event)) { pipeline.handleEvent(event).catch(() => {}); - results.push({ + results.add({ id: event.id, - kind: event.kind, + kind: event.kind as K, pubkey: event.pubkey, content: event.content, tags: event.tags, @@ -30,21 +35,21 @@ function getEvents(filters: Filter[], opts: GetEventsOpts = sig: event.sig, }); } - if (typeof opts.limit === 'number' && results.length >= opts.limit) { + if (typeof opts.limit === 'number' && results.size >= opts.limit) { unsub(); - resolve(results as Event[]); + resolve([...results]); } }, undefined, () => { unsub(); - resolve(results as Event[]); + resolve([...results]); }, ); opts.signal?.addEventListener('abort', () => { unsub(); - resolve(results as Event[]); + resolve([...results]); }); }); } @@ -59,6 +64,7 @@ function storeEvent(event: Event, opts: StoreEventOpts = {}): Promise { } const client: EventStore = { + supportedNips: [1], getEvents, storeEvent, countEvents: () => Promise.reject(new Error('COUNT not implemented')), diff --git a/src/config.ts b/src/config.ts index 246bf5a0..8c2626a4 100644 --- a/src/config.ts +++ b/src/config.ts @@ -42,6 +42,10 @@ const Conf = { const { protocol, host } = Conf.url; return `${protocol === 'https:' ? 'wss:' : 'ws:'}//${host}/relay`; }, + /** Relay to use for NIP-50 `search` queries. */ + get searchRelay() { + return Deno.env.get('SEARCH_RELAY'); + }, /** Origin of the Ditto server, including the protocol and port. */ get localDomain() { return Deno.env.get('LOCAL_DOMAIN') || 'http://localhost:8000'; diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 8316ede4..34518cda 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -1,12 +1,12 @@ import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { eventsDB } from '@/db/events.ts'; import { insertUser } from '@/db/users.ts'; import { findReplyTag, nip19, z } from '@/deps.ts'; import { type DittoFilter } from '@/filter.ts'; import { getAuthor, getFollowedPubkeys } from '@/queries.ts'; import { booleanParamSchema, fileSchema } from '@/schema.ts'; import { jsonMetaContentSchema } from '@/schemas/nostr.ts'; +import { eventsDB } from '@/storages.ts'; import { addTag, deleteTag, getTagSet } from '@/tags.ts'; import { uploadFile } from '@/upload.ts'; import { lookupAccount, nostrNow } from '@/utils.ts'; diff --git a/src/controllers/api/blocks.ts b/src/controllers/api/blocks.ts index 2ff4f3e4..b45e1c6d 100644 --- a/src/controllers/api/blocks.ts +++ b/src/controllers/api/blocks.ts @@ -1,5 +1,5 @@ import { type AppController } from '@/app.ts'; -import { eventsDB } from '@/db/events.ts'; +import { eventsDB } from '@/storages.ts'; import { getTagSet } from '@/tags.ts'; import { renderAccounts } from '@/views.ts'; diff --git a/src/controllers/api/bookmarks.ts b/src/controllers/api/bookmarks.ts index 78edff0e..81cd011d 100644 --- a/src/controllers/api/bookmarks.ts +++ b/src/controllers/api/bookmarks.ts @@ -1,5 +1,5 @@ import { type AppController } from '@/app.ts'; -import { eventsDB } from '@/db/events.ts'; +import { eventsDB } from '@/storages.ts'; import { getTagSet } from '@/tags.ts'; import { renderStatuses } from '@/views.ts'; diff --git a/src/controllers/api/notifications.ts b/src/controllers/api/notifications.ts index f50f8910..43c974ac 100644 --- a/src/controllers/api/notifications.ts +++ b/src/controllers/api/notifications.ts @@ -1,5 +1,5 @@ import { type AppController } from '@/app.ts'; -import { eventsDB } from '@/db/events.ts'; +import { eventsDB } from '@/storages.ts'; import { paginated, paginationSchema } from '@/utils/api.ts'; import { renderNotification } from '@/views/mastodon/notifications.ts'; diff --git a/src/controllers/api/pleroma.ts b/src/controllers/api/pleroma.ts index 3b842273..04370e43 100644 --- a/src/controllers/api/pleroma.ts +++ b/src/controllers/api/pleroma.ts @@ -1,7 +1,7 @@ import { type AppController } from '@/app.ts'; -import { eventsDB } from '@/db/events.ts'; import { z } from '@/deps.ts'; import { configSchema, elixirTupleSchema } from '@/schemas/pleroma-api.ts'; +import { eventsDB } from '@/storages.ts'; import { createAdminEvent } from '@/utils/api.ts'; import { Conf } from '@/config.ts'; diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index a514d666..d18694ca 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -1,9 +1,9 @@ import { AppController } from '@/app.ts'; -import { eventsDB } from '@/db/events.ts'; import { type Event, nip19, z } from '@/deps.ts'; import { type DittoFilter } from '@/filter.ts'; import { booleanParamSchema } from '@/schema.ts'; import { nostrIdSchema } from '@/schemas/nostr.ts'; +import { searchStore } from '@/storages.ts'; import { dedupeEvents } from '@/utils.ts'; import { lookupNip05Cached } from '@/utils/nip05.ts'; import { renderAccount } from '@/views/mastodon/accounts.ts'; @@ -30,9 +30,11 @@ const searchController: AppController = async (c) => { return c.json({ error: 'Bad request', schema: result.error }, 422); } + const signal = AbortSignal.timeout(1000); + const [event, events] = await Promise.all([ - lookupEvent(result.data), - searchEvents(result.data), + lookupEvent(result.data, signal), + searchEvents(result.data, signal), ]); if (event) { @@ -62,7 +64,7 @@ const searchController: AppController = async (c) => { }; /** Get events for the search params. */ -function searchEvents({ q, type, limit, account_id }: SearchQuery): Promise { +function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: AbortSignal): Promise { if (type === 'hashtags') return Promise.resolve([]); const filter: DittoFilter = { @@ -76,7 +78,7 @@ function searchEvents({ q, type, limit, account_id }: SearchQuery): Promise { +async function lookupEvent(query: SearchQuery, signal: AbortSignal): Promise { const filters = await getLookupFilters(query); - const [event] = await eventsDB.getEvents(filters, { limit: 1, signal }); + const [event] = await searchStore.getEvents(filters, { limit: 1, signal }); return event; } diff --git a/src/controllers/api/timelines.ts b/src/controllers/api/timelines.ts index 6d932801..47496a00 100644 --- a/src/controllers/api/timelines.ts +++ b/src/controllers/api/timelines.ts @@ -1,4 +1,4 @@ -import { eventsDB } from '@/db/events.ts'; +import { eventsDB } from '@/storages.ts'; import { z } from '@/deps.ts'; import { type DittoFilter } from '@/filter.ts'; import { getFeedPubkeys } from '@/queries.ts'; diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index a1f4de0e..f1c35111 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -1,5 +1,5 @@ import { relayInfoController } from '@/controllers/nostr/relay-info.ts'; -import { eventsDB } from '@/db/events.ts'; +import { eventsDB } from '@/storages.ts'; import * as pipeline from '@/pipeline.ts'; import { jsonSchema } from '@/schema.ts'; import { diff --git a/src/db/events.test.ts b/src/db/events.test.ts deleted file mode 100644 index 7a5537e8..00000000 --- a/src/db/events.test.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { assertEquals, assertRejects } from '@/deps-test.ts'; -import { buildUserEvent } from '@/db/users.ts'; - -import event0 from '~/fixtures/events/event-0.json' assert { type: 'json' }; -import event1 from '~/fixtures/events/event-1.json' assert { type: 'json' }; - -import { eventsDB as db } from './events.ts'; - -Deno.test('count filters', async () => { - assertEquals(await db.countEvents([{ kinds: [1] }]), 0); - await db.storeEvent(event1); - assertEquals(await db.countEvents([{ kinds: [1] }]), 1); -}); - -Deno.test('insert and filter events', async () => { - await db.storeEvent(event1); - - assertEquals(await db.getEvents([{ kinds: [1] }]), [event1]); - assertEquals(await db.getEvents([{ kinds: [3] }]), []); - assertEquals(await db.getEvents([{ since: 1691091000 }]), [event1]); - assertEquals(await db.getEvents([{ until: 1691091000 }]), []); - assertEquals( - await db.getEvents([{ '#proxy': ['https://gleasonator.com/objects/8f6fac53-4f66-4c6e-ac7d-92e5e78c3e79'] }]), - [event1], - ); -}); - -Deno.test('delete events', async () => { - await db.storeEvent(event1); - assertEquals(await db.getEvents([{ kinds: [1] }]), [event1]); - await db.deleteEvents([{ kinds: [1] }]); - assertEquals(await db.getEvents([{ kinds: [1] }]), []); -}); - -Deno.test('query events with local filter', async () => { - await db.storeEvent(event1); - - assertEquals(await db.getEvents([{}]), [event1]); - assertEquals(await db.getEvents([{ local: true }]), []); - assertEquals(await db.getEvents([{ local: false }]), [event1]); - - const userEvent = await buildUserEvent({ - username: 'alex', - pubkey: event1.pubkey, - inserted_at: new Date(), - admin: false, - }); - await db.storeEvent(userEvent); - - assertEquals(await db.getEvents([{ kinds: [1], local: true }]), [event1]); - assertEquals(await db.getEvents([{ kinds: [1], local: false }]), []); -}); - -Deno.test('inserting replaceable events', async () => { - assertEquals(await db.countEvents([{ kinds: [0], authors: [event0.pubkey] }]), 0); - - await db.storeEvent(event0); - await assertRejects(() => db.storeEvent(event0)); - assertEquals(await db.countEvents([{ kinds: [0], authors: [event0.pubkey] }]), 1); - - const changeEvent = { ...event0, id: '123', created_at: event0.created_at + 1 }; - await db.storeEvent(changeEvent); - assertEquals(await db.getEvents([{ kinds: [0] }]), [changeEvent]); -}); diff --git a/src/db/events.ts b/src/db/events.ts deleted file mode 100644 index 7764a8da..00000000 --- a/src/db/events.ts +++ /dev/null @@ -1,410 +0,0 @@ -import { Conf } from '@/config.ts'; -import { db, type DittoDB } from '@/db.ts'; -import { Debug, type Event, Kysely, type SelectQueryBuilder } from '@/deps.ts'; -import { type DittoFilter } from '@/filter.ts'; -import { isDittoInternalKind, isParameterizedReplaceableKind, isReplaceableKind } from '@/kinds.ts'; -import { jsonMetaContentSchema } from '@/schemas/nostr.ts'; -import { type DittoEvent, EventStore, type GetEventsOpts, type StoreEventOpts } from '@/store.ts'; -import { isNostrId, isURL } from '@/utils.ts'; - -const debug = Debug('ditto:db:events'); - -/** Function to decide whether or not to index a tag. */ -type TagCondition = ({ event, count, value }: { - event: Event; - opts: StoreEventOpts; - count: number; - value: string; -}) => boolean; - -/** Conditions for when to index certain tags. */ -const tagConditions: Record = { - 'd': ({ event, count }) => count === 0 && isParameterizedReplaceableKind(event.kind), - 'e': ({ event, count, value, opts }) => ((opts.data?.user && event.kind === 10003) || count < 15) && isNostrId(value), - 'media': ({ count, value, opts }) => (opts.data?.user || count < 4) && isURL(value), - 'p': ({ event, count, value }) => (count < 15 || event.kind === 3) && isNostrId(value), - 'proxy': ({ count, value }) => count === 0 && isURL(value), - 'q': ({ event, count, value }) => count === 0 && event.kind === 1 && isNostrId(value), - 't': ({ count, value }) => count < 5 && value.length < 50, - 'name': ({ event, count }) => event.kind === 30361 && count === 0, - 'role': ({ event, count }) => event.kind === 30361 && count === 0, -}; - -/** Insert an event (and its tags) into the database. */ -async function storeEvent(event: Event, opts: StoreEventOpts = {}): Promise { - debug('EVENT', JSON.stringify(event)); - - if (isDittoInternalKind(event.kind) && event.pubkey !== Conf.pubkey) { - throw new Error('Internal events can only be stored by the server keypair'); - } - - return await db.transaction().execute(async (trx) => { - /** Insert the event into the database. */ - async function addEvent() { - await trx.insertInto('events') - .values({ ...event, tags: JSON.stringify(event.tags) }) - .execute(); - } - - /** Add search data to the FTS table. */ - async function indexSearch() { - const searchContent = buildSearchContent(event); - if (!searchContent) return; - await trx.insertInto('events_fts') - .values({ id: event.id, content: searchContent.substring(0, 1000) }) - .execute(); - } - - /** Index event tags depending on the conditions defined above. */ - async function indexTags() { - const tags = filterIndexableTags(event, opts); - const rows = tags.map(([tag, value]) => ({ event_id: event.id, tag, value })); - - if (!tags.length) return; - await trx.insertInto('tags') - .values(rows) - .execute(); - } - - if (isReplaceableKind(event.kind)) { - const prevEvents = await getFilterQuery(trx, { kinds: [event.kind], authors: [event.pubkey] }).execute(); - for (const prevEvent of prevEvents) { - if (prevEvent.created_at >= event.created_at) { - throw new Error('Cannot replace an event with an older event'); - } - } - await deleteEventsTrx(trx, [{ kinds: [event.kind], authors: [event.pubkey] }]); - } - - if (isParameterizedReplaceableKind(event.kind)) { - const d = event.tags.find(([tag]) => tag === 'd')?.[1]; - if (d) { - const prevEvents = await getFilterQuery(trx, { kinds: [event.kind], authors: [event.pubkey], '#d': [d] }) - .execute(); - for (const prevEvent of prevEvents) { - if (prevEvent.created_at >= event.created_at) { - throw new Error('Cannot replace an event with an older event'); - } - } - await deleteEventsTrx(trx, [{ kinds: [event.kind], authors: [event.pubkey], '#d': [d] }]); - } - } - - // Run the queries. - await Promise.all([ - addEvent(), - indexTags(), - indexSearch(), - ]); - }).catch((error) => { - // Don't throw for duplicate events. - if (error.message.includes('UNIQUE constraint failed')) { - return; - } else { - throw error; - } - }); -} - -type EventQuery = SelectQueryBuilder; - -/** Build the query for a filter. */ -function getFilterQuery(db: Kysely, filter: DittoFilter): EventQuery { - let query = db - .selectFrom('events') - .select([ - 'events.id', - 'events.kind', - 'events.pubkey', - 'events.content', - 'events.tags', - 'events.created_at', - 'events.sig', - ]) - .orderBy('events.created_at', 'desc'); - - for (const [key, value] of Object.entries(filter)) { - if (value === undefined) continue; - - switch (key as keyof DittoFilter) { - case 'ids': - query = query.where('events.id', 'in', filter.ids!); - break; - case 'kinds': - query = query.where('events.kind', 'in', filter.kinds!); - break; - case 'authors': - query = query.where('events.pubkey', 'in', filter.authors!); - break; - case 'since': - query = query.where('events.created_at', '>=', filter.since!); - break; - case 'until': - query = query.where('events.created_at', '<=', filter.until!); - break; - case 'limit': - query = query.limit(filter.limit!); - break; - } - - if (key.startsWith('#')) { - const tag = key.replace(/^#/, ''); - const value = filter[key as `#${string}`] as string[]; - query = query - .leftJoin('tags', 'tags.event_id', 'events.id') - .where('tags.tag', '=', tag) - .where('tags.value', 'in', value); - } - } - - if (typeof filter.local === 'boolean') { - query = query - .leftJoin(usersQuery, (join) => join.onRef('users.d_tag', '=', 'events.pubkey')) - .where('users.d_tag', filter.local ? 'is not' : 'is', null); - } - - if (filter.relations?.includes('author')) { - query = query - .leftJoin( - (eb) => - eb - .selectFrom('events') - .selectAll() - .where('kind', '=', 0) - .groupBy('pubkey') - .as('authors'), - (join) => join.onRef('authors.pubkey', '=', 'events.pubkey'), - ) - .select([ - 'authors.id as author_id', - 'authors.kind as author_kind', - 'authors.pubkey as author_pubkey', - 'authors.content as author_content', - 'authors.tags as author_tags', - 'authors.created_at as author_created_at', - 'authors.sig as author_sig', - ]); - } - - if (filter.relations?.includes('author_stats')) { - query = query - .leftJoin('author_stats', 'author_stats.pubkey', 'events.pubkey') - .select((eb) => [ - eb.fn.coalesce('author_stats.followers_count', eb.val(0)).as('author_stats_followers_count'), - eb.fn.coalesce('author_stats.following_count', eb.val(0)).as('author_stats_following_count'), - eb.fn.coalesce('author_stats.notes_count', eb.val(0)).as('author_stats_notes_count'), - ]); - } - - if (filter.relations?.includes('event_stats')) { - query = query - .leftJoin('event_stats', 'event_stats.event_id', 'events.id') - .select((eb) => [ - eb.fn.coalesce('event_stats.replies_count', eb.val(0)).as('stats_replies_count'), - eb.fn.coalesce('event_stats.reposts_count', eb.val(0)).as('stats_reposts_count'), - eb.fn.coalesce('event_stats.reactions_count', eb.val(0)).as('stats_reactions_count'), - ]); - } - - if (filter.search) { - query = query - .innerJoin('events_fts', 'events_fts.id', 'events.id') - .where('events_fts.content', 'match', JSON.stringify(filter.search)); - } - - return query; -} - -/** Combine filter queries into a single union query. */ -function getEventsQuery(filters: DittoFilter[]) { - return filters - .map((filter) => db.selectFrom(() => getFilterQuery(db, filter).as('events')).selectAll()) - .reduce((result, query) => result.unionAll(query)); -} - -/** Query to get user events, joined by tags. */ -function usersQuery() { - return getFilterQuery(db, { kinds: [30361], authors: [Conf.pubkey] }) - .leftJoin('tags', 'tags.event_id', 'events.id') - .where('tags.tag', '=', 'd') - .select('tags.value as d_tag') - .as('users'); -} - -/** Get events for filters from the database. */ -async function getEvents( - filters: DittoFilter[], - opts: GetEventsOpts = {}, -): Promise[]> { - if (opts.signal?.aborted) return Promise.resolve([]); - if (!filters.length) return Promise.resolve([]); - debug('REQ', JSON.stringify(filters)); - let query = getEventsQuery(filters); - - if (typeof opts.limit === 'number') { - query = query.limit(opts.limit); - } - - return (await query.execute()).map((row) => { - const event: DittoEvent = { - id: row.id, - kind: row.kind as K, - pubkey: row.pubkey, - content: row.content, - created_at: row.created_at, - tags: JSON.parse(row.tags), - sig: row.sig, - }; - - if (row.author_id) { - event.author = { - id: row.author_id, - kind: row.author_kind! as 0, - pubkey: row.author_pubkey!, - content: row.author_content!, - created_at: row.author_created_at!, - tags: JSON.parse(row.author_tags!), - sig: row.author_sig!, - }; - } - - if (typeof row.author_stats_followers_count === 'number') { - event.author_stats = { - followers_count: row.author_stats_followers_count, - following_count: row.author_stats_following_count!, - notes_count: row.author_stats_notes_count!, - }; - } - - if (typeof row.stats_replies_count === 'number') { - event.event_stats = { - replies_count: row.stats_replies_count, - reposts_count: row.stats_reposts_count!, - reactions_count: row.stats_reactions_count!, - }; - } - - return event; - }); -} - -/** Delete events from each table. Should be run in a transaction! */ -async function deleteEventsTrx(db: Kysely, filters: DittoFilter[]) { - if (!filters.length) return Promise.resolve(); - debug('DELETE', JSON.stringify(filters)); - - const query = getEventsQuery(filters).clearSelect().select('id'); - - await db.deleteFrom('events_fts') - .where('id', 'in', () => query) - .execute(); - - return db.deleteFrom('events') - .where('id', 'in', () => query) - .execute(); -} - -/** Delete events based on filters from the database. */ -async function deleteEvents(filters: DittoFilter[]): Promise { - if (!filters.length) return Promise.resolve(); - debug('DELETE', JSON.stringify(filters)); - - await db.transaction().execute((trx) => deleteEventsTrx(trx, filters)); -} - -/** Get number of events that would be returned by filters. */ -async function countEvents(filters: DittoFilter[]): Promise { - if (!filters.length) return Promise.resolve(0); - debug('COUNT', JSON.stringify(filters)); - const query = getEventsQuery(filters); - - const [{ count }] = await query - .clearSelect() - .select((eb) => eb.fn.count('id').as('count')) - .execute(); - - return Number(count); -} - -/** Return only the tags that should be indexed. */ -function filterIndexableTags(event: Event, opts: StoreEventOpts): string[][] { - const tagCounts: Record = {}; - - function getCount(name: string) { - return tagCounts[name] || 0; - } - - function incrementCount(name: string) { - tagCounts[name] = getCount(name) + 1; - } - - function checkCondition(name: string, value: string, condition: TagCondition) { - return condition({ - event, - opts, - count: getCount(name), - value, - }); - } - - return event.tags.reduce((results, tag) => { - const [name, value] = tag; - const condition = tagConditions[name] as TagCondition | undefined; - - if (value && condition && value.length < 200 && checkCondition(name, value, condition)) { - results.push(tag); - } - - incrementCount(name); - return results; - }, []); -} - -/** Build a search index from the event. */ -function buildSearchContent(event: Event): string { - switch (event.kind) { - case 0: - return buildUserSearchContent(event as Event<0>); - case 1: - return event.content; - default: - return ''; - } -} - -/** Build search content for a user. */ -function buildUserSearchContent(event: Event<0>): string { - const { name, nip05, about } = jsonMetaContentSchema.parse(event.content); - return [name, nip05, about].filter(Boolean).join('\n'); -} - -/** SQLite database storage adapter for Nostr events. */ -const eventsDB: EventStore = { - getEvents, - storeEvent, - countEvents, - deleteEvents, -}; - -export { eventsDB }; diff --git a/src/db/memorelay.ts b/src/db/memorelay.ts deleted file mode 100644 index 57714a0c..00000000 --- a/src/db/memorelay.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { Debug, type Event, type Filter, LRUCache } from '@/deps.ts'; -import { getFilterId, getMicroFilters, isMicrofilter } from '@/filter.ts'; -import { type EventStore, type GetEventsOpts } from '@/store.ts'; - -const debug = Debug('ditto:memorelay'); - -const events = new LRUCache({ - max: 3000, - maxEntrySize: 5000, - sizeCalculation: (event) => JSON.stringify(event).length, -}); - -/** Get events from memory. */ -function getEvents(filters: Filter[], opts: GetEventsOpts = {}): Promise[]> { - if (opts.signal?.aborted) return Promise.resolve([]); - if (!filters.length) return Promise.resolve([]); - debug('REQ', JSON.stringify(filters)); - - const results: Event[] = []; - - for (const filter of filters) { - if (isMicrofilter(filter)) { - const event = events.get(getFilterId(filter)); - if (event) { - results.push(event as Event); - } - } - } - - return Promise.resolve(results); -} - -/** Insert an event into memory. */ -function storeEvent(event: Event): Promise { - for (const microfilter of getMicroFilters(event)) { - const filterId = getFilterId(microfilter); - const existing = events.get(filterId); - if (!existing || event.created_at > existing.created_at) { - events.set(filterId, event); - } - } - return Promise.resolve(); -} - -/** Count events in memory for the filters. */ -async function countEvents(filters: Filter[]): Promise { - const events = await getEvents(filters); - return events.length; -} - -/** Delete events from memory. */ -function deleteEvents(filters: Filter[]): Promise { - for (const filter of filters) { - if (isMicrofilter(filter)) { - events.delete(getFilterId(filter)); - } - } - return Promise.resolve(); -} - -/** In-memory data store for events using microfilters. */ -const memorelay: EventStore = { - getEvents, - storeEvent, - countEvents, - deleteEvents, -}; - -export { memorelay }; diff --git a/src/db/users.ts b/src/db/users.ts index 6b7f9efd..62340e1e 100644 --- a/src/db/users.ts +++ b/src/db/users.ts @@ -1,8 +1,8 @@ import { Conf } from '@/config.ts'; import { Debug, type Filter } from '@/deps.ts'; -import { eventsDB } from '@/db/events.ts'; import * as pipeline from '@/pipeline.ts'; import { signAdminEvent } from '@/sign.ts'; +import { eventsDB } from '@/storages.ts'; const debug = Debug('ditto:users'); diff --git a/src/deps.ts b/src/deps.ts index 3e763a73..164af3ad 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -18,6 +18,7 @@ export { getEventHash, getPublicKey, getSignature, + matchFilter, matchFilters, nip04, nip05, diff --git a/src/filter.test.ts b/src/filter.test.ts index efc00d7c..b1ea8729 100644 --- a/src/filter.test.ts +++ b/src/filter.test.ts @@ -4,7 +4,7 @@ import { assertEquals } from '@/deps-test.ts'; import event0 from '~/fixtures/events/event-0.json' assert { type: 'json' }; import event1 from '~/fixtures/events/event-1.json' assert { type: 'json' }; -import { eventToMicroFilter, getFilterId, getMicroFilters, isMicrofilter } from './filter.ts'; +import { eventToMicroFilter, getFilterId, getFilterLimit, getMicroFilters, isMicrofilter } from './filter.ts'; Deno.test('getMicroFilters', () => { const event = event0 as Event<0>; @@ -35,3 +35,13 @@ Deno.test('getFilterId', () => { '{"authors":["79c2cae114ea28a981e7559b4fe7854a473521a8d22a66bbab9fa248eb820ff6"],"kinds":[0]}', ); }); + +Deno.test('getFilterLimit', () => { + assertEquals(getFilterLimit({ ids: [event0.id] }), 1); + assertEquals(getFilterLimit({ ids: [event0.id], limit: 2 }), 1); + assertEquals(getFilterLimit({ ids: [event0.id], limit: 0 }), 0); + assertEquals(getFilterLimit({ ids: [event0.id], limit: -1 }), 0); + assertEquals(getFilterLimit({ kinds: [0], authors: [event0.pubkey] }), 1); + assertEquals(getFilterLimit({ kinds: [1], authors: [event0.pubkey] }), Infinity); + assertEquals(getFilterLimit({}), Infinity); +}); diff --git a/src/filter.ts b/src/filter.ts index 430a8d3f..f59d6c17 100644 --- a/src/filter.ts +++ b/src/filter.ts @@ -2,6 +2,7 @@ import { Conf } from '@/config.ts'; import { type Event, type Filter, matchFilters, stringifyStable, z } from '@/deps.ts'; import { nostrIdSchema } from '@/schemas/nostr.ts'; import { type EventData } from '@/types.ts'; +import { isReplaceableKind } from '@/kinds.ts'; /** Additional properties that may be added by Ditto to events. */ type Relation = 'author' | 'author_stats' | 'event_stats'; @@ -82,15 +83,50 @@ function isMicrofilter(filter: Filter): filter is MicroFilter { return microFilterSchema.safeParse(filter).success; } +/** Calculate the intrinsic limit of a filter. */ +function getFilterLimit(filter: Filter): number { + if (filter.ids && !filter.ids.length) return 0; + if (filter.kinds && !filter.kinds.length) return 0; + if (filter.authors && !filter.authors.length) return 0; + + return Math.min( + Math.max(0, filter.limit ?? Infinity), + filter.ids?.length ?? Infinity, + filter.authors?.length && + filter.kinds?.every((kind) => isReplaceableKind(kind)) + ? filter.authors.length * filter.kinds.length + : Infinity, + ); +} + +/** Returns true if the filter could potentially return any stored events at all. */ +function canFilter(filter: Filter): boolean { + return getFilterLimit(filter) > 0; +} + +/** Normalize the `limit` of each filter, and remove filters that can't produce any events. */ +function normalizeFilters(filters: F[]): F[] { + return filters.reduce((acc, filter) => { + const limit = getFilterLimit(filter); + if (limit > 0) { + acc.push(limit === Infinity ? filter : { ...filter, limit }); + } + return acc; + }, []); +} + export { type AuthorMicrofilter, + canFilter, type DittoFilter, eventToMicroFilter, getFilterId, + getFilterLimit, getMicroFilters, type IdMicrofilter, isMicrofilter, matchDittoFilters, type MicroFilter, + normalizeFilters, type Relation, }; diff --git a/src/pipeline.ts b/src/pipeline.ts index 69ae9412..a237b72b 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,7 +1,5 @@ import { client } from '@/client.ts'; import { Conf } from '@/config.ts'; -import { eventsDB } from '@/db/events.ts'; -import { memorelay } from '@/db/memorelay.ts'; import { addRelays } from '@/db/relays.ts'; import { deleteAttachedMedia } from '@/db/unattached-media.ts'; import { findUser } from '@/db/users.ts'; @@ -10,6 +8,7 @@ import { isEphemeralKind } from '@/kinds.ts'; import { isLocallyFollowed } from '@/queries.ts'; import { reqmeister } from '@/reqmeister.ts'; import { updateStats } from '@/stats.ts'; +import { eventsDB, memorelay } from '@/storages.ts'; import { Sub } from '@/subs.ts'; import { getTagSet } from '@/tags.ts'; import { type EventData } from '@/types.ts'; diff --git a/src/queries.ts b/src/queries.ts index cfeeb360..56a8b679 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -1,9 +1,8 @@ -import { eventsDB } from '@/db/events.ts'; -import { memorelay } from '@/db/memorelay.ts'; +import { eventsDB, memorelay } from '@/storages.ts'; import { Debug, type Event, findReplyTag } from '@/deps.ts'; import { type AuthorMicrofilter, type DittoFilter, type IdMicrofilter, type Relation } from '@/filter.ts'; import { reqmeister } from '@/reqmeister.ts'; -import { type DittoEvent } from '@/store.ts'; +import { type DittoEvent } from '@/storages/types.ts'; import { getTagSet } from '@/tags.ts'; const debug = Debug('ditto:queries'); diff --git a/src/reqmeister.ts b/src/reqmeister.ts index 27a97daa..48b1370d 100644 --- a/src/reqmeister.ts +++ b/src/reqmeister.ts @@ -1,6 +1,14 @@ import { client } from '@/client.ts'; import { Debug, type Event, EventEmitter, type Filter } from '@/deps.ts'; -import { AuthorMicrofilter, eventToMicroFilter, getFilterId, IdMicrofilter, type MicroFilter } from '@/filter.ts'; +import { + AuthorMicrofilter, + eventToMicroFilter, + getFilterId, + IdMicrofilter, + isMicrofilter, + type MicroFilter, +} from '@/filter.ts'; +import { type EventStore, GetEventsOpts } from '@/storages/types.ts'; import { Time } from '@/utils/time.ts'; const debug = Debug('ditto:reqmeister'); @@ -18,12 +26,14 @@ interface ReqmeisterReqOpts { type ReqmeisterQueueItem = [string, MicroFilter, WebSocket['url'][]]; /** Batches requests to Nostr relays using microfilters. */ -class Reqmeister extends EventEmitter<{ [filterId: string]: (event: Event) => any }> { +class Reqmeister extends EventEmitter<{ [filterId: string]: (event: Event) => any }> implements EventStore { #opts: ReqmeisterOpts; #queue: ReqmeisterQueueItem[] = []; #promise!: Promise; #resolve!: () => void; + supportedNips = []; + constructor(opts: ReqmeisterOpts = {}) { super(); this.#opts = opts; @@ -119,6 +129,33 @@ class Reqmeister extends EventEmitter<{ [filterId: string]: (event: Event) => an const filterId = getFilterId(eventToMicroFilter(event)); return this.#queue.some(([id]) => id === filterId); } + + getEvents(filters: Filter[], opts?: GetEventsOpts | undefined): Promise[]> { + if (opts?.signal?.aborted) return Promise.resolve([]); + if (!filters.length) return Promise.resolve([]); + + const promises = filters.reduce>[]>((result, filter) => { + if (isMicrofilter(filter)) { + result.push(this.req(filter) as Promise>); + } + return result; + }, []); + + return Promise.all(promises); + } + + storeEvent(event: Event): Promise { + this.encounter(event); + return Promise.resolve(); + } + + countEvents(_filters: Filter[]): Promise { + throw new Error('COUNT not implemented.'); + } + + deleteEvents(_filters: Filter[]): Promise { + throw new Error('DELETE not implemented.'); + } } const reqmeister = new Reqmeister({ diff --git a/src/stats.ts b/src/stats.ts index e09791fd..fd08cf52 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -1,6 +1,6 @@ import { type AuthorStatsRow, db, type DittoDB, type EventStatsRow } from '@/db.ts'; -import { eventsDB } from '@/db/events.ts'; import { Debug, type Event, findReplyTag, type InsertQueryBuilder } from '@/deps.ts'; +import { eventsDB } from '@/storages.ts'; type AuthorStat = keyof Omit; type EventStat = keyof Omit; diff --git a/src/storages.ts b/src/storages.ts new file mode 100644 index 00000000..db00b519 --- /dev/null +++ b/src/storages.ts @@ -0,0 +1,28 @@ +import { Conf } from '@/config.ts'; +import { db } from '@/db.ts'; +import { EventsDB } from '@/storages/events-db.ts'; +import { Memorelay } from '@/storages/memorelay.ts'; +import { Optimizer } from '@/storages/optimizer.ts'; +import { SearchStore } from '@/storages/search-store.ts'; +import { reqmeister } from '@/reqmeister.ts'; + +/** SQLite database to store events this Ditto server cares about. */ +const eventsDB = new EventsDB(db); + +/** In-memory data store for cached events. */ +const memorelay = new Memorelay({ max: 3000 }); + +/** Main Ditto storage adapter */ +const optimizer = new Optimizer({ + db: eventsDB, + cache: memorelay, + client: reqmeister, +}); + +/** Storage to use for remote search. */ +const searchStore = new SearchStore({ + relay: Conf.searchRelay, + fallback: optimizer, +}); + +export { eventsDB, memorelay, optimizer, searchStore }; diff --git a/src/storages/events-db.test.ts b/src/storages/events-db.test.ts new file mode 100644 index 00000000..349f82b9 --- /dev/null +++ b/src/storages/events-db.test.ts @@ -0,0 +1,67 @@ +import { db } from '@/db.ts'; +import { buildUserEvent } from '@/db/users.ts'; +import { assertEquals, assertRejects } from '@/deps-test.ts'; + +import event0 from '~/fixtures/events/event-0.json' assert { type: 'json' }; +import event1 from '~/fixtures/events/event-1.json' assert { type: 'json' }; + +import { EventsDB } from './events-db.ts'; + +const eventsDB = new EventsDB(db); + +Deno.test('count filters', async () => { + assertEquals(await eventsDB.countEvents([{ kinds: [1] }]), 0); + await eventsDB.storeEvent(event1); + assertEquals(await eventsDB.countEvents([{ kinds: [1] }]), 1); +}); + +Deno.test('insert and filter events', async () => { + await eventsDB.storeEvent(event1); + + assertEquals(await eventsDB.getEvents([{ kinds: [1] }]), [event1]); + assertEquals(await eventsDB.getEvents([{ kinds: [3] }]), []); + assertEquals(await eventsDB.getEvents([{ since: 1691091000 }]), [event1]); + assertEquals(await eventsDB.getEvents([{ until: 1691091000 }]), []); + assertEquals( + await eventsDB.getEvents([{ '#proxy': ['https://gleasonator.com/objects/8f6fac53-4f66-4c6e-ac7d-92e5e78c3e79'] }]), + [event1], + ); +}); + +Deno.test('delete events', async () => { + await eventsDB.storeEvent(event1); + assertEquals(await eventsDB.getEvents([{ kinds: [1] }]), [event1]); + await eventsDB.deleteEvents([{ kinds: [1] }]); + assertEquals(await eventsDB.getEvents([{ kinds: [1] }]), []); +}); + +Deno.test('query events with local filter', async () => { + await eventsDB.storeEvent(event1); + + assertEquals(await eventsDB.getEvents([{}]), [event1]); + assertEquals(await eventsDB.getEvents([{ local: true }]), []); + assertEquals(await eventsDB.getEvents([{ local: false }]), [event1]); + + const userEvent = await buildUserEvent({ + username: 'alex', + pubkey: event1.pubkey, + inserted_at: new Date(), + admin: false, + }); + await eventsDB.storeEvent(userEvent); + + assertEquals(await eventsDB.getEvents([{ kinds: [1], local: true }]), [event1]); + assertEquals(await eventsDB.getEvents([{ kinds: [1], local: false }]), []); +}); + +Deno.test('inserting replaceable events', async () => { + assertEquals(await eventsDB.countEvents([{ kinds: [0], authors: [event0.pubkey] }]), 0); + + await eventsDB.storeEvent(event0); + await assertRejects(() => eventsDB.storeEvent(event0)); + assertEquals(await eventsDB.countEvents([{ kinds: [0], authors: [event0.pubkey] }]), 1); + + const changeEvent = { ...event0, id: '123', created_at: event0.created_at + 1 }; + await eventsDB.storeEvent(changeEvent); + assertEquals(await eventsDB.getEvents([{ kinds: [0] }]), [changeEvent]); +}); diff --git a/src/storages/events-db.ts b/src/storages/events-db.ts new file mode 100644 index 00000000..143f4f6e --- /dev/null +++ b/src/storages/events-db.ts @@ -0,0 +1,414 @@ +import { Conf } from '@/config.ts'; +import { type DittoDB } from '@/db.ts'; +import { Debug, type Event, Kysely, type SelectQueryBuilder } from '@/deps.ts'; +import { type DittoFilter } from '@/filter.ts'; +import { isDittoInternalKind, isParameterizedReplaceableKind, isReplaceableKind } from '@/kinds.ts'; +import { jsonMetaContentSchema } from '@/schemas/nostr.ts'; +import { isNostrId, isURL } from '@/utils.ts'; + +import { type DittoEvent, EventStore, type GetEventsOpts, type StoreEventOpts } from './types.ts'; + +/** Function to decide whether or not to index a tag. */ +type TagCondition = ({ event, count, value }: { + event: Event; + opts: StoreEventOpts; + count: number; + value: string; +}) => boolean; + +/** Conditions for when to index certain tags. */ +const tagConditions: Record = { + 'd': ({ event, count }) => count === 0 && isParameterizedReplaceableKind(event.kind), + 'e': ({ event, count, value, opts }) => ((opts.data?.user && event.kind === 10003) || count < 15) && isNostrId(value), + 'media': ({ count, value, opts }) => (opts.data?.user || count < 4) && isURL(value), + 'p': ({ event, count, value }) => (count < 15 || event.kind === 3) && isNostrId(value), + 'proxy': ({ count, value }) => count === 0 && isURL(value), + 'q': ({ event, count, value }) => count === 0 && event.kind === 1 && isNostrId(value), + 't': ({ count, value }) => count < 5 && value.length < 50, + 'name': ({ event, count }) => event.kind === 30361 && count === 0, + 'role': ({ event, count }) => event.kind === 30361 && count === 0, +}; + +type EventQuery = SelectQueryBuilder; + +/** SQLite database storage adapter for Nostr events. */ +class EventsDB implements EventStore { + #db: Kysely; + #debug = Debug('ditto:db:events'); + + /** NIPs supported by this storage method. */ + supportedNips = [1, 45, 50]; + + constructor(db: Kysely) { + this.#db = db; + } + + /** Insert an event (and its tags) into the database. */ + async storeEvent(event: Event, opts: StoreEventOpts = {}): Promise { + this.#debug('EVENT', JSON.stringify(event)); + + if (isDittoInternalKind(event.kind) && event.pubkey !== Conf.pubkey) { + throw new Error('Internal events can only be stored by the server keypair'); + } + + return await this.#db.transaction().execute(async (trx) => { + /** Insert the event into the database. */ + async function addEvent() { + await trx.insertInto('events') + .values({ ...event, tags: JSON.stringify(event.tags) }) + .execute(); + } + + /** Add search data to the FTS table. */ + async function indexSearch() { + const searchContent = buildSearchContent(event); + if (!searchContent) return; + await trx.insertInto('events_fts') + .values({ id: event.id, content: searchContent.substring(0, 1000) }) + .execute(); + } + + /** Index event tags depending on the conditions defined above. */ + async function indexTags() { + const tags = filterIndexableTags(event, opts); + const rows = tags.map(([tag, value]) => ({ event_id: event.id, tag, value })); + + if (!tags.length) return; + await trx.insertInto('tags') + .values(rows) + .execute(); + } + + if (isReplaceableKind(event.kind)) { + const prevEvents = await this.getFilterQuery(trx, { kinds: [event.kind], authors: [event.pubkey] }).execute(); + for (const prevEvent of prevEvents) { + if (prevEvent.created_at >= event.created_at) { + throw new Error('Cannot replace an event with an older event'); + } + } + await this.deleteEventsTrx(trx, [{ kinds: [event.kind], authors: [event.pubkey] }]); + } + + if (isParameterizedReplaceableKind(event.kind)) { + const d = event.tags.find(([tag]) => tag === 'd')?.[1]; + if (d) { + const prevEvents = await this.getFilterQuery(trx, { kinds: [event.kind], authors: [event.pubkey], '#d': [d] }) + .execute(); + for (const prevEvent of prevEvents) { + if (prevEvent.created_at >= event.created_at) { + throw new Error('Cannot replace an event with an older event'); + } + } + await this.deleteEventsTrx(trx, [{ kinds: [event.kind], authors: [event.pubkey], '#d': [d] }]); + } + } + + // Run the queries. + await Promise.all([ + addEvent(), + indexTags(), + indexSearch(), + ]); + }).catch((error) => { + // Don't throw for duplicate events. + if (error.message.includes('UNIQUE constraint failed')) { + return; + } else { + throw error; + } + }); + } + + /** Build the query for a filter. */ + getFilterQuery(db: Kysely, filter: DittoFilter): EventQuery { + let query = db + .selectFrom('events') + .select([ + 'events.id', + 'events.kind', + 'events.pubkey', + 'events.content', + 'events.tags', + 'events.created_at', + 'events.sig', + ]) + .orderBy('events.created_at', 'desc'); + + for (const [key, value] of Object.entries(filter)) { + if (value === undefined) continue; + + switch (key as keyof DittoFilter) { + case 'ids': + query = query.where('events.id', 'in', filter.ids!); + break; + case 'kinds': + query = query.where('events.kind', 'in', filter.kinds!); + break; + case 'authors': + query = query.where('events.pubkey', 'in', filter.authors!); + break; + case 'since': + query = query.where('events.created_at', '>=', filter.since!); + break; + case 'until': + query = query.where('events.created_at', '<=', filter.until!); + break; + case 'limit': + query = query.limit(filter.limit!); + break; + } + + if (key.startsWith('#')) { + const tag = key.replace(/^#/, ''); + const value = filter[key as `#${string}`] as string[]; + query = query + .leftJoin('tags', 'tags.event_id', 'events.id') + .where('tags.tag', '=', tag) + .where('tags.value', 'in', value); + } + } + + if (typeof filter.local === 'boolean') { + query = query + .leftJoin(() => this.usersQuery(), (join) => join.onRef('users.d_tag', '=', 'events.pubkey')) + .where('users.d_tag', filter.local ? 'is not' : 'is', null); + } + + if (filter.relations?.includes('author')) { + query = query + .leftJoin( + (eb) => + eb + .selectFrom('events') + .selectAll() + .where('kind', '=', 0) + .groupBy('pubkey') + .as('authors'), + (join) => join.onRef('authors.pubkey', '=', 'events.pubkey'), + ) + .select([ + 'authors.id as author_id', + 'authors.kind as author_kind', + 'authors.pubkey as author_pubkey', + 'authors.content as author_content', + 'authors.tags as author_tags', + 'authors.created_at as author_created_at', + 'authors.sig as author_sig', + ]); + } + + if (filter.relations?.includes('author_stats')) { + query = query + .leftJoin('author_stats', 'author_stats.pubkey', 'events.pubkey') + .select((eb) => [ + eb.fn.coalesce('author_stats.followers_count', eb.val(0)).as('author_stats_followers_count'), + eb.fn.coalesce('author_stats.following_count', eb.val(0)).as('author_stats_following_count'), + eb.fn.coalesce('author_stats.notes_count', eb.val(0)).as('author_stats_notes_count'), + ]); + } + + if (filter.relations?.includes('event_stats')) { + query = query + .leftJoin('event_stats', 'event_stats.event_id', 'events.id') + .select((eb) => [ + eb.fn.coalesce('event_stats.replies_count', eb.val(0)).as('stats_replies_count'), + eb.fn.coalesce('event_stats.reposts_count', eb.val(0)).as('stats_reposts_count'), + eb.fn.coalesce('event_stats.reactions_count', eb.val(0)).as('stats_reactions_count'), + ]); + } + + if (filter.search) { + query = query + .innerJoin('events_fts', 'events_fts.id', 'events.id') + .where('events_fts.content', 'match', JSON.stringify(filter.search)); + } + + return query; + } + + /** Combine filter queries into a single union query. */ + getEventsQuery(filters: DittoFilter[]) { + return filters + .map((filter) => this.#db.selectFrom(() => this.getFilterQuery(this.#db, filter).as('events')).selectAll()) + .reduce((result, query) => result.unionAll(query)); + } + + /** Query to get user events, joined by tags. */ + usersQuery() { + return this.getFilterQuery(this.#db, { kinds: [30361], authors: [Conf.pubkey] }) + .leftJoin('tags', 'tags.event_id', 'events.id') + .where('tags.tag', '=', 'd') + .select('tags.value as d_tag') + .as('users'); + } + + /** Get events for filters from the database. */ + async getEvents( + filters: DittoFilter[], + opts: GetEventsOpts = {}, + ): Promise[]> { + if (opts.signal?.aborted) return Promise.resolve([]); + if (!filters.length) return Promise.resolve([]); + this.#debug('REQ', JSON.stringify(filters)); + let query = this.getEventsQuery(filters); + + if (typeof opts.limit === 'number') { + query = query.limit(opts.limit); + } + + return (await query.execute()).map((row) => { + const event: DittoEvent = { + id: row.id, + kind: row.kind as K, + pubkey: row.pubkey, + content: row.content, + created_at: row.created_at, + tags: JSON.parse(row.tags), + sig: row.sig, + }; + + if (row.author_id) { + event.author = { + id: row.author_id, + kind: row.author_kind! as 0, + pubkey: row.author_pubkey!, + content: row.author_content!, + created_at: row.author_created_at!, + tags: JSON.parse(row.author_tags!), + sig: row.author_sig!, + }; + } + + if (typeof row.author_stats_followers_count === 'number') { + event.author_stats = { + followers_count: row.author_stats_followers_count, + following_count: row.author_stats_following_count!, + notes_count: row.author_stats_notes_count!, + }; + } + + if (typeof row.stats_replies_count === 'number') { + event.event_stats = { + replies_count: row.stats_replies_count, + reposts_count: row.stats_reposts_count!, + reactions_count: row.stats_reactions_count!, + }; + } + + return event; + }); + } + + /** Delete events from each table. Should be run in a transaction! */ + async deleteEventsTrx(db: Kysely, filters: DittoFilter[]) { + if (!filters.length) return Promise.resolve(); + this.#debug('DELETE', JSON.stringify(filters)); + + const query = this.getEventsQuery(filters).clearSelect().select('id'); + + await db.deleteFrom('events_fts') + .where('id', 'in', () => query) + .execute(); + + return db.deleteFrom('events') + .where('id', 'in', () => query) + .execute(); + } + + /** Delete events based on filters from the database. */ + async deleteEvents(filters: DittoFilter[]): Promise { + if (!filters.length) return Promise.resolve(); + this.#debug('DELETE', JSON.stringify(filters)); + + await this.#db.transaction().execute((trx) => this.deleteEventsTrx(trx, filters)); + } + + /** Get number of events that would be returned by filters. */ + async countEvents(filters: DittoFilter[]): Promise { + if (!filters.length) return Promise.resolve(0); + this.#debug('COUNT', JSON.stringify(filters)); + const query = this.getEventsQuery(filters); + + const [{ count }] = await query + .clearSelect() + .select((eb) => eb.fn.count('id').as('count')) + .execute(); + + return Number(count); + } +} + +/** Return only the tags that should be indexed. */ +function filterIndexableTags(event: Event, opts: StoreEventOpts): string[][] { + const tagCounts: Record = {}; + + function getCount(name: string) { + return tagCounts[name] || 0; + } + + function incrementCount(name: string) { + tagCounts[name] = getCount(name) + 1; + } + + function checkCondition(name: string, value: string, condition: TagCondition) { + return condition({ + event, + opts, + count: getCount(name), + value, + }); + } + + return event.tags.reduce((results, tag) => { + const [name, value] = tag; + const condition = tagConditions[name] as TagCondition | undefined; + + if (value && condition && value.length < 200 && checkCondition(name, value, condition)) { + results.push(tag); + } + + incrementCount(name); + return results; + }, []); +} + +/** Build a search index from the event. */ +function buildSearchContent(event: Event): string { + switch (event.kind) { + case 0: + return buildUserSearchContent(event as Event<0>); + case 1: + return event.content; + default: + return ''; + } +} + +/** Build search content for a user. */ +function buildUserSearchContent(event: Event<0>): string { + const { name, nip05, about } = jsonMetaContentSchema.parse(event.content); + return [name, nip05, about].filter(Boolean).join('\n'); +} + +export { EventsDB }; diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts new file mode 100644 index 00000000..4d8edcba --- /dev/null +++ b/src/storages/hydrate.ts @@ -0,0 +1,27 @@ +import { type DittoFilter } from '@/filter.ts'; +import { type DittoEvent, type EventStore } from '@/storages/types.ts'; + +interface HydrateEventOpts { + events: DittoEvent[]; + filters: DittoFilter[]; + storage: EventStore; + signal?: AbortSignal; +} + +/** Hydrate event relationships using the provided storage. */ +async function hydrateEvents(opts: HydrateEventOpts): Promise[]> { + const { events, filters, storage, signal } = opts; + + if (filters.some((filter) => filter.relations?.includes('author'))) { + const pubkeys = new Set([...events].map((event) => event.pubkey)); + const authors = await storage.getEvents([{ kinds: [0], authors: [...pubkeys] }], { signal }); + + for (const event of events) { + event.author = authors.find((author) => author.pubkey === event.pubkey); + } + } + + return events; +} + +export { hydrateEvents }; diff --git a/src/db/memorelay.test.ts b/src/storages/memorelay.test.ts similarity index 71% rename from src/db/memorelay.test.ts rename to src/storages/memorelay.test.ts index 27c545d1..fbf3ba32 100644 --- a/src/db/memorelay.test.ts +++ b/src/storages/memorelay.test.ts @@ -2,7 +2,13 @@ import { assertEquals } from '@/deps-test.ts'; import event1 from '~/fixtures/events/event-1.json' assert { type: 'json' }; -import { memorelay } from './memorelay.ts'; +import { Memorelay } from './memorelay.ts'; + +const memorelay = new Memorelay({ + max: 3000, + maxEntrySize: 5000, + sizeCalculation: (event) => JSON.stringify(event).length, +}); Deno.test('memorelay', async () => { assertEquals(await memorelay.countEvents([{ ids: [event1.id] }]), 0); diff --git a/src/storages/memorelay.ts b/src/storages/memorelay.ts new file mode 100644 index 00000000..bfa2a320 --- /dev/null +++ b/src/storages/memorelay.ts @@ -0,0 +1,114 @@ +import { Debug, type Event, type Filter, LRUCache, matchFilter } from '@/deps.ts'; +import { normalizeFilters } from '@/filter.ts'; +import { EventSet } from '@/utils/event-set.ts'; + +import { type EventStore, type GetEventsOpts } from './types.ts'; + +/** In-memory data store for events. */ +class Memorelay implements EventStore { + #debug = Debug('ditto:memorelay'); + #cache: LRUCache; + + /** NIPs supported by this storage method. */ + supportedNips = [1, 45]; + + constructor(...args: ConstructorParameters>) { + this.#cache = new LRUCache(...args); + } + + /** Iterate stored events. */ + *#events(): Generator { + for (const event of this.#cache.values()) { + if (event && !(event instanceof Promise)) { + yield event; + } + } + } + + /** Get events from memory. */ + getEvents(filters: Filter[], opts: GetEventsOpts = {}): Promise[]> { + filters = normalizeFilters(filters); + + if (opts.signal?.aborted) return Promise.resolve([]); + if (!filters.length) return Promise.resolve([]); + + this.#debug('REQ', JSON.stringify(filters)); + + /** Event results to return. */ + const results = new EventSet>(); + + /** Number of times an event has been added to results for each filter. */ + const filterUsages: number[] = []; + + /** Check if all filters have been satisfied. */ + function checkSatisfied() { + return results.size >= (opts.limit ?? Infinity) || + filters.every((filter, index) => filter.limit && (filterUsages[index] >= filter.limit)); + } + + // Optimize for filters with IDs. + filters.forEach((filter, index) => { + if (filter.ids) { + for (const id of filter.ids) { + const event = this.#cache.get(id); + if (event && matchFilter(filter, event)) { + results.add(event as Event); + } + } + filterUsages[index] = Infinity; + } + }); + + // Return early if all filters are satisfied. + if (checkSatisfied()) { + return Promise.resolve([...results]); + } + + // Seek through all events in memory. + for (const event of this.#events()) { + filters.forEach((filter, index) => { + const limit = filter.limit ?? Infinity; + const usage = filterUsages[index] ?? 0; + + if (usage >= limit) { + return; + } else if (matchFilter(filter, event)) { + results.add(event as Event); + this.#cache.get(event.id); + filterUsages[index] = usage + 1; + } + + index++; + }); + + // Check after each event if we can return. + if (checkSatisfied()) { + break; + } + } + + return Promise.resolve([...results]); + } + + /** Insert an event into memory. */ + storeEvent(event: Event): Promise { + this.#cache.set(event.id, event); + return Promise.resolve(); + } + + /** Count events in memory for the filters. */ + async countEvents(filters: Filter[]): Promise { + const events = await this.getEvents(filters); + return events.length; + } + + /** Delete events from memory. */ + async deleteEvents(filters: Filter[]): Promise { + for (const event of await this.getEvents(filters)) { + this.#cache.delete(event.id); + } + return Promise.resolve(); + } +} + +export { Memorelay }; diff --git a/src/storages/optimizer.ts b/src/storages/optimizer.ts new file mode 100644 index 00000000..e0d18fb6 --- /dev/null +++ b/src/storages/optimizer.ts @@ -0,0 +1,111 @@ +import { Debug } from '@/deps.ts'; +import { type DittoFilter, normalizeFilters } from '@/filter.ts'; +import { EventSet } from '@/utils/event-set.ts'; + +import { type DittoEvent, type EventStore, type GetEventsOpts, type StoreEventOpts } from './types.ts'; + +interface OptimizerOpts { + db: EventStore; + cache: EventStore; + client: EventStore; +} + +class Optimizer implements EventStore { + #debug = Debug('ditto:optimizer'); + + #db: EventStore; + #cache: EventStore; + #client: EventStore; + + supportedNips = [1]; + + constructor(opts: OptimizerOpts) { + this.#db = opts.db; + this.#cache = opts.cache; + this.#client = opts.client; + } + + async storeEvent(event: DittoEvent, opts?: StoreEventOpts | undefined): Promise { + await Promise.all([ + this.#db.storeEvent(event, opts), + this.#cache.storeEvent(event, opts), + ]); + } + + async getEvents( + filters: DittoFilter[], + opts: GetEventsOpts | undefined = {}, + ): Promise[]> { + this.#debug('REQ', JSON.stringify(filters)); + + const { limit = Infinity } = opts; + filters = normalizeFilters(filters); + + if (opts?.signal?.aborted) return Promise.resolve([]); + if (!filters.length) return Promise.resolve([]); + + const results = new EventSet>(); + + // Filters with IDs are immutable, so we can take them straight from the cache if we have them. + for (let i = 0; i < filters.length; i++) { + const filter = filters[i]; + if (filter.ids) { + this.#debug(`Filter[${i}] is an IDs filter; querying cache...`); + const ids = new Set(filter.ids); + for (const event of await this.#cache.getEvents([filter], opts)) { + ids.delete(event.id); + results.add(event); + if (results.size >= limit) return getResults(); + } + filters[i] = { ...filter, ids: [...ids] }; + } + } + + filters = normalizeFilters(filters); + if (!filters.length) return getResults(); + + // Query the database for events. + this.#debug('Querying database...'); + for (const dbEvent of await this.#db.getEvents(filters, opts)) { + results.add(dbEvent); + if (results.size >= limit) return getResults(); + } + + // We already searched the DB, so stop if this is a search filter. + if (filters.some((filter) => typeof filter.search === 'string')) { + this.#debug(`Bailing early for search filter: "${filters[0]?.search}"`); + return getResults(); + } + + // Query the cache again. + this.#debug('Querying cache...'); + for (const cacheEvent of await this.#cache.getEvents(filters, opts)) { + results.add(cacheEvent); + if (results.size >= limit) return getResults(); + } + + // Finally, query the client. + this.#debug('Querying client...'); + for (const clientEvent of await this.#client.getEvents(filters, opts)) { + results.add(clientEvent); + if (results.size >= limit) return getResults(); + } + + /** Get return type from map. */ + function getResults() { + return [...results.values()]; + } + + return getResults(); + } + + countEvents(_filters: DittoFilter[]): Promise { + throw new Error('COUNT not implemented.'); + } + + deleteEvents(_filters: DittoFilter[]): Promise { + throw new Error('DELETE not implemented.'); + } +} + +export { Optimizer }; diff --git a/src/storages/search-store.ts b/src/storages/search-store.ts new file mode 100644 index 00000000..30e6ba16 --- /dev/null +++ b/src/storages/search-store.ts @@ -0,0 +1,85 @@ +import { NiceRelay } from 'https://gitlab.com/soapbox-pub/nostr-machina/-/raw/5f4fb59c90c092e5aa59c01e6556a4bec264c167/mod.ts'; + +import { Debug, type Event, type Filter } from '@/deps.ts'; +import { type DittoFilter, normalizeFilters } from '@/filter.ts'; +import { hydrateEvents } from '@/storages/hydrate.ts'; +import { type DittoEvent, type EventStore, type GetEventsOpts, type StoreEventOpts } from '@/storages/types.ts'; +import { EventSet } from '@/utils/event-set.ts'; + +interface SearchStoreOpts { + relay: string | undefined; + fallback: EventStore; + hydrator?: EventStore; +} + +class SearchStore implements EventStore { + #debug = Debug('ditto:storages:search'); + + #fallback: EventStore; + #hydrator: EventStore; + #relay: NiceRelay | undefined; + + supportedNips = [50]; + + constructor(opts: SearchStoreOpts) { + this.#fallback = opts.fallback; + this.#hydrator = opts.hydrator ?? this; + + if (opts.relay) { + this.#relay = new NiceRelay(opts.relay); + } + } + + storeEvent(_event: Event, _opts?: StoreEventOpts | undefined): Promise { + throw new Error('EVENT not implemented.'); + } + + async getEvents( + filters: DittoFilter[], + opts?: GetEventsOpts | undefined, + ): Promise[]> { + filters = normalizeFilters(filters); + + if (opts?.signal?.aborted) return Promise.resolve([]); + if (!filters.length) return Promise.resolve([]); + + this.#debug('REQ', JSON.stringify(filters)); + const query = filters[0]?.search; + + if (this.#relay) { + this.#debug(`Searching for "${query}" at ${this.#relay.socket.url}...`); + + const sub = this.#relay.req(filters, opts); + + const close = () => { + sub.close(); + opts?.signal?.removeEventListener('abort', close); + sub.eoseSignal.removeEventListener('abort', close); + }; + + opts?.signal?.addEventListener('abort', close, { once: true }); + sub.eoseSignal.addEventListener('abort', close, { once: true }); + + const events = new EventSet>(); + + for await (const event of sub) { + events.add(event); + } + + return hydrateEvents({ events: [...events], filters, storage: this.#hydrator, signal: opts?.signal }); + } else { + this.#debug(`Searching for "${query}" locally...`); + return this.#fallback.getEvents(filters, opts); + } + } + + countEvents(_filters: Filter[]): Promise { + throw new Error('COUNT not implemented.'); + } + + deleteEvents(_filters: Filter[]): Promise { + throw new Error('DELETE not implemented.'); + } +} + +export { SearchStore }; diff --git a/src/store.ts b/src/storages/types.ts similarity index 90% rename from src/store.ts rename to src/storages/types.ts index 55674064..c89e9cc9 100644 --- a/src/store.ts +++ b/src/storages/types.ts @@ -33,6 +33,8 @@ interface DittoEvent extends Event { /** Storage interface for Nostr events. */ interface EventStore { + /** Indicates NIPs supported by this data store, similar to NIP-11. For example, `50` would indicate support for `search` filters. */ + supportedNips: readonly number[]; /** Add an event to the store. */ storeEvent(event: Event, opts?: StoreEventOpts): Promise; /** Get events from filters. */ diff --git a/src/utils/api.ts b/src/utils/api.ts index 9a3eaf06..dbedc6c9 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -13,8 +13,8 @@ import { } from '@/deps.ts'; import * as pipeline from '@/pipeline.ts'; import { signAdminEvent, signEvent } from '@/sign.ts'; +import { eventsDB } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; -import { eventsDB } from '@/db/events.ts'; const debug = Debug('ditto:api'); diff --git a/src/utils/event-set.test.ts b/src/utils/event-set.test.ts new file mode 100644 index 00000000..b6e26b9c --- /dev/null +++ b/src/utils/event-set.test.ts @@ -0,0 +1,109 @@ +import { assertEquals } from '@/deps-test.ts'; + +import { EventSet } from './event-set.ts'; + +Deno.test('EventSet', () => { + const set = new EventSet(); + assertEquals(set.size, 0); + + const event = { id: '1', kind: 0, pubkey: 'abc', content: '', created_at: 0, sig: '', tags: [] }; + set.add(event); + assertEquals(set.size, 1); + assertEquals(set.has(event), true); + + set.add(event); + assertEquals(set.size, 1); + assertEquals(set.has(event), true); + + set.delete(event); + assertEquals(set.size, 0); + assertEquals(set.has(event), false); + + set.delete(event); + assertEquals(set.size, 0); + assertEquals(set.has(event), false); + + set.add(event); + assertEquals(set.size, 1); + assertEquals(set.has(event), true); + + set.clear(); + assertEquals(set.size, 0); + assertEquals(set.has(event), false); +}); + +Deno.test('EventSet.add (replaceable)', () => { + const event0 = { id: '1', kind: 0, pubkey: 'abc', content: '', created_at: 0, sig: '', tags: [] }; + const event1 = { id: '2', kind: 0, pubkey: 'abc', content: '', created_at: 1, sig: '', tags: [] }; + const event2 = { id: '3', kind: 0, pubkey: 'abc', content: '', created_at: 2, sig: '', tags: [] }; + + const set = new EventSet(); + set.add(event0); + assertEquals(set.size, 1); + assertEquals(set.has(event0), true); + + set.add(event1); + assertEquals(set.size, 1); + assertEquals(set.has(event0), false); + assertEquals(set.has(event1), true); + + set.add(event2); + assertEquals(set.size, 1); + assertEquals(set.has(event0), false); + assertEquals(set.has(event1), false); + assertEquals(set.has(event2), true); +}); + +Deno.test('EventSet.add (parameterized)', () => { + const event0 = { id: '1', kind: 30000, pubkey: 'abc', content: '', created_at: 0, sig: '', tags: [['d', 'a']] }; + const event1 = { id: '2', kind: 30000, pubkey: 'abc', content: '', created_at: 1, sig: '', tags: [['d', 'a']] }; + const event2 = { id: '3', kind: 30000, pubkey: 'abc', content: '', created_at: 2, sig: '', tags: [['d', 'a']] }; + + const set = new EventSet(); + set.add(event0); + assertEquals(set.size, 1); + assertEquals(set.has(event0), true); + + set.add(event1); + assertEquals(set.size, 1); + assertEquals(set.has(event0), false); + assertEquals(set.has(event1), true); + + set.add(event2); + assertEquals(set.size, 1); + assertEquals(set.has(event0), false); + assertEquals(set.has(event1), false); + assertEquals(set.has(event2), true); +}); + +Deno.test('EventSet.eventReplaces', () => { + const event0 = { id: '1', kind: 0, pubkey: 'abc', content: '', created_at: 0, sig: '', tags: [] }; + const event1 = { id: '2', kind: 0, pubkey: 'abc', content: '', created_at: 1, sig: '', tags: [] }; + const event2 = { id: '3', kind: 0, pubkey: 'abc', content: '', created_at: 2, sig: '', tags: [] }; + const event3 = { id: '4', kind: 0, pubkey: 'def', content: '', created_at: 0, sig: '', tags: [] }; + + assertEquals(EventSet.eventReplaces(event1, event0), true); + assertEquals(EventSet.eventReplaces(event2, event0), true); + assertEquals(EventSet.eventReplaces(event2, event1), true); + + assertEquals(EventSet.eventReplaces(event0, event1), false); + assertEquals(EventSet.eventReplaces(event0, event2), false); + assertEquals(EventSet.eventReplaces(event1, event2), false); + + assertEquals(EventSet.eventReplaces(event3, event1), false); + assertEquals(EventSet.eventReplaces(event1, event3), false); +}); + +Deno.test('EventSet.eventReplaces (parameterized)', () => { + const event0 = { id: '1', kind: 30000, pubkey: 'abc', content: '', created_at: 0, sig: '', tags: [['d', 'a']] }; + const event1 = { id: '2', kind: 30000, pubkey: 'abc', content: '', created_at: 1, sig: '', tags: [['d', 'a']] }; + const event2 = { id: '3', kind: 30000, pubkey: 'abc', content: '', created_at: 2, sig: '', tags: [['d', 'a']] }; + + assertEquals(EventSet.eventReplaces(event1, event0), true); + assertEquals(EventSet.eventReplaces(event2, event0), true); + assertEquals(EventSet.eventReplaces(event2, event1), true); + + assertEquals(EventSet.eventReplaces(event0, event1), false); + assertEquals(EventSet.eventReplaces(event0, event2), false); + assertEquals(EventSet.eventReplaces(event1, event2), false); +}); diff --git a/src/utils/event-set.ts b/src/utils/event-set.ts new file mode 100644 index 00000000..3fd06f0f --- /dev/null +++ b/src/utils/event-set.ts @@ -0,0 +1,77 @@ +import { type Event } from '@/deps.ts'; +import { isParameterizedReplaceableKind, isReplaceableKind } from '@/kinds.ts'; + +/** In-memory store for Nostr events with replaceable event functionality. */ +class EventSet implements Set { + #map = new Map(); + + get size() { + return this.#map.size; + } + + add(event: E): this { + if (isReplaceableKind(event.kind) || isParameterizedReplaceableKind(event.kind)) { + for (const e of this.values()) { + if (EventSet.eventReplaces(event, e)) { + this.delete(e); + } + } + } + this.#map.set(event.id, event); + return this; + } + + clear(): void { + this.#map.clear(); + } + + delete(event: E): boolean { + return this.#map.delete(event.id); + } + + forEach(callbackfn: (event: E, key: E, set: Set) => void, thisArg?: any): void { + return this.#map.forEach((event, _id) => callbackfn(event, event, this), thisArg); + } + + has(event: E): boolean { + return this.#map.has(event.id); + } + + *entries(): IterableIterator<[E, E]> { + for (const event of this.#map.values()) { + yield [event, event]; + } + } + + keys(): IterableIterator { + return this.#map.values(); + } + + values(): IterableIterator { + return this.#map.values(); + } + + [Symbol.iterator](): IterableIterator { + return this.#map.values(); + } + + [Symbol.toStringTag]: string = 'EventSet'; + + /** Returns true if both events are replaceable, belong to the same kind and pubkey (and `d` tag, for parameterized events), and the first event is newer than the second one. */ + static eventReplaces(event: Event, target: Event): boolean { + if (isReplaceableKind(event.kind)) { + return event.kind === target.kind && event.pubkey === target.pubkey && event.created_at > target.created_at; + } else if (isParameterizedReplaceableKind(event.kind)) { + const d = event.tags.find(([name]) => name === 'd')?.[1] || ''; + const d2 = target.tags.find(([name]) => name === 'd')?.[1] || ''; + + return event.kind === target.kind && + event.pubkey === target.pubkey && + d === d2 && + event.created_at > target.created_at; + } + return false; + } +} + +export { EventSet }; diff --git a/src/views.ts b/src/views.ts index c90f26d9..9ade6001 100644 --- a/src/views.ts +++ b/src/views.ts @@ -1,6 +1,6 @@ import { AppContext } from '@/app.ts'; -import { eventsDB } from '@/db/events.ts'; import { type Filter } from '@/deps.ts'; +import { eventsDB } from '@/storages.ts'; import { renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; import { paginated, paginationSchema } from '@/utils/api.ts'; diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index eae3dd76..620a1b5d 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -2,8 +2,8 @@ import { Conf } from '@/config.ts'; import { findUser } from '@/db/users.ts'; import { lodash, nip19, type UnsignedEvent } from '@/deps.ts'; import { jsonMetaContentSchema } from '@/schemas/nostr.ts'; +import { type DittoEvent } from '@/storages/types.ts'; import { verifyNip05Cached } from '@/utils/nip05.ts'; -import { type DittoEvent } from '@/store.ts'; import { Nip05, nostrDate, nostrNow, parseNip05 } from '@/utils.ts'; import { renderEmojis } from '@/views/mastodon/emojis.ts'; diff --git a/src/views/mastodon/relationships.ts b/src/views/mastodon/relationships.ts index ca2778b9..43ed7b83 100644 --- a/src/views/mastodon/relationships.ts +++ b/src/views/mastodon/relationships.ts @@ -1,4 +1,4 @@ -import { eventsDB } from '@/db/events.ts'; +import { eventsDB } from '@/storages.ts'; import { hasTag } from '@/tags.ts'; async function renderRelationship(sourcePubkey: string, targetPubkey: string) { diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index a951e0ae..9e45f495 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -1,12 +1,12 @@ import { isCWTag } from 'https://gitlab.com/soapbox-pub/mostr/-/raw/c67064aee5ade5e01597c6d23e22e53c628ef0e2/src/nostr/tags.ts'; import { Conf } from '@/config.ts'; -import { eventsDB } from '@/db/events.ts'; import { findReplyTag, nip19 } from '@/deps.ts'; import { getMediaLinks, parseNoteContent } from '@/note.ts'; import { getAuthor } from '@/queries.ts'; import { jsonMediaDataSchema } from '@/schemas/nostr.ts'; -import { DittoEvent } from '@/store.ts'; +import { eventsDB } from '@/storages.ts'; +import { type DittoEvent } from '@/storages/types.ts'; import { nostrDate } from '@/utils.ts'; import { unfurlCardCached } from '@/utils/unfurl.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts';