diff --git a/packages/db/DittoTables.ts b/packages/db/DittoTables.ts index 92226a84..12763c57 100644 --- a/packages/db/DittoTables.ts +++ b/packages/db/DittoTables.ts @@ -1,6 +1,8 @@ import type { NPostgresSchema } from '@nostrify/db'; import type { Generated } from 'kysely'; +import type { MastodonPreviewCard } from '@ditto/mastoapi/types'; + export interface DittoTables extends NPostgresSchema { auth_tokens: AuthTokenRow; author_stats: AuthorStatsRow; @@ -34,6 +36,7 @@ interface EventStatsRow { quotes_count: number; reactions: string; zaps_amount: number; + link_preview?: MastodonPreviewCard; } interface AuthTokenRow { diff --git a/packages/db/migrations/053_link_preview.ts b/packages/db/migrations/053_link_preview.ts new file mode 100644 index 00000000..99e56c68 --- /dev/null +++ b/packages/db/migrations/053_link_preview.ts @@ -0,0 +1,9 @@ +import type { Kysely } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema.alterTable('event_stats').addColumn('link_preview', 'jsonb').execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.alterTable('event_stats').dropColumn('link_preview').execute(); +} diff --git a/packages/ditto/controllers/api/trends.ts b/packages/ditto/controllers/api/trends.ts index f7a09b5f..4e5810ed 100644 --- a/packages/ditto/controllers/api/trends.ts +++ b/packages/ditto/controllers/api/trends.ts @@ -7,7 +7,6 @@ import { z } from 'zod'; import { AppController } from '@/app.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { generateDateRange, Time } from '@/utils/time.ts'; -import { unfurlCardCached } from '@/utils/unfurl.ts'; import { errorJson } from '@/utils/log.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; @@ -94,9 +93,8 @@ const trendingLinksController: AppController = async (c) => { async function getTrendingLinks(conf: DittoConf, relay: NStore): Promise { const trends = await getTrendingTags(relay, 'r', await conf.signer.getPublicKey()); - return Promise.all(trends.map(async (trend) => { + return Promise.all(trends.map((trend) => { const link = trend.value; - const card = await unfurlCardCached(link); const history = trend.history.map(({ day, authors, uses }) => ({ day: String(day), @@ -119,7 +117,6 @@ async function getTrendingLinks(conf: DittoConf, relay: NStore): Promise; zaps_amount: number; + link_preview?: MastodonPreviewCard; } /** Internal Event representation used by Ditto, including extra keys. */ diff --git a/packages/ditto/storages/DittoRelayStore.ts b/packages/ditto/storages/DittoRelayStore.ts index e8dc1b33..1d29aa6e 100644 --- a/packages/ditto/storages/DittoRelayStore.ts +++ b/packages/ditto/storages/DittoRelayStore.ts @@ -41,7 +41,7 @@ import { fetchFavicon, insertFavicon, queryFavicon } from '@/utils/favicon.ts'; import { lookupNip05 } from '@/utils/nip05.ts'; import { parseNoteContent, stripimeta } from '@/utils/note.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; -import { unfurlCardCached } from '@/utils/unfurl.ts'; +import { unfurlCard } from '@/utils/unfurl.ts'; import { renderWebPushNotification } from '@/views/mastodon/push.ts'; interface DittoRelayStoreOpts { @@ -217,10 +217,11 @@ export class DittoRelayStore implements NRelay { await relay.event(purifyEvent(event), { signal }); } finally { // This needs to run in steps, and should not block the API from responding. + const signal = AbortSignal.timeout(3000); Promise.allSettled([ this.handleZaps(event), this.updateAuthorData(event, signal), - this.prewarmLinkPreview(event, signal), + this.warmLinkPreview(event, signal), this.generateSetEvents(event), ]) .then(() => @@ -428,12 +429,34 @@ export class DittoRelayStore implements NRelay { } } - private async prewarmLinkPreview(event: NostrEvent, signal?: AbortSignal): Promise { + private async warmLinkPreview(event: NostrEvent, signal?: AbortSignal): Promise { + const { db, conf } = this.opts; + if (event.kind === 1) { const { firstUrl } = parseNoteContent(stripimeta(event.content, event.tags), [], this.opts); + console.log({ firstUrl }); + if (firstUrl) { - await unfurlCardCached(firstUrl, signal); + const linkPreview = await unfurlCard(firstUrl, { conf, signal }); + + console.log(linkPreview); + + if (linkPreview) { + await db.kysely.insertInto('event_stats') + .values({ + event_id: event.id, + replies_count: 0, + reposts_count: 0, + reactions_count: 0, + quotes_count: 0, + reactions: '{}', + zaps_amount: 0, + link_preview: linkPreview, + }) + .onConflict((oc) => oc.column('event_id').doUpdateSet({ link_preview: linkPreview })) + .execute(); + } } } } diff --git a/packages/ditto/storages/hydrate.ts b/packages/ditto/storages/hydrate.ts index 42d8b601..d2c64e90 100644 --- a/packages/ditto/storages/hydrate.ts +++ b/packages/ditto/storages/hydrate.ts @@ -411,6 +411,7 @@ async function gatherEventStats( quotes_count: Math.max(0, row.quotes_count), reactions: row.reactions, zaps_amount: Math.max(0, row.zaps_amount), + link_preview: row.link_preview, })); } diff --git a/packages/ditto/utils/unfurl.ts b/packages/ditto/utils/unfurl.ts index 31905a04..4d551d79 100644 --- a/packages/ditto/utils/unfurl.ts +++ b/packages/ditto/utils/unfurl.ts @@ -1,23 +1,27 @@ -import { cachedLinkPreviewSizeGauge } from '@ditto/metrics'; -import TTLCache from '@isaacs/ttlcache'; import { logi } from '@soapbox/logi'; import { safeFetch } from '@soapbox/safe-fetch'; import DOMPurify from 'isomorphic-dompurify'; import { unfurl } from 'unfurl.js'; -import { Conf } from '@/config.ts'; import { errorJson } from '@/utils/log.ts'; +import type { DittoConf } from '@ditto/conf'; import type { MastodonPreviewCard } from '@ditto/mastoapi/types'; -async function unfurlCard(url: string, signal: AbortSignal): Promise { +interface UnfurlCardOpts { + conf: DittoConf; + signal?: AbortSignal; +} + +export async function unfurlCard(url: string, opts: UnfurlCardOpts): Promise { + const { conf, signal } = opts; try { const result = await unfurl(url, { fetch: (url) => safeFetch(url, { headers: { 'Accept': 'text/html, application/xhtml+xml', - 'User-Agent': Conf.fetchUserAgent, + 'User-Agent': conf.fetchUserAgent, }, signal, }), @@ -54,19 +58,3 @@ async function unfurlCard(url: string, signal: AbortSignal): Promise>(Conf.caches.linkPreview); - -/** Unfurl card from cache if available, otherwise fetch it. */ -export function unfurlCardCached(url: string, signal = AbortSignal.timeout(1000)): Promise { - const cached = previewCardCache.get(url); - if (cached !== undefined) { - return cached; - } else { - const card = unfurlCard(url, signal); - previewCardCache.set(url, card); - cachedLinkPreviewSizeGauge.set(previewCardCache.size); - return card; - } -} diff --git a/packages/ditto/views/mastodon/statuses.ts b/packages/ditto/views/mastodon/statuses.ts index 065ac798..71b6b535 100644 --- a/packages/ditto/views/mastodon/statuses.ts +++ b/packages/ditto/views/mastodon/statuses.ts @@ -6,7 +6,6 @@ import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { nostrDate } from '@/utils.ts'; import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts'; import { findReplyTag } from '@/utils/tags.ts'; -import { unfurlCardCached } from '@/utils/unfurl.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { renderAttachment } from '@/views/mastodon/attachments.ts'; import { renderEmojis } from '@/views/mastodon/emojis.ts'; @@ -42,21 +41,17 @@ async function renderStatus( const mentions = event.mentions?.map((event) => renderMention(event)) ?? []; - const { html, links, firstUrl } = parseNoteContent(stripimeta(event.content, event.tags), mentions, { conf: Conf }); + const { html, links } = parseNoteContent(stripimeta(event.content, event.tags), mentions, { conf: Conf }); - const [card, relatedEvents] = await Promise - .all([ - firstUrl ? unfurlCardCached(firstUrl, AbortSignal.timeout(500)) : null, - viewerPubkey - ? await store.query([ - { kinds: [6], '#e': [event.id], authors: [viewerPubkey], limit: 1 }, - { kinds: [7], '#e': [event.id], authors: [viewerPubkey], limit: 1 }, - { kinds: [9734], '#e': [event.id], authors: [viewerPubkey], limit: 1 }, - { kinds: [10001], '#e': [event.id], authors: [viewerPubkey], limit: 1 }, - { kinds: [10003], '#e': [event.id], authors: [viewerPubkey], limit: 1 }, - ]) - : [], - ]); + const relatedEvents = viewerPubkey + ? await store.query([ + { kinds: [6], '#e': [event.id], authors: [viewerPubkey], limit: 1 }, + { kinds: [7], '#e': [event.id], authors: [viewerPubkey], limit: 1 }, + { kinds: [9734], '#e': [event.id], authors: [viewerPubkey], limit: 1 }, + { kinds: [10001], '#e': [event.id], authors: [viewerPubkey], limit: 1 }, + { kinds: [10003], '#e': [event.id], authors: [viewerPubkey], limit: 1 }, + ]) + : []; const reactionEvent = relatedEvents.find((event) => event.kind === 7); const repostEvent = relatedEvents.find((event) => event.kind === 6); @@ -93,10 +88,12 @@ async function renderStatus( const expiresAt = new Date(Number(event.tags.find(([name]) => name === 'expiration')?.[1]) * 1000); + console.log(event); + return { id: event.id, account, - card, + card: event.event_stats?.link_preview ?? null, content: compatMentions + html, created_at: nostrDate(event.created_at).toISOString(), in_reply_to_id: replyId ?? null,