From 7889ee5db42dec5f58be0a9f2349d04ef16c1738 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 4 Aug 2024 03:06:53 +0530 Subject: [PATCH 01/55] first version of opengraph functionality --- deno.json | 2 + deno.lock | 13 ++ src/app.ts | 3 +- src/middleware/opengraphMiddleware.ts | 250 ++++++++++++++++++++++++++ 4 files changed, 267 insertions(+), 1 deletion(-) create mode 100644 src/middleware/opengraphMiddleware.ts diff --git a/deno.json b/deno.json index 964490c6..d7eff020 100644 --- a/deno.json +++ b/deno.json @@ -44,6 +44,7 @@ "@std/json": "jsr:@std/json@^0.223.0", "@std/media-types": "jsr:@std/media-types@^0.224.1", "@std/streams": "jsr:@std/streams@^0.223.0", + "campfire.js": "https://esm.sh/campfire.js@4.0.0-alpha3", "comlink": "npm:comlink@^4.4.1", "comlink-async-generator": "npm:comlink-async-generator@^0.0.1", "deno-safe-fetch/load": "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts", @@ -64,6 +65,7 @@ "nostr-relaypool": "npm:nostr-relaypool2@0.6.34", "nostr-tools": "npm:nostr-tools@2.5.1", "nostr-wasm": "npm:nostr-wasm@^0.1.0", + "path-to-regexp": "npm:path-to-regexp@6.2.1", "postgres": "https://raw.githubusercontent.com/xyzshantaram/postgres.js/8a9bbce88b3f6425ecaacd99a80372338b157a53/deno/mod.js", "prom-client": "npm:prom-client@^15.1.2", "question-deno": "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/mod.ts", diff --git a/deno.lock b/deno.lock index 5b403d44..17ac0e5e 100644 --- a/deno.lock +++ b/deno.lock @@ -71,6 +71,8 @@ "npm:nostr-tools@^2.5.0": "npm:nostr-tools@2.5.1", "npm:nostr-tools@^2.7.0": "npm:nostr-tools@2.7.0", "npm:nostr-wasm@^0.1.0": "npm:nostr-wasm@0.1.0", + "npm:path-to-regexp@6.2.1": "npm:path-to-regexp@6.2.1", + "npm:path-to-regexp@7.1.0": "npm:path-to-regexp@7.1.0", "npm:postgres@3.4.4": "npm:postgres@3.4.4", "npm:prom-client@^15.1.2": "npm:prom-client@15.1.2", "npm:tldts@^6.0.14": "npm:tldts@6.1.18", @@ -933,6 +935,14 @@ "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", "dependencies": {} }, + "path-to-regexp@6.2.1": { + "integrity": "sha512-JLyh7xT1kizaEvcaXOQwOc2/Yhw6KZOvPf1S8401UyLk86CU79LN3vl7ztXGm/pZ+YjoyAJ4rxmHwbkBXJX+yw==", + "dependencies": {} + }, + "path-to-regexp@7.1.0": { + "integrity": "sha512-ZToe+MbUF4lBqk6dV8GKot4DKfzrxXsplOddH8zN3YK+qw9/McvP7+4ICjZvOne0jQhN4eJwHsX6tT0Ns19fvw==", + "dependencies": {} + }, "picomatch@2.3.1": { "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dependencies": {} @@ -1734,7 +1744,9 @@ "https://deno.land/x/postgres@v0.19.0/utils/deferred.ts": "5420531adb6c3ea29ca8aac57b9b59bd3e4b9a938a4996bbd0947a858f611080", "https://deno.land/x/postgres@v0.19.0/utils/utils.ts": "ca47193ea03ff5b585e487a06f106d367e509263a960b787197ce0c03113a738", "https://deno.land/x/sentry@7.112.2/index.mjs": "04382d5c2f4e233ba389611db46f77943b2a7f6efbeaaf31193f6e586f4366ef", + "https://esm.sh/campfire.js@4.0.0-alpha3": "81a8acce35c2b5dad01f8699d14671e0bef233908d86e2b3f12ebde3a3ac3436", "https://esm.sh/kysely@0.17.1/dist/esm/index-nodeless.js": "9c23bfd307118e3ccd3a9f0ec1261fc3451fb5301aa34aa6f28e05156818755a", + "https://esm.sh/v135/campfire.js@4.0.0-alpha3/denonext/campfire.mjs": "192fa94e07ff4356fe8afa75b59063a217eec1f0d68c26fe4b4547730e6fc3f2", "https://esm.sh/v135/kysely@0.17.1/denonext/dist/esm/index-nodeless.js": "6f73bbf2d73bc7e96cdabf941c4ae8c12f58fd7b441031edec44c029aed9532b", "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts": "3f74ab08cf97d4a3e6994cb79422e9b0069495e017416858121d5ff8ae04ac2a", "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/mod.ts": "5f505cd265aefbcb687cde6f98c79344d3292ee1dd978e85e5ffa84a617c6682", @@ -1841,6 +1853,7 @@ "npm:nostr-relaypool2@0.6.34", "npm:nostr-tools@2.5.1", "npm:nostr-wasm@^0.1.0", + "npm:path-to-regexp@6.2.1", "npm:prom-client@^15.1.2", "npm:tldts@^6.0.14", "npm:tseep@^1.2.1", diff --git a/src/app.ts b/src/app.ts index 66c33424..0b21bf15 100644 --- a/src/app.ts +++ b/src/app.ts @@ -121,6 +121,7 @@ import { requireSigner } from '@/middleware/requireSigner.ts'; import { signerMiddleware } from '@/middleware/signerMiddleware.ts'; import { storeMiddleware } from '@/middleware/storeMiddleware.ts'; import { uploaderMiddleware } from '@/middleware/uploaderMiddleware.ts'; +import { openGraphFrontendController } from '@/middleware/opengraphMiddleware.ts'; interface AppEnv extends HonoEnv { Variables: { @@ -314,7 +315,7 @@ app.use('/oauth/*', notImplementedController); const publicFiles = serveStatic({ root: './public/' }); const staticFiles = serveStatic({ root: './static/' }); -const frontendController = serveStatic({ path: './public/index.html' }); +const frontendController = openGraphFrontendController({ path: './public/index.html' }); // Known frontend routes app.get('/@:acct', frontendController); diff --git a/src/middleware/opengraphMiddleware.ts b/src/middleware/opengraphMiddleware.ts new file mode 100644 index 00000000..9efdd50a --- /dev/null +++ b/src/middleware/opengraphMiddleware.ts @@ -0,0 +1,250 @@ +import { Context, Env, MiddlewareHandler, Next } from '@hono/hono'; +import { serveStatic as baseServeStatic, ServeStaticOptions } from '@hono/hono/serve-static'; +import { match } from 'path-to-regexp'; +import { html, r } from 'campfire.js'; +import { nip05Cache } from '@/utils/nip05.ts'; +import { getAuthor, getEvent } from '@/queries.ts'; +import { nip19 } from 'nostr-tools'; +import { NostrMetadata, NSchema as n } from '@nostrify/nostrify'; +import { getInstanceMetadata } from '@/utils/instance.ts'; +import { Storages } from '@/storages.ts'; +import { Conf } from '@/config.ts'; + +/* + * TODO: implement caching for posts (LRUCache) + */ + +interface OpenGraphTemplateOpts { + title: string; + type: 'article' | 'profile' | 'website'; + url: string; + image?: StatusInfo['image']; + description: string; +} + +interface PathParams { + statusId?: string; + acct?: string; +} + +interface StatusInfo { + description: string; + image?: { + url: string; + w: number; + h: number; + alt?: string; + }; +} + +const store = await Storages.db(); + +const instanceName = async () => { + const meta = await getInstanceMetadata(store, AbortSignal.timeout(1000)); + return meta?.name || 'Ditto'; +}; + +const tpl = async ({ title, type, url, image, description }: OpenGraphTemplateOpts): Promise => + html`\ + + + + + + +${ + image + ? r(html` + + + +${image.alt ? r(html``) : ''} +`) + : '' + } + + + + +${ + image + ? r(html` + +${image.alt ? r(html``) : ''} +`) + : '' + } +`.replace(/\n+/g, '\n'); + +const BLANK_META = (url: string) => + tpl({ + title: 'Ditto', + type: 'website', + url, + description: 'Ditto, a decentralized, self-hosted social media server', + }); + +/** Placeholder to find & replace with metadata. */ +const OG_META_PLACEHOLDER = '' as const; + +/** URL routes to serve metadata on. */ +const SSR_ROUTES = [ + '/@:acct/posts/:statusId', + '/@:acct/:statusId', + '/@:acct', + '/users/:acct/statuses/:statusId', + '/users/:acct', + '/statuses/:statusId', + '/notice/:statusId', +] as const; + +const SSR_ROUTE_MATCHERS = SSR_ROUTES.map((route) => match(route, { decode: decodeURIComponent })); + +const getPathParams = (path: string) => { + for (const matcher of SSR_ROUTE_MATCHERS) { + const result = matcher(path); + if (result) return result.params as PathParams; + } +}; + +const normalizeHandle = async (handle: string) => { + const id = `${handle}`; + const parts = id.match(/(?:(.+))?@(.+)/); + + if (parts) { + const key = `${parts[1] || ''}@${parts[2]}`; + return await nip05Cache.fetch(key, { signal: AbortSignal.timeout(1000) }).then((res) => res.pubkey); + } else if (id.startsWith('npub1')) { + return nip19.decode(id as `npub1${string}`).data; + } + + // shouldn't ever happen for a well-formed link + return ''; +}; + +const getKind0 = async (handle: string | undefined): Promise>> => { + const id = await normalizeHandle(handle || ''); + const kind0 = await getAuthor(id); + + const short = nip19.npubEncode(id).substring(0, 8); + const blank = { name: short, about: `@${short}'s ditto profile` }; + if (!kind0) return blank; + + return Object.assign( + blank, + n.json().pipe(n.metadata()).parse(kind0.content), + ); +}; + +const truncate = (s: string, len: number, ellipsis = '...') => { + if (s.length <= len) return s; + return s.slice(0, len) + ellipsis; +}; + +const getStatus = async (id: string | undefined, handle?: string): Promise => { + const event = await getEvent(id || ''); + if (!event || !id) { + return { description: `A post on Ditto by @${handle}` }; + } + + const res: StatusInfo = { + description: event.content + .replace(/nostr:(npub1(?:[0-9]|[a-z]){58})/g, (_, key: string) => `@${key.slice(0, 8)}`), + }; + + let url: string; + let w: number; + let h: number; + + for (const [tag, ...values] of event.tags) { + if (tag !== 'imeta') continue; + for (const value of values) { + const [item, datum] = value.split(' '); + if (!['dim', 'url'].includes(item)) continue; + if (item === 'dim') { + [w, h] = datum.split('x').map(Number); + } else if (item === 'url') { + url = datum; + } + } + } + + // @ts-ignore conditional assign + if (url && w && h) { + res.image = { url, w, h }; + res.description = res.description.replace(url.trim(), ''); + } + + // needs to be done last incase the image url was surrounded by newlines + res.description = truncate(res.description.trim(), 140); + return res; +}; + +const buildMetaTags = async (params: PathParams, url: string): Promise => { + if (!params.acct && !params.statusId) return await BLANK_META(url); + + const kind0 = await getKind0(params.acct); + const { description, image } = await getStatus(params.statusId || ''); + + if (params.acct && params.statusId) { + return tpl({ + title: `View @${kind0.name}'s post on Ditto`, + type: 'article', + image, + description, + url, + }); + } else if (params.acct) { + return tpl({ + title: `View @${kind0.name}'s profile on Ditto`, + type: 'profile', + description: kind0.about, + url, + }); + } else if (params.statusId) { + return tpl({ + title: `View post on Ditto`, + type: 'profile', + description, + image, + url, + }); + } + + return await BLANK_META(url); +}; + +export const openGraphFrontendController = ( + options: ServeStaticOptions, +): MiddlewareHandler => { + // deno-lint-ignore require-await + return async function serveStatic(c: Context, next: Next) { + let file = ''; + const getContent = async (path: string) => { + try { + if (!file) file = await Deno.readTextFile(path); + if (!file) throw new Error(`File at ${path} was empty!`); + if (c.req.header('accept')?.includes('html') && file.includes(OG_META_PLACEHOLDER)) { + const params = getPathParams(c.req.path); + if (params) { + const meta = await buildMetaTags(params, Conf.local(c.req.path)); + return file.replace(OG_META_PLACEHOLDER, meta); + } + } + return file; + } catch (e) { + console.warn(`${e}`); + } + + return ''; + }; + const pathResolve = (path: string) => { + return `./${path}`; + }; + return baseServeStatic({ + ...options, + getContent, + pathResolve, + })(c, next); + }; +}; From a530313d15c5b9f05f3f4227357d2c1749cf39da Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 4 Aug 2024 03:15:04 +0530 Subject: [PATCH 02/55] update todo --- src/middleware/opengraphMiddleware.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/middleware/opengraphMiddleware.ts b/src/middleware/opengraphMiddleware.ts index 9efdd50a..4dd38d14 100644 --- a/src/middleware/opengraphMiddleware.ts +++ b/src/middleware/opengraphMiddleware.ts @@ -12,6 +12,7 @@ import { Conf } from '@/config.ts'; /* * TODO: implement caching for posts (LRUCache) + * TODO: use profile images if available */ interface OpenGraphTemplateOpts { From 23825037856a4b6cccc25b3bd64e1f4e4216d974 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 4 Aug 2024 03:20:57 +0530 Subject: [PATCH 03/55] flip the orders? --- src/middleware/opengraphMiddleware.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/middleware/opengraphMiddleware.ts b/src/middleware/opengraphMiddleware.ts index 4dd38d14..b725e2c5 100644 --- a/src/middleware/opengraphMiddleware.ts +++ b/src/middleware/opengraphMiddleware.ts @@ -47,19 +47,19 @@ const instanceName = async () => { const tpl = async ({ title, type, url, image, description }: OpenGraphTemplateOpts): Promise => html`\ - - - - - + + + + + ${ image ? r(html` - - - -${image.alt ? r(html``) : ''} + + + +${image.alt ? r(html``) : ''} `) : '' } @@ -71,7 +71,7 @@ ${ image ? r(html` -${image.alt ? r(html``) : ''} +${image.alt ? r(html``) : ''} `) : '' } From 4e522bd90f8dac0d190a0419d75ca203c9fab94f Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 4 Aug 2024 03:35:04 +0530 Subject: [PATCH 04/55] always use summary type for twitter --- src/middleware/opengraphMiddleware.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/middleware/opengraphMiddleware.ts b/src/middleware/opengraphMiddleware.ts index b725e2c5..7f05add5 100644 --- a/src/middleware/opengraphMiddleware.ts +++ b/src/middleware/opengraphMiddleware.ts @@ -64,7 +64,7 @@ ${image.alt ? r(html``) : ' : '' } - + ${ From b8eb549190264700aafec4b88d60ebdbf4a6d588 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 4 Aug 2024 04:35:56 +0530 Subject: [PATCH 05/55] opengraphFrontendController --> serveStaticWithOG --- src/app.ts | 4 ++-- src/middleware/opengraphMiddleware.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/app.ts b/src/app.ts index 0b21bf15..380820e4 100644 --- a/src/app.ts +++ b/src/app.ts @@ -121,7 +121,7 @@ import { requireSigner } from '@/middleware/requireSigner.ts'; import { signerMiddleware } from '@/middleware/signerMiddleware.ts'; import { storeMiddleware } from '@/middleware/storeMiddleware.ts'; import { uploaderMiddleware } from '@/middleware/uploaderMiddleware.ts'; -import { openGraphFrontendController } from '@/middleware/opengraphMiddleware.ts'; +import { serveStaticWithOG } from '@/middleware/opengraphMiddleware.ts'; interface AppEnv extends HonoEnv { Variables: { @@ -315,7 +315,7 @@ app.use('/oauth/*', notImplementedController); const publicFiles = serveStatic({ root: './public/' }); const staticFiles = serveStatic({ root: './static/' }); -const frontendController = openGraphFrontendController({ path: './public/index.html' }); +const frontendController = serveStaticWithOG({ path: './public/index.html' }); // Known frontend routes app.get('/@:acct', frontendController); diff --git a/src/middleware/opengraphMiddleware.ts b/src/middleware/opengraphMiddleware.ts index 7f05add5..3ce9a7e5 100644 --- a/src/middleware/opengraphMiddleware.ts +++ b/src/middleware/opengraphMiddleware.ts @@ -215,7 +215,7 @@ const buildMetaTags = async (params: PathParams, url: string): Promise = return await BLANK_META(url); }; -export const openGraphFrontendController = ( +export const serveStaticWithOG = ( options: ServeStaticOptions, ): MiddlewareHandler => { // deno-lint-ignore require-await From 5ee0c1e8529e4a157617fc2bdd166f3557e75bc6 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 4 Aug 2024 04:36:36 +0530 Subject: [PATCH 06/55] opengraphMiddleware.ts --> serveStaticWithOG.ts --- src/app.ts | 2 +- src/middleware/{opengraphMiddleware.ts => serveStaticWithOG.ts} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/middleware/{opengraphMiddleware.ts => serveStaticWithOG.ts} (100%) diff --git a/src/app.ts b/src/app.ts index 380820e4..cec7f131 100644 --- a/src/app.ts +++ b/src/app.ts @@ -121,7 +121,7 @@ import { requireSigner } from '@/middleware/requireSigner.ts'; import { signerMiddleware } from '@/middleware/signerMiddleware.ts'; import { storeMiddleware } from '@/middleware/storeMiddleware.ts'; import { uploaderMiddleware } from '@/middleware/uploaderMiddleware.ts'; -import { serveStaticWithOG } from '@/middleware/opengraphMiddleware.ts'; +import { serveStaticWithOG } from './middleware/serveStaticWithOG.ts'; interface AppEnv extends HonoEnv { Variables: { diff --git a/src/middleware/opengraphMiddleware.ts b/src/middleware/serveStaticWithOG.ts similarity index 100% rename from src/middleware/opengraphMiddleware.ts rename to src/middleware/serveStaticWithOG.ts From baa6e318f343c6847c78226ee445f573f5349efe Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 4 Aug 2024 04:43:47 +0530 Subject: [PATCH 07/55] remove premature optimization in serveStaticWithOG --- src/middleware/serveStaticWithOG.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/middleware/serveStaticWithOG.ts b/src/middleware/serveStaticWithOG.ts index 3ce9a7e5..70261854 100644 --- a/src/middleware/serveStaticWithOG.ts +++ b/src/middleware/serveStaticWithOG.ts @@ -222,11 +222,13 @@ export const serveStaticWithOG = ( return async function serveStatic(c: Context, next: Next) { let file = ''; const getContent = async (path: string) => { + console.log('here'); try { if (!file) file = await Deno.readTextFile(path); if (!file) throw new Error(`File at ${path} was empty!`); - if (c.req.header('accept')?.includes('html') && file.includes(OG_META_PLACEHOLDER)) { + if (file.includes(OG_META_PLACEHOLDER)) { const params = getPathParams(c.req.path); + console.log(params); if (params) { const meta = await buildMetaTags(params, Conf.local(c.req.path)); return file.replace(OG_META_PLACEHOLDER, meta); From 8c9facdad7e68f691c0440861c1dae6c4f780cfd Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 4 Aug 2024 04:49:24 +0530 Subject: [PATCH 08/55] remove unnecessary logs and use nip05s if avl --- src/middleware/serveStaticWithOG.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/middleware/serveStaticWithOG.ts b/src/middleware/serveStaticWithOG.ts index 70261854..d6f713ba 100644 --- a/src/middleware/serveStaticWithOG.ts +++ b/src/middleware/serveStaticWithOG.ts @@ -123,7 +123,7 @@ const normalizeHandle = async (handle: string) => { return ''; }; -const getKind0 = async (handle: string | undefined): Promise>> => { +const getKind0 = async (handle: string | undefined): Promise> => { const id = await normalizeHandle(handle || ''); const kind0 = await getAuthor(id); @@ -197,9 +197,9 @@ const buildMetaTags = async (params: PathParams, url: string): Promise = }); } else if (params.acct) { return tpl({ - title: `View @${kind0.name}'s profile on Ditto`, + title: `View @${kind0.nip05 || kind0.name || 'npub1xxx'}'s profile on Ditto`, type: 'profile', - description: kind0.about, + description: kind0.about || '', url, }); } else if (params.statusId) { @@ -222,13 +222,11 @@ export const serveStaticWithOG = ( return async function serveStatic(c: Context, next: Next) { let file = ''; const getContent = async (path: string) => { - console.log('here'); try { if (!file) file = await Deno.readTextFile(path); if (!file) throw new Error(`File at ${path} was empty!`); if (file.includes(OG_META_PLACEHOLDER)) { const params = getPathParams(c.req.path); - console.log(params); if (params) { const meta = await buildMetaTags(params, Conf.local(c.req.path)); return file.replace(OG_META_PLACEHOLDER, meta); From 80f8932513d53b4714e71a61acefbb28df8d6792 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 4 Aug 2024 04:54:50 +0530 Subject: [PATCH 09/55] send profile pictures where possible --- src/middleware/serveStaticWithOG.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/middleware/serveStaticWithOG.ts b/src/middleware/serveStaticWithOG.ts index d6f713ba..f8939229 100644 --- a/src/middleware/serveStaticWithOG.ts +++ b/src/middleware/serveStaticWithOG.ts @@ -123,7 +123,7 @@ const normalizeHandle = async (handle: string) => { return ''; }; -const getKind0 = async (handle: string | undefined): Promise> => { +const getKind0 = async (handle: string | undefined): Promise => { const id = await normalizeHandle(handle || ''); const kind0 = await getAuthor(id); @@ -186,10 +186,11 @@ const buildMetaTags = async (params: PathParams, url: string): Promise = const kind0 = await getKind0(params.acct); const { description, image } = await getStatus(params.statusId || ''); + const handle = kind0.nip05 || kind0.name || 'npub1xxx'; if (params.acct && params.statusId) { return tpl({ - title: `View @${kind0.name}'s post on Ditto`, + title: `View @${handle}'s post on Ditto`, type: 'article', image, description, @@ -197,10 +198,18 @@ const buildMetaTags = async (params: PathParams, url: string): Promise = }); } else if (params.acct) { return tpl({ - title: `View @${kind0.nip05 || kind0.name || 'npub1xxx'}'s profile on Ditto`, + title: `View @${handle}'s profile on Ditto`, type: 'profile', description: kind0.about || '', url, + image: kind0.picture + ? { + url: kind0.picture, + // Time will tell if this is fine. + h: 150, + w: 150, + } + : undefined, }); } else if (params.statusId) { return tpl({ From 7aaa1bc0304fb9295f0ee8927f999feb4106202d Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 4 Aug 2024 05:52:24 +0530 Subject: [PATCH 10/55] generate metadata for bech32 encoded entities --- src/middleware/serveStaticWithOG.ts | 31 +++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/middleware/serveStaticWithOG.ts b/src/middleware/serveStaticWithOG.ts index f8939229..97764b8c 100644 --- a/src/middleware/serveStaticWithOG.ts +++ b/src/middleware/serveStaticWithOG.ts @@ -23,10 +23,7 @@ interface OpenGraphTemplateOpts { description: string; } -interface PathParams { - statusId?: string; - acct?: string; -} +type PathParams = Partial>; interface StatusInfo { description: string; @@ -97,6 +94,11 @@ const SSR_ROUTES = [ '/users/:acct', '/statuses/:statusId', '/notice/:statusId', + '/posts/:statusId', + '/note:note', + '/nevent:nevent', + '/nprofile:nprofile', + '/npub:npub', ] as const; const SSR_ROUTE_MATCHERS = SSR_ROUTES.map((route) => match(route, { decode: decodeURIComponent })); @@ -104,19 +106,36 @@ const SSR_ROUTE_MATCHERS = SSR_ROUTES.map((route) => match(route, { decode: deco const getPathParams = (path: string) => { for (const matcher of SSR_ROUTE_MATCHERS) { const result = matcher(path); - if (result) return result.params as PathParams; + if (!result) continue; + const params = result.params as PathParams; + if (params.nevent) { + const decoded = nip19.decode(`nevent${params.nevent}`).data as nip19.EventPointer; + params.statusId = decoded.id; + } else if (params.note) { + params.statusId = nip19.decode(`note${params.note}`).data as string; + } + + if (params.nprofile) { + const decoded = nip19.decode(`nprofile${params.nprofile}`).data as nip19.ProfilePointer; + params.acct = decoded.pubkey; + } else if (params.npub) { + params.acct = nip19.decode(`npub${params.npub}`).data as string; + } + console.log(params); + return params; } }; const normalizeHandle = async (handle: string) => { const id = `${handle}`; const parts = id.match(/(?:(.+))?@(.+)/); - if (parts) { const key = `${parts[1] || ''}@${parts[2]}`; return await nip05Cache.fetch(key, { signal: AbortSignal.timeout(1000) }).then((res) => res.pubkey); } else if (id.startsWith('npub1')) { return nip19.decode(id as `npub1${string}`).data; + } else if (/(?:[0-9]|[a-f]){64}/.test(id)) { + return id; } // shouldn't ever happen for a well-formed link From 435ca60b0ddedc11c39a443956a6c27b5af78f28 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 4 Aug 2024 05:52:51 +0530 Subject: [PATCH 11/55] update todo --- src/middleware/serveStaticWithOG.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/middleware/serveStaticWithOG.ts b/src/middleware/serveStaticWithOG.ts index 97764b8c..0630a6b0 100644 --- a/src/middleware/serveStaticWithOG.ts +++ b/src/middleware/serveStaticWithOG.ts @@ -12,7 +12,6 @@ import { Conf } from '@/config.ts'; /* * TODO: implement caching for posts (LRUCache) - * TODO: use profile images if available */ interface OpenGraphTemplateOpts { From 77d73c47eeb2a17d6538cfc4502d4ba4867ceb02 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 4 Aug 2024 11:07:29 +0530 Subject: [PATCH 12/55] move a bunch of utils to their own file --- src/middleware/serveStaticWithOG.ts | 180 ++++------------------------ src/utils/og-metadata.ts | 148 +++++++++++++++++++++++ 2 files changed, 170 insertions(+), 158 deletions(-) create mode 100644 src/utils/og-metadata.ts diff --git a/src/middleware/serveStaticWithOG.ts b/src/middleware/serveStaticWithOG.ts index 0630a6b0..3adf64f3 100644 --- a/src/middleware/serveStaticWithOG.ts +++ b/src/middleware/serveStaticWithOG.ts @@ -1,53 +1,35 @@ import { Context, Env, MiddlewareHandler, Next } from '@hono/hono'; import { serveStatic as baseServeStatic, ServeStaticOptions } from '@hono/hono/serve-static'; -import { match } from 'path-to-regexp'; import { html, r } from 'campfire.js'; -import { nip05Cache } from '@/utils/nip05.ts'; -import { getAuthor, getEvent } from '@/queries.ts'; -import { nip19 } from 'nostr-tools'; -import { NostrMetadata, NSchema as n } from '@nostrify/nostrify'; -import { getInstanceMetadata } from '@/utils/instance.ts'; -import { Storages } from '@/storages.ts'; import { Conf } from '@/config.ts'; +import { + getInstanceName, + getPathParams, + getProfileInfo, + getStatusInfo, + OpenGraphTemplateOpts, + PathParams, +} from '@/utils/og-metadata.ts'; + +/** Placeholder to find & replace with metadata. */ +const OG_META_PLACEHOLDER = '' as const; /* * TODO: implement caching for posts (LRUCache) */ -interface OpenGraphTemplateOpts { - title: string; - type: 'article' | 'profile' | 'website'; - url: string; - image?: StatusInfo['image']; - description: string; -} - -type PathParams = Partial>; - -interface StatusInfo { - description: string; - image?: { - url: string; - w: number; - h: number; - alt?: string; - }; -} - -const store = await Storages.db(); - -const instanceName = async () => { - const meta = await getInstanceMetadata(store, AbortSignal.timeout(1000)); - return meta?.name || 'Ditto'; -}; - +/** + * Builds a series of meta tags from supplied metadata for injection into the served HTML page. + * @param opts the metadata to use to fill the template. + * @returns the built OpenGraph metadata. + */ const tpl = async ({ title, type, url, image, description }: OpenGraphTemplateOpts): Promise => html`\ - + ${ image @@ -81,129 +63,11 @@ const BLANK_META = (url: string) => description: 'Ditto, a decentralized, self-hosted social media server', }); -/** Placeholder to find & replace with metadata. */ -const OG_META_PLACEHOLDER = '' as const; - -/** URL routes to serve metadata on. */ -const SSR_ROUTES = [ - '/@:acct/posts/:statusId', - '/@:acct/:statusId', - '/@:acct', - '/users/:acct/statuses/:statusId', - '/users/:acct', - '/statuses/:statusId', - '/notice/:statusId', - '/posts/:statusId', - '/note:note', - '/nevent:nevent', - '/nprofile:nprofile', - '/npub:npub', -] as const; - -const SSR_ROUTE_MATCHERS = SSR_ROUTES.map((route) => match(route, { decode: decodeURIComponent })); - -const getPathParams = (path: string) => { - for (const matcher of SSR_ROUTE_MATCHERS) { - const result = matcher(path); - if (!result) continue; - const params = result.params as PathParams; - if (params.nevent) { - const decoded = nip19.decode(`nevent${params.nevent}`).data as nip19.EventPointer; - params.statusId = decoded.id; - } else if (params.note) { - params.statusId = nip19.decode(`note${params.note}`).data as string; - } - - if (params.nprofile) { - const decoded = nip19.decode(`nprofile${params.nprofile}`).data as nip19.ProfilePointer; - params.acct = decoded.pubkey; - } else if (params.npub) { - params.acct = nip19.decode(`npub${params.npub}`).data as string; - } - console.log(params); - return params; - } -}; - -const normalizeHandle = async (handle: string) => { - const id = `${handle}`; - const parts = id.match(/(?:(.+))?@(.+)/); - if (parts) { - const key = `${parts[1] || ''}@${parts[2]}`; - return await nip05Cache.fetch(key, { signal: AbortSignal.timeout(1000) }).then((res) => res.pubkey); - } else if (id.startsWith('npub1')) { - return nip19.decode(id as `npub1${string}`).data; - } else if (/(?:[0-9]|[a-f]){64}/.test(id)) { - return id; - } - - // shouldn't ever happen for a well-formed link - return ''; -}; - -const getKind0 = async (handle: string | undefined): Promise => { - const id = await normalizeHandle(handle || ''); - const kind0 = await getAuthor(id); - - const short = nip19.npubEncode(id).substring(0, 8); - const blank = { name: short, about: `@${short}'s ditto profile` }; - if (!kind0) return blank; - - return Object.assign( - blank, - n.json().pipe(n.metadata()).parse(kind0.content), - ); -}; - -const truncate = (s: string, len: number, ellipsis = '...') => { - if (s.length <= len) return s; - return s.slice(0, len) + ellipsis; -}; - -const getStatus = async (id: string | undefined, handle?: string): Promise => { - const event = await getEvent(id || ''); - if (!event || !id) { - return { description: `A post on Ditto by @${handle}` }; - } - - const res: StatusInfo = { - description: event.content - .replace(/nostr:(npub1(?:[0-9]|[a-z]){58})/g, (_, key: string) => `@${key.slice(0, 8)}`), - }; - - let url: string; - let w: number; - let h: number; - - for (const [tag, ...values] of event.tags) { - if (tag !== 'imeta') continue; - for (const value of values) { - const [item, datum] = value.split(' '); - if (!['dim', 'url'].includes(item)) continue; - if (item === 'dim') { - [w, h] = datum.split('x').map(Number); - } else if (item === 'url') { - url = datum; - } - } - } - - // @ts-ignore conditional assign - if (url && w && h) { - res.image = { url, w, h }; - res.description = res.description.replace(url.trim(), ''); - } - - // needs to be done last incase the image url was surrounded by newlines - res.description = truncate(res.description.trim(), 140); - return res; -}; - const buildMetaTags = async (params: PathParams, url: string): Promise => { if (!params.acct && !params.statusId) return await BLANK_META(url); - const kind0 = await getKind0(params.acct); - const { description, image } = await getStatus(params.statusId || ''); + const kind0 = await getProfileInfo(params.acct); + const { description, image } = await getStatusInfo(params.statusId || ''); const handle = kind0.nip05 || kind0.name || 'npub1xxx'; if (params.acct && params.statusId) { @@ -242,9 +106,9 @@ const buildMetaTags = async (params: PathParams, url: string): Promise = return await BLANK_META(url); }; -export const serveStaticWithOG = ( +export function serveStaticWithOG( options: ServeStaticOptions, -): MiddlewareHandler => { +): MiddlewareHandler { // deno-lint-ignore require-await return async function serveStatic(c: Context, next: Next) { let file = ''; @@ -275,4 +139,4 @@ export const serveStaticWithOG = ( pathResolve, })(c, next); }; -}; +} diff --git a/src/utils/og-metadata.ts b/src/utils/og-metadata.ts new file mode 100644 index 00000000..94cee4ca --- /dev/null +++ b/src/utils/og-metadata.ts @@ -0,0 +1,148 @@ +import { Storages } from '@/storages.ts'; +import { getInstanceMetadata } from '@/utils/instance.ts'; +import { nip19 } from 'nostr-tools'; +import { match } from 'path-to-regexp'; +import { nip05Cache } from '@/utils/nip05.ts'; +import { NostrMetadata, NSchema as n } from '@nostrify/nostrify'; +import { getAuthor, getEvent } from '@/queries.ts'; + +export interface OpenGraphTemplateOpts { + title: string; + type: 'article' | 'profile' | 'website'; + url: string; + image?: StatusInfo['image']; + description: string; +} + +export type PathParams = Partial>; + +interface StatusInfo { + description: string; + image?: { + url: string; + w: number; + h: number; + alt?: string; + }; +} + +const store = await Storages.db(); +export const getInstanceName = async () => { + const meta = await getInstanceMetadata(store, AbortSignal.timeout(1000)); + return meta?.name || 'Ditto'; +}; + +/** URL routes to serve metadata on. */ +const SSR_ROUTES = [ + '/@:acct/posts/:statusId', + '/@:acct/:statusId', + '/@:acct', + '/users/:acct/statuses/:statusId', + '/users/:acct', + '/statuses/:statusId', + '/notice/:statusId', + '/posts/:statusId', + '/note:note', + '/nevent:nevent', + '/nprofile:nprofile', + '/npub:npub', +] as const; + +const SSR_ROUTE_MATCHERS = SSR_ROUTES.map((route) => match(route, { decode: decodeURIComponent })); + +export function getPathParams(path: string) { + for (const matcher of SSR_ROUTE_MATCHERS) { + const result = matcher(path); + if (!result) continue; + const params = result.params as PathParams; + if (params.nevent) { + const decoded = nip19.decode(`nevent${params.nevent}`).data as nip19.EventPointer; + params.statusId = decoded.id; + } else if (params.note) { + params.statusId = nip19.decode(`note${params.note}`).data as string; + } + + if (params.nprofile) { + const decoded = nip19.decode(`nprofile${params.nprofile}`).data as nip19.ProfilePointer; + params.acct = decoded.pubkey; + } else if (params.npub) { + params.acct = nip19.decode(`npub${params.npub}`).data as string; + } + console.log(params); + return params; + } +} + +async function urlParamToPubkey(handle: string) { + const id = `${handle}`; + const parts = id.match(/(?:(.+))?@(.+)/); + if (parts) { + const key = `${parts[1] || ''}@${parts[2]}`; + return await nip05Cache.fetch(key, { signal: AbortSignal.timeout(1000) }).then((res) => res.pubkey); + } else if (id.startsWith('npub1')) { + return nip19.decode(id as `npub1${string}`).data; + } else if (/(?:[0-9]|[a-f]){64}/.test(id)) { + return id; + } + + // shouldn't ever happen for a well-formed link + return ''; +} + +export async function getProfileInfo(handle: string | undefined): Promise { + const id = await urlParamToPubkey(handle || ''); + const kind0 = await getAuthor(id); + + const short = nip19.npubEncode(id).substring(0, 8); + const blank = { name: short, about: `@${short}'s ditto profile` }; + if (!kind0) return blank; + + return Object.assign( + blank, + n.json().pipe(n.metadata()).parse(kind0.content), + ); +} + +const truncate = (s: string, len: number, ellipsis = '...') => { + if (s.length <= len) return s; + return s.slice(0, len) + ellipsis; +}; + +export async function getStatusInfo(id: string | undefined, handle?: string): Promise { + const event = await getEvent(id || ''); + if (!event || !id) { + return { description: `A post on Ditto by @${handle}` }; + } + + const res: StatusInfo = { + description: event.content + .replace(/nostr:(npub1(?:[0-9]|[a-z]){58})/g, (_, key: string) => `@${key.slice(0, 8)}`), + }; + + let url: string; + let w: number; + let h: number; + + for (const [tag, ...values] of event.tags) { + if (tag !== 'imeta') continue; + for (const value of values) { + const [item, datum] = value.split(' '); + if (!['dim', 'url'].includes(item)) continue; + if (item === 'dim') { + [w, h] = datum.split('x').map(Number); + } else if (item === 'url') { + url = datum; + } + } + } + + // @ts-ignore conditional assign + if (url && w && h) { + res.image = { url, w, h }; + res.description = res.description.replace(url.trim(), ''); + } + + // needs to be done last incase the image url was surrounded by newlines + res.description = truncate(res.description.trim(), 140); + return res; +} From 6aa49c12e162aef3816de1b11ddd384f07249c12 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 4 Aug 2024 11:10:33 +0530 Subject: [PATCH 13/55] steal html util from campfire, remove external dependency on cf --- deno.json | 1 - src/middleware/serveStaticWithOG.ts | 2 +- src/utils/html.ts | 58 +++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 src/utils/html.ts diff --git a/deno.json b/deno.json index d7eff020..317826f5 100644 --- a/deno.json +++ b/deno.json @@ -44,7 +44,6 @@ "@std/json": "jsr:@std/json@^0.223.0", "@std/media-types": "jsr:@std/media-types@^0.224.1", "@std/streams": "jsr:@std/streams@^0.223.0", - "campfire.js": "https://esm.sh/campfire.js@4.0.0-alpha3", "comlink": "npm:comlink@^4.4.1", "comlink-async-generator": "npm:comlink-async-generator@^0.0.1", "deno-safe-fetch/load": "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts", diff --git a/src/middleware/serveStaticWithOG.ts b/src/middleware/serveStaticWithOG.ts index 3adf64f3..8f679a90 100644 --- a/src/middleware/serveStaticWithOG.ts +++ b/src/middleware/serveStaticWithOG.ts @@ -1,6 +1,6 @@ import { Context, Env, MiddlewareHandler, Next } from '@hono/hono'; import { serveStatic as baseServeStatic, ServeStaticOptions } from '@hono/hono/serve-static'; -import { html, r } from 'campfire.js'; +import { html, r } from '@/utils/html.ts'; import { Conf } from '@/config.ts'; import { getInstanceName, diff --git a/src/utils/html.ts b/src/utils/html.ts new file mode 100644 index 00000000..5e9ff918 --- /dev/null +++ b/src/utils/html.ts @@ -0,0 +1,58 @@ +interface RawHtml { + raw: true; + contents: string; +} + +/** + * Options for r() + */ +interface RawHtmlOptions { + joiner?: string; +} + +export function escape(str: string) { + if (!str) return ''; + + return str.replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +/** + * Prevent values from being escaped by html``. + * @param val Any value. + * @returns An object that tells html`` to not escape `val` while building the HTML string. + */ +export function r(val: any, options?: RawHtmlOptions): RawHtml { + return { + raw: true, + contents: Array.isArray(val) ? val.join(options?.joiner ?? ' ') : val.toString(), + }; +} + +/** + * @param strings The constant portions of the template string. + * @param values The templated values. + * @returns The built HTML. + * @example + * ``` + * const unsafe = `oops `; + * testing.innerHTML = html`foo bar baz ${unsafe}`; + * console.assert(testing === "foo bar baz oops%20%3Cscript%3Ealert%281%29%3C/script%3E"); + * ``` + */ +export function html(strings: TemplateStringsArray, ...values: (string | number | RawHtml)[]) { + const built = []; + for (let i = 0; i < strings.length; i++) { + built.push(strings[i] || ''); + const val = values[i]; + if (typeof val !== 'undefined' && typeof val !== 'object') { + built.push(escape((val || '').toString())); + } else { + built.push(val?.contents || ''); + } + } + return built.join(''); +} From b95e31cd5d8a33609ce0d8b13ec2d9a8c3a23dcc Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 4 Aug 2024 12:07:24 +0530 Subject: [PATCH 14/55] rework kind 0 handling --- src/utils/og-metadata.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/utils/og-metadata.ts b/src/utils/og-metadata.ts index 94cee4ca..c368a07d 100644 --- a/src/utils/og-metadata.ts +++ b/src/utils/og-metadata.ts @@ -68,16 +68,15 @@ export function getPathParams(path: string) { } else if (params.npub) { params.acct = nip19.decode(`npub${params.npub}`).data as string; } - console.log(params); return params; } } async function urlParamToPubkey(handle: string) { const id = `${handle}`; - const parts = id.match(/(?:(.+))?@(.+)/); + const parts = id.match(/(?:(.+)@)?(.+)/); if (parts) { - const key = `${parts[1] || ''}@${parts[2]}`; + const key = `${parts[1] ? (parts[1] + '@') : ''}${parts[2]}`; return await nip05Cache.fetch(key, { signal: AbortSignal.timeout(1000) }).then((res) => res.pubkey); } else if (id.startsWith('npub1')) { return nip19.decode(id as `npub1${string}`).data; From 8b8a801da8f51e783d9071b3ec32a97569fc2713 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 4 Aug 2024 12:21:31 +0530 Subject: [PATCH 15/55] handle underscores in nip05 correctly --- src/middleware/serveStaticWithOG.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/middleware/serveStaticWithOG.ts b/src/middleware/serveStaticWithOG.ts index 8f679a90..0d22186b 100644 --- a/src/middleware/serveStaticWithOG.ts +++ b/src/middleware/serveStaticWithOG.ts @@ -68,7 +68,7 @@ const buildMetaTags = async (params: PathParams, url: string): Promise = const kind0 = await getProfileInfo(params.acct); const { description, image } = await getStatusInfo(params.statusId || ''); - const handle = kind0.nip05 || kind0.name || 'npub1xxx'; + const handle = kind0.nip05?.replace(/^@_/, '') || kind0.name || 'npub1xxx'; if (params.acct && params.statusId) { return tpl({ From 95ef308d0e20790d7b5a2a36b03b1c137dd43b7c Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 4 Aug 2024 12:25:30 +0530 Subject: [PATCH 16/55] fix typo in nip05 cleaner regex --- src/middleware/serveStaticWithOG.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/middleware/serveStaticWithOG.ts b/src/middleware/serveStaticWithOG.ts index 0d22186b..486ef76b 100644 --- a/src/middleware/serveStaticWithOG.ts +++ b/src/middleware/serveStaticWithOG.ts @@ -67,8 +67,10 @@ const buildMetaTags = async (params: PathParams, url: string): Promise = if (!params.acct && !params.statusId) return await BLANK_META(url); const kind0 = await getProfileInfo(params.acct); + console.log(params.acct); const { description, image } = await getStatusInfo(params.statusId || ''); - const handle = kind0.nip05?.replace(/^@_/, '') || kind0.name || 'npub1xxx'; + const handle = kind0.nip05?.replace(/^_@/, '') || kind0.name || 'npub1xxx'; + console.log({ n: kind0.nip05, handle }); if (params.acct && params.statusId) { return tpl({ From 4c83d6d1b221eb4767e2f3283242de05b19e43c5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 4 Aug 2024 13:26:05 -0500 Subject: [PATCH 17/55] Do frontendController like I want to --- src/app.ts | 3 +- .../frontend.ts} | 52 ++++++------------- src/utils/og-metadata.ts | 1 + 3 files changed, 19 insertions(+), 37 deletions(-) rename src/{middleware/serveStaticWithOG.ts => controllers/frontend.ts} (72%) diff --git a/src/app.ts b/src/app.ts index cec7f131..a99fd255 100644 --- a/src/app.ts +++ b/src/app.ts @@ -108,6 +108,7 @@ import { trendingTagsController, } from '@/controllers/api/trends.ts'; import { errorHandler } from '@/controllers/error.ts'; +import { frontendController } from '@/controllers/frontend.ts'; import { metricsController } from '@/controllers/metrics.ts'; import { indexController } from '@/controllers/site.ts'; import '@/startup.ts'; @@ -121,7 +122,6 @@ import { requireSigner } from '@/middleware/requireSigner.ts'; import { signerMiddleware } from '@/middleware/signerMiddleware.ts'; import { storeMiddleware } from '@/middleware/storeMiddleware.ts'; import { uploaderMiddleware } from '@/middleware/uploaderMiddleware.ts'; -import { serveStaticWithOG } from './middleware/serveStaticWithOG.ts'; interface AppEnv extends HonoEnv { Variables: { @@ -315,7 +315,6 @@ app.use('/oauth/*', notImplementedController); const publicFiles = serveStatic({ root: './public/' }); const staticFiles = serveStatic({ root: './static/' }); -const frontendController = serveStaticWithOG({ path: './public/index.html' }); // Known frontend routes app.get('/@:acct', frontendController); diff --git a/src/middleware/serveStaticWithOG.ts b/src/controllers/frontend.ts similarity index 72% rename from src/middleware/serveStaticWithOG.ts rename to src/controllers/frontend.ts index 486ef76b..8a1c5c0c 100644 --- a/src/middleware/serveStaticWithOG.ts +++ b/src/controllers/frontend.ts @@ -1,5 +1,3 @@ -import { Context, Env, MiddlewareHandler, Next } from '@hono/hono'; -import { serveStatic as baseServeStatic, ServeStaticOptions } from '@hono/hono/serve-static'; import { html, r } from '@/utils/html.ts'; import { Conf } from '@/config.ts'; import { @@ -10,6 +8,7 @@ import { OpenGraphTemplateOpts, PathParams, } from '@/utils/og-metadata.ts'; +import { AppMiddleware } from '@/app.ts'; /** Placeholder to find & replace with metadata. */ const OG_META_PLACEHOLDER = '' as const; @@ -108,37 +107,20 @@ const buildMetaTags = async (params: PathParams, url: string): Promise = return await BLANK_META(url); }; -export function serveStaticWithOG( - options: ServeStaticOptions, -): MiddlewareHandler { - // deno-lint-ignore require-await - return async function serveStatic(c: Context, next: Next) { - let file = ''; - const getContent = async (path: string) => { - try { - if (!file) file = await Deno.readTextFile(path); - if (!file) throw new Error(`File at ${path} was empty!`); - if (file.includes(OG_META_PLACEHOLDER)) { - const params = getPathParams(c.req.path); - if (params) { - const meta = await buildMetaTags(params, Conf.local(c.req.path)); - return file.replace(OG_META_PLACEHOLDER, meta); - } - } - return file; - } catch (e) { - console.warn(`${e}`); - } +export const frontendController: AppMiddleware = async (c, next) => { + try { + const content = await Deno.readTextFile(new URL('../../public/index.html', import.meta.url)); + if (content.includes(OG_META_PLACEHOLDER)) { + const params = getPathParams(c.req.path); - return ''; - }; - const pathResolve = (path: string) => { - return `./${path}`; - }; - return baseServeStatic({ - ...options, - getContent, - pathResolve, - })(c, next); - }; -} + if (params) { + const meta = await buildMetaTags(params, Conf.local(c.req.path)); + return c.html(content.replace(OG_META_PLACEHOLDER, meta)); + } + } + return c.html(content); + } catch (e) { + console.log(e); + await next(); + } +}; diff --git a/src/utils/og-metadata.ts b/src/utils/og-metadata.ts index c368a07d..a9685961 100644 --- a/src/utils/og-metadata.ts +++ b/src/utils/og-metadata.ts @@ -27,6 +27,7 @@ interface StatusInfo { } const store = await Storages.db(); + export const getInstanceName = async () => { const meta = await getInstanceMetadata(store, AbortSignal.timeout(1000)); return meta?.name || 'Ditto'; From 2cafc2014f9f17304a5fa9f7855d4407f145163a Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 15:13:09 +0530 Subject: [PATCH 18/55] remove cf from lockfile --- deno.lock | 2 -- 1 file changed, 2 deletions(-) diff --git a/deno.lock b/deno.lock index 17ac0e5e..dc6ecaa2 100644 --- a/deno.lock +++ b/deno.lock @@ -1744,9 +1744,7 @@ "https://deno.land/x/postgres@v0.19.0/utils/deferred.ts": "5420531adb6c3ea29ca8aac57b9b59bd3e4b9a938a4996bbd0947a858f611080", "https://deno.land/x/postgres@v0.19.0/utils/utils.ts": "ca47193ea03ff5b585e487a06f106d367e509263a960b787197ce0c03113a738", "https://deno.land/x/sentry@7.112.2/index.mjs": "04382d5c2f4e233ba389611db46f77943b2a7f6efbeaaf31193f6e586f4366ef", - "https://esm.sh/campfire.js@4.0.0-alpha3": "81a8acce35c2b5dad01f8699d14671e0bef233908d86e2b3f12ebde3a3ac3436", "https://esm.sh/kysely@0.17.1/dist/esm/index-nodeless.js": "9c23bfd307118e3ccd3a9f0ec1261fc3451fb5301aa34aa6f28e05156818755a", - "https://esm.sh/v135/campfire.js@4.0.0-alpha3/denonext/campfire.mjs": "192fa94e07ff4356fe8afa75b59063a217eec1f0d68c26fe4b4547730e6fc3f2", "https://esm.sh/v135/kysely@0.17.1/denonext/dist/esm/index-nodeless.js": "6f73bbf2d73bc7e96cdabf941c4ae8c12f58fd7b441031edec44c029aed9532b", "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts": "3f74ab08cf97d4a3e6994cb79422e9b0069495e017416858121d5ff8ae04ac2a", "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/mod.ts": "5f505cd265aefbcb687cde6f98c79344d3292ee1dd978e85e5ffa84a617c6682", From c0a6d48d555667d5728835538de5ff4829f4bba1 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 15:14:51 +0530 Subject: [PATCH 19/55] rename OG_META_PLACEHOLDER --> META_PLACEHOLDER --- src/controllers/frontend.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index 8a1c5c0c..6fc3050a 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -11,7 +11,7 @@ import { import { AppMiddleware } from '@/app.ts'; /** Placeholder to find & replace with metadata. */ -const OG_META_PLACEHOLDER = '' as const; +const META_PLACEHOLDER = '' as const; /* * TODO: implement caching for posts (LRUCache) @@ -110,12 +110,12 @@ const buildMetaTags = async (params: PathParams, url: string): Promise = export const frontendController: AppMiddleware = async (c, next) => { try { const content = await Deno.readTextFile(new URL('../../public/index.html', import.meta.url)); - if (content.includes(OG_META_PLACEHOLDER)) { + if (content.includes(META_PLACEHOLDER)) { const params = getPathParams(c.req.path); if (params) { const meta = await buildMetaTags(params, Conf.local(c.req.path)); - return c.html(content.replace(OG_META_PLACEHOLDER, meta)); + return c.html(content.replace(META_PLACEHOLDER, meta)); } } return c.html(content); From c863655b4d7a45a37aaadfa2d180a4ab0aafae5d Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 15:15:48 +0530 Subject: [PATCH 20/55] sort imports by path --- src/controllers/frontend.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index 6fc3050a..30186845 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -1,5 +1,6 @@ -import { html, r } from '@/utils/html.ts'; +import { AppMiddleware } from '@/app.ts'; import { Conf } from '@/config.ts'; +import { html, r } from '@/utils/html.ts'; import { getInstanceName, getPathParams, @@ -8,7 +9,6 @@ import { OpenGraphTemplateOpts, PathParams, } from '@/utils/og-metadata.ts'; -import { AppMiddleware } from '@/app.ts'; /** Placeholder to find & replace with metadata. */ const META_PLACEHOLDER = '' as const; From 0706f53b9fd82683b62c73b882df46df3e05f06a Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 15:18:17 +0530 Subject: [PATCH 21/55] sort imports by path --- src/utils/og-metadata.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/utils/og-metadata.ts b/src/utils/og-metadata.ts index a9685961..04e3b250 100644 --- a/src/utils/og-metadata.ts +++ b/src/utils/og-metadata.ts @@ -1,10 +1,10 @@ import { Storages } from '@/storages.ts'; -import { getInstanceMetadata } from '@/utils/instance.ts'; -import { nip19 } from 'nostr-tools'; -import { match } from 'path-to-regexp'; -import { nip05Cache } from '@/utils/nip05.ts'; import { NostrMetadata, NSchema as n } from '@nostrify/nostrify'; import { getAuthor, getEvent } from '@/queries.ts'; +import { getInstanceMetadata } from '@/utils/instance.ts'; +import { nip05Cache } from '@/utils/nip05.ts'; +import { match } from 'path-to-regexp'; +import { nip19 } from 'nostr-tools'; export interface OpenGraphTemplateOpts { title: string; From 7e2217ccd8a783a4e1ed56bd5162cec76cbeca12 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 15:42:32 +0530 Subject: [PATCH 22/55] use entities escape instead of shitty custom one --- src/utils/html.ts | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/utils/html.ts b/src/utils/html.ts index 5e9ff918..557945dd 100644 --- a/src/utils/html.ts +++ b/src/utils/html.ts @@ -1,3 +1,5 @@ +import { escape } from 'entities'; + interface RawHtml { raw: true; contents: string; @@ -10,16 +12,6 @@ interface RawHtmlOptions { joiner?: string; } -export function escape(str: string) { - if (!str) return ''; - - return str.replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"') - .replace(/'/g, '''); -} - /** * Prevent values from being escaped by html``. * @param val Any value. @@ -40,7 +32,7 @@ export function r(val: any, options?: RawHtmlOptions): RawHtml { * ``` * const unsafe = `oops `; * testing.innerHTML = html`foo bar baz ${unsafe}`; - * console.assert(testing === "foo bar baz oops%20%3Cscript%3Ealert%281%29%3C/script%3E"); + * console.assert(testing === "foo bar baz oops<script>alert(1)</script>"); * ``` */ export function html(strings: TemplateStringsArray, ...values: (string | number | RawHtml)[]) { From 29d7495c39e964414c3d6fbca2d36e415b0af5a2 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 15:43:56 +0530 Subject: [PATCH 23/55] add `site` property to OpenGraphTemplateOpts --- src/utils/og-metadata.ts | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/utils/og-metadata.ts b/src/utils/og-metadata.ts index 04e3b250..ba3ebb8f 100644 --- a/src/utils/og-metadata.ts +++ b/src/utils/og-metadata.ts @@ -1,7 +1,5 @@ -import { Storages } from '@/storages.ts'; import { NostrMetadata, NSchema as n } from '@nostrify/nostrify'; import { getAuthor, getEvent } from '@/queries.ts'; -import { getInstanceMetadata } from '@/utils/instance.ts'; import { nip05Cache } from '@/utils/nip05.ts'; import { match } from 'path-to-regexp'; import { nip19 } from 'nostr-tools'; @@ -12,6 +10,7 @@ export interface OpenGraphTemplateOpts { url: string; image?: StatusInfo['image']; description: string; + site: string; } export type PathParams = Partial>; @@ -26,13 +25,6 @@ interface StatusInfo { }; } -const store = await Storages.db(); - -export const getInstanceName = async () => { - const meta = await getInstanceMetadata(store, AbortSignal.timeout(1000)); - return meta?.name || 'Ditto'; -}; - /** URL routes to serve metadata on. */ const SSR_ROUTES = [ '/@:acct/posts/:statusId', From c7e5aed6790fe6be2bb93637f4574e0a6d2b6eb7 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 18:12:53 +0530 Subject: [PATCH 24/55] move parseAndVerifyNip05 to utils/nip05 --- src/controllers/frontend.ts | 81 ++++++++++++++++------------------ src/utils/nip05.ts | 21 ++++++++- src/views/mastodon/accounts.ts | 20 +-------- 3 files changed, 59 insertions(+), 63 deletions(-) diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index 30186845..773ffb7d 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -1,14 +1,15 @@ import { AppMiddleware } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { html, r } from '@/utils/html.ts'; +import { html } from '@/utils/html.ts'; +import { Storages } from '@/storages.ts'; import { - getInstanceName, getPathParams, getProfileInfo, getStatusInfo, OpenGraphTemplateOpts, PathParams, } from '@/utils/og-metadata.ts'; +import { getInstanceMetadata } from '@/utils/instance.ts'; /** Placeholder to find & replace with metadata. */ const META_PLACEHOLDER = '' as const; @@ -22,54 +23,45 @@ const META_PLACEHOLDER = '' as const; * @param opts the metadata to use to fill the template. * @returns the built OpenGraph metadata. */ -const tpl = async ({ title, type, url, image, description }: OpenGraphTemplateOpts): Promise => - html`\ - - - - - +const tpl = ({ title, type, url, image, description, site }: OpenGraphTemplateOpts): string => { + const res = []; + res.push(html`\ + + + + + + + + + `); -${ - image - ? r(html` - - - -${image.alt ? r(html``) : ''} -`) - : '' + if (image) { + res.push(html`\ + + + + + `); + if (image.alt) { + res.push(html``); + res.push(html``); + } } - - - -${ - image - ? r(html` - -${image.alt ? r(html``) : ''} -`) - : '' - } -`.replace(/\n+/g, '\n'); + return res.join('\n').replace(/\n+/g, '\n').replace(/^[ ]+/gm, ''); +}; -const BLANK_META = (url: string) => - tpl({ - title: 'Ditto', - type: 'website', - url, - description: 'Ditto, a decentralized, self-hosted social media server', - }); +const store = await Storages.db(); -const buildMetaTags = async (params: PathParams, url: string): Promise => { - if (!params.acct && !params.statusId) return await BLANK_META(url); +async function buildMetaTags(params: PathParams, url: string): Promise { + // should never happen + if (!params.acct && !params.statusId) return ''; + const meta = await getInstanceMetadata(store); const kind0 = await getProfileInfo(params.acct); - console.log(params.acct); const { description, image } = await getStatusInfo(params.statusId || ''); const handle = kind0.nip05?.replace(/^_@/, '') || kind0.name || 'npub1xxx'; - console.log({ n: kind0.nip05, handle }); if (params.acct && params.statusId) { return tpl({ @@ -78,6 +70,7 @@ const buildMetaTags = async (params: PathParams, url: string): Promise = image, description, url, + site: meta.name, }); } else if (params.acct) { return tpl({ @@ -85,6 +78,7 @@ const buildMetaTags = async (params: PathParams, url: string): Promise = type: 'profile', description: kind0.about || '', url, + site: meta.name, image: kind0.picture ? { url: kind0.picture, @@ -101,11 +95,12 @@ const buildMetaTags = async (params: PathParams, url: string): Promise = description, image, url, + site: meta.name, }); } - return await BLANK_META(url); -}; + return ''; +} export const frontendController: AppMiddleware = async (c, next) => { try { diff --git a/src/utils/nip05.ts b/src/utils/nip05.ts index a579da6c..e9dd78cc 100644 --- a/src/utils/nip05.ts +++ b/src/utils/nip05.ts @@ -1,12 +1,13 @@ +import { nip19 } from 'nostr-tools'; import { NIP05, NStore } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; -import { nip19 } from 'nostr-tools'; import tldts from 'tldts'; import { Conf } from '@/config.ts'; +import { Storages } from '@/storages.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; import { Time } from '@/utils/time.ts'; -import { Storages } from '@/storages.ts'; +import { Nip05, parseNip05 } from '@/utils.ts'; import { fetchWorker } from '@/workers/fetch.ts'; const debug = Debug('ditto:nip05'); @@ -60,4 +61,20 @@ async function localNip05Lookup(store: NStore, localpart: string): Promise { + if (!nip05) return; + try { + const result = await nip05Cache.fetch(nip05, { signal }); + if (result.pubkey === pubkey) { + return parseNip05(nip05); + } + } catch (_e) { + // do nothing + } +} + export { localNip05Lookup, nip05Cache }; diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index 5abb1aca..eb2d6891 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -6,9 +6,9 @@ import { Conf } from '@/config.ts'; import { MastodonAccount } from '@/entities/MastodonAccount.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { getLnurl } from '@/utils/lnurl.ts'; -import { nip05Cache } from '@/utils/nip05.ts'; +import { parseAndVerifyNip05 } from '@/utils/nip05.ts'; import { getTagSet } from '@/utils/tags.ts'; -import { Nip05, nostrDate, nostrNow, parseNip05 } from '@/utils.ts'; +import { nostrDate, nostrNow } from '@/utils.ts'; import { renderEmojis } from '@/views/mastodon/emojis.ts'; interface ToAccountOpts { @@ -113,20 +113,4 @@ function accountFromPubkey(pubkey: string, opts: ToAccountOpts = {}): Promise { - if (!nip05) return; - try { - const result = await nip05Cache.fetch(nip05, { signal }); - if (result.pubkey === pubkey) { - return parseNip05(nip05); - } - } catch (_e) { - // do nothing - } -} - export { accountFromPubkey, renderAccount }; From 5785f070526efbb8e5f62d0e440c3dbfe780114e Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 20:02:03 +0530 Subject: [PATCH 25/55] tpl --> metadataView --- src/views/meta.ts | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/views/meta.ts diff --git a/src/views/meta.ts b/src/views/meta.ts new file mode 100644 index 00000000..f4f2b470 --- /dev/null +++ b/src/views/meta.ts @@ -0,0 +1,36 @@ +import { html } from '@/utils/html.ts'; +import { OpenGraphTemplateOpts } from '@/utils/og-metadata.ts'; + +/** + * Builds a series of meta tags from supplied metadata for injection into the served HTML page. + * @param opts the metadata to use to fill the template. + * @returns the built OpenGraph metadata. + */ +export const metadataView = ({ title, type, url, image, description, site }: OpenGraphTemplateOpts): string => { + const res = []; + res.push(html`\ + + + + + + + + + `); + + if (image) { + res.push(html`\ + + + + + `); + if (image.alt) { + res.push(html``); + res.push(html``); + } + } + + return res.join('\n').replace(/\n+/g, '\n').replace(/^[ ]+/gm, ''); +}; From 33da9a41b27d7b89b05ca4c197baff830f836d8f Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 20:04:26 +0530 Subject: [PATCH 26/55] rewrite metadata generation --- src/controllers/frontend.ts | 101 +++++++++--------------------------- src/utils/og-metadata.ts | 93 +++++++++++++++------------------ 2 files changed, 65 insertions(+), 129 deletions(-) diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index 773ffb7d..4b01329f 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -1,8 +1,8 @@ import { AppMiddleware } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { html } from '@/utils/html.ts'; import { Storages } from '@/storages.ts'; import { + getHandle, getPathParams, getProfileInfo, getStatusInfo, @@ -10,6 +10,7 @@ import { PathParams, } from '@/utils/og-metadata.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; +import { metadataView } from '@/views/meta.ts'; /** Placeholder to find & replace with metadata. */ const META_PLACEHOLDER = '' as const; @@ -18,88 +19,34 @@ const META_PLACEHOLDER = '' as const; * TODO: implement caching for posts (LRUCache) */ -/** - * Builds a series of meta tags from supplied metadata for injection into the served HTML page. - * @param opts the metadata to use to fill the template. - * @returns the built OpenGraph metadata. - */ -const tpl = ({ title, type, url, image, description, site }: OpenGraphTemplateOpts): string => { - const res = []; - res.push(html`\ - - - - - - - - - `); - - if (image) { - res.push(html`\ - - - - - `); - if (image.alt) { - res.push(html``); - res.push(html``); - } - } - - return res.join('\n').replace(/\n+/g, '\n').replace(/^[ ]+/gm, ''); -}; - const store = await Storages.db(); -async function buildMetaTags(params: PathParams, url: string): Promise { - // should never happen - if (!params.acct && !params.statusId) return ''; - +async function buildTemplateOpts(params: PathParams, url: string): Promise { const meta = await getInstanceMetadata(store); - const kind0 = await getProfileInfo(params.acct); - const { description, image } = await getStatusInfo(params.statusId || ''); - const handle = kind0.nip05?.replace(/^_@/, '') || kind0.name || 'npub1xxx'; + const res: OpenGraphTemplateOpts = { + title: `View this page on ${meta.name}`, + type: 'article', + description: meta.about, + url, + site: meta.name, + }; - if (params.acct && params.statusId) { - return tpl({ - title: `View @${handle}'s post on Ditto`, - type: 'article', - image, - description, - url, - site: meta.name, - }); - } else if (params.acct) { - return tpl({ - title: `View @${handle}'s profile on Ditto`, - type: 'profile', - description: kind0.about || '', - url, - site: meta.name, - image: kind0.picture - ? { - url: kind0.picture, - // Time will tell if this is fine. - h: 150, - w: 150, - } - : undefined, - }); + if (params.acct && !params.statusId) { + const profile = await getProfileInfo(params.acct); + res.type = 'profile'; + res.title = `View @${await getHandle(params.acct)}'s profile on Ditto`; + res.description = profile.about; + if (profile.picture) { + res.image = { url: profile.picture, h: 150, w: 150 }; + } } else if (params.statusId) { - return tpl({ - title: `View post on Ditto`, - type: 'profile', - description, - image, - url, - site: meta.name, - }); + const { description, image, title } = await getStatusInfo(params.statusId); + res.description = description; + res.image = image; + res.title = title; } - return ''; + return res; } export const frontendController: AppMiddleware = async (c, next) => { @@ -109,7 +56,7 @@ export const frontendController: AppMiddleware = async (c, next) => { const params = getPathParams(c.req.path); if (params) { - const meta = await buildMetaTags(params, Conf.local(c.req.path)); + const meta = metadataView(await buildTemplateOpts(params, Conf.local(c.req.path))); return c.html(content.replace(META_PLACEHOLDER, meta)); } } diff --git a/src/utils/og-metadata.ts b/src/utils/og-metadata.ts index ba3ebb8f..33a2e201 100644 --- a/src/utils/og-metadata.ts +++ b/src/utils/og-metadata.ts @@ -1,8 +1,9 @@ import { NostrMetadata, NSchema as n } from '@nostrify/nostrify'; -import { getAuthor, getEvent } from '@/queries.ts'; -import { nip05Cache } from '@/utils/nip05.ts'; +import { getEvent } from '@/queries.ts'; import { match } from 'path-to-regexp'; import { nip19 } from 'nostr-tools'; +import { lookupAccount, lookupPubkey } from '@/utils/lookup.ts'; +import { parseAndVerifyNip05 } from '@/utils/nip05.ts'; export interface OpenGraphTemplateOpts { title: string; @@ -16,6 +17,7 @@ export interface OpenGraphTemplateOpts { export type PathParams = Partial>; interface StatusInfo { + title: string; description: string; image?: { url: string; @@ -65,70 +67,57 @@ export function getPathParams(path: string) { } } -async function urlParamToPubkey(handle: string) { - const id = `${handle}`; - const parts = id.match(/(?:(.+)@)?(.+)/); - if (parts) { - const key = `${parts[1] ? (parts[1] + '@') : ''}${parts[2]}`; - return await nip05Cache.fetch(key, { signal: AbortSignal.timeout(1000) }).then((res) => res.pubkey); - } else if (id.startsWith('npub1')) { - return nip19.decode(id as `npub1${string}`).data; - } else if (/(?:[0-9]|[a-f]){64}/.test(id)) { - return id; - } +type ProfileInfo = { name: string; about: string } & NostrMetadata; - // shouldn't ever happen for a well-formed link - return ''; +/** + * Look up the name and bio of a user for use in generating OpenGraph metadata. + * + * @param handle The bech32 / nip05 identifier for the user, obtained from the URL. + * @returns An object containing the `name` and `about` fields of the user's kind 0, + * or sensible defaults if the kind 0 has those values missing. + */ +export async function getProfileInfo(handle: string | undefined): Promise { + const acc = await lookupAccount(handle || ''); + if (!acc) throw new Error('Invalid handle specified, or account not found.'); + + const short = nip19.npubEncode(acc.id).slice(0, 8); + const { name = short, about = `@${short}'s Nostr profile` } = n.json().pipe(n.metadata()).parse(acc.content); + + return { name, about }; } -export async function getProfileInfo(handle: string | undefined): Promise { - const id = await urlParamToPubkey(handle || ''); - const kind0 = await getAuthor(id); - - const short = nip19.npubEncode(id).substring(0, 8); - const blank = { name: short, about: `@${short}'s ditto profile` }; - if (!kind0) return blank; - - return Object.assign( - blank, - n.json().pipe(n.metadata()).parse(kind0.content), - ); -} - -const truncate = (s: string, len: number, ellipsis = '...') => { +function truncate(s: string, len: number, ellipsis = '…') { if (s.length <= len) return s; return s.slice(0, len) + ellipsis; -}; +} -export async function getStatusInfo(id: string | undefined, handle?: string): Promise { - const event = await getEvent(id || ''); - if (!event || !id) { - return { description: `A post on Ditto by @${handle}` }; - } +export async function getHandle(id: string, name?: string | undefined) { + const pubkey = /[a-z][0-9]{64}/.test(id) ? id : await lookupPubkey(id); + if (!pubkey) throw new Error('Invalid user identifier'); + const parsed = await parseAndVerifyNip05(id, pubkey); + return parsed?.handle || name || 'npub1xxx'; +} +export async function getStatusInfo(id: string): Promise { + const event = await getEvent(id); + if (!id || !event) throw new Error('Invalid post id supplied'); + + const handle = await getHandle(event.pubkey); const res: StatusInfo = { + title: `View @${handle}'s post on Ditto`, description: event.content .replace(/nostr:(npub1(?:[0-9]|[a-z]){58})/g, (_, key: string) => `@${key.slice(0, 8)}`), }; - let url: string; - let w: number; - let h: number; + const data: string[][] = event.tags + .find(([name]) => name === 'imeta')?.slice(1) + .map((entry: string) => entry.split(' ')) ?? []; - for (const [tag, ...values] of event.tags) { - if (tag !== 'imeta') continue; - for (const value of values) { - const [item, datum] = value.split(' '); - if (!['dim', 'url'].includes(item)) continue; - if (item === 'dim') { - [w, h] = datum.split('x').map(Number); - } else if (item === 'url') { - url = datum; - } - } - } + const url = data.find(([name]) => name === 'url')?.[1]; + const dim = data.find(([name]) => name === 'dim')?.[1]; + + const [w, h] = dim?.split('x').map(Number) ?? [null, null]; - // @ts-ignore conditional assign if (url && w && h) { res.image = { url, w, h }; res.description = res.description.replace(url.trim(), ''); From 2ad81d3fbf07668f314e469ed7aebc106c078437 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 20:07:08 +0530 Subject: [PATCH 27/55] sort imports --- src/utils/og-metadata.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/og-metadata.ts b/src/utils/og-metadata.ts index 33a2e201..3631a362 100644 --- a/src/utils/og-metadata.ts +++ b/src/utils/og-metadata.ts @@ -1,7 +1,8 @@ import { NostrMetadata, NSchema as n } from '@nostrify/nostrify'; import { getEvent } from '@/queries.ts'; -import { match } from 'path-to-regexp'; import { nip19 } from 'nostr-tools'; +import { match } from 'path-to-regexp'; + import { lookupAccount, lookupPubkey } from '@/utils/lookup.ts'; import { parseAndVerifyNip05 } from '@/utils/nip05.ts'; From 85b54e81beea491e9e69ea7dc0a7466820946850 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 20:14:43 +0530 Subject: [PATCH 28/55] ignore OG metadata errors --- src/controllers/frontend.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index 4b01329f..b6c36014 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -54,10 +54,14 @@ export const frontendController: AppMiddleware = async (c, next) => { const content = await Deno.readTextFile(new URL('../../public/index.html', import.meta.url)); if (content.includes(META_PLACEHOLDER)) { const params = getPathParams(c.req.path); - if (params) { - const meta = metadataView(await buildTemplateOpts(params, Conf.local(c.req.path))); - return c.html(content.replace(META_PLACEHOLDER, meta)); + try { + const meta = metadataView(await buildTemplateOpts(params, Conf.local(c.req.path))); + return c.html(content.replace(META_PLACEHOLDER, meta)); + } + catch { + return c.html(content); + } } } return c.html(content); From 7c56b9dad8df13246fd882ccfdd4c8294f72695e Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 20:18:16 +0530 Subject: [PATCH 29/55] use stickynotes, move storages.db to inside buildTemplateOpts --- src/controllers/frontend.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index b6c36014..72f778ac 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -1,5 +1,6 @@ import { AppMiddleware } from '@/app.ts'; import { Conf } from '@/config.ts'; +import { Stickynotes } from '@soapbox/stickynotes'; import { Storages } from '@/storages.ts'; import { getHandle, @@ -12,6 +13,8 @@ import { import { getInstanceMetadata } from '@/utils/instance.ts'; import { metadataView } from '@/views/meta.ts'; +const console = new Stickynotes('ditto:frontend'); + /** Placeholder to find & replace with metadata. */ const META_PLACEHOLDER = '' as const; @@ -19,9 +22,8 @@ const META_PLACEHOLDER = '' as const; * TODO: implement caching for posts (LRUCache) */ -const store = await Storages.db(); - async function buildTemplateOpts(params: PathParams, url: string): Promise { + const store = await Storages.db(); const meta = await getInstanceMetadata(store); const res: OpenGraphTemplateOpts = { title: `View this page on ${meta.name}`, From e4f24c3b261f527558b9a7836cec5a18ee64f787 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 20:18:49 +0530 Subject: [PATCH 30/55] format --- src/controllers/frontend.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index 72f778ac..45c63c3d 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -60,8 +60,7 @@ export const frontendController: AppMiddleware = async (c, next) => { try { const meta = metadataView(await buildTemplateOpts(params, Conf.local(c.req.path))); return c.html(content.replace(META_PLACEHOLDER, meta)); - } - catch { + } catch { return c.html(content); } } From 9175596d5f624b440f9686f757197cbd8b11fe43 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 20:24:03 +0530 Subject: [PATCH 31/55] use nip27 replacer instead of janky regex --- src/utils/og-metadata.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/utils/og-metadata.ts b/src/utils/og-metadata.ts index 3631a362..0d7a1e61 100644 --- a/src/utils/og-metadata.ts +++ b/src/utils/og-metadata.ts @@ -1,6 +1,6 @@ import { NostrMetadata, NSchema as n } from '@nostrify/nostrify'; import { getEvent } from '@/queries.ts'; -import { nip19 } from 'nostr-tools'; +import { nip19, nip27 } from 'nostr-tools'; import { match } from 'path-to-regexp'; import { lookupAccount, lookupPubkey } from '@/utils/lookup.ts'; @@ -106,8 +106,10 @@ export async function getStatusInfo(id: string): Promise { const handle = await getHandle(event.pubkey); const res: StatusInfo = { title: `View @${handle}'s post on Ditto`, - description: event.content - .replace(/nostr:(npub1(?:[0-9]|[a-z]){58})/g, (_, key: string) => `@${key.slice(0, 8)}`), + description: nip27.replaceAll( + event.content, + ({ decoded, value }) => decoded.type === 'npub' ? value.slice(0, 8) : '', + ), }; const data: string[][] = event.tags From a3012d341ba39709315e32b32ccef2f57e8be787 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 21:32:44 +0530 Subject: [PATCH 32/55] log stuff --- src/controllers/frontend.ts | 3 ++- src/utils/og-metadata.ts | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index 45c63c3d..6e39a266 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -60,7 +60,8 @@ export const frontendController: AppMiddleware = async (c, next) => { try { const meta = metadataView(await buildTemplateOpts(params, Conf.local(c.req.path))); return c.html(content.replace(META_PLACEHOLDER, meta)); - } catch { + } catch (e) { + console.debug(e); return c.html(content); } } diff --git a/src/utils/og-metadata.ts b/src/utils/og-metadata.ts index 0d7a1e61..956d855d 100644 --- a/src/utils/og-metadata.ts +++ b/src/utils/og-metadata.ts @@ -3,9 +3,12 @@ import { getEvent } from '@/queries.ts'; import { nip19, nip27 } from 'nostr-tools'; import { match } from 'path-to-regexp'; +import { Stickynotes } from '@soapbox/stickynotes'; import { lookupAccount, lookupPubkey } from '@/utils/lookup.ts'; import { parseAndVerifyNip05 } from '@/utils/nip05.ts'; +const console = new Stickynotes('ditto:frontend'); + export interface OpenGraphTemplateOpts { title: string; type: 'article' | 'profile' | 'website'; @@ -78,6 +81,7 @@ type ProfileInfo = { name: string; about: string } & NostrMetadata; * or sensible defaults if the kind 0 has those values missing. */ export async function getProfileInfo(handle: string | undefined): Promise { + console.debug(handle); const acc = await lookupAccount(handle || ''); if (!acc) throw new Error('Invalid handle specified, or account not found.'); From 2f9dd7e9a2ba51ea7c818e0856d13906c7d2a384 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 21:34:20 +0530 Subject: [PATCH 33/55] more logging --- src/utils/og-metadata.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils/og-metadata.ts b/src/utils/og-metadata.ts index 956d855d..b1fe2f85 100644 --- a/src/utils/og-metadata.ts +++ b/src/utils/og-metadata.ts @@ -97,6 +97,7 @@ function truncate(s: string, len: number, ellipsis = '…') { } export async function getHandle(id: string, name?: string | undefined) { + console.debug({ id, name }); const pubkey = /[a-z][0-9]{64}/.test(id) ? id : await lookupPubkey(id); if (!pubkey) throw new Error('Invalid user identifier'); const parsed = await parseAndVerifyNip05(id, pubkey); From 00fe609ef191358b1101947f44d97e37bf4edcd2 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 21:36:49 +0530 Subject: [PATCH 34/55] fix naked pubkey test regex --- src/utils/og-metadata.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/og-metadata.ts b/src/utils/og-metadata.ts index b1fe2f85..07b7ad04 100644 --- a/src/utils/og-metadata.ts +++ b/src/utils/og-metadata.ts @@ -98,7 +98,7 @@ function truncate(s: string, len: number, ellipsis = '…') { export async function getHandle(id: string, name?: string | undefined) { console.debug({ id, name }); - const pubkey = /[a-z][0-9]{64}/.test(id) ? id : await lookupPubkey(id); + const pubkey = /[a-z0-9]{64}/.test(id) ? id : await lookupPubkey(id); if (!pubkey) throw new Error('Invalid user identifier'); const parsed = await parseAndVerifyNip05(id, pubkey); return parsed?.handle || name || 'npub1xxx'; From 18e0c990e4bce841c106f7351be40c14905c29c2 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 22:35:55 +0530 Subject: [PATCH 35/55] add ditto favicon to default metadata --- src/controllers/frontend.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index 6e39a266..bc4f149c 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -31,6 +31,11 @@ async function buildTemplateOpts(params: PathParams, url: string): Promise Date: Mon, 5 Aug 2024 22:36:48 +0530 Subject: [PATCH 36/55] catch errors while building opengraph metadata to allow _some_ metadata to make it out still --- src/controllers/frontend.ts | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index bc4f149c..e19e5a1e 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -37,20 +37,25 @@ async function buildTemplateOpts(params: PathParams, url: string): Promise { const meta = metadataView(await buildTemplateOpts(params, Conf.local(c.req.path))); return c.html(content.replace(META_PLACEHOLDER, meta)); } catch (e) { - console.debug(e); + console.log(e); return c.html(content); } } From c109d06a6e43e806a2bd1eaa9f89e011c95c72c6 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 23:00:08 +0530 Subject: [PATCH 37/55] better getHandle --- src/controllers/frontend.ts | 13 +++--- src/utils/og-metadata.ts | 80 ++++++++++++++++++++++++++----------- 2 files changed, 63 insertions(+), 30 deletions(-) diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index e19e5a1e..f287f851 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -3,9 +3,9 @@ import { Conf } from '@/config.ts'; import { Stickynotes } from '@soapbox/stickynotes'; import { Storages } from '@/storages.ts'; import { + fetchProfile, getHandle, getPathParams, - getProfileInfo, getStatusInfo, OpenGraphTemplateOpts, PathParams, @@ -39,12 +39,13 @@ async function buildTemplateOpts(params: PathParams, url: string): Promise>, +): Promise { + if (!handle && !pubkey) { + throw new Error('Tried to fetch kind 0 with no args'); + } -/** - * Look up the name and bio of a user for use in generating OpenGraph metadata. - * - * @param handle The bech32 / nip05 identifier for the user, obtained from the URL. - * @returns An object containing the `name` and `about` fields of the user's kind 0, - * or sensible defaults if the kind 0 has those values missing. - */ -export async function getProfileInfo(handle: string | undefined): Promise { - console.debug(handle); - const acc = await lookupAccount(handle || ''); - if (!acc) throw new Error('Invalid handle specified, or account not found.'); + if (handle) pubkey = await lookupPubkey(handle); + if (!pubkey) throw new Error('NIP-05 or bech32 specified and no pubkey found'); - const short = nip19.npubEncode(acc.id).slice(0, 8); - const { name = short, about = `@${short}'s Nostr profile` } = n.json().pipe(n.metadata()).parse(acc.content); + const author = await getAuthor(pubkey); + if (!author) throw new Error(`Author not found for pubkey ${pubkey}`); - return { name, about }; + return { + meta: n.json() + .pipe(n.metadata()) + .parse(author.content), + event: author, + }; } +type ProfileInfo = { meta: NostrMetadata; event: NostrEvent }; + function truncate(s: string, len: number, ellipsis = '…') { if (s.length <= len) return s; return s.slice(0, len) + ellipsis; } -export async function getHandle(id: string, name?: string | undefined) { +/** + * @param id A nip-05 identifier, bech32 encoded npub/nprofile + * @param acc A ProfileInfo object, if you've already fetched it then this is used to build a handle. + * @returns The handle + */ +export async function getHandle(id: string, acc?: ProfileInfo) { console.debug({ id, name }); - const pubkey = /[a-z0-9]{64}/.test(id) ? id : await lookupPubkey(id); - if (!pubkey) throw new Error('Invalid user identifier'); - const parsed = await parseAndVerifyNip05(id, pubkey); - return parsed?.handle || name || 'npub1xxx'; + let handle: string | undefined = ''; + + const handlePubkey = async (pubkey: string) => { + const author = acc || await fetchProfile({ pubkey }); + if (author.meta.nip05) return parseNip05(author.meta.nip05).handle; + else if (author.meta.name) return author.meta.name; + }; + + if (/[a-z0-9]{64}/.test(id)) { + await handlePubkey(id); + } else if (n.bech32().safeParse(id).success) { + if (id.startsWith('npub')) { + handle = await handlePubkey(nip19.decode(id as `npub1${string}`).data) || id.slice(0, 8); + } else if (id.startsWith('nprofile')) { + const decoded = nip19.decode(id as `nprofile1${string}`).data.pubkey; + handle = await handlePubkey(decoded); + if (!handle) handle = nip19.npubEncode(decoded).slice(0, 8); + } else { + throw new Error('non-nprofile or -npub bech32 passed to getHandle()'); + } + } else { + const pubkey = await lookupPubkey(id); + if (!pubkey) throw new Error('Invalid user identifier'); + const parsed = await parseAndVerifyNip05(id, pubkey); + handle = parsed?.handle; + } + + return handle || name || 'npub1xxx'; } export async function getStatusInfo(id: string): Promise { const event = await getEvent(id); if (!id || !event) throw new Error('Invalid post id supplied'); - const handle = await getHandle(event.pubkey); const res: StatusInfo = { title: `View @${handle}'s post on Ditto`, From f6bd7fc3a5eb2ab4b74b84a558e48f6866a56df9 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 5 Aug 2024 23:23:13 +0530 Subject: [PATCH 38/55] fail gracefully if getHandle fails to fetch profile --- src/utils/og-metadata.ts | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/utils/og-metadata.ts b/src/utils/og-metadata.ts index d42423e4..db0b6b06 100644 --- a/src/utils/og-metadata.ts +++ b/src/utils/og-metadata.ts @@ -101,7 +101,7 @@ function truncate(s: string, len: number, ellipsis = '…') { } /** - * @param id A nip-05 identifier, bech32 encoded npub/nprofile + * @param id A nip-05 identifier, bech32 encoded npub/nprofile, or a pubkey * @param acc A ProfileInfo object, if you've already fetched it then this is used to build a handle. * @returns The handle */ @@ -110,20 +110,25 @@ export async function getHandle(id: string, acc?: ProfileInfo) { let handle: string | undefined = ''; const handlePubkey = async (pubkey: string) => { - const author = acc || await fetchProfile({ pubkey }); - if (author.meta.nip05) return parseNip05(author.meta.nip05).handle; - else if (author.meta.name) return author.meta.name; + const fallback = nip19.npubEncode(pubkey).slice(0, 8); + try { + const author = acc || await fetchProfile({ pubkey }); + if (author.meta.nip05) return parseNip05(author.meta.nip05).handle; + else if (author.meta.name) return author.meta.name; + } catch (e) { + console.debug('Error in getHandle: ', e); + } + return fallback; }; if (/[a-z0-9]{64}/.test(id)) { await handlePubkey(id); } else if (n.bech32().safeParse(id).success) { if (id.startsWith('npub')) { - handle = await handlePubkey(nip19.decode(id as `npub1${string}`).data) || id.slice(0, 8); + handle = await handlePubkey(nip19.decode(id as `npub1${string}`).data); } else if (id.startsWith('nprofile')) { const decoded = nip19.decode(id as `nprofile1${string}`).data.pubkey; handle = await handlePubkey(decoded); - if (!handle) handle = nip19.npubEncode(decoded).slice(0, 8); } else { throw new Error('non-nprofile or -npub bech32 passed to getHandle()'); } @@ -140,9 +145,15 @@ export async function getHandle(id: string, acc?: ProfileInfo) { export async function getStatusInfo(id: string): Promise { const event = await getEvent(id); if (!id || !event) throw new Error('Invalid post id supplied'); - const handle = await getHandle(event.pubkey); + let title = 'View post on Ditto'; + try { + const handle = await getHandle(event.pubkey); + title = `View @${handle}'s post on Ditto`; + } catch (e) { + console.log(e); + } const res: StatusInfo = { - title: `View @${handle}'s post on Ditto`, + title, description: nip27.replaceAll( event.content, ({ decoded, value }) => decoded.type === 'npub' ? value.slice(0, 8) : '', From aa0727a96ddbb5734903f666449f01f8254a0439 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 6 Aug 2024 00:20:05 +0530 Subject: [PATCH 39/55] acct can be a bare pubkey --- src/controllers/frontend.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index f287f851..2b5f0152 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -39,7 +39,8 @@ async function buildTemplateOpts(params: PathParams, url: string): Promise Date: Tue, 6 Aug 2024 00:20:58 +0530 Subject: [PATCH 40/55] typo --- src/utils/og-metadata.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/utils/og-metadata.ts b/src/utils/og-metadata.ts index db0b6b06..c6df021e 100644 --- a/src/utils/og-metadata.ts +++ b/src/utils/og-metadata.ts @@ -106,7 +106,6 @@ function truncate(s: string, len: number, ellipsis = '…') { * @returns The handle */ export async function getHandle(id: string, acc?: ProfileInfo) { - console.debug({ id, name }); let handle: string | undefined = ''; const handlePubkey = async (pubkey: string) => { @@ -122,7 +121,7 @@ export async function getHandle(id: string, acc?: ProfileInfo) { }; if (/[a-z0-9]{64}/.test(id)) { - await handlePubkey(id); + handle = await handlePubkey(id); } else if (n.bech32().safeParse(id).success) { if (id.startsWith('npub')) { handle = await handlePubkey(nip19.decode(id as `npub1${string}`).data); From 4f8ebb95f360454f3dbb6e793932bcd7b1620940 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 6 Aug 2024 00:26:04 +0530 Subject: [PATCH 41/55] hex regex typo fix --- src/controllers/frontend.ts | 2 +- src/utils/og-metadata.ts | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index 2b5f0152..7adc0926 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -39,7 +39,7 @@ async function buildTemplateOpts(params: PathParams, url: string): Promise { + const pubkeyToHandle = async (pubkey: string) => { const fallback = nip19.npubEncode(pubkey).slice(0, 8); try { const author = acc || await fetchProfile({ pubkey }); @@ -120,14 +120,14 @@ export async function getHandle(id: string, acc?: ProfileInfo) { return fallback; }; - if (/[a-z0-9]{64}/.test(id)) { - handle = await handlePubkey(id); + if (/[a-f0-9]{64}/.test(id)) { + handle = await pubkeyToHandle(id); } else if (n.bech32().safeParse(id).success) { if (id.startsWith('npub')) { - handle = await handlePubkey(nip19.decode(id as `npub1${string}`).data); + handle = await pubkeyToHandle(nip19.decode(id as `npub1${string}`).data); } else if (id.startsWith('nprofile')) { const decoded = nip19.decode(id as `nprofile1${string}`).data.pubkey; - handle = await handlePubkey(decoded); + handle = await pubkeyToHandle(decoded); } else { throw new Error('non-nprofile or -npub bech32 passed to getHandle()'); } From af70065d842b84e36d94fbc55788c59ae22460c2 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 6 Aug 2024 00:32:13 +0530 Subject: [PATCH 42/55] yet another typo fix --- src/controllers/frontend.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index 7adc0926..39d4d5ce 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -39,7 +39,8 @@ async function buildTemplateOpts(params: PathParams, url: string): Promise Date: Tue, 6 Aug 2024 00:37:14 +0530 Subject: [PATCH 43/55] use Object.assign to fill template opts --- src/controllers/frontend.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index 39d4d5ce..e806b546 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -43,17 +43,20 @@ async function buildTemplateOpts(params: PathParams, url: string): Promise Date: Tue, 6 Aug 2024 00:45:28 +0530 Subject: [PATCH 44/55] regex guard around opengraph routes --- src/config.ts | 4 ++++ src/controllers/frontend.ts | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/config.ts b/src/config.ts index 56825cc0..86cb05cd 100644 --- a/src/config.ts +++ b/src/config.ts @@ -250,6 +250,10 @@ class Conf { static get cronEnabled(): boolean { return optionalBooleanSchema.parse(Deno.env.get('CRON_ENABLED')) ?? true; } + static get opengraphRouteRegex(): string { + return Deno.env.get('OPENGRAPH_ROUTE_REGEX') || + 'googlebot|bingbot|yandex|baiduspider|twitterbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterestbot|slackbot|vkShare|W3C_Validator|whatsapp|mastodon|pleroma|Discordbot|AhrefsBot|SEMrushBot|MJ12bot|SeekportBot|Synapse|Matrix'; + } /** Path to the custom policy module. Must be an absolute path, https:, npm:, or jsr: URI. */ static get policy(): string { return Deno.env.get('DITTO_POLICY') || new URL('../data/policy.ts', import.meta.url).pathname; diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index e806b546..726cadfc 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -18,10 +18,6 @@ const console = new Stickynotes('ditto:frontend'); /** Placeholder to find & replace with metadata. */ const META_PLACEHOLDER = '' as const; -/* - * TODO: implement caching for posts (LRUCache) - */ - async function buildTemplateOpts(params: PathParams, url: string): Promise { const store = await Storages.db(); const meta = await getInstanceMetadata(store); @@ -70,6 +66,10 @@ async function buildTemplateOpts(params: PathParams, url: string): Promise { try { const content = await Deno.readTextFile(new URL('../../public/index.html', import.meta.url)); + const shouldInject = new RegExp(Conf.opengraphRouteRegex, 'i'); + if (!shouldInject) { + return c.html(content); + } if (content.includes(META_PLACEHOLDER)) { const params = getPathParams(c.req.path); if (params) { From 166106b12b9a1da038b075dd7b3e0d4e71e59476 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 6 Aug 2024 00:56:27 +0530 Subject: [PATCH 45/55] even better kind 0 handling --- src/controllers/frontend.ts | 29 ++++++++++++++++++++--------- src/utils/og-metadata.ts | 4 ++-- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index 726cadfc..75d09627 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -37,18 +37,26 @@ async function buildTemplateOpts(params: PathParams, url: string): Promise { try { const content = await Deno.readTextFile(new URL('../../public/index.html', import.meta.url)); - const shouldInject = new RegExp(Conf.opengraphRouteRegex, 'i'); - if (!shouldInject) { + const ua = c.req.header('User-Agent'); + console.debug('got ua', ua); + if (!SHOULD_INJECT_RE.test(ua || '')) { return c.html(content); } if (content.includes(META_PLACEHOLDER)) { diff --git a/src/utils/og-metadata.ts b/src/utils/og-metadata.ts index 88097f2e..62717113 100644 --- a/src/utils/og-metadata.ts +++ b/src/utils/og-metadata.ts @@ -112,8 +112,8 @@ export async function getHandle(id: string, acc?: ProfileInfo) { const fallback = nip19.npubEncode(pubkey).slice(0, 8); try { const author = acc || await fetchProfile({ pubkey }); - if (author.meta.nip05) return parseNip05(author.meta.nip05).handle; - else if (author.meta.name) return author.meta.name; + if (author?.meta?.nip05) return parseNip05(author.meta.nip05).handle; + else if (author?.meta?.name) return author.meta.name; } catch (e) { console.debug('Error in getHandle: ', e); } From 16f404860425ae38c56e14d595401de6ec68e2fb Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 6 Aug 2024 01:25:39 +0530 Subject: [PATCH 46/55] way better logging --- src/controllers/frontend.ts | 9 ++++----- src/utils/og-metadata.ts | 7 ++++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index 75d09627..72e3931d 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -36,7 +36,6 @@ async function buildTemplateOpts(params: PathParams, url: string): Promise { try { const content = await Deno.readTextFile(new URL('../../public/index.html', import.meta.url)); const ua = c.req.header('User-Agent'); - console.debug('got ua', ua); + console.debug('ua', ua); if (!SHOULD_INJECT_RE.test(ua || '')) { return c.html(content); } @@ -88,7 +87,7 @@ export const frontendController: AppMiddleware = async (c, next) => { const meta = metadataView(await buildTemplateOpts(params, Conf.local(c.req.path))); return c.html(content.replace(META_PLACEHOLDER, meta)); } catch (e) { - console.log(e); + console.log(`Error in building meta tags: ${e}`); return c.html(content); } } diff --git a/src/utils/og-metadata.ts b/src/utils/og-metadata.ts index 62717113..f1b65a57 100644 --- a/src/utils/og-metadata.ts +++ b/src/utils/og-metadata.ts @@ -115,7 +115,8 @@ export async function getHandle(id: string, acc?: ProfileInfo) { if (author?.meta?.nip05) return parseNip05(author.meta.nip05).handle; else if (author?.meta?.name) return author.meta.name; } catch (e) { - console.debug('Error in getHandle: ', e); + console.debug('Error fetching profile/parsing nip-05 in getHandle'); + console.debug(e); } return fallback; }; @@ -148,8 +149,8 @@ export async function getStatusInfo(id: string): Promise { try { const handle = await getHandle(event.pubkey); title = `View @${handle}'s post on Ditto`; - } catch (e) { - console.log(e); + } catch (_) { + console.log(`Error getting status info for ${id}: proceeding silently`); } const res: StatusInfo = { title, From 612c845f957fc5b6ce3450e5909eff0c9b915591 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 6 Aug 2024 01:51:43 +0530 Subject: [PATCH 47/55] get rid of r() and RawHtml from html.ts --- src/utils/html.ts | 32 ++------------------------------ 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/src/utils/html.ts b/src/utils/html.ts index 557945dd..56edd5d8 100644 --- a/src/utils/html.ts +++ b/src/utils/html.ts @@ -1,29 +1,5 @@ import { escape } from 'entities'; -interface RawHtml { - raw: true; - contents: string; -} - -/** - * Options for r() - */ -interface RawHtmlOptions { - joiner?: string; -} - -/** - * Prevent values from being escaped by html``. - * @param val Any value. - * @returns An object that tells html`` to not escape `val` while building the HTML string. - */ -export function r(val: any, options?: RawHtmlOptions): RawHtml { - return { - raw: true, - contents: Array.isArray(val) ? val.join(options?.joiner ?? ' ') : val.toString(), - }; -} - /** * @param strings The constant portions of the template string. * @param values The templated values. @@ -35,16 +11,12 @@ export function r(val: any, options?: RawHtmlOptions): RawHtml { * console.assert(testing === "foo bar baz oops<script>alert(1)</script>"); * ``` */ -export function html(strings: TemplateStringsArray, ...values: (string | number | RawHtml)[]) { +export function html(strings: TemplateStringsArray, ...values: (string | number)[]) { const built = []; for (let i = 0; i < strings.length; i++) { built.push(strings[i] || ''); const val = values[i]; - if (typeof val !== 'undefined' && typeof val !== 'object') { - built.push(escape((val || '').toString())); - } else { - built.push(val?.contents || ''); - } + built.push(escape((val || '').toString())); } return built.join(''); } From 020736fd47cd749fad9f79458ee4c73505da6f07 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 7 Aug 2024 18:19:39 -0500 Subject: [PATCH 48/55] Escape @ in SSR routes --- src/utils/og-metadata.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils/og-metadata.ts b/src/utils/og-metadata.ts index f1b65a57..7c748b5b 100644 --- a/src/utils/og-metadata.ts +++ b/src/utils/og-metadata.ts @@ -34,9 +34,9 @@ interface StatusInfo { /** URL routes to serve metadata on. */ const SSR_ROUTES = [ - '/@:acct/posts/:statusId', - '/@:acct/:statusId', - '/@:acct', + '/\\@:acct/posts/:statusId', + '/\\@:acct/:statusId', + '/\\@:acct', '/users/:acct/statuses/:statusId', '/users/:acct', '/statuses/:statusId', From 72970bf4807ddcda190344d7e825001d5938b20b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 7 Aug 2024 18:50:32 -0500 Subject: [PATCH 49/55] Code style changes --- src/config.ts | 5 ++-- src/controllers/frontend.ts | 46 ++++++++++++++++++++++--------------- src/utils/og-metadata.ts | 14 +++++------ src/views/meta.ts | 44 +++++++++++++++++++---------------- 4 files changed, 61 insertions(+), 48 deletions(-) diff --git a/src/config.ts b/src/config.ts index 86cb05cd..9d89d026 100644 --- a/src/config.ts +++ b/src/config.ts @@ -250,8 +250,9 @@ class Conf { static get cronEnabled(): boolean { return optionalBooleanSchema.parse(Deno.env.get('CRON_ENABLED')) ?? true; } - static get opengraphRouteRegex(): string { - return Deno.env.get('OPENGRAPH_ROUTE_REGEX') || + /** Crawler User-Agent regex to render link previews to. */ + static get crawlerRegex(): string { + return Deno.env.get('CRAWLER_REGEX') || 'googlebot|bingbot|yandex|baiduspider|twitterbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterestbot|slackbot|vkShare|W3C_Validator|whatsapp|mastodon|pleroma|Discordbot|AhrefsBot|SEMrushBot|MJ12bot|SeekportBot|Synapse|Matrix'; } /** Path to the custom policy module. Must be an absolute path, https:, npm:, or jsr: URI. */ diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index 72e3931d..5c7e4c62 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -21,30 +21,43 @@ const META_PLACEHOLDER = '' as const; async function buildTemplateOpts(params: PathParams, url: string): Promise { const store = await Storages.db(); const meta = await getInstanceMetadata(store); + const res: OpenGraphTemplateOpts = { - title: `View this page on ${meta.name}`, + title: meta.name, type: 'article', description: meta.about, url, site: meta.name, image: { url: Conf.local('/favicon.ico'), - w: 48, - h: 48, }, }; + try { - if (params.acct && !params.statusId) { + if (params.statusId) { + const { description, image, title } = await getStatusInfo(params.statusId); + + res.description = description; + res.title = title; + + if (res.image) { + res.image = image; + } + } else if (params.acct) { const key = /^[a-f0-9]{64}$/.test(params.acct) ? 'pubkey' : 'handle'; let handle = ''; try { const profile = await fetchProfile({ [key]: params.acct }); handle = await getHandle(params.acct, profile); - res.description = profile.meta.about || `@${handle}'s Nostr profile`; + + res.description = profile.meta.about; + if (profile.meta.picture) { - res.image = { url: profile.meta.picture, h: 150, w: 150 }; + res.image = { + url: profile.meta.picture, + }; } - } catch (_) { + } catch { console.debug(`couldn't find kind 0 for ${params.acct}`); // @ts-ignore we don't want getHandle trying to do a lookup here // but we do want it to give us a nice pretty npub @@ -52,14 +65,8 @@ async function buildTemplateOpts(params: PathParams, url: string): Promise { try { const content = await Deno.readTextFile(new URL('../../public/index.html', import.meta.url)); + const ua = c.req.header('User-Agent'); console.debug('ua', ua); - if (!SHOULD_INJECT_RE.test(ua || '')) { + + if (!new RegExp(Conf.crawlerRegex, 'i').test(ua ?? '')) { return c.html(content); } + if (content.includes(META_PLACEHOLDER)) { const params = getPathParams(c.req.path); if (params) { @@ -87,7 +95,7 @@ export const frontendController: AppMiddleware = async (c, next) => { const meta = metadataView(await buildTemplateOpts(params, Conf.local(c.req.path))); return c.html(content.replace(META_PLACEHOLDER, meta)); } catch (e) { - console.log(`Error in building meta tags: ${e}`); + console.log(`Error building meta tags: ${e}`); return c.html(content); } } diff --git a/src/utils/og-metadata.ts b/src/utils/og-metadata.ts index 7c748b5b..beaaac95 100644 --- a/src/utils/og-metadata.ts +++ b/src/utils/og-metadata.ts @@ -1,9 +1,9 @@ import { NostrEvent, NostrMetadata, NSchema as n } from '@nostrify/nostrify'; -import { getAuthor, getEvent } from '@/queries.ts'; +import { Stickynotes } from '@soapbox/stickynotes'; import { nip19, nip27 } from 'nostr-tools'; import { match } from 'path-to-regexp'; -import { Stickynotes } from '@soapbox/stickynotes'; +import { getAuthor, getEvent } from '@/queries.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; import { parseAndVerifyNip05 } from '@/utils/nip05.ts'; import { parseNip05 } from '@/utils.ts'; @@ -15,7 +15,7 @@ export interface OpenGraphTemplateOpts { type: 'article' | 'profile' | 'website'; url: string; image?: StatusInfo['image']; - description: string; + description?: string; site: string; } @@ -26,8 +26,8 @@ interface StatusInfo { description: string; image?: { url: string; - w: number; - h: number; + w?: number; + h?: number; alt?: string; }; } @@ -50,7 +50,7 @@ const SSR_ROUTES = [ const SSR_ROUTE_MATCHERS = SSR_ROUTES.map((route) => match(route, { decode: decodeURIComponent })); -export function getPathParams(path: string) { +export function getPathParams(path: string): PathParams | undefined { for (const matcher of SSR_ROUTE_MATCHERS) { const result = matcher(path); if (!result) continue; @@ -144,7 +144,7 @@ export async function getHandle(id: string, acc?: ProfileInfo) { export async function getStatusInfo(id: string): Promise { const event = await getEvent(id); - if (!id || !event) throw new Error('Invalid post id supplied'); + if (!event) throw new Error('Invalid post id supplied'); let title = 'View post on Ditto'; try { const handle = await getHandle(event.pubkey); diff --git a/src/views/meta.ts b/src/views/meta.ts index f4f2b470..5cdc9a4d 100644 --- a/src/views/meta.ts +++ b/src/views/meta.ts @@ -6,31 +6,35 @@ import { OpenGraphTemplateOpts } from '@/utils/og-metadata.ts'; * @param opts the metadata to use to fill the template. * @returns the built OpenGraph metadata. */ -export const metadataView = ({ title, type, url, image, description, site }: OpenGraphTemplateOpts): string => { - const res = []; - res.push(html`\ - - - - - - - - - `); +export function metadataView({ title, type, url, image, description, site }: OpenGraphTemplateOpts): string { + const res: string[] = [ + html` `, + html` `, + html` `, + html` `, + html` `, + html` `, + ]; + + if (description) { + res.push(html``); + res.push(html``); + } if (image) { - res.push(html`\ - - - - - `); + res.push(html``); + res.push(html``); + + if (image.w && image.h) { + res.push(html``); + res.push(html``); + } + if (image.alt) { res.push(html``); res.push(html``); } } - return res.join('\n').replace(/\n+/g, '\n').replace(/^[ ]+/gm, ''); -}; + return res.join(''); +} From ba241f0431f448a307b6f79fad6d22dd796faf19 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 7 Aug 2024 20:47:53 -0500 Subject: [PATCH 50/55] Rework opengraph --- src/config.ts | 9 +- src/controllers/api/search.ts | 4 +- src/controllers/frontend.ts | 106 +++++++----------- src/entities/MastodonStatus.ts | 11 +- src/utils/lookup.ts | 2 +- src/utils/og-metadata.ts | 190 +++++++-------------------------- src/utils/outbox.ts | 2 +- src/views/meta.ts | 64 ++++++----- 8 files changed, 133 insertions(+), 255 deletions(-) diff --git a/src/config.ts b/src/config.ts index 9d89d026..d5b94da3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -251,9 +251,12 @@ class Conf { return optionalBooleanSchema.parse(Deno.env.get('CRON_ENABLED')) ?? true; } /** Crawler User-Agent regex to render link previews to. */ - static get crawlerRegex(): string { - return Deno.env.get('CRAWLER_REGEX') || - 'googlebot|bingbot|yandex|baiduspider|twitterbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterestbot|slackbot|vkShare|W3C_Validator|whatsapp|mastodon|pleroma|Discordbot|AhrefsBot|SEMrushBot|MJ12bot|SeekportBot|Synapse|Matrix'; + static get crawlerRegex(): RegExp { + return new RegExp( + Deno.env.get('CRAWLER_REGEX') || + 'googlebot|bingbot|yandex|baiduspider|twitterbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterestbot|slackbot|vkShare|W3C_Validator|whatsapp|mastodon|pleroma|Discordbot|AhrefsBot|SEMrushBot|MJ12bot|SeekportBot|Synapse|Matrix', + 'i', + ); } /** Path to the custom policy module. Must be an absolute path, https:, npm:, or jsr: URI. */ static get policy(): string { diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 23eba60f..9bddc336 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -155,7 +155,7 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort } return filters; } catch { - // do nothing + // fall through } try { @@ -164,7 +164,7 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort return [{ kinds: [0], authors: [pubkey] }]; } } catch { - // do nothing + // fall through } return []; diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index 5c7e4c62..9f52a27e 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -2,81 +2,19 @@ import { AppMiddleware } from '@/app.ts'; import { Conf } from '@/config.ts'; import { Stickynotes } from '@soapbox/stickynotes'; import { Storages } from '@/storages.ts'; -import { - fetchProfile, - getHandle, - getPathParams, - getStatusInfo, - OpenGraphTemplateOpts, - PathParams, -} from '@/utils/og-metadata.ts'; +import { getPathParams, MetadataEntities } from '@/utils/og-metadata.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; -import { metadataView } from '@/views/meta.ts'; +import { lookupPubkey } from '@/utils/lookup.ts'; +import { renderMetadata } from '@/views/meta.ts'; +import { getAuthor, getEvent } from '@/queries.ts'; +import { renderStatus } from '@/views/mastodon/statuses.ts'; +import { renderAccount } from '@/views/mastodon/accounts.ts'; const console = new Stickynotes('ditto:frontend'); /** Placeholder to find & replace with metadata. */ const META_PLACEHOLDER = '' as const; -async function buildTemplateOpts(params: PathParams, url: string): Promise { - const store = await Storages.db(); - const meta = await getInstanceMetadata(store); - - const res: OpenGraphTemplateOpts = { - title: meta.name, - type: 'article', - description: meta.about, - url, - site: meta.name, - image: { - url: Conf.local('/favicon.ico'), - }, - }; - - try { - if (params.statusId) { - const { description, image, title } = await getStatusInfo(params.statusId); - - res.description = description; - res.title = title; - - if (res.image) { - res.image = image; - } - } else if (params.acct) { - const key = /^[a-f0-9]{64}$/.test(params.acct) ? 'pubkey' : 'handle'; - let handle = ''; - try { - const profile = await fetchProfile({ [key]: params.acct }); - handle = await getHandle(params.acct, profile); - - res.description = profile.meta.about; - - if (profile.meta.picture) { - res.image = { - url: profile.meta.picture, - }; - } - } catch { - console.debug(`couldn't find kind 0 for ${params.acct}`); - // @ts-ignore we don't want getHandle trying to do a lookup here - // but we do want it to give us a nice pretty npub - handle = await getHandle(params.acct, {}); - res.description = `@${handle}'s Nostr profile`; - } - - res.type = 'profile'; - res.title = `View @${handle}'s profile on Ditto`; - } - } catch (e) { - console.debug('Error getting OpenGraph metadata information:'); - console.debug(e); - console.trace(); - } - - return res; -} - export const frontendController: AppMiddleware = async (c, next) => { try { const content = await Deno.readTextFile(new URL('../../public/index.html', import.meta.url)); @@ -84,7 +22,7 @@ export const frontendController: AppMiddleware = async (c, next) => { const ua = c.req.header('User-Agent'); console.debug('ua', ua); - if (!new RegExp(Conf.crawlerRegex, 'i').test(ua ?? '')) { + if (!Conf.crawlerRegex.test(ua ?? '')) { return c.html(content); } @@ -92,7 +30,8 @@ export const frontendController: AppMiddleware = async (c, next) => { const params = getPathParams(c.req.path); if (params) { try { - const meta = metadataView(await buildTemplateOpts(params, Conf.local(c.req.path))); + const entities = await getEntities(params); + const meta = renderMetadata(c.req.url, entities); return c.html(content.replace(META_PLACEHOLDER, meta)); } catch (e) { console.log(`Error building meta tags: ${e}`); @@ -106,3 +45,30 @@ export const frontendController: AppMiddleware = async (c, next) => { await next(); } }; + +async function getEntities(params: { acct?: string; statusId?: string }): Promise { + const store = await Storages.db(); + + const entities: MetadataEntities = { + instance: await getInstanceMetadata(store), + }; + + if (params.statusId) { + const event = await getEvent(params.statusId, { kind: 1 }); + if (event) { + entities.status = await renderStatus(event, {}); + entities.account = entities.status?.account; + } + return entities; + } + + if (params.acct) { + const pubkey = await lookupPubkey(params.acct); + const event = pubkey ? await getAuthor(pubkey) : undefined; + if (event) { + entities.account = await renderAccount(event); + } + } + + return entities; +} diff --git a/src/entities/MastodonStatus.ts b/src/entities/MastodonStatus.ts index 20c52438..e446eb11 100644 --- a/src/entities/MastodonStatus.ts +++ b/src/entities/MastodonStatus.ts @@ -24,7 +24,16 @@ export interface MastodonStatus { pinned: boolean; reblog: MastodonStatus | null; application: unknown; - media_attachments: unknown[]; + media_attachments: { + type: string; + preview_url?: string; + meta?: { + original?: { + width?: number; + height?: number; + }; + }; + }[]; mentions: unknown[]; tags: unknown[]; emojis: unknown[]; diff --git a/src/utils/lookup.ts b/src/utils/lookup.ts index a824949a..8b082abd 100644 --- a/src/utils/lookup.ts +++ b/src/utils/lookup.ts @@ -75,7 +75,7 @@ export function extractIdentifier(value: string): string | undefined { } } } catch { - // do nothing + // fall through } value = value.replace(/^@/, ''); diff --git a/src/utils/og-metadata.ts b/src/utils/og-metadata.ts index beaaac95..c0e5756c 100644 --- a/src/utils/og-metadata.ts +++ b/src/utils/og-metadata.ts @@ -1,35 +1,20 @@ -import { NostrEvent, NostrMetadata, NSchema as n } from '@nostrify/nostrify'; -import { Stickynotes } from '@soapbox/stickynotes'; -import { nip19, nip27 } from 'nostr-tools'; +import { nip19 } from 'nostr-tools'; import { match } from 'path-to-regexp'; -import { getAuthor, getEvent } from '@/queries.ts'; -import { lookupPubkey } from '@/utils/lookup.ts'; -import { parseAndVerifyNip05 } from '@/utils/nip05.ts'; -import { parseNip05 } from '@/utils.ts'; +import { MastodonAccount } from '@/entities/MastodonAccount.ts'; +import { MastodonStatus } from '@/entities/MastodonStatus.ts'; +import { InstanceMetadata } from '@/utils/instance.ts'; -const console = new Stickynotes('ditto:frontend'); - -export interface OpenGraphTemplateOpts { - title: string; - type: 'article' | 'profile' | 'website'; - url: string; - image?: StatusInfo['image']; - description?: string; - site: string; +export interface MetadataEntities { + status?: MastodonStatus; + account?: MastodonAccount; + instance: InstanceMetadata; } -export type PathParams = Partial>; - -interface StatusInfo { - title: string; - description: string; - image?: { - url: string; - w?: number; - h?: number; - alt?: string; - }; +export interface MetadataPathParams { + statusId?: string; + acct?: string; + bech32?: string; } /** URL routes to serve metadata on. */ @@ -42,139 +27,40 @@ const SSR_ROUTES = [ '/statuses/:statusId', '/notice/:statusId', '/posts/:statusId', - '/note:note', - '/nevent:nevent', - '/nprofile:nprofile', - '/npub:npub', + '/:bech32', ] as const; -const SSR_ROUTE_MATCHERS = SSR_ROUTES.map((route) => match(route, { decode: decodeURIComponent })); +const SSR_ROUTE_MATCHERS = SSR_ROUTES.map((route) => match(route)); -export function getPathParams(path: string): PathParams | undefined { +export function getPathParams(path: string): MetadataPathParams | undefined { for (const matcher of SSR_ROUTE_MATCHERS) { const result = matcher(path); if (!result) continue; - const params = result.params as PathParams; - if (params.nevent) { - const decoded = nip19.decode(`nevent${params.nevent}`).data as nip19.EventPointer; - params.statusId = decoded.id; - } else if (params.note) { - params.statusId = nip19.decode(`note${params.note}`).data as string; + + const params: MetadataPathParams = result.params; + + if (params.bech32) { + try { + const decoded = nip19.decode(params.bech32); + switch (decoded.type) { + case 'nevent': + params.statusId = decoded.data.id; + break; + case 'note': + params.statusId = decoded.data; + break; + case 'nprofile': + params.acct = decoded.data.pubkey; + break; + case 'npub': + params.acct = decoded.data; + break; + } + } catch { + // fall through + } } - if (params.nprofile) { - const decoded = nip19.decode(`nprofile${params.nprofile}`).data as nip19.ProfilePointer; - params.acct = decoded.pubkey; - } else if (params.npub) { - params.acct = nip19.decode(`npub${params.npub}`).data as string; - } return params; } } - -export async function fetchProfile( - { pubkey, handle }: Partial>, -): Promise { - if (!handle && !pubkey) { - throw new Error('Tried to fetch kind 0 with no args'); - } - - if (handle) pubkey = await lookupPubkey(handle); - if (!pubkey) throw new Error('NIP-05 or bech32 specified and no pubkey found'); - - const author = await getAuthor(pubkey); - if (!author) throw new Error(`Author not found for pubkey ${pubkey}`); - - return { - meta: n.json() - .pipe(n.metadata()) - .parse(author.content), - event: author, - }; -} - -type ProfileInfo = { meta: NostrMetadata; event: NostrEvent }; - -function truncate(s: string, len: number, ellipsis = '…') { - if (s.length <= len) return s; - return s.slice(0, len) + ellipsis; -} - -/** - * @param id A nip-05 identifier, bech32 encoded npub/nprofile, or a pubkey - * @param acc A ProfileInfo object, if you've already fetched it then this is used to build a handle. - * @returns The handle - */ -export async function getHandle(id: string, acc?: ProfileInfo) { - let handle: string | undefined = ''; - - const pubkeyToHandle = async (pubkey: string) => { - const fallback = nip19.npubEncode(pubkey).slice(0, 8); - try { - const author = acc || await fetchProfile({ pubkey }); - if (author?.meta?.nip05) return parseNip05(author.meta.nip05).handle; - else if (author?.meta?.name) return author.meta.name; - } catch (e) { - console.debug('Error fetching profile/parsing nip-05 in getHandle'); - console.debug(e); - } - return fallback; - }; - - if (/[a-f0-9]{64}/.test(id)) { - handle = await pubkeyToHandle(id); - } else if (n.bech32().safeParse(id).success) { - if (id.startsWith('npub')) { - handle = await pubkeyToHandle(nip19.decode(id as `npub1${string}`).data); - } else if (id.startsWith('nprofile')) { - const decoded = nip19.decode(id as `nprofile1${string}`).data.pubkey; - handle = await pubkeyToHandle(decoded); - } else { - throw new Error('non-nprofile or -npub bech32 passed to getHandle()'); - } - } else { - const pubkey = await lookupPubkey(id); - if (!pubkey) throw new Error('Invalid user identifier'); - const parsed = await parseAndVerifyNip05(id, pubkey); - handle = parsed?.handle; - } - - return handle || name || 'npub1xxx'; -} - -export async function getStatusInfo(id: string): Promise { - const event = await getEvent(id); - if (!event) throw new Error('Invalid post id supplied'); - let title = 'View post on Ditto'; - try { - const handle = await getHandle(event.pubkey); - title = `View @${handle}'s post on Ditto`; - } catch (_) { - console.log(`Error getting status info for ${id}: proceeding silently`); - } - const res: StatusInfo = { - title, - description: nip27.replaceAll( - event.content, - ({ decoded, value }) => decoded.type === 'npub' ? value.slice(0, 8) : '', - ), - }; - - const data: string[][] = event.tags - .find(([name]) => name === 'imeta')?.slice(1) - .map((entry: string) => entry.split(' ')) ?? []; - - const url = data.find(([name]) => name === 'url')?.[1]; - const dim = data.find(([name]) => name === 'dim')?.[1]; - - const [w, h] = dim?.split('x').map(Number) ?? [null, null]; - - if (url && w && h) { - res.image = { url, w, h }; - res.description = res.description.replace(url.trim(), ''); - } - - // needs to be done last incase the image url was surrounded by newlines - res.description = truncate(res.description.trim(), 140); - return res; -} diff --git a/src/utils/outbox.ts b/src/utils/outbox.ts index 72b83388..891cccb8 100644 --- a/src/utils/outbox.ts +++ b/src/utils/outbox.ts @@ -18,7 +18,7 @@ export async function getRelays(store: NStore, pubkey: string): Promise`, - html` `, - html` `, - html` `, - html` `, - html` `, - ]; +export function renderMetadata(url: string, { account, status, instance }: MetadataEntities): string { + const tags: string[] = []; + + const title = account ? `${account.display_name} (@${account.acct})` : instance.name; + const attachment = status?.media_attachments?.find((a) => a.type === 'image'); + const description = status?.content || account?.note || instance.tagline; + const image = attachment?.preview_url || account?.avatar_static || instance.picture; + const siteName = instance?.name; + const width = attachment?.meta?.original?.width; + const height = attachment?.meta?.original?.height; + + if (title) { + tags.push(html`${title}`); + tags.push(html``); + tags.push(html``); + } if (description) { - res.push(html``); - res.push(html``); + tags.push(html``); + tags.push(html``); + tags.push(html``); } if (image) { - res.push(html``); - res.push(html``); - - if (image.w && image.h) { - res.push(html``); - res.push(html``); - } - - if (image.alt) { - res.push(html``); - res.push(html``); - } + tags.push(html``); + tags.push(html``); } - return res.join(''); + if (typeof width === 'number' && typeof height === 'number') { + tags.push(html``); + tags.push(html``); + } + + if (siteName) { + tags.push(html``); + } + + // Extra tags (always present if other tags exist). + if (tags.length > 0) { + tags.push(html``); + tags.push(''); + tags.push(''); + } + + return tags.join(''); } From 5f1b87c3b6b9c34d9d6aa484f85de709464538f7 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 7 Aug 2024 21:00:07 -0500 Subject: [PATCH 51/55] Pretend to be WhatsApp when fetching links --- src/utils/unfurl.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/utils/unfurl.ts b/src/utils/unfurl.ts index 41ffcd3a..a0ab1d7b 100644 --- a/src/utils/unfurl.ts +++ b/src/utils/unfurl.ts @@ -13,7 +13,11 @@ async function unfurlCard(url: string, signal: AbortSignal): Promise fetchWorker(url, { signal }), + fetch: (url) => + fetchWorker(url, { + headers: { 'User-Agent': 'WhatsApp/2' }, + signal, + }), }); const { oEmbed, title, description, canonical_url, open_graph } = result; From 2f2fda2ac3d0ba6677e044b2c54b7b602b27d833 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 7 Aug 2024 21:06:51 -0500 Subject: [PATCH 52/55] Render OG data on every page --- src/controllers/frontend.ts | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/controllers/frontend.ts b/src/controllers/frontend.ts index 9f52a27e..8a598d08 100644 --- a/src/controllers/frontend.ts +++ b/src/controllers/frontend.ts @@ -28,15 +28,13 @@ export const frontendController: AppMiddleware = async (c, next) => { if (content.includes(META_PLACEHOLDER)) { const params = getPathParams(c.req.path); - if (params) { - try { - const entities = await getEntities(params); - const meta = renderMetadata(c.req.url, entities); - return c.html(content.replace(META_PLACEHOLDER, meta)); - } catch (e) { - console.log(`Error building meta tags: ${e}`); - return c.html(content); - } + try { + const entities = await getEntities(params ?? {}); + const meta = renderMetadata(c.req.url, entities); + return c.html(content.replace(META_PLACEHOLDER, meta)); + } catch (e) { + console.log(`Error building meta tags: ${e}`); + return c.html(content); } } return c.html(content); From 15c46ec3f028c4737065935cd95a18455c5990bf Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 7 Aug 2024 21:08:03 -0500 Subject: [PATCH 53/55] Rework nginx file again for link previews --- installation/ditto.conf | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/installation/ditto.conf b/installation/ditto.conf index 256498f4..9eddebf3 100644 --- a/installation/ditto.conf +++ b/installation/ditto.conf @@ -31,31 +31,12 @@ server { root /opt/ditto/public; - location @spa { - try_files /index.html /dev/null; - } - - location @frontend { - try_files $uri @ditto-static; - } - - location @ditto-static { - root /opt/ditto/static; - try_files $uri @spa; - } - location /packs { add_header Cache-Control "public, max-age=31536000, immutable"; add_header Strict-Transport-Security "max-age=31536000" always; root /opt/ditto/public; } - location /metrics { - allow 127.0.0.1; - deny all; - proxy_pass http://ditto; - } - location ~ ^/(instance|sw\.js$|sw\.js\.map$) { root /opt/ditto/public; try_files $uri =404; @@ -66,11 +47,13 @@ server { try_files $uri =404; } - location ~ ^/(api|relay|oauth|manifest.json|nodeinfo|.well-known/(nodeinfo|nostr.json)) { + location /metrics { + allow 127.0.0.1; + deny all; proxy_pass http://ditto; } location / { - try_files /dev/null @frontend; + proxy_pass http://ditto; } } From efc121a4aee6a44f215978585d6ced52fcad3264 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 7 Aug 2024 21:10:31 -0500 Subject: [PATCH 54/55] Fallback to favicon.ico in link previews --- src/views/meta.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/views/meta.ts b/src/views/meta.ts index b88e864a..37dd35b5 100644 --- a/src/views/meta.ts +++ b/src/views/meta.ts @@ -1,3 +1,4 @@ +import { Conf } from '@/config.ts'; import { html } from '@/utils/html.ts'; import { MetadataEntities } from '@/utils/og-metadata.ts'; @@ -12,7 +13,7 @@ export function renderMetadata(url: string, { account, status, instance }: Metad const title = account ? `${account.display_name} (@${account.acct})` : instance.name; const attachment = status?.media_attachments?.find((a) => a.type === 'image'); const description = status?.content || account?.note || instance.tagline; - const image = attachment?.preview_url || account?.avatar_static || instance.picture; + const image = attachment?.preview_url || account?.avatar_static || instance.picture || Conf.local('/favicon.ico'); const siteName = instance?.name; const width = attachment?.meta?.original?.width; const height = attachment?.meta?.original?.height; From 313c37564c8f4afb4e36b14474b25382d17cd656 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 7 Aug 2024 21:19:04 -0500 Subject: [PATCH 55/55] Add MastodonAttachment type --- src/entities/MastodonAttachment.ts | 15 +++++++++++++++ src/entities/MastodonStatus.ts | 12 ++---------- src/views/mastodon/attachments.ts | 5 ++++- src/views/mastodon/statuses.ts | 5 ++++- 4 files changed, 25 insertions(+), 12 deletions(-) create mode 100644 src/entities/MastodonAttachment.ts diff --git a/src/entities/MastodonAttachment.ts b/src/entities/MastodonAttachment.ts new file mode 100644 index 00000000..7660c913 --- /dev/null +++ b/src/entities/MastodonAttachment.ts @@ -0,0 +1,15 @@ +export interface MastodonAttachment { + id: string; + type: string; + url: string; + preview_url?: string; + remote_url?: string | null; + description?: string; + blurhash?: string | null; + meta?: { + original?: { + width?: number; + height?: number; + }; + }; +} diff --git a/src/entities/MastodonStatus.ts b/src/entities/MastodonStatus.ts index e446eb11..3bc15f55 100644 --- a/src/entities/MastodonStatus.ts +++ b/src/entities/MastodonStatus.ts @@ -1,4 +1,5 @@ import { MastodonAccount } from '@/entities/MastodonAccount.ts'; +import { MastodonAttachment } from '@/entities/MastodonAttachment.ts'; import { PreviewCard } from '@/entities/PreviewCard.ts'; export interface MastodonStatus { @@ -24,16 +25,7 @@ export interface MastodonStatus { pinned: boolean; reblog: MastodonStatus | null; application: unknown; - media_attachments: { - type: string; - preview_url?: string; - meta?: { - original?: { - width?: number; - height?: number; - }; - }; - }[]; + media_attachments: MastodonAttachment[]; mentions: unknown[]; tags: unknown[]; emojis: unknown[]; diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 0be61cba..9320f604 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -1,7 +1,10 @@ +import { MastodonAttachment } from '@/entities/MastodonAttachment.ts'; import { getUrlMediaType } from '@/utils/media.ts'; /** Render Mastodon media attachment. */ -function renderAttachment(media: { id?: string; data: string[][] }) { +function renderAttachment( + media: { id?: string; data: string[][] }, +): (MastodonAttachment & { cid?: string }) | undefined { const { id, data: tags } = media; const url = tags.find(([name]) => name === 'url')?.[1]; diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index f5d8d5bb..d1c02f1e 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -2,6 +2,7 @@ import { NostrEvent } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; import { Conf } from '@/config.ts'; +import { MastodonAttachment } from '@/entities/MastodonAttachment.ts'; import { MastodonMention } from '@/entities/MastodonMention.ts'; import { MastodonStatus } from '@/entities/MastodonStatus.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; @@ -119,7 +120,9 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< pinned: Boolean(pinEvent), reblog: null, application: null, - media_attachments: media.map((m) => renderAttachment({ data: m })).filter(Boolean), + media_attachments: media.map((m) => renderAttachment({ data: m })).filter((m): m is MastodonAttachment => + Boolean(m) + ), mentions, tags: [], emojis: renderEmojis(event),