Merge branch 'opengraph-metadata' into 'main'

OpenGraph metadata support

Closes #179

See merge request soapbox-pub/ditto!444

installation/ditto.conf

@@ -31,31 +31,12 @@ server {
 
   root /opt/ditto/public;
 
-  location @spa {
-    try_files /index.html /dev/null;
-  }
-
-  location @frontend {
-    try_files $uri @ditto-static;
-  }
-
-  location @ditto-static {
-    root /opt/ditto/static;
-    try_files $uri @spa;
-  }
-
   location /packs {
     add_header Cache-Control "public, max-age=31536000, immutable";
     add_header Strict-Transport-Security "max-age=31536000" always;
     root /opt/ditto/public;
   }
 
-  location /metrics {
-    allow 127.0.0.1;
-    deny all;
-    proxy_pass http://ditto;
-  }
-
   location ~ ^/(instance|sw\.js$|sw\.js\.map$) {
     root /opt/ditto/public;
     try_files $uri =404;
@@ -66,11 +47,13 @@ server {
     try_files $uri =404;
   }
 
-  location ~ ^/(api|relay|oauth|manifest.json|nodeinfo|.well-known/(nodeinfo|nostr.json)) {
+  location /metrics {
+    allow 127.0.0.1;
+    deny all;
     proxy_pass http://ditto;
   }
 
   location / {
-    try_files /dev/null @frontend;
+    proxy_pass http://ditto;
   }
 }

src/app.ts

@@ -110,6 +110,7 @@ import {
   trendingTagsController,
 } from '@/controllers/api/trends.ts';
 import { errorHandler } from '@/controllers/error.ts';
+import { frontendController } from '@/controllers/frontend.ts';
 import { metricsController } from '@/controllers/metrics.ts';
 import { indexController } from '@/controllers/site.ts';
 import '@/startup.ts';
@@ -324,7 +325,6 @@ app.use('/oauth/*', notImplementedController);
 
 const publicFiles = serveStatic({ root: './public/' });
 const staticFiles = serveStatic({ root: './static/' });
-const frontendController = serveStatic({ path: './public/index.html' });
 
 // Known frontend routes
 app.get('/@:acct', frontendController);

src/config.ts

@@ -250,6 +250,14 @@ class Conf {
   static get cronEnabled(): boolean {
     return optionalBooleanSchema.parse(Deno.env.get('CRON_ENABLED')) ?? true;
   }
+  /** Crawler User-Agent regex to render link previews to. */
+  static get crawlerRegex(): RegExp {
+    return new RegExp(
+      Deno.env.get('CRAWLER_REGEX') ||
+        'googlebot|bingbot|yandex|baiduspider|twitterbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterestbot|slackbot|vkShare|W3C_Validator|whatsapp|mastodon|pleroma|Discordbot|AhrefsBot|SEMrushBot|MJ12bot|SeekportBot|Synapse|Matrix',
+      'i',
+    );
+  }
   /** Path to the custom policy module. Must be an absolute path, https:, npm:, or jsr: URI. */
   static get policy(): string {
     return Deno.env.get('DITTO_POLICY') || new URL('../data/policy.ts', import.meta.url).pathname;

src/controllers/api/search.ts

@@ -155,7 +155,7 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort
     }
     return filters;
   } catch {
-    // do nothing
+    // fall through
   }
 
   try {
@@ -164,7 +164,7 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort
       return [{ kinds: [0], authors: [pubkey] }];
     }
   } catch {
-    // do nothing
+    // fall through
   }
 
   return [];

src/controllers/frontend.ts

@@ -0,0 +1,72 @@
+import { AppMiddleware } from '@/app.ts';
+import { Conf } from '@/config.ts';
+import { Stickynotes } from '@soapbox/stickynotes';
+import { Storages } from '@/storages.ts';
+import { getPathParams, MetadataEntities } from '@/utils/og-metadata.ts';
+import { getInstanceMetadata } from '@/utils/instance.ts';
+import { lookupPubkey } from '@/utils/lookup.ts';
+import { renderMetadata } from '@/views/meta.ts';
+import { getAuthor, getEvent } from '@/queries.ts';
+import { renderStatus } from '@/views/mastodon/statuses.ts';
+import { renderAccount } from '@/views/mastodon/accounts.ts';
+
+const console = new Stickynotes('ditto:frontend');
+
+/** Placeholder to find & replace with metadata. */
+const META_PLACEHOLDER = '<!--server-generated-meta-->' as const;
+
+export const frontendController: AppMiddleware = async (c, next) => {
+  try {
+    const content = await Deno.readTextFile(new URL('../../public/index.html', import.meta.url));
+
+    const ua = c.req.header('User-Agent');
+    console.debug('ua', ua);
+
+    if (!Conf.crawlerRegex.test(ua ?? '')) {
+      return c.html(content);
+    }
+
+    if (content.includes(META_PLACEHOLDER)) {
+      const params = getPathParams(c.req.path);
+      try {
+        const entities = await getEntities(params ?? {});
+        const meta = renderMetadata(c.req.url, entities);
+        return c.html(content.replace(META_PLACEHOLDER, meta));
+      } catch (e) {
+        console.log(`Error building meta tags: ${e}`);
+        return c.html(content);
+      }
+    }
+    return c.html(content);
+  } catch (e) {
+    console.log(e);
+    await next();
+  }
+};
+
+async function getEntities(params: { acct?: string; statusId?: string }): Promise<MetadataEntities> {
+  const store = await Storages.db();
+
+  const entities: MetadataEntities = {
+    instance: await getInstanceMetadata(store),
+  };
+
+  if (params.statusId) {
+    const event = await getEvent(params.statusId, { kind: 1 });
+    if (event) {
+      entities.status = await renderStatus(event, {});
+      entities.account = entities.status?.account;
+    }
+    return entities;
+  }
+
+  if (params.acct) {
+    const pubkey = await lookupPubkey(params.acct);
+    const event = pubkey ? await getAuthor(pubkey) : undefined;
+    if (event) {
+      entities.account = await renderAccount(event);
+    }
+  }
+
+  return entities;
+}

src/entities/MastodonAttachment.ts

@@ -0,0 +1,15 @@
+export interface MastodonAttachment {
+  id: string;
+  type: string;
+  url: string;
+  preview_url?: string;
+  remote_url?: string | null;
+  description?: string;
+  blurhash?: string | null;
+  meta?: {
+    original?: {
+      width?: number;
+      height?: number;
+    };
+  };
+}

src/entities/MastodonStatus.ts

@@ -1,4 +1,5 @@
 import { MastodonAccount } from '@/entities/MastodonAccount.ts';
+import { MastodonAttachment } from '@/entities/MastodonAttachment.ts';
 import { PreviewCard } from '@/entities/PreviewCard.ts';
 
 export interface MastodonStatus {
@@ -24,7 +25,7 @@ export interface MastodonStatus {
   pinned: boolean;
   reblog: MastodonStatus | null;
   application: unknown;
-  media_attachments: unknown[];
+  media_attachments: MastodonAttachment[];
   mentions: unknown[];
   tags: unknown[];
   emojis: unknown[];

src/utils/html.ts

@@ -0,0 +1,22 @@
+import { escape } from 'entities';
+
+/**
+ * @param strings The constant portions of the template string.
+ * @param values The templated values.
+ * @returns The built HTML.
+ * @example
+ * ```
+ * const unsafe = `oops <script>alert(1)</script>`;
+ * testing.innerHTML = html`foo bar baz ${unsafe}`;
+ * console.assert(testing === "foo bar baz oops&lt;script&gt;alert(1)&lt;/script&gt;");
+ * ```
+ */
+export function html(strings: TemplateStringsArray, ...values: (string | number)[]) {
+  const built = [];
+  for (let i = 0; i < strings.length; i++) {
+    built.push(strings[i] || '');
+    const val = values[i];
+    built.push(escape((val || '').toString()));
+  }
+  return built.join('');
+}

src/utils/lookup.ts

@@ -75,7 +75,7 @@ export function extractIdentifier(value: string): string | undefined {
       }
     }
   } catch {
-    // do nothing
+    // fall through
   }
 
   value = value.replace(/^@/, '');

src/utils/nip05.ts

@@ -1,12 +1,13 @@
+import { nip19 } from 'nostr-tools';
 import { NIP05, NStore } from '@nostrify/nostrify';
 import Debug from '@soapbox/stickynotes/debug';
-import { nip19 } from 'nostr-tools';
 import tldts from 'tldts';
 
 import { Conf } from '@/config.ts';
+import { Storages } from '@/storages.ts';
 import { SimpleLRU } from '@/utils/SimpleLRU.ts';
 import { Time } from '@/utils/time.ts';
-import { Storages } from '@/storages.ts';
+import { Nip05, parseNip05 } from '@/utils.ts';
 import { fetchWorker } from '@/workers/fetch.ts';
 
 const debug = Debug('ditto:nip05');
@@ -60,4 +61,20 @@ async function localNip05Lookup(store: NStore, localpart: string): Promise<nip19
   }
 }
 
+export async function parseAndVerifyNip05(
+  nip05: string | undefined,
+  pubkey: string,
+  signal = AbortSignal.timeout(3000),
+): Promise<Nip05 | undefined> {
+  if (!nip05) return;
+  try {
+    const result = await nip05Cache.fetch(nip05, { signal });
+    if (result.pubkey === pubkey) {
+      return parseNip05(nip05);
+    }
+  } catch (_e) {
+    // do nothing
+  }
+}
+
 export { localNip05Lookup, nip05Cache };

src/utils/og-metadata.ts

@@ -0,0 +1,66 @@
+import { nip19 } from 'nostr-tools';
+import { match } from 'path-to-regexp';
+
+import { MastodonAccount } from '@/entities/MastodonAccount.ts';
+import { MastodonStatus } from '@/entities/MastodonStatus.ts';
+import { InstanceMetadata } from '@/utils/instance.ts';
+
+export interface MetadataEntities {
+  status?: MastodonStatus;
+  account?: MastodonAccount;
+  instance: InstanceMetadata;
+}
+
+export interface MetadataPathParams {
+  statusId?: string;
+  acct?: string;
+  bech32?: string;
+}
+
+/** URL routes to serve metadata on. */
+const SSR_ROUTES = [
+  '/\\@:acct/posts/:statusId',
+  '/\\@:acct/:statusId',
+  '/\\@:acct',
+  '/users/:acct/statuses/:statusId',
+  '/users/:acct',
+  '/statuses/:statusId',
+  '/notice/:statusId',
+  '/posts/:statusId',
+  '/:bech32',
+] as const;
+
+const SSR_ROUTE_MATCHERS = SSR_ROUTES.map((route) => match(route));
+
+export function getPathParams(path: string): MetadataPathParams | undefined {
+  for (const matcher of SSR_ROUTE_MATCHERS) {
+    const result = matcher(path);
+    if (!result) continue;
+
+    const params: MetadataPathParams = result.params;
+
+    if (params.bech32) {
+      try {
+        const decoded = nip19.decode(params.bech32);
+        switch (decoded.type) {
+          case 'nevent':
+            params.statusId = decoded.data.id;
+            break;
+          case 'note':
+            params.statusId = decoded.data;
+            break;
+          case 'nprofile':
+            params.acct = decoded.data.pubkey;
+            break;
+          case 'npub':
+            params.acct = decoded.data;
+            break;
+        }
+      } catch {
+        // fall through
+      }
+    }
+
+    return params;
+  }
+}

src/utils/outbox.ts

@@ -18,7 +18,7 @@ export async function getRelays(store: NStore, pubkey: string): Promise<Set<stri
             relays.add(url.toString() as `wss://${string}`);
           }
         } catch (_e) {
-          // do nothing
+          // fall through
         }
       }
     }

src/utils/unfurl.ts

@@ -13,7 +13,11 @@ async function unfurlCard(url: string, signal: AbortSignal): Promise<PreviewCard
   debug(`Unfurling ${url}...`);
   try {
     const result = await unfurl(url, {
-      fetch: (url) => fetchWorker(url, { signal }),
+      fetch: (url) =>
+        fetchWorker(url, {
+          headers: { 'User-Agent': 'WhatsApp/2' },
+          signal,
+        }),
     });
 
     const { oEmbed, title, description, canonical_url, open_graph } = result;

src/views/mastodon/accounts.ts

@@ -6,9 +6,9 @@ import { Conf } from '@/config.ts';
 import { MastodonAccount } from '@/entities/MastodonAccount.ts';
 import { type DittoEvent } from '@/interfaces/DittoEvent.ts';
 import { getLnurl } from '@/utils/lnurl.ts';
-import { nip05Cache } from '@/utils/nip05.ts';
+import { parseAndVerifyNip05 } from '@/utils/nip05.ts';
 import { getTagSet } from '@/utils/tags.ts';
-import { Nip05, nostrDate, nostrNow, parseNip05 } from '@/utils.ts';
+import { nostrDate, nostrNow } from '@/utils.ts';
 import { renderEmojis } from '@/views/mastodon/emojis.ts';
 
 interface ToAccountOpts {
@@ -115,20 +115,4 @@ function accountFromPubkey(pubkey: string, opts: ToAccountOpts = {}): Promise<Ma
   return renderAccount(event, opts);
 }
 
-async function parseAndVerifyNip05(
-  nip05: string | undefined,
-  pubkey: string,
-  signal = AbortSignal.timeout(3000),
-): Promise<Nip05 | undefined> {
-  if (!nip05) return;
-  try {
-    const result = await nip05Cache.fetch(nip05, { signal });
-    if (result.pubkey === pubkey) {
-      return parseNip05(nip05);
-    }
-  } catch (_e) {
-    // do nothing
-  }
-}
-
 export { accountFromPubkey, renderAccount };

src/views/mastodon/attachments.ts

@@ -1,7 +1,10 @@
+import { MastodonAttachment } from '@/entities/MastodonAttachment.ts';
 import { getUrlMediaType } from '@/utils/media.ts';
 
 /** Render Mastodon media attachment. */
-function renderAttachment(media: { id?: string; data: string[][] }) {
+function renderAttachment(
+  media: { id?: string; data: string[][] },
+): (MastodonAttachment & { cid?: string }) | undefined {
   const { id, data: tags } = media;
 
   const url = tags.find(([name]) => name === 'url')?.[1];

src/views/mastodon/statuses.ts

@@ -2,6 +2,7 @@ import { NostrEvent } from '@nostrify/nostrify';
 import { nip19 } from 'nostr-tools';
 
 import { Conf } from '@/config.ts';
+import { MastodonAttachment } from '@/entities/MastodonAttachment.ts';
 import { MastodonMention } from '@/entities/MastodonMention.ts';
 import { MastodonStatus } from '@/entities/MastodonStatus.ts';
 import { type DittoEvent } from '@/interfaces/DittoEvent.ts';
@@ -119,7 +120,9 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
     pinned: Boolean(pinEvent),
     reblog: null,
     application: null,
-    media_attachments: media.map((m) => renderAttachment({ data: m })).filter(Boolean),
+    media_attachments: media.map((m) => renderAttachment({ data: m })).filter((m): m is MastodonAttachment =>
+      Boolean(m)
+    ),
     mentions,
     tags: [],
     emojis: renderEmojis(event),

src/views/meta.ts

@@ -0,0 +1,55 @@
+import { Conf } from '@/config.ts';
+import { html } from '@/utils/html.ts';
+import { MetadataEntities } from '@/utils/og-metadata.ts';
+
+/**
+ * Builds a series of meta tags from supplied metadata for injection into the served HTML page.
+ * @param opts the metadata to use to fill the template.
+ * @returns the built OpenGraph metadata.
+ */
+export function renderMetadata(url: string, { account, status, instance }: MetadataEntities): string {
+  const tags: string[] = [];
+
+  const title = account ? `${account.display_name} (@${account.acct})` : instance.name;
+  const attachment = status?.media_attachments?.find((a) => a.type === 'image');
+  const description = status?.content || account?.note || instance.tagline;
+  const image = attachment?.preview_url || account?.avatar_static || instance.picture || Conf.local('/favicon.ico');
+  const siteName = instance?.name;
+  const width = attachment?.meta?.original?.width;
+  const height = attachment?.meta?.original?.height;
+
+  if (title) {
+    tags.push(html`<title>${title}</title>`);
+    tags.push(html`<meta property="og:title" content="${title}">`);
+    tags.push(html`<meta name="twitter:title" content="${title}">`);
+  }
+
+  if (description) {
+    tags.push(html`<meta name="description" content="${description}">`);
+    tags.push(html`<meta property="og:description" content="${description}">`);
+    tags.push(html`<meta name="twitter:description" content="${description}">`);
+  }
+
+  if (image) {
+    tags.push(html`<meta property="og:image" content="${image}">`);
+    tags.push(html`<meta name="twitter:image" content="${image}">`);
+  }
+
+  if (typeof width === 'number' && typeof height === 'number') {
+    tags.push(html`<meta property="og:image:width" content="${width}">`);
+    tags.push(html`<meta property="og:image:height" content="${height}">`);
+  }
+
+  if (siteName) {
+    tags.push(html`<meta property="og:site_name" content="${siteName}">`);
+  }
+
+  // Extra tags (always present if other tags exist).
+  if (tags.length > 0) {
+    tags.push(html`<meta property="og:url" content="${url}">`);
+    tags.push('<meta property="og:type" content="website">');
+    tags.push('<meta name="twitter:card" content="summary">');
+  }
+
+  return tags.join('');
+}