Always inject og metadata, but add generous cache headers

src/app.ts

@@ -126,7 +126,6 @@ import { translateController } from '@/controllers/api/translate.ts';
 import { errorHandler } from '@/controllers/error.ts';
 import { frontendController } from '@/controllers/frontend.ts';
 import { metricsController } from '@/controllers/metrics.ts';
-import { indexController } from '@/controllers/site.ts';
 import { manifestController } from '@/controllers/manifest.ts';
 import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts';
 import { nostrController } from '@/controllers/well-known/nostr.ts';
@@ -490,10 +489,7 @@ app.get(
   publicFiles,
 );
 
-// Site index
-app.get('/', ratelimit, frontendController, indexController);
-
-// Fallback
+app.get('/', ratelimit, frontendController);
 app.get('*', publicFiles, staticFiles, ratelimit, frontendController);
 
 app.onError(errorHandler);

src/config.ts

@@ -267,14 +267,6 @@ class Conf {
   static get cronEnabled(): boolean {
     return optionalBooleanSchema.parse(Deno.env.get('CRON_ENABLED')) ?? true;
   }
-  /** Crawler User-Agent regex to render link previews to. */
-  static get crawlerRegex(): RegExp {
-    return new RegExp(
-      Deno.env.get('CRAWLER_REGEX') ||
-        'googlebot|bingbot|yandex|baiduspider|twitterbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterestbot|slackbot|vkShare|W3C_Validator|whatsapp|mastodon|pleroma|Discordbot|AhrefsBot|SEMrushBot|MJ12bot|SeekportBot|Synapse|Matrix',
-      'i',
-    );
-  }
   /** User-Agent to use when fetching link previews. Pretend to be Facebook by default. */
   static get fetchUserAgent(): string {
     return Deno.env.get('DITTO_FETCH_USER_AGENT') ?? 'facebookexternalhit';

src/controllers/frontend.ts

@@ -1,5 +1,4 @@
 import { AppMiddleware } from '@/app.ts';
-import { Conf } from '@/config.ts';
 import { Stickynotes } from '@soapbox/stickynotes';
 import { Storages } from '@/storages.ts';
 import { getPathParams, MetadataEntities } from '@/utils/og-metadata.ts';
@@ -15,23 +14,17 @@ const console = new Stickynotes('ditto:frontend');
 /** Placeholder to find & replace with metadata. */
 const META_PLACEHOLDER = '<!--server-generated-meta-->' as const;
 
-export const frontendController: AppMiddleware = async (c, next) => {
+export const frontendController: AppMiddleware = async (c) => {
+  c.header('Cache-Control', 'max-age=86400, s-maxage=30, public, stale-if-error=604800');
+
   try {
     const content = await Deno.readTextFile(new URL('../../public/index.html', import.meta.url));
 
-    const ua = c.req.header('User-Agent');
-    console.debug('ua', ua);
-
-    if (!Conf.crawlerRegex.test(ua ?? '')) {
-      return c.html(content);
-    }
-
     if (content.includes(META_PLACEHOLDER)) {
       const params = getPathParams(c.req.path);
       try {
         const entities = await getEntities(params ?? {});
         const meta = renderMetadata(c.req.url, entities);
-        c.header('Cache-Control', 'max-age=30, public, stale-while-revalidate=30');
         return c.html(content.replace(META_PLACEHOLDER, meta));
       } catch (e) {
         console.log(`Error building meta tags: ${e}`);
@@ -39,9 +32,8 @@ export const frontendController: AppMiddleware = async (c, next) => {
       }
     }
     return c.html(content);
-  } catch (e) {
-    console.log(e);
-    await next();
+  } catch {
+    return c.notFound();
   }
 };
 

src/controllers/site.ts

@@ -1,17 +0,0 @@
-import { Conf } from '@/config.ts';
-
-import type { AppController } from '@/app.ts';
-
-/** Landing page controller. */
-const indexController: AppController = (c) => {
-  const { origin } = Conf.url;
-
-  return c.text(`Please connect with a Mastodon client:
-
-    ${origin}
-
-Ditto <https://gitlab.com/soapbox-pub/ditto>
-`);
-};
-
-export { indexController };