Merge remote-tracking branch 'origin/main' into ipfs-image-metadata

src/app.ts

@@ -49,6 +49,7 @@ import {
   nameRequestController,
   nameRequestsController,
   statusZapSplitsController,
+  updateInstanceController,
   updateZapSplitsController,
 } from '@/controllers/api/ditto.ts';
 import { emptyArrayController, notImplementedController } from '@/controllers/api/fallback.ts';
@@ -169,6 +170,11 @@ const app = new Hono<AppEnv>({ strict: false });
 
 const debug = Debug('ditto:http');
 
+/** User-provided files in the gitignored `public/` directory. */
+const publicFiles = serveStatic({ root: './public/' });
+/** Static files provided by the Ditto repo, checked into git. */
+const staticFiles = serveStatic({ root: './static/' });
+
 app.use('*', rateLimitMiddleware(300, Time.minutes(5)));
 
 app.use('/api/*', metricsMiddleware, paginationMiddleware, logger(debug));
@@ -303,6 +309,8 @@ app.delete('/api/v1/pleroma/admin/statuses/:id', requireRole('admin'), pleromaAd
 app.get('/api/v1/admin/ditto/relays', requireRole('admin'), adminRelaysController);
 app.put('/api/v1/admin/ditto/relays', requireRole('admin'), adminSetRelaysController);
 
+app.put('/api/v1/admin/ditto/instance', requireRole('admin'), updateInstanceController);
+
 app.post('/api/v1/ditto/names', requireSigner, nameRequestController);
 app.get('/api/v1/ditto/names', requireSigner, nameRequestsController);
 
@@ -362,13 +370,10 @@ app.get('/api/v1/conversations', emptyArrayController);
 app.get('/api/v1/lists', emptyArrayController);
 
 app.use('/api/*', notImplementedController);
-app.use('/.well-known/*', notImplementedController);
+app.use('/.well-known/*', publicFiles, notImplementedController);
 app.use('/nodeinfo/*', notImplementedController);
 app.use('/oauth/*', notImplementedController);
 
-const publicFiles = serveStatic({ root: './public/' });
-const staticFiles = serveStatic({ root: './static/' });
-
 // Known frontend routes
 app.get('/:acct{@.*}', frontendController);
 app.get('/:acct{@.*}/*', frontendController);

src/controllers/api/accounts.ts

@@ -258,8 +258,8 @@ const accountStatusesController: AppController = async (c) => {
 };
 
 const updateCredentialsSchema = z.object({
-  display_name: z.string().optional(),
+  display_name: z.coerce.string().optional(),
-  note: z.string().optional(),
+  note: z.coerce.string().optional(),
   avatar: fileSchema.or(z.literal('')).optional(),
   header: fileSchema.or(z.literal('')).optional(),
   locked: z.boolean().optional(),

src/controllers/api/ditto.ts

@@ -1,19 +1,24 @@
 import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify';
+import { HTTPException } from '@hono/hono/http-exception';
 import { z } from 'zod';
 
-import { accountFromPubkey } from '@/views/mastodon/accounts.ts';
 import { AppController } from '@/app.ts';
-import { addTag } from '@/utils/tags.ts';
-import { AdminSigner } from '@/signers/AdminSigner.ts';
-import { booleanParamSchema, percentageSchema } from '@/schema.ts';
 import { Conf } from '@/config.ts';
-import { createEvent, paginated, parseBody } from '@/utils/api.ts';
+import { dittoUploads } from '@/DittoUploads.ts';
+import { addTag } from '@/utils/tags.ts';
+import { getAuthor } from '@/queries.ts';
+import { createEvent, paginated, parseBody, updateEvent } from '@/utils/api.ts';
+import { getInstanceMetadata } from '@/utils/instance.ts';
 import { deleteTag } from '@/utils/tags.ts';
 import { DittoEvent } from '@/interfaces/DittoEvent.ts';
 import { DittoZapSplits, getZapSplits } from '@/utils/zap-split.ts';
-import { getAuthor } from '@/queries.ts';
+import { AdminSigner } from '@/signers/AdminSigner.ts';
+import { screenshotsSchema } from '@/schemas/nostr.ts';
+import { booleanParamSchema, percentageSchema } from '@/schema.ts';
 import { hydrateEvents } from '@/storages/hydrate.ts';
 import { renderNameRequest } from '@/views/ditto.ts';
+import { accountFromPubkey } from '@/views/mastodon/accounts.ts';
+import { renderAttachment } from '@/views/mastodon/attachments.ts';
 import { renderAccount } from '@/views/mastodon/accounts.ts';
 import { Storages } from '@/storages.ts';
 import { updateListAdminEvent } from '@/utils/api.ts';
@@ -287,3 +292,90 @@ export const statusZapSplitsController: AppController = async (c) => {
 
   return c.json(zapSplits, 200);
 };
+
+const updateInstanceSchema = z.object({
+  title: z.string().optional(),
+  description: z.string().optional(),
+  /** Mastodon doesn't have this field. */
+  short_description: z.string().optional(),
+  /** Mastodon doesn't have this field. */
+  screenshot_ids: z.string().array().nullish(),
+  /** Mastodon doesn't have this field. */
+  thumbnail_id: z.string().optional(),
+}).strict();
+
+export const updateInstanceController: AppController = async (c) => {
+  const body = await parseBody(c.req.raw);
+  const result = updateInstanceSchema.safeParse(body);
+  const pubkey = Conf.pubkey;
+
+  if (!result.success) {
+    return c.json(result.error, 422);
+  }
+
+  await updateEvent(
+    { kinds: [0], authors: [pubkey], limit: 1 },
+    async (_) => {
+      const meta = await getInstanceMetadata(await Storages.db(), c.req.raw.signal);
+      const {
+        title,
+        description,
+        short_description,
+        screenshot_ids,
+        thumbnail_id,
+      } = result.data;
+
+      const thumbnailUrl: string | undefined = (() => {
+        if (!thumbnail_id) {
+          return undefined;
+        }
+
+        const upload = dittoUploads.get(thumbnail_id);
+
+        if (!upload) {
+          throw new HTTPException(422, { message: 'Uploaded attachment is no longer available.' });
+        }
+        return upload.url;
+      })();
+
+      const screenshots: z.infer<typeof screenshotsSchema> = (screenshot_ids ?? []).map((id) => {
+        const upload = dittoUploads.get(id);
+
+        if (!upload) {
+          throw new HTTPException(422, { message: 'Uploaded attachment is no longer available.' });
+        }
+
+        const data = renderAttachment(upload);
+
+        if (!data?.url || !data.meta?.original) {
+          throw new HTTPException(422, { message: 'Image must have an URL and size dimensions.' });
+        }
+
+        const screenshot = {
+          src: data.url,
+          label: data.description,
+          sizes: `${data?.meta?.original?.width}x${data?.meta?.original?.height}`,
+          type: data?.type, // FIX-ME, I BEG YOU: Returns just `image` instead of a valid MIME type
+        };
+
+        return screenshot;
+      });
+
+      meta.name = title ?? meta.name;
+      meta.about = description ?? meta.about;
+      meta.tagline = short_description ?? meta.tagline;
+      meta.screenshots = screenshot_ids ? screenshots : meta.screenshots;
+      meta.picture = thumbnailUrl ?? meta.picture;
+      delete meta.event;
+
+      return {
+        kind: 0,
+        content: JSON.stringify(meta),
+        tags: [],
+      };
+    },
+    c,
+  );
+
+  return c.json(204);
+};

src/controllers/api/fallback.ts

@@ -1,4 +1,4 @@
-import { Context } from '@hono/hono';
+import { type Context } from '@hono/hono';
 
 const emptyArrayController = (c: Context) => c.json([]);
 const notImplementedController = (c: Context) => Promise.resolve(c.json({ error: 'Not implemented' }, 404));

src/controllers/api/instance.ts

@@ -96,6 +96,7 @@ const instanceV2Controller: AppController = async (c) => {
         '@2x': meta.picture,
       },
     },
+    screenshots: meta.screenshots,
     languages: [
       'en',
     ],

src/controllers/api/push.ts

@@ -1,3 +1,4 @@
+import { HTTPException } from '@hono/hono/http-exception';
 import { nip19 } from 'nostr-tools';
 import { z } from 'zod';
 
@@ -62,7 +63,7 @@ export const pushSubscribeController: AppController = async (c) => {
   const { subscription, data } = result.data;
 
   const pubkey = await signer.getPublicKey();
-  const tokenHash = await getTokenHash(accessToken as `token1${string}`);
+  const tokenHash = await getTokenHash(accessToken);
 
   const { id } = await kysely.transaction().execute(async (trx) => {
     await trx
@@ -105,13 +106,17 @@ export const getSubscriptionController: AppController = async (c) => {
   const accessToken = getAccessToken(c.req.raw);
 
   const kysely = await Storages.kysely();
-  const tokenHash = await getTokenHash(accessToken as `token1${string}`);
+  const tokenHash = await getTokenHash(accessToken);
 
   const row = await kysely
     .selectFrom('push_subscriptions')
     .selectAll()
     .where('token_hash', '=', tokenHash)
-    .executeTakeFirstOrThrow();
+    .executeTakeFirst();
+
+  if (!row) {
+    return c.json({ error: 'Record not found' }, 404);
+  }
 
   return c.json(
     {
@@ -124,8 +129,11 @@ export const getSubscriptionController: AppController = async (c) => {
   );
 };
 
-/** Get access token from HTTP headers, but only if it's a `token1`. Otherwise return undefined. */
+/**
-function getAccessToken(request: Request): `token1${string}` | undefined {
+ * Get access token from HTTP headers, but only if it's a `token1`.
+ * Otherwise throw an `HTTPException` with a 401.
+ */
+function getAccessToken(request: Request): `token1${string}` {
   const BEARER_REGEX = new RegExp(`^Bearer (${nip19.BECH32_REGEX.source})$`);
 
   const authorization = request.headers.get('authorization');
@@ -136,4 +144,6 @@ function getAccessToken(request: Request): `token1${string}` | undefined {
   if (accessToken?.startsWith('token1')) {
     return accessToken as `token1${string}`;
   }
+
+  throw new HTTPException(401, { message: 'The access token is invalid' });
 }

src/controllers/api/search.ts

@@ -11,7 +11,7 @@ import { nip05Cache } from '@/utils/nip05.ts';
 import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts';
 import { renderStatus } from '@/views/mastodon/statuses.ts';
 import { getFollowedPubkeys } from '@/queries.ts';
-import { getPubkeysBySearch } from '@/utils/search.ts';
+import { getIdsBySearch, getPubkeysBySearch } from '@/utils/search.ts';
 
 const searchQuerySchema = z.object({
   q: z.string().transform(decodeURIComponent),
@@ -94,10 +94,10 @@ async function searchEvents(
     limit,
   };
 
+  const kysely = await Storages.kysely();
+
   // For account search, use a special index, and prioritize followed accounts.
   if (type === 'accounts') {
-    const kysely = await Storages.kysely();
-
     const followedPubkeys = viewerPubkey ? await getFollowedPubkeys(viewerPubkey) : new Set<string>();
     const searchPubkeys = await getPubkeysBySearch(kysely, { q, limit, offset, followedPubkeys });
 
@@ -105,6 +105,13 @@ async function searchEvents(
     filter.search = undefined;
   }
 
+  // For status search, use a specific query so it supports offset and is open to customizations.
+  if (type === 'statuses') {
+    const ids = await getIdsBySearch(kysely, { q, limit, offset });
+    filter.ids = [...ids];
+    filter.search = undefined;
+  }
+
   // Results should only be shown from one author.
   if (account_id) {
     filter.authors = [account_id];

src/controllers/api/streaming.ts

@@ -1,6 +1,6 @@
 import TTLCache from '@isaacs/ttlcache';
 import { NostrEvent, NostrFilter } from '@nostrify/nostrify';
-import Debug from '@soapbox/stickynotes/debug';
+import { Stickynotes } from '@soapbox/stickynotes';
 import { z } from 'zod';
 
 import { type AppController } from '@/app.ts';
@@ -19,7 +19,7 @@ import { bech32ToPubkey, Time } from '@/utils.ts';
 import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts';
 import { renderNotification } from '@/views/mastodon/notifications.ts';
 
-const debug = Debug('ditto:streaming');
+const console = new Stickynotes('ditto:streaming');
 
 /**
  * Streaming timelines/categories.
@@ -100,7 +100,7 @@ const streamingController: AppController = async (c) => {
 
   function send(e: StreamingEvent) {
     if (socket.readyState === WebSocket.OPEN) {
-      debug('send', e.event, e.payload);
+      console.debug('send', e.event, e.payload);
       streamingServerMessagesCounter.inc();
       socket.send(JSON.stringify(e));
     }
@@ -129,7 +129,7 @@ const streamingController: AppController = async (c) => {
         }
       }
     } catch (e) {
-      debug('streaming error:', e);
+      console.debug('streaming error:', e);
     }
   }
 

src/controllers/manifest.ts

@@ -20,6 +20,7 @@ export const manifestController: AppController = async (c) => {
     scope: '/',
     short_name: meta.name,
     start_url: '/',
+    screenshots: meta.screenshots,
   };
 
   return c.json(manifest, {

src/db/migrations/041_pg_notify_id_only.ts

@@ -0,0 +1,27 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await sql`DROP TRIGGER IF EXISTS nostr_event_trigger ON nostr_events`.execute(db);
+
+  await sql`
+    CREATE OR REPLACE FUNCTION notify_nostr_event()
+    RETURNS TRIGGER AS $$
+    BEGIN
+        PERFORM pg_notify('nostr_event', NEW.id::text);
+
+        RETURN NEW;
+    END;
+    $$ LANGUAGE plpgsql;
+  `.execute(db);
+
+  await sql`
+    CREATE TRIGGER nostr_event_trigger
+    AFTER INSERT OR UPDATE ON nostr_events
+    FOR EACH ROW EXECUTE FUNCTION notify_nostr_event()
+  `.execute(db);
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await sql`DROP TRIGGER nostr_event_trigger ON nostr_events`.execute(db);
+  await sql`DROP FUNCTION notify_nostr_event()`.execute(db);
+}

src/notify.ts

@@ -1,5 +1,6 @@
 import { Semaphore } from '@lambdalisue/async';
 
+import { Conf } from '@/config.ts';
 import * as pipeline from '@/pipeline.ts';
 import { Storages } from '@/storages.ts';
 
@@ -7,12 +8,18 @@ const sem = new Semaphore(1);
 
 export async function startNotify(): Promise<void> {
   const { listen } = await Storages.database();
+  const store = await Storages.db();
 
   listen('nostr_event', (payload) => {
     sem.lock(async () => {
       try {
-        const event = JSON.parse(payload);
+        const id = payload;
-        await pipeline.handleEvent(event, AbortSignal.timeout(5000));
+        const timeout = Conf.db.timeouts.default;
+
+        const [event] = await store.query([{ ids: [id], limit: 1 }], { signal: AbortSignal.timeout(timeout) });
+        if (event) {
+          await pipeline.handleEvent(event, AbortSignal.timeout(timeout));
+        }
       } catch (e) {
         console.warn(e);
       }

src/pipeline.ts

@@ -33,31 +33,65 @@ const console = new Stickynotes('ditto:pipeline');
 async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise<void> {
   // Integer max value for Postgres. TODO: switch to a bigint in 2038.
   if (event.created_at >= 2_147_483_647) {
-    throw new RelayError('blocked', 'event too far in the future');
+    throw new RelayError('invalid', 'event too far in the future');
   }
+  // Integer max value for Postgres.
   if (event.kind >= 2_147_483_647) {
-    throw new RelayError('blocked', 'event kind too large');
+    throw new RelayError('invalid', 'event kind too large');
   }
-  if (!(await verifyEventWorker(event))) return;
+  // The only point of ephemeral events is to stream them,
-  if (encounterEvent(event)) return;
+  // so throw an error if we're not even going to do that.
-
+  if (NKinds.ephemeral(event.kind) && !isFresh(event)) {
-  console.info(`NostrEvent<${event.kind}> ${event.id}`);
+    throw new RelayError('invalid', 'event too old');
-  pipelineEventsCounter.inc({ kind: event.kind });
+  }
-
+  // Block NIP-70 events, because we have no way to `AUTH`.
   if (isProtectedEvent(event)) {
     throw new RelayError('invalid', 'protected event');
   }
+  // Validate the event's signature.
+  if (!(await verifyEventWorker(event))) {
+    throw new RelayError('invalid', 'invalid signature');
+  }
+  // Skip events that have been recently encountered.
+  // We must do this after verifying the signature.
+  if (encounterEvent(event)) {
+    throw new RelayError('duplicate', 'already have this event');
+  }
 
-  if (event.kind !== 24133 && event.pubkey !== Conf.pubkey) {
+  // Log the event.
+  console.info(`NostrEvent<${event.kind}> ${event.id}`);
+  pipelineEventsCounter.inc({ kind: event.kind });
+
+  // NIP-46 events get special treatment.
+  // They are exempt from policies and other side-effects, and should be streamed out immediately.
+  // If streaming fails, an error should be returned.
+  if (event.kind === 24133) {
+    await streamOut(event);
+    return;
+  }
+
+  // Ensure the event doesn't violate the policy.
+  if (event.pubkey !== Conf.pubkey) {
     await policyFilter(event, signal);
   }
 
+  // Prepare the event for additional checks.
+  // FIXME: This is kind of hacky. Should be reorganized to fetch only what's needed for each stage.
   await hydrateEvent(event, signal);
 
+  // Ensure that the author is not banned.
   const n = getTagSet(event.user?.tags ?? [], 'n');
-
   if (n.has('disabled')) {
-    throw new RelayError('blocked', 'user is disabled');
+    throw new RelayError('blocked', 'author is blocked');
+  }
+
+  // Ephemeral events must throw if they are not streamed out.
+  if (NKinds.ephemeral(event.kind)) {
+    await Promise.all([
+      streamOut(event),
+      webPush(event),
+    ]);
+    return;
   }
 
   const kysely = await Storages.kysely();
@@ -130,7 +164,7 @@ async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise<voi
 }
 
 /** Maybe store the event, if eligible. */
-async function storeEvent(event: DittoEvent, signal?: AbortSignal): Promise<undefined> {
+async function storeEvent(event: NostrEvent, signal?: AbortSignal): Promise<undefined> {
   if (NKinds.ephemeral(event.kind)) return;
   const store = await Storages.db();
 
@@ -217,20 +251,22 @@ async function setLanguage(event: NostrEvent): Promise<void> {
 
 /** Determine if the event is being received in a timely manner. */
 function isFresh(event: NostrEvent): boolean {
-  return eventAge(event) < Time.seconds(10);
+  return eventAge(event) < Time.minutes(1);
 }
 
 /** Distribute the event through active subscriptions. */
 async function streamOut(event: NostrEvent): Promise<void> {
-  if (isFresh(event)) {
+  if (!isFresh(event)) {
-    const pubsub = await Storages.pubsub();
+    throw new RelayError('invalid', 'event too old');
-    await pubsub.event(event);
   }
+
+  const pubsub = await Storages.pubsub();
+  await pubsub.event(event);
 }
 
 async function webPush(event: NostrEvent): Promise<void> {
   if (!isFresh(event)) {
-    return;
+    throw new RelayError('invalid', 'event too old');
   }
 
   const kysely = await Storages.kysely();

src/schema.test.ts

@@ -1,6 +1,6 @@
 import { assertEquals } from '@std/assert';
 
-import { percentageSchema } from '@/schema.ts';
+import { percentageSchema, sizesSchema } from '@/schema.ts';
 
 Deno.test('Value is any percentage from 1 to 100', () => {
   assertEquals(percentageSchema.safeParse('latvia' as unknown).success, false);
@@ -20,3 +20,12 @@ Deno.test('Value is any percentage from 1 to 100', () => {
 
   assertEquals(percentageSchema.safeParse('1e1').success, true);
 });
+
+Deno.test('Size or sizes has correct format', () => {
+  assertEquals(sizesSchema.safeParse('orphan' as unknown).success, false);
+  assertEquals(sizesSchema.safeParse('0000x 20x20' as unknown).success, false);
+  assertEquals(sizesSchema.safeParse('0000x10 20X20 1x22' as unknown).success, false);
+  assertEquals(sizesSchema.safeParse('1000x10 20X20 1x22' as unknown).success, true);
+  assertEquals(sizesSchema.safeParse('3333X6666 1x22 f' as unknown).success, false);
+  assertEquals(sizesSchema.safeParse('11xxxxxxx0 20X20 1x22' as unknown).success, false);
+});

src/schema.ts

@@ -65,6 +65,11 @@ const localeSchema = z.string().transform<Intl.Locale>((val, ctx) => {
   }
 });
 
+/** White-space separated list of sizes, each in the format <number with up to 4 digits>x<number with up to 4 digits> or with "X" in upper case. */
+const sizesSchema = z.string().refine((value) =>
+  value.split(' ').every((v) => /^[1-9]\d{0,3}[xX][1-9]\d{0,3}$/.test(v))
+);
+
 export {
   booleanParamSchema,
   decode64Schema,
@@ -75,4 +80,5 @@ export {
   localeSchema,
   percentageSchema,
   safeUrlSchema,
+  sizesSchema,
 };

src/schemas/mastodon.ts

@@ -0,0 +1,13 @@
+import { z } from 'zod';
+
+/** https://docs.joinmastodon.org/entities/Instance/#thumbnail */
+const thumbnailSchema = z.object({
+  url: z.string().url(),
+  blurhash: z.string().optional(),
+  versions: z.object({
+    '@1x': z.string().url().optional(),
+    '@2x': z.string().url().optional(),
+  }).optional(),
+});
+
+export { thumbnailSchema };

src/schemas/nostr.ts

@@ -2,17 +2,48 @@ import { NSchema as n } from '@nostrify/nostrify';
 import { getEventHash, verifyEvent } from 'nostr-tools';
 import { z } from 'zod';
 
-import { safeUrlSchema } from '@/schema.ts';
+import { safeUrlSchema, sizesSchema } from '@/schema.ts';
 
 /** Nostr event schema that also verifies the event's signature. */
 const signedEventSchema = n.event()
   .refine((event) => event.id === getEventHash(event), 'Event ID does not match hash')
   .refine(verifyEvent, 'Event signature is invalid');
 
+/**
+ * Stored in the kind 0 content.
+ * https://developer.mozilla.org/en-US/docs/Web/Manifest/screenshots
+ */
+const screenshotsSchema = z.array(z.object({
+  form_factor: z.enum(['narrow', 'wide']).optional(),
+  label: z.string().optional(),
+  platform: z.enum([
+    'android',
+    'chromeos',
+    'ipados',
+    'ios',
+    'kaios',
+    'macos',
+    'windows',
+    'xbox',
+    'chrome_web_store',
+    'itunes',
+    'microsoft-inbox',
+    'microsoft-store',
+    'play',
+  ]).optional(),
+  /** https://developer.mozilla.org/en-US/docs/Web/Manifest/screenshots#sizes */
+  sizes: sizesSchema,
+  /** Absolute URL. */
+  src: z.string().url(),
+  /** MIME type of the image. */
+  type: z.string().optional(),
+}));
+
 /** Kind 0 content schema for the Ditto server admin user. */
 const serverMetaSchema = n.metadata().and(z.object({
   tagline: z.string().optional().catch(undefined),
   email: z.string().optional().catch(undefined),
+  screenshots: screenshotsSchema.optional(),
 }));
 
 /** NIP-11 Relay Information Document. */
@@ -32,4 +63,4 @@ const emojiTagSchema = z.tuple([z.literal('emoji'), z.string(), z.string().url()
 /** NIP-30 custom emoji tag. */
 type EmojiTag = z.infer<typeof emojiTagSchema>;
 
-export { type EmojiTag, emojiTagSchema, relayInfoDocSchema, serverMetaSchema, signedEventSchema };
+export { type EmojiTag, emojiTagSchema, relayInfoDocSchema, screenshotsSchema, serverMetaSchema, signedEventSchema };

src/storages/EventsDB.ts

@@ -321,13 +321,13 @@ class EventsDB extends NPostgres {
         }
 
         if (domains.size) {
-          const query = this.opts.kysely
+          let query = this.opts.kysely
             .selectFrom('pubkey_domains')
             .select('pubkey')
             .where('domain', 'in', [...domains]);
 
           if (filter.authors) {
-            query.where('pubkey', 'in', filter.authors);
+            query = query.where('pubkey', 'in', filter.authors);
           }
 
           const pubkeys = await query.execute().then((rows) => rows.map((row) => row.pubkey));

src/utils/instance.ts

@@ -1,7 +1,8 @@
 import { NostrEvent, NostrMetadata, NSchema as n, NStore } from '@nostrify/nostrify';
+import { z } from 'zod';
 
 import { Conf } from '@/config.ts';
-import { serverMetaSchema } from '@/schemas/nostr.ts';
+import { screenshotsSchema, serverMetaSchema } from '@/schemas/nostr.ts';
 
 /** Like NostrMetadata, but some fields are required and also contains some extra fields. */
 export interface InstanceMetadata extends NostrMetadata {
@@ -11,6 +12,7 @@ export interface InstanceMetadata extends NostrMetadata {
   picture: string;
   tagline: string;
   event?: NostrEvent;
+  screenshots: z.infer<typeof screenshotsSchema>;
 }
 
 /** Get and parse instance metadata from the kind 0 of the admin user. */
@@ -34,5 +36,6 @@ export async function getInstanceMetadata(store: NStore, signal?: AbortSignal):
     email: meta.email ?? `postmaster@${Conf.url.host}`,
     picture: meta.picture ?? Conf.local('/images/thumbnail.png'),
     event,
+    screenshots: meta.screenshots ?? [],
   };
 }

src/utils/lookup.ts

@@ -83,7 +83,9 @@ export function extractIdentifier(value: string): string | undefined {
     return value;
   }
 
-  if (tldts.parse(value).isIcann) {
+  const { isIcann, domain } = tldts.parse(value);
+
+  if (isIcann && domain) {
     return value;
   }
 }

src/utils/search.test.ts

@@ -1,7 +1,7 @@
 import { assertEquals } from '@std/assert';
 
-import { createTestDB } from '@/test.ts';
+import { createTestDB, genEvent } from '@/test.ts';
-import { getPubkeysBySearch } from '@/utils/search.ts';
+import { getIdsBySearch, getPubkeysBySearch } from '@/utils/search.ts';
 
 Deno.test('fuzzy search works', async () => {
   await using db = await createTestDB();
@@ -48,3 +48,45 @@ Deno.test('fuzzy search works with offset', async () => {
     new Set(),
   );
 });
+
+Deno.test('Searching for posts work', async () => {
+  await using db = await createTestDB();
+
+  const event = genEvent({ content: "I'm not an orphan. Death is my importance", kind: 1 });
+  await db.store.event(event);
+  await db.kysely.updateTable('nostr_events').set('language', 'en').where('id', '=', event.id).execute();
+
+  const event2 = genEvent({ content: 'The more I explore is the more I fall in love with the music I make.', kind: 1 });
+  await db.store.event(event2);
+  await db.kysely.updateTable('nostr_events').set('language', 'en').where('id', '=', event2.id).execute();
+
+  assertEquals(
+    await getIdsBySearch(db.kysely, { q: 'Death is my importance', limit: 1, offset: 0 }), // ordered words
+    new Set([event.id]),
+  );
+
+  assertEquals(
+    await getIdsBySearch(db.kysely, { q: 'make I music', limit: 1, offset: 0 }), // reversed words
+    new Set([event2.id]),
+  );
+
+  assertEquals(
+    await getIdsBySearch(db.kysely, { q: 'language:en make I music', limit: 10, offset: 0 }), // reversed words, english
+    new Set([event2.id]),
+  );
+
+  assertEquals(
+    await getIdsBySearch(db.kysely, { q: 'language:en an orphan', limit: 10, offset: 0 }), // all posts in english plus search
+    new Set([event.id]),
+  );
+
+  assertEquals(
+    await getIdsBySearch(db.kysely, { q: 'language:en', limit: 10, offset: 0 }), // all posts in english
+    new Set([event.id, event2.id]),
+  );
+
+  assertEquals(
+    await getIdsBySearch(db.kysely, { q: '', limit: 10, offset: 0 }),
+    new Set(),
+  );
+});

src/utils/search.ts

@@ -1,6 +1,7 @@
 import { Kysely, sql } from 'kysely';
 
 import { DittoTables } from '@/db/DittoTables.ts';
+import { NIP50 } from '@nostrify/nostrify';
 
 /** Get pubkeys whose name and NIP-05 is similar to 'q' */
 export async function getPubkeysBySearch(
@@ -32,3 +33,77 @@ export async function getPubkeysBySearch(
 
   return new Set(Array.from(followingPubkeys.union(pubkeys)));
 }
+
+/**
+ * Get kind 1 ids whose content matches `q`.
+ * It supports NIP-50 extensions.
+ */
+export async function getIdsBySearch(
+  kysely: Kysely<DittoTables>,
+  opts: { q: string; limit: number; offset: number },
+): Promise<Set<string>> {
+  const { q, limit, offset } = opts;
+
+  const [lexemes] = (await sql<{ phraseto_tsquery: 'string' }>`SELECT phraseto_tsquery(${q})`.execute(kysely)).rows;
+
+  // if it's just stop words, don't bother making a request to the database
+  if (!lexemes.phraseto_tsquery) {
+    return new Set();
+  }
+
+  const tokens = NIP50.parseInput(q);
+  const parsedSearch = tokens.filter((t) => typeof t === 'string').join(' ');
+
+  let query = kysely
+    .selectFrom('nostr_events')
+    .select('id')
+    .where('kind', '=', 1)
+    .orderBy(['created_at desc'])
+    .limit(limit)
+    .offset(offset);
+
+  const languages = new Set<string>();
+  const domains = new Set<string>();
+
+  for (const token of tokens) {
+    if (typeof token === 'object' && token.key === 'language') {
+      languages.add(token.value);
+    }
+    if (typeof token === 'object' && token.key === 'domain') {
+      domains.add(token.value);
+    }
+  }
+
+  if (languages.size) {
+    query = query.where('language', 'in', [...languages]);
+  }
+
+  if (domains.size) {
+    const pubkeys = kysely
+      .selectFrom('pubkey_domains')
+      .select('pubkey')
+      .where('domain', 'in', [...domains]);
+
+    query = query.where('pubkey', 'in', pubkeys);
+  }
+
+  let fallbackQuery = query;
+  if (parsedSearch) {
+    query = query.where('search', '@@', sql`phraseto_tsquery(${parsedSearch})`);
+  }
+
+  const ids = new Set((await query.execute()).map(({ id }) => id));
+
+  // If there is no ids, fallback to `plainto_tsquery`
+  if (!ids.size) {
+    fallbackQuery = fallbackQuery.where(
+      'search',
+      '@@',
+      sql`plainto_tsquery(${parsedSearch})`,
+    );
+    const ids = new Set((await fallbackQuery.execute()).map(({ id }) => id));
+    return ids;
+  }
+
+  return ids;
+}

src/views/mastodon/accounts.ts

@@ -1,5 +1,4 @@
 import { NSchema as n } from '@nostrify/nostrify';
-import { escape } from 'entities';
 import { nip19, UnsignedEvent } from 'nostr-tools';
 
 import { Conf } from '@/config.ts';
@@ -7,6 +6,7 @@ import { MastodonAccount } from '@/entities/MastodonAccount.ts';
 import { type DittoEvent } from '@/interfaces/DittoEvent.ts';
 import { getLnurl } from '@/utils/lnurl.ts';
 import { parseAndVerifyNip05 } from '@/utils/nip05.ts';
+import { parseNoteContent } from '@/utils/note.ts';
 import { getTagSet } from '@/utils/tags.ts';
 import { faviconCache } from '@/utils/favicon.ts';
 import { nostrDate, nostrNow } from '@/utils.ts';
@@ -57,6 +57,7 @@ async function renderAccount(
       favicon = new URL('/favicon.ico', `https://${parsed05.domain}/`);
     }
   }
+  const { html } = parseNoteContent(about || '', []);
 
   return {
     id: pubkey,
@@ -77,7 +78,7 @@ async function renderAccount(
     header_static: banner,
     last_status_at: null,
     locked: false,
-    note: about ? escape(about) : '',
+    note: html,
     roles: [],
     source: opts.withSource
       ? {