diff --git a/src/app.ts b/src/app.ts index fdcacf29..29886c89 100644 --- a/src/app.ts +++ b/src/app.ts @@ -49,6 +49,7 @@ import { nameRequestController, nameRequestsController, statusZapSplitsController, + updateInstanceController, updateZapSplitsController, } from '@/controllers/api/ditto.ts'; import { emptyArrayController, notImplementedController } from '@/controllers/api/fallback.ts'; @@ -169,6 +170,11 @@ const app = new Hono({ strict: false }); const debug = Debug('ditto:http'); +/** User-provided files in the gitignored `public/` directory. */ +const publicFiles = serveStatic({ root: './public/' }); +/** Static files provided by the Ditto repo, checked into git. */ +const staticFiles = serveStatic({ root: './static/' }); + app.use('*', rateLimitMiddleware(300, Time.minutes(5))); app.use('/api/*', metricsMiddleware, paginationMiddleware, logger(debug)); @@ -303,6 +309,8 @@ app.delete('/api/v1/pleroma/admin/statuses/:id', requireRole('admin'), pleromaAd app.get('/api/v1/admin/ditto/relays', requireRole('admin'), adminRelaysController); app.put('/api/v1/admin/ditto/relays', requireRole('admin'), adminSetRelaysController); +app.put('/api/v1/admin/ditto/instance', requireRole('admin'), updateInstanceController); + app.post('/api/v1/ditto/names', requireSigner, nameRequestController); app.get('/api/v1/ditto/names', requireSigner, nameRequestsController); @@ -362,13 +370,10 @@ app.get('/api/v1/conversations', emptyArrayController); app.get('/api/v1/lists', emptyArrayController); app.use('/api/*', notImplementedController); -app.use('/.well-known/*', notImplementedController); +app.use('/.well-known/*', publicFiles, notImplementedController); app.use('/nodeinfo/*', notImplementedController); app.use('/oauth/*', notImplementedController); -const publicFiles = serveStatic({ root: './public/' }); -const staticFiles = serveStatic({ root: './static/' }); - // Known frontend routes app.get('/:acct{@.*}', frontendController); app.get('/:acct{@.*}/*', frontendController); diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 4f5b55ff..27cf7590 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -258,8 +258,8 @@ const accountStatusesController: AppController = async (c) => { }; const updateCredentialsSchema = z.object({ - display_name: z.string().optional(), - note: z.string().optional(), + display_name: z.coerce.string().optional(), + note: z.coerce.string().optional(), avatar: fileSchema.or(z.literal('')).optional(), header: fileSchema.or(z.literal('')).optional(), locked: z.boolean().optional(), diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index b9be027f..5d51027e 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -1,19 +1,24 @@ import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; +import { HTTPException } from '@hono/hono/http-exception'; import { z } from 'zod'; -import { accountFromPubkey } from '@/views/mastodon/accounts.ts'; import { AppController } from '@/app.ts'; -import { addTag } from '@/utils/tags.ts'; -import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { booleanParamSchema, percentageSchema } from '@/schema.ts'; import { Conf } from '@/config.ts'; -import { createEvent, paginated, parseBody } from '@/utils/api.ts'; +import { dittoUploads } from '@/DittoUploads.ts'; +import { addTag } from '@/utils/tags.ts'; +import { getAuthor } from '@/queries.ts'; +import { createEvent, paginated, parseBody, updateEvent } from '@/utils/api.ts'; +import { getInstanceMetadata } from '@/utils/instance.ts'; import { deleteTag } from '@/utils/tags.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { DittoZapSplits, getZapSplits } from '@/utils/zap-split.ts'; -import { getAuthor } from '@/queries.ts'; +import { AdminSigner } from '@/signers/AdminSigner.ts'; +import { screenshotsSchema } from '@/schemas/nostr.ts'; +import { booleanParamSchema, percentageSchema } from '@/schema.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { renderNameRequest } from '@/views/ditto.ts'; +import { accountFromPubkey } from '@/views/mastodon/accounts.ts'; +import { renderAttachment } from '@/views/mastodon/attachments.ts'; import { renderAccount } from '@/views/mastodon/accounts.ts'; import { Storages } from '@/storages.ts'; import { updateListAdminEvent } from '@/utils/api.ts'; @@ -287,3 +292,90 @@ export const statusZapSplitsController: AppController = async (c) => { return c.json(zapSplits, 200); }; + +const updateInstanceSchema = z.object({ + title: z.string().optional(), + description: z.string().optional(), + /** Mastodon doesn't have this field. */ + short_description: z.string().optional(), + /** Mastodon doesn't have this field. */ + screenshot_ids: z.string().array().nullish(), + /** Mastodon doesn't have this field. */ + thumbnail_id: z.string().optional(), +}).strict(); + +export const updateInstanceController: AppController = async (c) => { + const body = await parseBody(c.req.raw); + const result = updateInstanceSchema.safeParse(body); + const pubkey = Conf.pubkey; + + if (!result.success) { + return c.json(result.error, 422); + } + + await updateEvent( + { kinds: [0], authors: [pubkey], limit: 1 }, + async (_) => { + const meta = await getInstanceMetadata(await Storages.db(), c.req.raw.signal); + const { + title, + description, + short_description, + screenshot_ids, + thumbnail_id, + } = result.data; + + const thumbnailUrl: string | undefined = (() => { + if (!thumbnail_id) { + return undefined; + } + + const upload = dittoUploads.get(thumbnail_id); + + if (!upload) { + throw new HTTPException(422, { message: 'Uploaded attachment is no longer available.' }); + } + return upload.url; + })(); + + const screenshots: z.infer = (screenshot_ids ?? []).map((id) => { + const upload = dittoUploads.get(id); + + if (!upload) { + throw new HTTPException(422, { message: 'Uploaded attachment is no longer available.' }); + } + + const data = renderAttachment(upload); + + if (!data?.url || !data.meta?.original) { + throw new HTTPException(422, { message: 'Image must have an URL and size dimensions.' }); + } + + const screenshot = { + src: data.url, + label: data.description, + sizes: `${data?.meta?.original?.width}x${data?.meta?.original?.height}`, + type: data?.type, // FIX-ME, I BEG YOU: Returns just `image` instead of a valid MIME type + }; + + return screenshot; + }); + + meta.name = title ?? meta.name; + meta.about = description ?? meta.about; + meta.tagline = short_description ?? meta.tagline; + meta.screenshots = screenshot_ids ? screenshots : meta.screenshots; + meta.picture = thumbnailUrl ?? meta.picture; + delete meta.event; + + return { + kind: 0, + content: JSON.stringify(meta), + tags: [], + }; + }, + c, + ); + + return c.json(204); +}; diff --git a/src/controllers/api/fallback.ts b/src/controllers/api/fallback.ts index 9e170093..0e98ac79 100644 --- a/src/controllers/api/fallback.ts +++ b/src/controllers/api/fallback.ts @@ -1,4 +1,4 @@ -import { Context } from '@hono/hono'; +import { type Context } from '@hono/hono'; const emptyArrayController = (c: Context) => c.json([]); const notImplementedController = (c: Context) => Promise.resolve(c.json({ error: 'Not implemented' }, 404)); diff --git a/src/controllers/api/instance.ts b/src/controllers/api/instance.ts index 78a72dd4..9f504cad 100644 --- a/src/controllers/api/instance.ts +++ b/src/controllers/api/instance.ts @@ -96,6 +96,7 @@ const instanceV2Controller: AppController = async (c) => { '@2x': meta.picture, }, }, + screenshots: meta.screenshots, languages: [ 'en', ], diff --git a/src/controllers/api/push.ts b/src/controllers/api/push.ts index 2f0d9844..0fa7c107 100644 --- a/src/controllers/api/push.ts +++ b/src/controllers/api/push.ts @@ -1,3 +1,4 @@ +import { HTTPException } from '@hono/hono/http-exception'; import { nip19 } from 'nostr-tools'; import { z } from 'zod'; @@ -62,7 +63,7 @@ export const pushSubscribeController: AppController = async (c) => { const { subscription, data } = result.data; const pubkey = await signer.getPublicKey(); - const tokenHash = await getTokenHash(accessToken as `token1${string}`); + const tokenHash = await getTokenHash(accessToken); const { id } = await kysely.transaction().execute(async (trx) => { await trx @@ -105,13 +106,17 @@ export const getSubscriptionController: AppController = async (c) => { const accessToken = getAccessToken(c.req.raw); const kysely = await Storages.kysely(); - const tokenHash = await getTokenHash(accessToken as `token1${string}`); + const tokenHash = await getTokenHash(accessToken); const row = await kysely .selectFrom('push_subscriptions') .selectAll() .where('token_hash', '=', tokenHash) - .executeTakeFirstOrThrow(); + .executeTakeFirst(); + + if (!row) { + return c.json({ error: 'Record not found' }, 404); + } return c.json( { @@ -124,8 +129,11 @@ export const getSubscriptionController: AppController = async (c) => { ); }; -/** Get access token from HTTP headers, but only if it's a `token1`. Otherwise return undefined. */ -function getAccessToken(request: Request): `token1${string}` | undefined { +/** + * Get access token from HTTP headers, but only if it's a `token1`. + * Otherwise throw an `HTTPException` with a 401. + */ +function getAccessToken(request: Request): `token1${string}` { const BEARER_REGEX = new RegExp(`^Bearer (${nip19.BECH32_REGEX.source})$`); const authorization = request.headers.get('authorization'); @@ -136,4 +144,6 @@ function getAccessToken(request: Request): `token1${string}` | undefined { if (accessToken?.startsWith('token1')) { return accessToken as `token1${string}`; } + + throw new HTTPException(401, { message: 'The access token is invalid' }); } diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index e8ec6057..4c3aa75f 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -11,7 +11,7 @@ import { nip05Cache } from '@/utils/nip05.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; import { getFollowedPubkeys } from '@/queries.ts'; -import { getPubkeysBySearch } from '@/utils/search.ts'; +import { getIdsBySearch, getPubkeysBySearch } from '@/utils/search.ts'; const searchQuerySchema = z.object({ q: z.string().transform(decodeURIComponent), @@ -94,10 +94,10 @@ async function searchEvents( limit, }; + const kysely = await Storages.kysely(); + // For account search, use a special index, and prioritize followed accounts. if (type === 'accounts') { - const kysely = await Storages.kysely(); - const followedPubkeys = viewerPubkey ? await getFollowedPubkeys(viewerPubkey) : new Set(); const searchPubkeys = await getPubkeysBySearch(kysely, { q, limit, offset, followedPubkeys }); @@ -105,6 +105,13 @@ async function searchEvents( filter.search = undefined; } + // For status search, use a specific query so it supports offset and is open to customizations. + if (type === 'statuses') { + const ids = await getIdsBySearch(kysely, { q, limit, offset }); + filter.ids = [...ids]; + filter.search = undefined; + } + // Results should only be shown from one author. if (account_id) { filter.authors = [account_id]; diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index 9693a16c..5e90085d 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -1,6 +1,6 @@ import TTLCache from '@isaacs/ttlcache'; import { NostrEvent, NostrFilter } from '@nostrify/nostrify'; -import Debug from '@soapbox/stickynotes/debug'; +import { Stickynotes } from '@soapbox/stickynotes'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; @@ -19,7 +19,7 @@ import { bech32ToPubkey, Time } from '@/utils.ts'; import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts'; import { renderNotification } from '@/views/mastodon/notifications.ts'; -const debug = Debug('ditto:streaming'); +const console = new Stickynotes('ditto:streaming'); /** * Streaming timelines/categories. @@ -100,7 +100,7 @@ const streamingController: AppController = async (c) => { function send(e: StreamingEvent) { if (socket.readyState === WebSocket.OPEN) { - debug('send', e.event, e.payload); + console.debug('send', e.event, e.payload); streamingServerMessagesCounter.inc(); socket.send(JSON.stringify(e)); } @@ -129,7 +129,7 @@ const streamingController: AppController = async (c) => { } } } catch (e) { - debug('streaming error:', e); + console.debug('streaming error:', e); } } diff --git a/src/controllers/manifest.ts b/src/controllers/manifest.ts index 60e2a2ac..2e75de04 100644 --- a/src/controllers/manifest.ts +++ b/src/controllers/manifest.ts @@ -20,6 +20,7 @@ export const manifestController: AppController = async (c) => { scope: '/', short_name: meta.name, start_url: '/', + screenshots: meta.screenshots, }; return c.json(manifest, { diff --git a/src/db/migrations/041_pg_notify_id_only.ts b/src/db/migrations/041_pg_notify_id_only.ts new file mode 100644 index 00000000..192dd42f --- /dev/null +++ b/src/db/migrations/041_pg_notify_id_only.ts @@ -0,0 +1,27 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await sql`DROP TRIGGER IF EXISTS nostr_event_trigger ON nostr_events`.execute(db); + + await sql` + CREATE OR REPLACE FUNCTION notify_nostr_event() + RETURNS TRIGGER AS $$ + BEGIN + PERFORM pg_notify('nostr_event', NEW.id::text); + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + `.execute(db); + + await sql` + CREATE TRIGGER nostr_event_trigger + AFTER INSERT OR UPDATE ON nostr_events + FOR EACH ROW EXECUTE FUNCTION notify_nostr_event() + `.execute(db); +} + +export async function down(db: Kysely): Promise { + await sql`DROP TRIGGER nostr_event_trigger ON nostr_events`.execute(db); + await sql`DROP FUNCTION notify_nostr_event()`.execute(db); +} diff --git a/src/notify.ts b/src/notify.ts index 4033d6c6..69480875 100644 --- a/src/notify.ts +++ b/src/notify.ts @@ -1,5 +1,6 @@ import { Semaphore } from '@lambdalisue/async'; +import { Conf } from '@/config.ts'; import * as pipeline from '@/pipeline.ts'; import { Storages } from '@/storages.ts'; @@ -7,12 +8,18 @@ const sem = new Semaphore(1); export async function startNotify(): Promise { const { listen } = await Storages.database(); + const store = await Storages.db(); listen('nostr_event', (payload) => { sem.lock(async () => { try { - const event = JSON.parse(payload); - await pipeline.handleEvent(event, AbortSignal.timeout(5000)); + const id = payload; + const timeout = Conf.db.timeouts.default; + + const [event] = await store.query([{ ids: [id], limit: 1 }], { signal: AbortSignal.timeout(timeout) }); + if (event) { + await pipeline.handleEvent(event, AbortSignal.timeout(timeout)); + } } catch (e) { console.warn(e); } diff --git a/src/pipeline.ts b/src/pipeline.ts index 867f2c22..ab303709 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -33,31 +33,65 @@ const console = new Stickynotes('ditto:pipeline'); async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { // Integer max value for Postgres. TODO: switch to a bigint in 2038. if (event.created_at >= 2_147_483_647) { - throw new RelayError('blocked', 'event too far in the future'); + throw new RelayError('invalid', 'event too far in the future'); } + // Integer max value for Postgres. if (event.kind >= 2_147_483_647) { - throw new RelayError('blocked', 'event kind too large'); + throw new RelayError('invalid', 'event kind too large'); } - if (!(await verifyEventWorker(event))) return; - if (encounterEvent(event)) return; - - console.info(`NostrEvent<${event.kind}> ${event.id}`); - pipelineEventsCounter.inc({ kind: event.kind }); - + // The only point of ephemeral events is to stream them, + // so throw an error if we're not even going to do that. + if (NKinds.ephemeral(event.kind) && !isFresh(event)) { + throw new RelayError('invalid', 'event too old'); + } + // Block NIP-70 events, because we have no way to `AUTH`. if (isProtectedEvent(event)) { throw new RelayError('invalid', 'protected event'); } + // Validate the event's signature. + if (!(await verifyEventWorker(event))) { + throw new RelayError('invalid', 'invalid signature'); + } + // Skip events that have been recently encountered. + // We must do this after verifying the signature. + if (encounterEvent(event)) { + throw new RelayError('duplicate', 'already have this event'); + } - if (event.kind !== 24133 && event.pubkey !== Conf.pubkey) { + // Log the event. + console.info(`NostrEvent<${event.kind}> ${event.id}`); + pipelineEventsCounter.inc({ kind: event.kind }); + + // NIP-46 events get special treatment. + // They are exempt from policies and other side-effects, and should be streamed out immediately. + // If streaming fails, an error should be returned. + if (event.kind === 24133) { + await streamOut(event); + return; + } + + // Ensure the event doesn't violate the policy. + if (event.pubkey !== Conf.pubkey) { await policyFilter(event, signal); } + // Prepare the event for additional checks. + // FIXME: This is kind of hacky. Should be reorganized to fetch only what's needed for each stage. await hydrateEvent(event, signal); + // Ensure that the author is not banned. const n = getTagSet(event.user?.tags ?? [], 'n'); - if (n.has('disabled')) { - throw new RelayError('blocked', 'user is disabled'); + throw new RelayError('blocked', 'author is blocked'); + } + + // Ephemeral events must throw if they are not streamed out. + if (NKinds.ephemeral(event.kind)) { + await Promise.all([ + streamOut(event), + webPush(event), + ]); + return; } const kysely = await Storages.kysely(); @@ -130,7 +164,7 @@ async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { +async function storeEvent(event: NostrEvent, signal?: AbortSignal): Promise { if (NKinds.ephemeral(event.kind)) return; const store = await Storages.db(); @@ -217,20 +251,22 @@ async function setLanguage(event: NostrEvent): Promise { /** Determine if the event is being received in a timely manner. */ function isFresh(event: NostrEvent): boolean { - return eventAge(event) < Time.seconds(10); + return eventAge(event) < Time.minutes(1); } /** Distribute the event through active subscriptions. */ async function streamOut(event: NostrEvent): Promise { - if (isFresh(event)) { - const pubsub = await Storages.pubsub(); - await pubsub.event(event); + if (!isFresh(event)) { + throw new RelayError('invalid', 'event too old'); } + + const pubsub = await Storages.pubsub(); + await pubsub.event(event); } async function webPush(event: NostrEvent): Promise { if (!isFresh(event)) { - return; + throw new RelayError('invalid', 'event too old'); } const kysely = await Storages.kysely(); diff --git a/src/schema.test.ts b/src/schema.test.ts index c6b577de..66809c53 100644 --- a/src/schema.test.ts +++ b/src/schema.test.ts @@ -1,6 +1,6 @@ import { assertEquals } from '@std/assert'; -import { percentageSchema } from '@/schema.ts'; +import { percentageSchema, sizesSchema } from '@/schema.ts'; Deno.test('Value is any percentage from 1 to 100', () => { assertEquals(percentageSchema.safeParse('latvia' as unknown).success, false); @@ -20,3 +20,12 @@ Deno.test('Value is any percentage from 1 to 100', () => { assertEquals(percentageSchema.safeParse('1e1').success, true); }); + +Deno.test('Size or sizes has correct format', () => { + assertEquals(sizesSchema.safeParse('orphan' as unknown).success, false); + assertEquals(sizesSchema.safeParse('0000x 20x20' as unknown).success, false); + assertEquals(sizesSchema.safeParse('0000x10 20X20 1x22' as unknown).success, false); + assertEquals(sizesSchema.safeParse('1000x10 20X20 1x22' as unknown).success, true); + assertEquals(sizesSchema.safeParse('3333X6666 1x22 f' as unknown).success, false); + assertEquals(sizesSchema.safeParse('11xxxxxxx0 20X20 1x22' as unknown).success, false); +}); diff --git a/src/schema.ts b/src/schema.ts index a9dd56e3..6147f562 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -65,6 +65,11 @@ const localeSchema = z.string().transform((val, ctx) => { } }); +/** White-space separated list of sizes, each in the format x or with "X" in upper case. */ +const sizesSchema = z.string().refine((value) => + value.split(' ').every((v) => /^[1-9]\d{0,3}[xX][1-9]\d{0,3}$/.test(v)) +); + export { booleanParamSchema, decode64Schema, @@ -75,4 +80,5 @@ export { localeSchema, percentageSchema, safeUrlSchema, + sizesSchema, }; diff --git a/src/schemas/mastodon.ts b/src/schemas/mastodon.ts new file mode 100644 index 00000000..bedd1aad --- /dev/null +++ b/src/schemas/mastodon.ts @@ -0,0 +1,13 @@ +import { z } from 'zod'; + +/** https://docs.joinmastodon.org/entities/Instance/#thumbnail */ +const thumbnailSchema = z.object({ + url: z.string().url(), + blurhash: z.string().optional(), + versions: z.object({ + '@1x': z.string().url().optional(), + '@2x': z.string().url().optional(), + }).optional(), +}); + +export { thumbnailSchema }; diff --git a/src/schemas/nostr.ts b/src/schemas/nostr.ts index d8aa29a4..46f68a34 100644 --- a/src/schemas/nostr.ts +++ b/src/schemas/nostr.ts @@ -2,17 +2,48 @@ import { NSchema as n } from '@nostrify/nostrify'; import { getEventHash, verifyEvent } from 'nostr-tools'; import { z } from 'zod'; -import { safeUrlSchema } from '@/schema.ts'; +import { safeUrlSchema, sizesSchema } from '@/schema.ts'; /** Nostr event schema that also verifies the event's signature. */ const signedEventSchema = n.event() .refine((event) => event.id === getEventHash(event), 'Event ID does not match hash') .refine(verifyEvent, 'Event signature is invalid'); +/** + * Stored in the kind 0 content. + * https://developer.mozilla.org/en-US/docs/Web/Manifest/screenshots + */ +const screenshotsSchema = z.array(z.object({ + form_factor: z.enum(['narrow', 'wide']).optional(), + label: z.string().optional(), + platform: z.enum([ + 'android', + 'chromeos', + 'ipados', + 'ios', + 'kaios', + 'macos', + 'windows', + 'xbox', + 'chrome_web_store', + 'itunes', + 'microsoft-inbox', + 'microsoft-store', + 'play', + ]).optional(), + /** https://developer.mozilla.org/en-US/docs/Web/Manifest/screenshots#sizes */ + sizes: sizesSchema, + /** Absolute URL. */ + src: z.string().url(), + /** MIME type of the image. */ + type: z.string().optional(), +})); + /** Kind 0 content schema for the Ditto server admin user. */ const serverMetaSchema = n.metadata().and(z.object({ tagline: z.string().optional().catch(undefined), email: z.string().optional().catch(undefined), + screenshots: screenshotsSchema.optional(), })); /** NIP-11 Relay Information Document. */ @@ -32,4 +63,4 @@ const emojiTagSchema = z.tuple([z.literal('emoji'), z.string(), z.string().url() /** NIP-30 custom emoji tag. */ type EmojiTag = z.infer; -export { type EmojiTag, emojiTagSchema, relayInfoDocSchema, serverMetaSchema, signedEventSchema }; +export { type EmojiTag, emojiTagSchema, relayInfoDocSchema, screenshotsSchema, serverMetaSchema, signedEventSchema }; diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 591a1376..208e0c72 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -321,13 +321,13 @@ class EventsDB extends NPostgres { } if (domains.size) { - const query = this.opts.kysely + let query = this.opts.kysely .selectFrom('pubkey_domains') .select('pubkey') .where('domain', 'in', [...domains]); if (filter.authors) { - query.where('pubkey', 'in', filter.authors); + query = query.where('pubkey', 'in', filter.authors); } const pubkeys = await query.execute().then((rows) => rows.map((row) => row.pubkey)); diff --git a/src/utils/instance.ts b/src/utils/instance.ts index f46541c2..c0b9c0d4 100644 --- a/src/utils/instance.ts +++ b/src/utils/instance.ts @@ -1,7 +1,8 @@ import { NostrEvent, NostrMetadata, NSchema as n, NStore } from '@nostrify/nostrify'; +import { z } from 'zod'; import { Conf } from '@/config.ts'; -import { serverMetaSchema } from '@/schemas/nostr.ts'; +import { screenshotsSchema, serverMetaSchema } from '@/schemas/nostr.ts'; /** Like NostrMetadata, but some fields are required and also contains some extra fields. */ export interface InstanceMetadata extends NostrMetadata { @@ -11,6 +12,7 @@ export interface InstanceMetadata extends NostrMetadata { picture: string; tagline: string; event?: NostrEvent; + screenshots: z.infer; } /** Get and parse instance metadata from the kind 0 of the admin user. */ @@ -34,5 +36,6 @@ export async function getInstanceMetadata(store: NStore, signal?: AbortSignal): email: meta.email ?? `postmaster@${Conf.url.host}`, picture: meta.picture ?? Conf.local('/images/thumbnail.png'), event, + screenshots: meta.screenshots ?? [], }; } diff --git a/src/utils/lookup.ts b/src/utils/lookup.ts index 29133f80..48c6ba81 100644 --- a/src/utils/lookup.ts +++ b/src/utils/lookup.ts @@ -83,7 +83,9 @@ export function extractIdentifier(value: string): string | undefined { return value; } - if (tldts.parse(value).isIcann) { + const { isIcann, domain } = tldts.parse(value); + + if (isIcann && domain) { return value; } } diff --git a/src/utils/search.test.ts b/src/utils/search.test.ts index 056c2927..d7073a39 100644 --- a/src/utils/search.test.ts +++ b/src/utils/search.test.ts @@ -1,7 +1,7 @@ import { assertEquals } from '@std/assert'; -import { createTestDB } from '@/test.ts'; -import { getPubkeysBySearch } from '@/utils/search.ts'; +import { createTestDB, genEvent } from '@/test.ts'; +import { getIdsBySearch, getPubkeysBySearch } from '@/utils/search.ts'; Deno.test('fuzzy search works', async () => { await using db = await createTestDB(); @@ -48,3 +48,45 @@ Deno.test('fuzzy search works with offset', async () => { new Set(), ); }); + +Deno.test('Searching for posts work', async () => { + await using db = await createTestDB(); + + const event = genEvent({ content: "I'm not an orphan. Death is my importance", kind: 1 }); + await db.store.event(event); + await db.kysely.updateTable('nostr_events').set('language', 'en').where('id', '=', event.id).execute(); + + const event2 = genEvent({ content: 'The more I explore is the more I fall in love with the music I make.', kind: 1 }); + await db.store.event(event2); + await db.kysely.updateTable('nostr_events').set('language', 'en').where('id', '=', event2.id).execute(); + + assertEquals( + await getIdsBySearch(db.kysely, { q: 'Death is my importance', limit: 1, offset: 0 }), // ordered words + new Set([event.id]), + ); + + assertEquals( + await getIdsBySearch(db.kysely, { q: 'make I music', limit: 1, offset: 0 }), // reversed words + new Set([event2.id]), + ); + + assertEquals( + await getIdsBySearch(db.kysely, { q: 'language:en make I music', limit: 10, offset: 0 }), // reversed words, english + new Set([event2.id]), + ); + + assertEquals( + await getIdsBySearch(db.kysely, { q: 'language:en an orphan', limit: 10, offset: 0 }), // all posts in english plus search + new Set([event.id]), + ); + + assertEquals( + await getIdsBySearch(db.kysely, { q: 'language:en', limit: 10, offset: 0 }), // all posts in english + new Set([event.id, event2.id]), + ); + + assertEquals( + await getIdsBySearch(db.kysely, { q: '', limit: 10, offset: 0 }), + new Set(), + ); +}); diff --git a/src/utils/search.ts b/src/utils/search.ts index 29ecefd9..420b0812 100644 --- a/src/utils/search.ts +++ b/src/utils/search.ts @@ -1,6 +1,7 @@ import { Kysely, sql } from 'kysely'; import { DittoTables } from '@/db/DittoTables.ts'; +import { NIP50 } from '@nostrify/nostrify'; /** Get pubkeys whose name and NIP-05 is similar to 'q' */ export async function getPubkeysBySearch( @@ -32,3 +33,77 @@ export async function getPubkeysBySearch( return new Set(Array.from(followingPubkeys.union(pubkeys))); } + +/** + * Get kind 1 ids whose content matches `q`. + * It supports NIP-50 extensions. + */ +export async function getIdsBySearch( + kysely: Kysely, + opts: { q: string; limit: number; offset: number }, +): Promise> { + const { q, limit, offset } = opts; + + const [lexemes] = (await sql<{ phraseto_tsquery: 'string' }>`SELECT phraseto_tsquery(${q})`.execute(kysely)).rows; + + // if it's just stop words, don't bother making a request to the database + if (!lexemes.phraseto_tsquery) { + return new Set(); + } + + const tokens = NIP50.parseInput(q); + const parsedSearch = tokens.filter((t) => typeof t === 'string').join(' '); + + let query = kysely + .selectFrom('nostr_events') + .select('id') + .where('kind', '=', 1) + .orderBy(['created_at desc']) + .limit(limit) + .offset(offset); + + const languages = new Set(); + const domains = new Set(); + + for (const token of tokens) { + if (typeof token === 'object' && token.key === 'language') { + languages.add(token.value); + } + if (typeof token === 'object' && token.key === 'domain') { + domains.add(token.value); + } + } + + if (languages.size) { + query = query.where('language', 'in', [...languages]); + } + + if (domains.size) { + const pubkeys = kysely + .selectFrom('pubkey_domains') + .select('pubkey') + .where('domain', 'in', [...domains]); + + query = query.where('pubkey', 'in', pubkeys); + } + + let fallbackQuery = query; + if (parsedSearch) { + query = query.where('search', '@@', sql`phraseto_tsquery(${parsedSearch})`); + } + + const ids = new Set((await query.execute()).map(({ id }) => id)); + + // If there is no ids, fallback to `plainto_tsquery` + if (!ids.size) { + fallbackQuery = fallbackQuery.where( + 'search', + '@@', + sql`plainto_tsquery(${parsedSearch})`, + ); + const ids = new Set((await fallbackQuery.execute()).map(({ id }) => id)); + return ids; + } + + return ids; +} diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index 663aff2c..6d0e0446 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -1,5 +1,4 @@ import { NSchema as n } from '@nostrify/nostrify'; -import { escape } from 'entities'; import { nip19, UnsignedEvent } from 'nostr-tools'; import { Conf } from '@/config.ts'; @@ -7,6 +6,7 @@ import { MastodonAccount } from '@/entities/MastodonAccount.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { getLnurl } from '@/utils/lnurl.ts'; import { parseAndVerifyNip05 } from '@/utils/nip05.ts'; +import { parseNoteContent } from '@/utils/note.ts'; import { getTagSet } from '@/utils/tags.ts'; import { faviconCache } from '@/utils/favicon.ts'; import { nostrDate, nostrNow } from '@/utils.ts'; @@ -57,6 +57,7 @@ async function renderAccount( favicon = new URL('/favicon.ico', `https://${parsed05.domain}/`); } } + const { html } = parseNoteContent(about || '', []); return { id: pubkey, @@ -77,7 +78,7 @@ async function renderAccount( header_static: banner, last_status_at: null, locked: false, - note: about ? escape(about) : '', + note: html, roles: [], source: opts.withSource ? {