mirror/ditto
mirror/ditto
1
0
Fork 0
mirror of https://gitlab.com/soapbox-pub/ditto.git synced 2025-12-06 11:29:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'search-fixes' into 'main'

Browse source 
search: parse bech32 ids from pasted URLs, improve Mastodon API compat

See merge request soapbox-pub/ditto!450
This commit is contained in:
Alex Gleason 2024-08-07 23:11:20 +00:00
commit e264d6116c
11 changed files with 205 additions and 171 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
deno.json
View file

@ -65,6 +65,7 @@
"nostr-relaypool": "npm:nostr-relaypool2@0.6.34",
"nostr-tools": "npm:nostr-tools@2.5.1",
"nostr-wasm": "npm:nostr-wasm@^0.1.0",
"path-to-regexp": "npm:path-to-regexp@^7.1.0",
"postgres": "https://raw.githubusercontent.com/xyzshantaram/postgres.js/8a9bbce88b3f6425ecaacd99a80372338b157a53/deno/mod.js",
"prom-client": "npm:prom-client@^15.1.2",
"question-deno": "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/mod.ts",

6
deno.lock generated
View file

@ -73,6 +73,7 @@
"npm:nostr-tools@^2.5.0": "npm:nostr-tools@2.5.1",
"npm:nostr-tools@^2.7.0": "npm:nostr-tools@2.7.0",
"npm:nostr-wasm@^0.1.0": "npm:nostr-wasm@0.1.0",
"npm:path-to-regexp@^7.1.0": "npm:path-to-regexp@7.1.0",
"npm:postgres@3.4.4": "npm:postgres@3.4.4",
"npm:prom-client@^15.1.2": "npm:prom-client@15.1.2",
"npm:tldts@^6.0.14": "npm:tldts@6.1.18",
@ -947,6 +948,10 @@
"integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==",
"dependencies": {}
},
"path-to-regexp@7.1.0": {
"integrity": "sha512-ZToe+MbUF4lBqk6dV8GKot4DKfzrxXsplOddH8zN3YK+qw9/McvP7+4ICjZvOne0jQhN4eJwHsX6tT0Ns19fvw==",
"dependencies": {}
},
"picomatch@2.3.1": {
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
"dependencies": {}
@ -1856,6 +1861,7 @@
"npm:nostr-relaypool2@0.6.34",
"npm:nostr-tools@2.5.1",
"npm:nostr-wasm@^0.1.0",
"npm:path-to-regexp@^7.1.0",
"npm:prom-client@^15.1.2",
"npm:tldts@^6.0.14",
"npm:tseep@^1.2.1",

45
src/controllers/api/accounts.ts
View file

@ -10,7 +10,7 @@ import { Storages } from '@/storages.ts';
import { uploadFile } from '@/utils/upload.ts';
import { nostrNow } from '@/utils.ts';
import { createEvent, paginated, parseBody, updateListEvent } from '@/utils/api.ts';
import { lookupAccount } from '@/utils/lookup.ts';
import { extractIdentifier, lookupAccount, lookupPubkey } from '@/utils/lookup.ts';
import { renderAccounts, renderEventAccounts, renderStatuses } from '@/views.ts';
import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts';
import { renderRelationship } from '@/views/mastodon/relationships.ts';
@ -110,45 +110,38 @@ const accountSearchQuerySchema = z.object({
q: z.string().transform(decodeURIComponent),
resolve: booleanParamSchema.optional().transform(Boolean),
following: z.boolean().default(false),
limit: z.coerce.number().catch(20).transform((value) => Math.min(Math.max(value, 0), 40)),
});
const accountSearchController: AppController = async (c) => {
const result = accountSearchQuerySchema.safeParse(c.req.query());
const { signal } = c.req.raw;
const { limit } = c.get('pagination');
const result = accountSearchQuerySchema.safeParse(c.req.query());
if (!result.success) {
return c.json({ error: 'Bad request', schema: result.error }, 422);
}
const { q, limit } = result.data;
const query = decodeURIComponent(q);
const query = decodeURIComponent(result.data.q);
const store = await Storages.search();
const [event, events] = await Promise.all([
lookupAccount(query),
store.query([{ kinds: [0], search: query, limit }], { signal }),
]);
const lookup = extractIdentifier(query);
const event = await lookupAccount(lookup ?? query);
const results = await hydrateEvents({
events: event ? [event, ...events] : events,
store,
signal,
});
if ((results.length < 1) && query.match(/npub1\w+/)) {
const possibleNpub = query;
try {
const npubHex = nip19.decode(possibleNpub);
return c.json([await accountFromPubkey(String(npubHex.data))]);
} catch (e) {
console.log(e);
return c.json([]);
}
if (!event && lookup) {
const pubkey = await lookupPubkey(lookup);
return c.json(pubkey ? [await accountFromPubkey(pubkey)] : []);
}
const accounts = await Promise.all(results.map((event) => renderAccount(event)));
const events = event ? [event] : await store.query([{ kinds: [0], search: query, limit }], { signal });
const accounts = await hydrateEvents({ events, store, signal }).then(
(events) =>
Promise.all(
events.map((event) => renderAccount(event)),
),
);
return c.json(accounts);
};

97
src/controllers/api/search.ts
View file

@ -5,14 +5,11 @@ import { z } from 'zod';
import { AppController } from '@/app.ts';
import { booleanParamSchema } from '@/schema.ts';
import { Storages } from '@/storages.ts';
import { dedupeEvents } from '@/utils.ts';
import { hydrateEvents } from '@/storages/hydrate.ts';
import { extractIdentifier, lookupPubkey } from '@/utils/lookup.ts';
import { nip05Cache } from '@/utils/nip05.ts';
import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts';
import { renderStatus } from '@/views/mastodon/statuses.ts';
import { hydrateEvents } from '@/storages/hydrate.ts';
/** Matches NIP-05 names with or without an @ in front. */
const ACCT_REGEX = /^@?(?:([\w.+-]+)@)?([\w.-]+)$/;
const searchQuerySchema = z.object({
q: z.string().transform(decodeURIComponent),
@ -33,43 +30,44 @@ const searchController: AppController = async (c) => {
return c.json({ error: 'Bad request', schema: result.error }, 422);
}
const [event, events] = await Promise.all([
lookupEvent(result.data, signal),
searchEvents(result.data, signal),
]);
const event = await lookupEvent(result.data, signal);
const lookup = extractIdentifier(result.data.q);
if (event) {
events.push(event);
// Render account from pubkey.
if (!event && lookup) {
const pubkey = await lookupPubkey(lookup);
return c.json({
accounts: pubkey ? [await accountFromPubkey(pubkey)] : [],
statuses: [],
hashtags: [],
});
}
let events: NostrEvent[] = [];
if (event) {
events = [event];
} else {
events = await searchEvents(result.data, signal);
}
const results = dedupeEvents(events);
const viewerPubkey = await c.get('signer')?.getPublicKey();
const [accounts, statuses] = await Promise.all([
Promise.all(
results
events
.filter((event) => event.kind === 0)
.map((event) => renderAccount(event))
.filter(Boolean),
),
Promise.all(
results
events
.filter((event) => event.kind === 1)
.map((event) => renderStatus(event, { viewerPubkey }))
.filter(Boolean),
),
]);
if ((result.data.type === 'accounts') && (accounts.length < 1) && (result.data.q.match(/npub1\w+/))) {
const possibleNpub = result.data.q;
try {
const npubHex = nip19.decode(possibleNpub);
accounts.push(await accountFromPubkey(String(npubHex.data)));
} catch (e) {
console.log(e);
}
}
return c.json({
accounts,
statuses,
@ -121,18 +119,26 @@ async function lookupEvent(query: SearchQuery, signal: AbortSignal): Promise<Nos
/** Get filters to lookup the input value. */
async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: AbortSignal): Promise<NostrFilter[]> {
const filters: NostrFilter[] = [];
const accounts = !type || type === 'accounts';
const statuses = !type || type === 'statuses';
if (!resolve || type === 'hashtags') {
return [];
}
if (n.id().safeParse(q).success) {
const filters: NostrFilter[] = [];
if (accounts) filters.push({ kinds: [0], authors: [q] });
if (statuses) filters.push({ kinds: [1], ids: [q] });
return filters;
}
if (new RegExp(`^${nip19.BECH32_REGEX.source}$`).test(q)) {
const lookup = extractIdentifier(q);
if (!lookup) return [];
try {
const result = nip19.decode(q);
const result = nip19.decode(lookup);
const filters: NostrFilter[] = [];
switch (result.type) {
case 'npub':
if (accounts) filters.push({ kinds: [0], authors: [result.data] });
@ -141,34 +147,27 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort
if (accounts) filters.push({ kinds: [0], authors: [result.data.pubkey] });
break;
case 'note':
if (statuses) {
filters.push({ kinds: [1], ids: [result.data] });
}
if (statuses) filters.push({ kinds: [1], ids: [result.data] });
break;
case 'nevent':
if (statuses) {
filters.push({ kinds: [1], ids: [result.data.id] });
}
if (statuses) filters.push({ kinds: [1], ids: [result.data.id] });
break;
}
} catch (_e) {
return filters;
} catch {
// do nothing
}
} else if (/^[0-9a-f]{64}$/.test(q)) {
if (accounts) filters.push({ kinds: [0], authors: [q] });
if (statuses) filters.push({ kinds: [1], ids: [q] });
} else if (accounts && ACCT_REGEX.test(q)) {
try {
const { pubkey } = await nip05Cache.fetch(q, { signal });
if (pubkey) {
filters.push({ kinds: [0], authors: [pubkey] });
}
} catch (_e) {
// do nothing
}
}
return filters;
try {
const { pubkey } = await nip05Cache.fetch(lookup, { signal });
if (pubkey) {
return [{ kinds: [0], authors: [pubkey] }];
}
} catch {
// do nothing
}
return [];
}
export { searchController };

1
src/entities/MastodonAccount.ts
View file

@ -36,6 +36,7 @@ export interface MastodonAccount {
};
};
statuses_count: number;
uri: string;
url: string;
username: string;
ditto: {

21
src/utils.ts
View file

@ -19,7 +19,7 @@ function bech32ToPubkey(bech32: string): string | undefined {
case 'npub':
return decoded.data;
}
} catch (_) {
} catch {
//
}
}
@ -78,11 +78,6 @@ async function sha256(message: string): Promise<string> {
return hashHex;
}
/** Deduplicate events by ID. */
function dedupeEvents(events: NostrEvent[]): NostrEvent[] {
return [...new Map(events.map((event) => [event.id, event])).values()];
}
/** Test whether the value is a Nostr ID. */
function isNostrId(value: unknown): boolean {
return n.id().safeParse(value).success;
@ -93,18 +88,6 @@ function isURL(value: unknown): boolean {
return z.string().url().safeParse(value).success;
}
export {
bech32ToPubkey,
dedupeEvents,
eventAge,
findTag,
isNostrId,
isURL,
type Nip05,
nostrDate,
nostrNow,
parseNip05,
sha256,
};
export { bech32ToPubkey, eventAge, findTag, isNostrId, isURL, type Nip05, nostrDate, nostrNow, parseNip05, sha256 };
export { Time } from '@/utils/time.ts';

53
src/utils/lookup.ts
View file

@ -1,4 +1,6 @@
import { NIP05, NostrEvent, NSchema as n } from '@nostrify/nostrify';
import { nip19 } from 'nostr-tools';
import { match } from 'path-to-regexp';
import { getAuthor } from '@/queries.ts';
import { bech32ToPubkey } from '@/utils.ts';
@ -35,3 +37,54 @@ export async function lookupPubkey(value: string, signal?: AbortSignal): Promise
}
}
}
/** Extract an acct or bech32 identifier out of a URL or of itself. */
export function extractIdentifier(value: string): string | undefined {
value = value.trim();
try {
const uri = new URL(value);
switch (uri.protocol) {
// Extract from NIP-19 URI, eg `nostr:npub1q3sle0kvfsehgsuexttt3ugjd8xdklxfwwkh559wxckmzddywnws6cd26p`.
case 'nostr:':
value = uri.pathname;
break;
// Extract from URL, eg `https://njump.me/npub1q3sle0kvfsehgsuexttt3ugjd8xdklxfwwkh559wxckmzddywnws6cd26p`.
case 'http:':
case 'https:': {
const accountUriMatch = match<{ acct: string }>('/users/:acct')(uri.pathname);
const accountUrlMatch = match<{ acct: string }>('/\\@:acct')(uri.pathname);
const statusUriMatch = match<{ acct: string; id: string }>('/users/:acct/statuses/:id')(uri.pathname);
const statusUrlMatch = match<{ acct: string; id: string }>('/\\@:acct/:id')(uri.pathname);
const soapboxMatch = match<{ acct: string; id: string }>('/\\@:acct/posts/:id')(uri.pathname);
const nostrMatch = match<{ bech32: string }>('/:bech32')(uri.pathname);
if (accountUriMatch) {
value = accountUriMatch.params.acct;
} else if (accountUrlMatch) {
value = accountUrlMatch.params.acct;
} else if (statusUriMatch) {
value = nip19.noteEncode(statusUriMatch.params.id);
} else if (statusUrlMatch) {
value = nip19.noteEncode(statusUrlMatch.params.id);
} else if (soapboxMatch) {
value = nip19.noteEncode(soapboxMatch.params.id);
} else if (nostrMatch) {
value = nostrMatch.params.bech32;
}
break;
}
}
} catch {
// do nothing
}
value = value.replace(/^@/, '');
if (n.bech32().safeParse(value).success) {
return value;
}
if (NIP05.regex().test(value)) {
return value;
}
}

2
src/utils/note.test.ts
View file

@ -4,7 +4,7 @@ import { eventFixture } from '@/test.ts';
import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts';
Deno.test('parseNoteContent', () => {
const { html, links, firstUrl } = parseNoteContent('Hello, world!');
const { html, links, firstUrl } = parseNoteContent('Hello, world!', []);
assertEquals(html, 'Hello, world!');
assertEquals(links, []);
assertEquals(firstUrl, undefined);

42
src/utils/note.ts
View file

@ -4,12 +4,27 @@ import linkify from 'linkifyjs';
import { nip19, nip21, nip27 } from 'nostr-tools';
import { Conf } from '@/config.ts';
import { MastodonMention } from '@/entities/MastodonMention.ts';
import { getUrlMediaType, isPermittedMediaType } from '@/utils/media.ts';
linkify.registerCustomProtocol('nostr', true);
linkify.registerCustomProtocol('wss');
const linkifyOpts: linkify.Opts = {
type Link = ReturnType<typeof linkify.find>[0];
interface ParsedNoteContent {
html: string;
links: Link[];
/** First non-media URL - eligible for a preview card. */
firstUrl: string | undefined;
}
/** Convert Nostr content to Mastodon API HTML. Also return parsed data. */
function parseNoteContent(content: string, mentions: MastodonMention[]): ParsedNoteContent {
const links = linkify.find(content).filter(isLinkURL);
const firstUrl = links.find(isNonMediaLink)?.href;
const html = linkifyStr(content, {
render: {
hashtag: ({ content }) => {
const tag = content.replace(/^#/, '');
@ -21,8 +36,11 @@ const linkifyOpts: linkify.Opts = {
const { decoded } = nip21.parse(content);
const pubkey = getDecodedPubkey(decoded);
if (pubkey) {
const name = pubkey.substring(0, 8);
const href = Conf.local(`/users/${pubkey}`);
const mention = mentions.find((m) => m.id === pubkey);
const npub = nip19.npubEncode(pubkey);
const acct = mention?.acct ?? npub;
const name = mention?.acct ?? npub.substring(0, 8);
const href = mention?.url ?? Conf.local(`/@${acct}`);
return `<span class="h-card"><a class="u-url mention" href="${href}" rel="ugc">@<span>${name}</span></a></span>`;
} else {
return '';
@ -36,23 +54,7 @@ const linkifyOpts: linkify.Opts = {
}
},
},
};
type Link = ReturnType<typeof linkify.find>[0];
interface ParsedNoteContent {
html: string;
links: Link[];
/** First non-media URL - eligible for a preview card. */
firstUrl: string | undefined;
}
/** Convert Nostr content to Mastodon API HTML. Also return parsed data. */
function parseNoteContent(content: string): ParsedNoteContent {
// Parsing twice is ineffecient, but I don't know how to do only once.
const html = linkifyStr(content, linkifyOpts).replace(/\n+$/, '');
const links = linkify.find(content).filter(isLinkURL);
const firstUrl = links.find(isNonMediaLink)?.href;
}).replace(/\n+$/, '');
return {
html,

6
src/views/mastodon/accounts.ts
View file

@ -42,10 +42,11 @@ async function renderAccount(
const npub = nip19.npubEncode(pubkey);
const parsed05 = await parseAndVerifyNip05(nip05, pubkey);
const acct = parsed05?.handle || npub;
return {
id: pubkey,
acct: parsed05?.handle || npub,
acct,
avatar: picture,
avatar_static: picture,
bot: false,
@ -78,7 +79,8 @@ async function renderAccount(
}
: undefined,
statuses_count: event.author_stats?.notes_count ?? 0,
url: Conf.local(`/users/${pubkey}`),
uri: Conf.local(`/users/${acct}`),
url: Conf.local(`/@${acct}`),
username: parsed05?.nickname || npub.substring(0, 8),
ditto: {
accepts_zaps: Boolean(getLnurl({ lud06, lud16 })),

40
src/views/mastodon/statuses.ts
View file

@ -46,13 +46,14 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
[{ kinds: [0], authors: mentionedPubkeys, limit: mentionedPubkeys.length }],
);
const { html, links, firstUrl } = parseNoteContent(stripimeta(event.content, event.tags));
const mentions = await Promise.all(
mentionedPubkeys.map((pubkey) => renderMention(pubkey, mentionedProfiles.find((event) => event.pubkey === pubkey))),
);
const [mentions, card, relatedEvents] = await Promise
const { html, links, firstUrl } = parseNoteContent(stripimeta(event.content, event.tags), mentions);
const [card, relatedEvents] = await Promise
.all([
Promise.all(
mentionedPubkeys.map((pubkey) => toMention(pubkey, mentionedProfiles.find((event) => event.pubkey === pubkey))),
),
firstUrl ? unfurlCardCached(firstUrl) : null,
viewerPubkey
? await store.query([
@ -71,7 +72,11 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
const bookmarkEvent = relatedEvents.find((event) => event.kind === 10003);
const zapEvent = relatedEvents.find((event) => event.kind === 9734);
const content = buildInlineRecipients(mentions) + html;
const compatMentions = buildInlineRecipients(mentions.filter((m) => {
if (m.id === account.id) return false;
if (html.includes(m.url)) return false;
return true;
}));
const cw = event.tags.find(([name]) => name === 'content-warning');
const subject = event.tags.find(([name]) => name === 'subject');
@ -95,7 +100,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
id: event.id,
account,
card,
content,
content: compatMentions + html,
created_at: nostrDate(event.created_at).toISOString(),
in_reply_to_id: replyId ?? null,
in_reply_to_account_id: null,
@ -121,8 +126,8 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
poll: null,
quote: !event.quote ? null : await renderStatus(event.quote, { depth: depth + 1 }),
quote_id: event.quote?.id ?? null,
uri: Conf.local(`/${note}`),
url: Conf.local(`/${note}`),
uri: Conf.local(`/users/${account.acct}/statuses/${event.id}`),
url: Conf.local(`/@${account.acct}/${event.id}`),
zapped: Boolean(zapEvent),
ditto: {
external_url: Conf.external(note),
@ -152,25 +157,14 @@ async function renderReblog(event: DittoEvent, opts: RenderStatusOpts): Promise<
};
}
async function toMention(pubkey: string, event?: NostrEvent): Promise<MastodonMention> {
const account = event ? await renderAccount(event) : undefined;
if (account) {
async function renderMention(pubkey: string, event?: NostrEvent): Promise<MastodonMention> {
const account = event ? await renderAccount(event) : await accountFromPubkey(pubkey);
return {
id: account.id,
acct: account.acct,
username: account.username,
url: account.url,
};
} else {
const npub = nip19.npubEncode(pubkey);
return {
id: pubkey,
acct: npub,
username: npub.substring(0, 8),
url: Conf.local(`/users/${pubkey}`),
};
}
}
function buildInlineRecipients(mentions: MastodonMention[]): string {
@ -178,7 +172,7 @@ function buildInlineRecipients(mentions: MastodonMention[]): string {
const elements = mentions.reduce<string[]>((acc, { url, username }) => {
const name = nip19.BECH32_REGEX.test(username) ? username.substring(0, 8) : username;
acc.push(`<a href="${url}" class="u-url mention" rel="ugc">@<span>${name}</span></a>`);
acc.push(`<span class="h-card"><a class="u-url mention" href="${url}" rel="ugc">@<span>${name}</span></a></span>`);
return acc;
}, []);