mirror/ditto
mirror/ditto
1
0
Fork 0
mirror of https://gitlab.com/soapbox-pub/ditto.git synced 2025-12-06 11:29:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'main' into ditto-as-a-service

Browse source 
need the commander dependency from db:export
This commit is contained in:
Siddharth Singh 2024-09-02 19:11:09 +05:30
commit 71965f53aa
No known key found for this signature in database
12 changed files with 367 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
deno.json
View file

@ -34,7 +34,7 @@
"@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1",
"@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0",
"@nostrify/db": "jsr:@nostrify/db@^0.31.2",
"@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.30.0",
"@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.30.1",
"@scure/base": "npm:@scure/base@^1.1.6",
"@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs",
"@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0",
@ -50,6 +50,7 @@
"@std/streams": "jsr:@std/streams@^0.223.0",
"comlink": "npm:comlink@^4.4.1",
"comlink-async-generator": "npm:comlink-async-generator@^0.0.1",
"commander": "npm:commander@12.1.0",
"deno-safe-fetch/load": "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts",
"deno.json": "./deno.json",
"entities": "npm:entities@^4.5.0",

40
deno.lock generated
View file

@ -12,13 +12,14 @@
"jsr:@gleasonator/policy@0.4.0": "jsr:@gleasonator/policy@0.4.0",
"jsr:@gleasonator/policy@0.4.1": "jsr:@gleasonator/policy@0.4.1",
"jsr:@gleasonator/policy@0.4.2": "jsr:@gleasonator/policy@0.4.2",
"jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.5",
"jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.9",
"jsr:@lambdalisue/async@^2.1.1": "jsr:@lambdalisue/async@2.1.1",
"jsr:@nostrify/db@^0.31.2": "jsr:@nostrify/db@0.31.2",
"jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5",
"jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4",
"jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5",
"jsr:@nostrify/nostrify@^0.30.0": "jsr:@nostrify/nostrify@0.30.0",
"jsr:@nostrify/nostrify@^0.30.0": "jsr:@nostrify/nostrify@0.30.1",
"jsr:@nostrify/nostrify@^0.30.1": "jsr:@nostrify/nostrify@0.30.1",
"jsr:@nostrify/types@^0.30.0": "jsr:@nostrify/types@0.30.0",
"jsr:@soapbox/kysely-deno-sqlite@^2.1.0": "jsr:@soapbox/kysely-deno-sqlite@2.2.0",
"jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0",
@ -30,6 +31,7 @@
"jsr:@std/bytes@^0.224.0": "jsr:@std/bytes@0.224.0",
"jsr:@std/bytes@^1.0.0-rc.3": "jsr:@std/bytes@1.0.0",
"jsr:@std/bytes@^1.0.1-rc.3": "jsr:@std/bytes@1.0.2",
"jsr:@std/bytes@^1.0.2": "jsr:@std/bytes@1.0.2",
"jsr:@std/bytes@^1.0.2-rc.3": "jsr:@std/bytes@1.0.2",
"jsr:@std/crypto@^0.224.0": "jsr:@std/crypto@0.224.0",
"jsr:@std/dotenv@^0.224.0": "jsr:@std/dotenv@0.224.2",
@ -43,7 +45,7 @@
"jsr:@std/fs@^0.221.0": "jsr:@std/fs@0.221.0",
"jsr:@std/fs@^0.229.3": "jsr:@std/fs@0.229.3",
"jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.1",
"jsr:@std/io@^0.224": "jsr:@std/io@0.224.4",
"jsr:@std/io@^0.224": "jsr:@std/io@0.224.6",
"jsr:@std/json@^0.223.0": "jsr:@std/json@0.223.0",
"jsr:@std/media-types@^0.224.1": "jsr:@std/media-types@0.224.1",
"jsr:@std/path@0.213.1": "jsr:@std/path@0.213.1",
@ -60,6 +62,7 @@
"npm:comlink-async-generator": "npm:comlink-async-generator@0.0.1",
"npm:comlink-async-generator@^0.0.1": "npm:comlink-async-generator@0.0.1",
"npm:comlink@^4.4.1": "npm:comlink@4.4.1",
"npm:commander@12.1.0": "npm:commander@12.1.0",
"npm:entities@^4.5.0": "npm:entities@4.5.0",
"npm:fast-stable-stringify@^1.0.0": "npm:fast-stable-stringify@1.0.0",
"npm:formdata-helper@^0.3.0": "npm:formdata-helper@0.3.0",
@ -171,6 +174,9 @@
"@hono/hono@4.5.5": {
"integrity": "e5a63b5f535475cd80974b65fed23a138d0cbb91fe1cc9a17a7c7278e835c308"
},
"@hono/hono@4.5.9": {
"integrity": "47f561e67aedbd6d1e21e3a1ae26c1b80ffdb62a51c161d502e75bee17ca40af"
},
"@lambdalisue/async@2.1.1": {
"integrity": "1fc9bc6f4ed50215cd2f7217842b18cea80f81c25744f88f8c5eb4be5a1c9ab4"
},
@ -227,6 +233,21 @@
"npm:zod@^3.23.8"
]
},
"@nostrify/nostrify@0.30.1": {
"integrity": "fcc923707e87a9fbecc82dbb18756d1d3d134cd0763f4b1254c4bce709e811eb",
"dependencies": [
"jsr:@nostrify/types@^0.30.0",
"jsr:@std/crypto@^0.224.0",
"jsr:@std/encoding@^0.224.1",
"npm:@scure/base@^1.1.6",
"npm:@scure/bip32@^1.4.0",
"npm:@scure/bip39@^1.3.0",
"npm:lru-cache@^10.2.0",
"npm:nostr-tools@^2.7.0",
"npm:websocket-ts@^2.1.5",
"npm:zod@^3.23.8"
]
},
"@nostrify/types@0.30.0": {
"integrity": "1f38fa849cff930bd709edbf94ef9ac02f46afb8b851f86c8736517b354616da"
},
@ -341,6 +362,12 @@
"jsr:@std/bytes@^1.0.2-rc.3"
]
},
"@std/io@0.224.6": {
"integrity": "eefe034a370be34daf066c8634dd645635d099bb21eccf110f0bdc28d9040891",
"dependencies": [
"jsr:@std/bytes@^1.0.2"
]
},
"@std/json@0.223.0": {
"integrity": "9a4a255931dd0397924c6b10bb6a72fe3e28ddd876b981ada2e3b8dd0764163f"
},
@ -543,6 +570,10 @@
"integrity": "sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==",
"dependencies": {}
},
"commander@12.1.0": {
"integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==",
"dependencies": {}
},
"cross-spawn@7.0.3": {
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
"dependencies": {
@ -1887,7 +1918,7 @@
"jsr:@hono/hono@^4.4.6",
"jsr:@lambdalisue/async@^2.1.1",
"jsr:@nostrify/db@^0.31.2",
"jsr:@nostrify/nostrify@^0.30.0",
"jsr:@nostrify/nostrify@^0.30.1",
"jsr:@soapbox/kysely-deno-sqlite@^2.1.0",
"jsr:@soapbox/stickynotes@^0.4.0",
"jsr:@std/assert@^0.225.1",
@ -1904,6 +1935,7 @@
"npm:@scure/base@^1.1.6",
"npm:comlink-async-generator@^0.0.1",
"npm:comlink@^4.4.1",
"npm:commander@12.1.0",
"npm:entities@^4.5.0",
"npm:fast-stable-stringify@^1.0.0",
"npm:formdata-helper@^0.3.0",

5
installation/ditto.conf
View file

@ -42,11 +42,6 @@ server {
try_files $uri =404;
}
location = /favicon.ico {
root /opt/ditto/static;
try_files $uri =404;
}
location /metrics {
allow 127.0.0.1;
deny all;

94
scripts/db-export.test.ts Normal file
View file

@ -0,0 +1,94 @@
import { assertEquals, assertThrows } from '@std/assert';
import { buildFilter } from './db-export.ts';
Deno.test('buildFilter should return an empty filter when no arguments are provided', () => {
const filter = buildFilter({});
assertEquals(Object.keys(filter).length, 0);
});
Deno.test('buildFilter should correctly handle valid authors', () => {
const filter = buildFilter({
authors: ['a'.repeat(64)],
});
assertEquals(filter.authors, ['a'.repeat(64)]);
});
Deno.test('buildFilter throws on invalid author pubkey', () => {
assertThrows(
() => {
buildFilter({
authors: ['invalid_pubkey'],
});
},
Error,
'ERROR: Invalid pubkey invalid_pubkey supplied.',
);
});
Deno.test('buildFilter should correctly handle valid ids', () => {
const filter = buildFilter({
ids: ['b'.repeat(64)],
});
assertEquals(filter.ids, ['b'.repeat(64)]);
});
Deno.test('buildFilter should throw on invalid event IDs', () => {
assertThrows(
() => {
buildFilter({
ids: ['invalid_id'],
});
},
Error,
'ERROR: Invalid event ID invalid_id supplied.',
);
});
Deno.test('buildFilter should correctly handle tag shortcuts', () => {
const filter = buildFilter({
d: 'value1',
e: 'a'.repeat(64),
p: 'b'.repeat(64),
});
assertEquals(filter['#d'], ['value1']);
assertEquals(filter['#e'], ['a'.repeat(64)]);
assertEquals(filter['#p'], ['b'.repeat(64)]);
});
Deno.test('buildFilter should correctly handle since and until args', () => {
const filter = buildFilter({
since: 1000,
until: 2000,
});
assertEquals(filter.since, 1000);
assertEquals(filter.until, 2000);
});
Deno.test('buildFilter should correctly handle search field', () => {
const filter = buildFilter({
search: 'search_term',
});
assertEquals(filter.search, 'search_term');
});
Deno.test('buildFilter should correctly handle tag k-v pairs', () => {
const filter = buildFilter({
tags: ['tag1=value1', 'tag2=value2'],
});
assertEquals(filter['#tag1'], ['value1']);
assertEquals(filter['#tag2'], ['value2']);
});
Deno.test('buildFilter should correctly handle limit specifier', () => {
const filter = buildFilter({
limit: 10,
});
assertEquals(filter.limit, 10);
});

174
scripts/db-export.ts
View file

@ -1,24 +1,158 @@
import { Storages } from '@/storages.ts';
import { NostrFilter } from '@nostrify/nostrify';
import { Command, InvalidOptionArgumentError } from 'commander';
const store = await Storages.db();
console.warn('Exporting events...');
let count = 0;
for await (const msg of store.req([{}])) {
if (msg[0] === 'EOSE') {
break;
}
if (msg[0] === 'EVENT') {
console.log(JSON.stringify(msg[2]));
count++;
}
if (msg[0] === 'CLOSED') {
console.error('Database closed unexpectedly');
break;
}
interface ExportFilter {
authors?: string[];
ids?: string[];
kinds?: number[];
limit?: number;
search?: string;
/**
* Array of `key=value` pairs.
*/
tags?: string[];
since?: number;
until?: number;
/**
* shortcut for `--tag d=<value>`
*/
d?: string;
/**
* shortcut for `--tag e=<value>`
*/
e?: string;
/**
* shortcut for `--tag p=<value>`
*/
p?: string;
}
console.warn(`Exported ${count} events`);
Deno.exit();
function safeParseInt(s: string) {
const n = parseInt(s);
if (isNaN(n)) throw new InvalidOptionArgumentError('Not a number.');
return n;
}
function findInvalid(arr: string[], predicate = (v: string) => !/[a-f0-9]{64}/.test(v)) {
return arr.find(predicate);
}
function die(code: number, ...args: any[]) {
console.error(...args);
Deno.exit(code);
}
function tagFilterShortcut(name: 'd' | 'e' | 'p', value: string) {
const val = [value];
if (findInvalid(val)) throw new Error(`ERROR: Invalid value supplied for ${name}-tag.`);
return val;
}
export function buildFilter(args: ExportFilter) {
const filter: NostrFilter = {};
const { authors, ids, kinds, d, e, limit, p, search, since, until, tags } = args;
if (since) {
filter.since = since;
}
if (until) {
filter.until = until;
}
if (authors && authors.length) {
const invalid = findInvalid(authors);
if (invalid) throw new Error(`ERROR: Invalid pubkey ${invalid} supplied.`);
filter.authors = authors;
}
if (ids) {
const invalid = findInvalid(ids);
if (invalid) throw new Error(`ERROR: Invalid event ID ${invalid} supplied.`);
filter.ids = ids;
}
if (kinds && kinds.length) {
filter.kinds = kinds;
}
if (d) {
filter['#d'] = [d];
}
if (e) {
filter['#e'] = tagFilterShortcut('e', e);
}
if (p) {
filter['#p'] = tagFilterShortcut('e', p);
}
if (search) {
filter.search = search;
}
if (limit) {
filter.limit = limit;
}
if (tags) {
for (const val of tags) {
const [name, ...values] = val.split('=');
filter[`#${name}`] = [values.join('=')];
}
}
return filter;
}
async function exportEvents(args: ExportFilter) {
const store = await Storages.db();
let filter: NostrFilter = {};
try {
filter = buildFilter(args);
} catch (e) {
die(1, e.message || e.toString());
}
let count = 0;
for await (const msg of store.req([filter])) {
if (msg[0] === 'EOSE') {
break;
}
if (msg[0] === 'EVENT') {
console.log(JSON.stringify(msg[2]));
count++;
}
if (msg[0] === 'CLOSED') {
console.error('Database closed unexpectedly');
break;
}
}
console.warn(`Exported ${count} events`);
}
if (import.meta.main) {
const exporter = new Command()
.name('db:export')
.description('Export the specified set of events from the Ditto database, in JSONL format.')
.version('0.1.0')
.showHelpAfterError();
exporter
.option('-a, --authors <authors...>', 'Pubkeys of authors whose events you want to export.', [])
.option('-i, --ids <ids...>', 'IDs of events you want to export.', [])
.option(
'-k --kinds <kinds...>',
'Event kinds you want to export.',
(v: string, arr: number[]) => arr.concat([safeParseInt(v)]),
[],
)
.option(
'-t --tags <tag pairs...>',
'A list of key=value pairs of tags to search for events using. For tag values with spaces etc, simply quote the entire item, like `deno task db:export -t "name=A string with spaces in it"`.',
[],
)
.option('--search <search string>', 'A string to full-text search the db for.')
.option('-s --since <number>', 'The oldest time an exported event should be from.', safeParseInt)
.option('-u --until <number>', 'The newest time an exported event should be from.', safeParseInt)
.option('--limit <number>', 'Maximum number of events to export.', safeParseInt)
.option('-d <string>', 'Shortcut for `--tag d=<value>`.')
.option('-e <string>', 'Shortcut for `--tag e=<value>`.')
.option('-p <string>', 'Shortcut for `--tag p=<value>`.')
.action(exportEvents);
await exporter.parseAsync(Deno.args, { from: 'user' });
}

6
src/controllers/api/ditto.ts
View file

@ -276,10 +276,12 @@ export const statusZapSplitsController: AppController = async (c) => {
const weight = percentageSchema.catch(0).parse(zapsTag.find((name) => name[1] === pubkey)![3]) ?? 0;
const message = zapsTag.find((name) => name[1] === pubkey)![4] ?? '';
return {
account,
message: '',
weight: weight,
message,
weight,
};
}))).filter((zapSplit) => zapSplit.weight > 0);

2
src/controllers/api/statuses.ts
View file

@ -177,7 +177,7 @@ const createStatusController: AppController = async (c) => {
let totalSplit = 0;
for (const pubkey in dittoZapSplit) {
totalSplit += dittoZapSplit[pubkey].weight;
tags.push(['zap', pubkey, Conf.relay, dittoZapSplit[pubkey].weight.toString()]);
tags.push(['zap', pubkey, Conf.relay, dittoZapSplit[pubkey].weight.toString(), dittoZapSplit[pubkey].message]);
}
if (totalSplit) {
tags.push(['zap', author?.pubkey as string, Conf.relay, Math.max(0, 100 - totalSplit).toString()]);

2
src/metrics.ts
View file

@ -9,7 +9,7 @@ export const httpRequestCounter = new Counter({
export const httpResponseCounter = new Counter({
name: 'http_responses_total',
help: 'Total number of HTTP responses',
labelNames: ['status', 'path'],
labelNames: ['method', 'path', 'status'],
});
export const streamingConnectionsGauge = new Gauge({

2
src/middleware/metricsMiddleware.ts
View file

@ -16,5 +16,5 @@ export const metricsMiddleware: MiddlewareHandler = async (c, next) => {
// Get a parameterized path name like `/posts/:id` instead of `/posts/1234`.
// Tries to find actual route names first before falling back on potential middleware handlers like `app.use('*')`.
const path = c.req.matchedRoutes.find((r) => r.method !== 'ALL')?.path ?? c.req.routePath;
httpResponseCounter.inc({ status, path });
httpResponseCounter.inc({ method, status, path });
};

2
src/utils/favicon.ts
View file

@ -1,4 +1,4 @@
import { DOMParser } from '@b-fuze/deno-dom/native';
import { DOMParser } from '@b-fuze/deno-dom';
import Debug from '@soapbox/stickynotes/debug';
import tldts from 'tldts';

39
src/utils/note.test.ts
View file

@ -10,6 +10,45 @@ Deno.test('parseNoteContent', () => {
assertEquals(firstUrl, undefined);
});
Deno.test('parseNoteContent parses URLs', () => {
const { html } = parseNoteContent('check out my website: https://alexgleason.me', []);
assertEquals(html, 'check out my website: <a href="https://alexgleason.me">https://alexgleason.me</a>');
});
Deno.test('parseNoteContent parses bare URLs', () => {
const { html } = parseNoteContent('have you seen ditto.pub?', []);
assertEquals(html, 'have you seen <a href="http://ditto.pub">ditto.pub</a>?');
});
Deno.test('parseNoteContent parses mentions with apostrophes', () => {
const { html } = parseNoteContent(
`did you see nostr:nprofile1qqsqgc0uhmxycvm5gwvn944c7yfxnnxm0nyh8tt62zhrvtd3xkj8fhgprdmhxue69uhkwmr9v9ek7mnpw3hhytnyv4mz7un9d3shjqgcwaehxw309ahx7umywf5hvefwv9c8qtmjv4kxz7gpzemhxue69uhhyetvv9ujumt0wd68ytnsw43z7s3al0v's speech?`,
[{
id: '0461fcbecc4c3374439932d6b8f11269ccdb7cc973ad7a50ae362db135a474dd',
username: 'alex',
acct: 'alex@gleasonator.dev',
url: 'https://gleasonator.dev/@alex',
}],
);
assertEquals(
html,
'did you see <span class="h-card"><a class="u-url mention" href="https://gleasonator.dev/@alex" rel="ugc">@<span>alex@gleasonator.dev</span></a></span>&apos;s speech?',
);
});
Deno.test("parseNoteContent doesn't parse invalid nostr URIs", () => {
const { html } = parseNoteContent('nip19 has URIs like nostr:npub and nostr:nevent, etc.', []);
assertEquals(html, 'nip19 has URIs like nostr:npub and nostr:nevent, etc.');
});
Deno.test('parseNoteContent renders empty for non-profile nostr URIs', () => {
const { html } = parseNoteContent(
'nostr:nevent1qgsr9cvzwc652r4m83d86ykplrnm9dg5gwdvzzn8ameanlvut35wy3gpz3mhxue69uhhztnnwashymtnw3ezucm0d5qzqru8mkz2q4gzsxg99q7pdneyx7n8p5u0afe3ntapj4sryxxmg4gpcdvgce',
[],
);
assertEquals(html, '');
});
Deno.test('getMediaLinks', () => {
const links = [
{ href: 'https://example.com/image.png' },

54
src/utils/note.ts
View file

@ -1,10 +1,11 @@
import 'linkify-plugin-hashtag';
import linkifyStr from 'linkify-string';
import linkify from 'linkifyjs';
import { nip19, nip21, nip27 } from 'nostr-tools';
import { nip19, nip27 } from 'nostr-tools';
import { Conf } from '@/config.ts';
import { MastodonMention } from '@/entities/MastodonMention.ts';
import { html } from '@/utils/html.ts';
import { getUrlMediaType, isPermittedMediaType } from '@/utils/media.ts';
linkify.registerCustomProtocol('nostr', true);
@ -24,40 +25,53 @@ function parseNoteContent(content: string, mentions: MastodonMention[]): ParsedN
const links = linkify.find(content).filter(isLinkURL);
const firstUrl = links.find(isNonMediaLink)?.href;
const html = linkifyStr(content, {
const result = linkifyStr(content, {
render: {
hashtag: ({ content }) => {
const tag = content.replace(/^#/, '');
const href = Conf.local(`/tags/${tag}`);
return `<a class=\"mention hashtag\" href=\"${href}\" rel=\"tag\"><span>#</span>${tag}</a>`;
return html`<a class="mention hashtag" href="${href}" rel="tag"><span>#</span>${tag}</a>`;
},
url: ({ attributes, content }) => {
try {
const { decoded } = nip21.parse(content);
const pubkey = getDecodedPubkey(decoded);
if (pubkey) {
const mention = mentions.find((m) => m.id === pubkey);
const npub = nip19.npubEncode(pubkey);
const acct = mention?.acct ?? npub;
const name = mention?.acct ?? npub.substring(0, 8);
const href = mention?.url ?? Conf.local(`/@${acct}`);
return `<span class="h-card"><a class="u-url mention" href="${href}" rel="ugc">@<span>${name}</span></a></span>`;
} else {
return '';
const { protocol, pathname } = new URL(content);
if (protocol === 'nostr:') {
const match = pathname.match(new RegExp(`^${nip19.BECH32_REGEX.source}`));
if (match) {
const bech32 = match[0];
const extra = pathname.slice(bech32.length);
const decoded = nip19.decode(bech32);
const pubkey = getDecodedPubkey(decoded);
if (pubkey) {
const mention = mentions.find((m) => m.id === pubkey);
const npub = nip19.npubEncode(pubkey);
const acct = mention?.acct ?? npub;
const name = mention?.acct ?? npub.substring(0, 8);
const href = mention?.url ?? Conf.local(`/@${acct}`);
return html`<span class="h-card"><a class="u-url mention" href="${href}" rel="ugc">@<span>${name}</span></a></span>${extra}`;
} else {
return '';
}
} else {
return content;
}
}
} catch {
const attr = Object.entries(attributes)
.map(([name, value]) => `${name}="${value}"`)
.join(' ');
return `<a ${attr}>${content}</a>`;
// fallthrough
}
const attr = Object.entries(attributes)
.map(([name, value]) => `${name}="${value}"`)
.join(' ');
return `<a ${attr}>${content}</a>`;
},
},
}).replace(/\n+$/, '');
return {
html,
html: result,
links,
firstUrl,
};