mirror/ditto
mirror/ditto
1
0
Fork 0
mirror of https://gitlab.com/soapbox-pub/ditto.git synced 2025-12-06 11:29:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'tag-queries' into 'main'

Browse source 
Vastly improve tag query performance

Closes #169

See merge request soapbox-pub/ditto!436
This commit is contained in:
Alex Gleason 2024-07-29 21:50:40 +00:00
commit c54d801dd0
10 changed files with 174 additions and 115 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
deno.json
View file

@ -27,7 +27,7 @@
"@hono/hono": "jsr:@hono/hono@^4.4.6",
"@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1",
"@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0",
"@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.26.3",
"@nostrify/nostrify": "jsr:@nostrify/nostrify@0.27.0",
"@scure/base": "npm:@scure/base@^1.1.6",
"@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs",
"@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0",

18
deno.lock generated
View file

@ -8,11 +8,11 @@
"jsr:@gleasonator/policy": "jsr:@gleasonator/policy@0.2.0",
"jsr:@gleasonator/policy@0.2.0": "jsr:@gleasonator/policy@0.2.0",
"jsr:@gleasonator/policy@0.4.0": "jsr:@gleasonator/policy@0.4.0",
"jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.0",
"jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.1",
"jsr:@nostrify/nostrify@0.27.0": "jsr:@nostrify/nostrify@0.27.0",
"jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5",
"jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4",
"jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5",
"jsr:@nostrify/nostrify@^0.26.3": "jsr:@nostrify/nostrify@0.26.3",
"jsr:@soapbox/kysely-deno-sqlite@^2.1.0": "jsr:@soapbox/kysely-deno-sqlite@2.2.0",
"jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0",
"jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0",
@ -30,7 +30,7 @@
"jsr:@std/fmt@^0.221.0": "jsr:@std/fmt@0.221.0",
"jsr:@std/fs@^0.221.0": "jsr:@std/fs@0.221.0",
"jsr:@std/fs@^0.229.3": "jsr:@std/fs@0.229.3",
"jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.0",
"jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.1",
"jsr:@std/io@^0.224": "jsr:@std/io@0.224.3",
"jsr:@std/media-types@^0.224.1": "jsr:@std/media-types@0.224.1",
"jsr:@std/path@0.217": "jsr:@std/path@0.217.0",
@ -113,6 +113,9 @@
"@hono/hono@4.5.0": {
"integrity": "4a410f7773ac4b5b0eb4520b26c7ab7795a271d57a9df7fa1953ded6b90ccaf7"
},
"@hono/hono@4.5.1": {
"integrity": "459748ed4d4146c6e4bdff0213ff1ac44749904066ae02e7550d6c7f28c9bc4c"
},
"@nostrify/nostrify@0.22.4": {
"integrity": "1c8a7847e5773213044b491e85fd7cafae2ad194ce59da4d957d2b27c776b42d",
"dependencies": [
@ -142,8 +145,8 @@
"npm:zod@^3.23.8"
]
},
"@nostrify/nostrify@0.26.3": {
"integrity": "3e13e30f4fa3f76dcbcf9178630a9b2871186eb1d226d66234c0cdfd4841f548",
"@nostrify/nostrify@0.27.0": {
"integrity": "c4461dd93ed78c7bd0f3a4fbc0d77ba68acafa05ffcf6b82f3f3962a3a7e9698",
"dependencies": [
"jsr:@std/crypto@^0.224.0",
"jsr:@std/encoding@^0.224.1",
@ -225,6 +228,9 @@
"@std/internal@1.0.0": {
"integrity": "ac6a6dfebf838582c4b4f61a6907374e27e05bedb6ce276e0f1608fe84e7cd9a"
},
"@std/internal@1.0.1": {
"integrity": "6f8c7544d06a11dd256c8d6ba54b11ed870aac6c5aeafff499892662c57673e6"
},
"@std/io@0.224.0": {
"integrity": "0aff885d21d829c050b8a08b1d71b54aed5841aecf227f8d77e99ec529a11e8e",
"dependencies": [
@ -1760,7 +1766,7 @@
"jsr:@bradenmacdonald/s3-lite-client@^0.7.4",
"jsr:@db/sqlite@^0.11.1",
"jsr:@hono/hono@^4.4.6",
"jsr:@nostrify/nostrify@^0.26.3",
"jsr:@nostrify/nostrify@0.27.0",
"jsr:@soapbox/kysely-deno-sqlite@^2.1.0",
"jsr:@soapbox/stickynotes@^0.4.0",
"jsr:@std/assert@^0.225.1",

26
src/controllers/api/trends.ts
View file

@ -9,11 +9,18 @@ import { generateDateRange, Time } from '@/utils/time.ts';
import { unfurlCardCached } from '@/utils/unfurl.ts';
import { renderStatus } from '@/views/mastodon/statuses.ts';
let trendingHashtagsCache = getTrendingHashtags();
let trendingHashtagsCache = getTrendingHashtags().catch((e) => {
console.error(`Failed to get trending hashtags: ${e}`);
return Promise.resolve([]);
});
Deno.cron('update trending hashtags cache', '35 * * * *', async () => {
const trends = await getTrendingHashtags();
trendingHashtagsCache = Promise.resolve(trends);
try {
const trends = await getTrendingHashtags();
trendingHashtagsCache = Promise.resolve(trends);
} catch (e) {
console.error(e);
}
});
const trendingTagsQuerySchema = z.object({
@ -48,11 +55,18 @@ async function getTrendingHashtags() {
});
}
let trendingLinksCache = getTrendingLinks();
let trendingLinksCache = getTrendingLinks().catch((e) => {
console.error(`Failed to get trending links: ${e}`);
return Promise.resolve([]);
});
Deno.cron('update trending links cache', '50 * * * *', async () => {
const trends = await getTrendingLinks();
trendingLinksCache = Promise.resolve(trends);
try {
const trends = await getTrendingLinks();
trendingLinksCache = Promise.resolve(trends);
} catch (e) {
console.error(e);
}
});
const trendingLinksController: AppController = async (c) => {

3
src/db/DittoTables.ts
View file

@ -46,6 +46,9 @@ interface TagRow {
event_id: string;
name: string;
value: string;
kind: number;
pubkey: string;
created_at: number;
}
interface NIP46TokenRow {

95
src/db/migrations/029_tag_queries.ts Normal file
View file

@ -0,0 +1,95 @@
import { Kysely, sql } from 'kysely';
export async function up(db: Kysely<any>): Promise<void> {
await db.schema
.createTable('nostr_tags_new')
.addColumn('event_id', 'text', (col) => col.notNull().references('nostr_events.id').onDelete('cascade'))
.addColumn('name', 'text', (col) => col.notNull())
.addColumn('value', 'text', (col) => col.notNull())
.addColumn('kind', 'integer', (col) => col.notNull())
.addColumn('pubkey', 'text', (col) => col.notNull())
.addColumn('created_at', 'integer', (col) => col.notNull())
.execute();
let iid: number | undefined;
const tid = setTimeout(() => {
console.warn(
'Recreating the tags table to boost performance. Depending on the size of your database, this could take a very long time, even as long as 2 days!',
);
const emojis = ['⚡', '🐛', '🔎', '😂', '😅', '😬', '😭', '🙃', '🤔', '🧐', '🧐', '🫠'];
iid = setInterval(() => {
const emoji = emojis[Math.floor(Math.random() * emojis.length)];
console.info(`Recreating tags table... ${emoji}`);
}, 60_000);
}, 10_000);
// Copy data to the new table.
await sql`
INSERT INTO
nostr_tags_new (name, value, event_id, kind, pubkey, created_at)
SELECT
t.name, t.value, t.event_id, e.kind, e.pubkey, e.created_at
FROM
nostr_tags as t LEFT JOIN nostr_events e on t.event_id = e.id;
`.execute(db);
clearTimeout(tid);
if (iid) clearInterval(iid);
// Drop the old table and rename it.
await db.schema.dropTable('nostr_tags').execute();
await db.schema.alterTable('nostr_tags_new').renameTo('nostr_tags').execute();
await db.schema
.createIndex('nostr_tags_created_at')
.on('nostr_tags')
.ifNotExists()
.columns(['value', 'name', 'created_at desc', 'event_id asc'])
.execute();
await db.schema
.createIndex('nostr_tags_kind_created_at')
.on('nostr_tags')
.ifNotExists()
.columns(['value', 'name', 'kind', 'created_at desc', 'event_id asc'])
.execute();
await db.schema
.createIndex('nostr_tags_kind_pubkey_created_at')
.on('nostr_tags')
.ifNotExists()
.columns(['value', 'name', 'kind', 'pubkey', 'created_at desc', 'event_id asc'])
.execute();
await db.schema
.createIndex('nostr_tags_trends')
.on('nostr_tags')
.ifNotExists()
.columns(['created_at', 'name', 'kind'])
.execute();
}
export async function down(db: Kysely<any>): Promise<void> {
await db.schema
.createTable('nostr_tags_old')
.addColumn('event_id', 'text', (col) => col.references('nostr_events.id').onDelete('cascade'))
.addColumn('name', 'text', (col) => col.notNull())
.addColumn('value', 'text', (col) => col.notNull())
.addColumn('kind', 'integer', (col) => col.notNull())
.addColumn('pubkey', 'text', (col) => col.notNull())
.addColumn('created_at', 'integer', (col) => col.notNull())
.execute();
await sql`
INSERT INTO
nostr_tags_old (name, value, event_id)
SELECT
name, value, event_id
FROM
nostr_tags;
`.execute(db);
await db.schema.dropTable('nostr_tags').execute();
await db.schema.alterTable('nostr_tags_old').renameTo('nostr_tags').execute();
await db.schema.createIndex('idx_tags_event_id').on('nostr_tags').ifNotExists().column('event_id').execute();
await db.schema.createIndex('idx_tags_name').on('nostr_tags').ifNotExists().column('name').execute();
await db.schema.createIndex('idx_tags_tag_value').on('nostr_tags').ifNotExists().columns(['name', 'value']).execute();
}

3
src/startup.ts
View file

@ -1,7 +1,6 @@
// Starts up applications required to run before the HTTP server is on.
import { Conf } from '@/config.ts';
import { seedZapSplits } from '@/utils/zap-split.ts';
import { cron } from '@/cron.ts';
import { startFirehose } from '@/firehose.ts';
@ -12,5 +11,3 @@ if (Conf.firehoseEnabled) {
if (Conf.cronEnabled) {
cron();
}
await seedZapSplits();

5
src/storages.ts
View file

@ -7,6 +7,7 @@ import { SearchStore } from '@/storages/search-store.ts';
import { InternalRelay } from '@/storages/InternalRelay.ts';
import { NPool, NRelay1 } from '@nostrify/nostrify';
import { getRelays } from '@/utils/outbox.ts';
import { seedZapSplits } from '@/utils/zap-split.ts';
export class Storages {
private static _db: Promise<EventsDB> | undefined;
@ -20,7 +21,9 @@ export class Storages {
if (!this._db) {
this._db = (async () => {
const kysely = await DittoDB.getInstance();
return new EventsDB(kysely);
const store = new EventsDB(kysely);
await seedZapSplits(store);
return store;
})();
}
return this._db;

65
src/trends.ts
View file

@ -22,24 +22,23 @@ export async function getTrendingTagValues(
): Promise<{ value: string; authors: number; uses: number }[]> {
let query = kysely
.selectFrom('nostr_tags')
.innerJoin('nostr_events', 'nostr_events.id', 'nostr_tags.event_id')
.select(({ fn }) => [
'nostr_tags.value',
fn.agg<number>('count', ['nostr_events.pubkey']).distinct().as('authors'),
fn.agg<number>('count', ['nostr_tags.pubkey']).distinct().as('authors'),
fn.countAll<number>().as('uses'),
])
.where('nostr_tags.name', 'in', tagNames)
.groupBy('nostr_tags.value')
.orderBy((c) => c.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc');
.orderBy((c) => c.fn.agg('count', ['nostr_tags.pubkey']).distinct(), 'desc');
if (filter.kinds) {
query = query.where('nostr_events.kind', 'in', filter.kinds);
query = query.where('nostr_tags.kind', 'in', filter.kinds);
}
if (typeof filter.since === 'number') {
query = query.where('nostr_events.created_at', '>=', filter.since);
query = query.where('nostr_tags.created_at', '>=', filter.since);
}
if (typeof filter.until === 'number') {
query = query.where('nostr_events.created_at', '<=', filter.until);
query = query.where('nostr_tags.created_at', '<=', filter.until);
}
if (typeof filter.limit === 'number') {
query = query.limit(filter.limit);
@ -72,33 +71,37 @@ export async function updateTrendingTags(
const tagNames = aliases ? [tagName, ...aliases] : [tagName];
const trends = await getTrendingTagValues(kysely, tagNames, {
kinds,
since: yesterday,
until: now,
limit,
});
try {
const trends = await getTrendingTagValues(kysely, tagNames, {
kinds,
since: yesterday,
until: now,
limit,
});
if (!trends.length) {
console.info(`No trending ${l} found. Skipping.`);
return;
if (!trends.length) {
console.info(`No trending ${l} found. Skipping.`);
return;
}
const signer = new AdminSigner();
const label = await signer.signEvent({
kind: 1985,
content: '',
tags: [
['L', 'pub.ditto.trends'],
['l', l, 'pub.ditto.trends'],
...trends.map(({ value, authors, uses }) => [tagName, value, extra, authors.toString(), uses.toString()]),
],
created_at: Math.floor(Date.now() / 1000),
});
await handleEvent(label, signal);
console.info(`Trending ${l} updated.`);
} catch (e) {
console.error(`Error updating trending ${l}: ${e.message}`);
}
const signer = new AdminSigner();
const label = await signer.signEvent({
kind: 1985,
content: '',
tags: [
['L', 'pub.ditto.trends'],
['l', l, 'pub.ditto.trends'],
...trends.map(({ value, authors, uses }) => [tagName, value, extra, authors.toString(), uses.toString()]),
],
created_at: Math.floor(Date.now() / 1000),
});
await handleEvent(label, signal);
console.info(`Trending ${l} updated.`);
}
/** Update trending pubkeys. */

60
src/utils/zap-split.test.ts
View file

@ -1,60 +0,0 @@
import { assertEquals } from '@std/assert';
import { generateSecretKey, getPublicKey } from 'nostr-tools';
import { genEvent } from '@/test.ts';
import { getZapSplits } from '@/utils/zap-split.ts';
import { getTestDB } from '@/test.ts';
Deno.test('Get zap splits in DittoZapSplits format', async () => {
const { store } = await getTestDB();
const sk = generateSecretKey();
const pubkey = getPublicKey(sk);
const event = genEvent({
kind: 30078,
tags: [
['d', 'pub.ditto.zapSplits'],
['p', '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', '2', 'Patrick developer'],
['p', '0461fcbecc4c3374439932d6b8f11269ccdb7cc973ad7a50ae362db135a474dd', '3', 'Alex creator of Ditto'],
],
}, sk);
await store.event(event);
const eventFromDb = await store.query([{ kinds: [30078], authors: [pubkey] }]);
assertEquals(eventFromDb.length, 1);
const zapSplits = await getZapSplits(store, pubkey);
assertEquals(zapSplits, {
'0461fcbecc4c3374439932d6b8f11269ccdb7cc973ad7a50ae362db135a474dd': { amount: 3, message: 'Alex creator of Ditto' },
'47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4': { amount: 2, message: 'Patrick developer' },
});
assertEquals(await getZapSplits(store, 'garbage'), undefined);
});
Deno.test('Zap split is empty', async () => {
const { store } = await getTestDB();
const sk = generateSecretKey();
const pubkey = getPublicKey(sk);
const event = genEvent({
kind: 30078,
tags: [
['d', 'pub.ditto.zapSplits'],
['p', 'baka'],
],
}, sk);
await store.event(event);
const eventFromDb = await store.query([{ kinds: [30078], authors: [pubkey] }]);
assertEquals(eventFromDb.length, 1);
const zapSplits = await getZapSplits(store, pubkey);
assertEquals(zapSplits, {});
});

12
src/utils/zap-split.ts
View file

@ -1,10 +1,8 @@
import { AdminSigner } from '@/signers/AdminSigner.ts';
import { Conf } from '@/config.ts';
import { handleEvent } from '@/pipeline.ts';
import { NSchema as n, NStore } from '@nostrify/nostrify';
import { nostrNow } from '@/utils.ts';
import { percentageSchema } from '@/schema.ts';
import { Storages } from '@/storages.ts';
type Pubkey = string;
type ExtraMessage = string;
@ -39,11 +37,10 @@ export async function getZapSplits(store: NStore, pubkey: string): Promise<Ditto
return zapSplits;
}
export async function seedZapSplits() {
const store = await Storages.admin();
export async function seedZapSplits(store: NStore) {
const zapSplit: DittoZapSplits | undefined = await getZapSplits(store, Conf.pubkey);
const zap_split: DittoZapSplits | undefined = await getZapSplits(store, Conf.pubkey);
if (!zap_split) {
if (!zapSplit) {
const dittoPubkey = '781a1527055f74c1f70230f10384609b34548f8ab6a0a6caa74025827f9fdae5';
const dittoMsg = 'Official Ditto Account';
@ -57,6 +54,7 @@ export async function seedZapSplits() {
['p', dittoPubkey, '5', dittoMsg],
],
});
await handleEvent(event, AbortSignal.timeout(5000));
await store.event(event);
}
}