From 91eac97d5cb351edc6d46979ce99b0bc0b62fda8 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 6 Aug 2023 12:54:00 -0500 Subject: [PATCH 01/34] Preliminary sqlite db setup --- src/controllers/activitypub/actor.ts | 4 +- src/controllers/well-known/nostr.ts | 31 ++++----- src/controllers/well-known/webfinger.ts | 39 ++++++------ src/db.ts | 84 +++++++++++++++++++++---- 4 files changed, 107 insertions(+), 51 deletions(-) diff --git a/src/controllers/activitypub/actor.ts b/src/controllers/activitypub/actor.ts index 5d78609e..0051c95e 100644 --- a/src/controllers/activitypub/actor.ts +++ b/src/controllers/activitypub/actor.ts @@ -7,7 +7,9 @@ import type { AppContext, AppController } from '@/app.ts'; const actorController: AppController = async (c) => { const username = c.req.param('username'); - const user = await db.users.findFirst({ where: { username } }); + + const user = db.getUserByUsername(username); + if (!user) return notFound(c); const event = await getAuthor(user.pubkey); if (!event) return notFound(c); diff --git a/src/controllers/well-known/nostr.ts b/src/controllers/well-known/nostr.ts index db46d559..1e6a6d28 100644 --- a/src/controllers/well-known/nostr.ts +++ b/src/controllers/well-known/nostr.ts @@ -10,25 +10,20 @@ const nameSchema = z.string().min(1).regex(/^\w+$/); * Serves NIP-05's nostr.json. * https://github.com/nostr-protocol/nips/blob/master/05.md */ -const nostrController: AppController = async (c) => { - try { - const name = nameSchema.parse(c.req.query('name')); - const user = await db.users.findFirst({ where: { username: name } }); - const relay = Conf.relay; +const nostrController: AppController = (c) => { + const name = nameSchema.safeParse(c.req.query('name')); + const user = name.success ? db.getUserByUsername(name.data) : null; - return c.json({ - names: { - [user.username]: user.pubkey, - }, - relays: relay - ? { - [user.pubkey]: [relay], - } - : {}, - }); - } catch (_e) { - return c.json({ names: {}, relays: {} }); - } + if (!user) return c.json({ names: {}, relays: {} }); + + return c.json({ + names: { + [user.username]: user.pubkey, + }, + relays: { + [user.pubkey]: [Conf.relay], + }, + }); }; export { nostrController }; diff --git a/src/controllers/well-known/webfinger.ts b/src/controllers/well-known/webfinger.ts index 8ee51d2d..0f65ef66 100644 --- a/src/controllers/well-known/webfinger.ts +++ b/src/controllers/well-known/webfinger.ts @@ -36,26 +36,27 @@ const acctSchema = z.custom((value) => value instanceof URL) path: ['resource', 'acct'], }); -async function handleAcct(c: AppContext, resource: URL): Promise { - try { - const [username, host] = acctSchema.parse(resource); - const user = await db.users.findFirst({ where: { username } }); - - const json = renderWebfinger({ - pubkey: user.pubkey, - username: user.username, - subject: `acct:${username}@${host}`, - }); - - c.header('content-type', 'application/jrd+json'); - return c.body(JSON.stringify(json)); - } catch (e) { - if (e instanceof z.ZodError) { - return c.json({ error: 'Invalid acct URI', schema: e }, 400); - } else { - return c.json({ error: 'Not found' }, 404); - } +function handleAcct(c: AppContext, resource: URL): Response { + const result = acctSchema.safeParse(resource); + if (!result.success) { + return c.json({ error: 'Invalid acct URI', schema: result.error }, 400); } + + const [username, host] = result.data; + const user = db.getUserByUsername(username); + + if (!user) { + return c.json({ error: 'Not found' }, 404); + } + + const json = renderWebfinger({ + pubkey: user.pubkey, + username: user.username, + subject: `acct:${username}@${host}`, + }); + + c.header('content-type', 'application/jrd+json'); + return c.body(JSON.stringify(json)); } interface RenderWebfingerOpts { diff --git a/src/db.ts b/src/db.ts index 8a22426d..ca195174 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,18 +1,76 @@ -import { createPentagon, z } from '@/deps.ts'; -import { hexIdSchema } from '@/schema.ts'; +import { Sqlite } from '@/deps.ts'; -const kv = await Deno.openKv(); +interface User { + pubkey: string; + username: string; + inserted_at: Date; +} -const userSchema = z.object({ - pubkey: hexIdSchema.describe('primary'), - username: z.string().regex(/^\w{1,30}$/).describe('unique'), - createdAt: z.date(), -}); +class DittoDB { + #db: Sqlite; -const db = createPentagon(kv, { - users: { - schema: userSchema, - }, -}); + constructor(db: Sqlite) { + this.#db = db; + this.#db.execute(` + CREATE TABLE events ( + id TEXT PRIMARY KEY, + kind INTEGER NOT NULL, + pubkey TEXT NOT NULL, + content TEXT NOT NULL, + created_at INTEGER NOT NULL, + tags TEXT NOT NULL, + sig TEXT NOT NULL + ); + + CREATE INDEX idx_events_kind ON events(kind); + CREATE INDEX idx_events_pubkey ON events(pubkey); + + CREATE TABLE tags ( + tag TEXT NOT NULL, + value_1 TEXT, + value_2 TEXT, + value_3 TEXT, + event_id TEXT NOT NULL, + FOREIGN KEY(event_id) REFERENCES events(id) ON DELETE CASCADE + ); + + CREATE INDEX idx_tags_tag ON tags(tag); + CREATE INDEX idx_tags_value_1 ON tags(value_1); + CREATE INDEX idx_tags_event_id ON tags(event_id); + + CREATE TABLE users ( + pubkey TEXT PRIMARY KEY, + username TEXT NOT NULL, + inserted_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL + ); + + CREATE UNIQUE INDEX idx_users_username ON users(username); + `); + } + + insertUser(user: Pick): void { + this.#db.query( + 'INSERT INTO users(pubkey, username) VALUES (?, ?)', + [user.pubkey, user.username], + ); + } + + getUserByUsername(username: string): User | null { + const result = this.#db.query<[string, string, Date]>( + 'SELECT pubkey, username, inserted_at FROM users WHERE username = ?', + [username], + )[0]; + if (!result) return null; + return { + pubkey: result[0], + username: result[1], + inserted_at: result[2], + }; + } +} + +const db = new DittoDB( + new Sqlite('data/db.sqlite3'), +); export { db }; From 8ceb63fc89c59b609eae8cfd06f167d46ce32f06 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 6 Aug 2023 15:03:29 -0500 Subject: [PATCH 02/34] Try storing events in new database --- src/db.ts | 51 ++++++++++++++++++++++++++++++++++++++++--------- src/loopback.ts | 8 +++++--- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/src/db.ts b/src/db.ts index ca195174..46115245 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,4 +1,5 @@ import { Sqlite } from '@/deps.ts'; +import { SignedEvent } from '@/event.ts'; interface User { pubkey: string; @@ -13,7 +14,7 @@ class DittoDB { this.#db = db; this.#db.execute(` - CREATE TABLE events ( + CREATE TABLE IF NOT EXISTS events ( id TEXT PRIMARY KEY, kind INTEGER NOT NULL, pubkey TEXT NOT NULL, @@ -23,10 +24,10 @@ class DittoDB { sig TEXT NOT NULL ); - CREATE INDEX idx_events_kind ON events(kind); - CREATE INDEX idx_events_pubkey ON events(pubkey); + CREATE INDEX IF NOT EXISTS idx_events_kind ON events(kind); + CREATE INDEX IF NOT EXISTS idx_events_pubkey ON events(pubkey); - CREATE TABLE tags ( + CREATE TABLE IF NOT EXISTS tags ( tag TEXT NOT NULL, value_1 TEXT, value_2 TEXT, @@ -35,17 +36,17 @@ class DittoDB { FOREIGN KEY(event_id) REFERENCES events(id) ON DELETE CASCADE ); - CREATE INDEX idx_tags_tag ON tags(tag); - CREATE INDEX idx_tags_value_1 ON tags(value_1); - CREATE INDEX idx_tags_event_id ON tags(event_id); + CREATE INDEX IF NOT EXISTS idx_tags_tag ON tags(tag); + CREATE INDEX IF NOT EXISTS idx_tags_value_1 ON tags(value_1); + CREATE INDEX IF NOT EXISTS idx_tags_event_id ON tags(event_id); - CREATE TABLE users ( + CREATE TABLE IF NOT EXISTS users ( pubkey TEXT PRIMARY KEY, username TEXT NOT NULL, inserted_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL ); - CREATE UNIQUE INDEX idx_users_username ON users(username); + CREATE UNIQUE INDEX IF NOT EXISTS idx_users_username ON users(username); `); } @@ -68,6 +69,38 @@ class DittoDB { inserted_at: result[2], }; } + + insertEvent(event: SignedEvent): void { + this.#db.transaction(() => { + this.#db.query( + ` + INSERT INTO events(id, kind, pubkey, content, created_at, tags, sig) + VALUES (?, ?, ?, ?, ?, ?, ?) + `, + [ + event.id, + event.kind, + event.pubkey, + event.content, + event.created_at, + JSON.stringify(event.tags), + event.sig, + ], + ); + + for (const [tag, value1, value2, value3] of event.tags) { + if (['p', 'e', 'q', 'd', 't', 'proxy'].includes(tag)) { + this.#db.query( + ` + INSERT INTO tags(event_id, tag, value_1, value_2, value_3) + VALUES (?, ?, ?, ?, ?) + `, + [event.id, tag, value1 || null, value2 || null, value3 || null], + ); + } + } + }); + } } const db = new DittoDB( diff --git a/src/loopback.ts b/src/loopback.ts index e9f13fcb..af69d918 100644 --- a/src/loopback.ts +++ b/src/loopback.ts @@ -1,9 +1,10 @@ import { Conf } from '@/config.ts'; +import { db } from '@/db.ts'; import { RelayPool } from '@/deps.ts'; import { trends } from '@/trends.ts'; import { nostrDate, nostrNow } from '@/utils.ts'; -import type { Event } from '@/event.ts'; +import type { SignedEvent } from '@/event.ts'; const relay = new RelayPool([Conf.relay]); @@ -19,13 +20,14 @@ relay.subscribe( ); /** Handle events through the loopback pipeline. */ -function handleEvent(event: Event): void { +function handleEvent(event: SignedEvent): void { console.info('loopback event:', event.id); + db.insertEvent(event); trackHashtags(event); } /** Track whenever a hashtag is used, for processing trending tags. */ -function trackHashtags(event: Event): void { +function trackHashtags(event: SignedEvent): void { const date = nostrDate(event.created_at); const tags = event.tags From d799075657c7a85e3ab70a2f76a74dd7b7ac8862 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 6 Aug 2023 20:14:11 -0500 Subject: [PATCH 03/34] Get Kysely adapter almost working... --- lib/kysely-deno-sqlite.ts | 88 +++++++++++++++++++++++++++++++++++++++ src/db.ts | 9 +++- src/db/builder.ts | 40 ++++++++++++++++++ src/deps.ts | 1 + 4 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 lib/kysely-deno-sqlite.ts create mode 100644 src/db/builder.ts diff --git a/lib/kysely-deno-sqlite.ts b/lib/kysely-deno-sqlite.ts new file mode 100644 index 00000000..4a0467ef --- /dev/null +++ b/lib/kysely-deno-sqlite.ts @@ -0,0 +1,88 @@ +import { + type DatabaseIntrospector, + type Dialect, + type DialectAdapter, + type Driver, + Kysely, + type QueryCompiler, + SqliteAdapter, + type SqliteDatabase, + type SqliteDialectConfig, + SqliteDriver, + SqliteIntrospector, + SqliteQueryCompiler, + type SqliteStatement, +} from 'npm:kysely@^0.25.0'; + +import type { DB as DenoSqlite } from 'https://deno.land/x/sqlite@v3.7.3/mod.ts'; + +class DenoSqliteDatabase implements SqliteDatabase { + #db: DenoSqlite; + + constructor(db: DenoSqlite) { + this.#db = db; + } + + close(): void { + this.#db.close(); + } + + prepare(sql: string): SqliteStatement { + return { + reader: true, + all: (parameters: ReadonlyArray) => { + console.log(sql); + return this.#db.query(sql, parameters as any); + }, + run: (parameters: ReadonlyArray) => { + this.#db.query(sql, parameters as any); + return { + changes: this.#db.changes, + lastInsertRowid: this.#db.lastInsertRowId, + }; + }, + }; + } +} + +interface DenoSqliteDialectConfig extends Omit { + database: DenoSqlite | (() => Promise); +} + +class DenoSqliteDriver extends SqliteDriver { + constructor(config: DenoSqliteDialectConfig) { + super({ + ...config, + database: async () => + new DenoSqliteDatabase( + typeof config.database === 'function' ? await config.database() : config.database, + ), + }); + } +} + +class DenoSqliteDialect implements Dialect { + readonly #config: DenoSqliteDialectConfig; + + constructor(config: DenoSqliteDialectConfig) { + this.#config = Object.freeze({ ...config }); + } + + createDriver(): Driver { + return new DenoSqliteDriver(this.#config); + } + + createQueryCompiler(): QueryCompiler { + return new SqliteQueryCompiler(); + } + + createAdapter(): DialectAdapter { + return new SqliteAdapter(); + } + + createIntrospector(db: Kysely): DatabaseIntrospector { + return new SqliteIntrospector(db); + } +} + +export { DenoSqliteDatabase, DenoSqliteDialect, type DenoSqliteDialectConfig, DenoSqliteDriver }; diff --git a/src/db.ts b/src/db.ts index 46115245..08f83413 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,4 +1,5 @@ -import { Sqlite } from '@/deps.ts'; +import { builder } from '@/db/builder.ts'; +import { type Filter, Sqlite } from '@/deps.ts'; import { SignedEvent } from '@/event.ts'; interface User { @@ -101,9 +102,15 @@ class DittoDB { } }); } + + getFilter(filter: Filter) { + } } const db = new DittoDB( new Sqlite('data/db.sqlite3'), ); + +console.log(await builder.selectFrom('events').selectAll().limit(1).execute()) + export { db }; diff --git a/src/db/builder.ts b/src/db/builder.ts new file mode 100644 index 00000000..dec5debe --- /dev/null +++ b/src/db/builder.ts @@ -0,0 +1,40 @@ +import { Kysely, Sqlite } from '@/deps.ts'; +import { DenoSqliteDialect } from '../../lib/kysely-deno-sqlite.ts'; + +interface Tables { + events: EventsTable; + tags: TagsTable; + users: UsersTable; +} + +interface EventsTable { + id: string; + kind: number; + pubkey: string; + content: string; + created_at: number; + tags: string; + sig: string; +} + +interface TagsTable { + tag: string; + value_1: string | null; + value_2: string | null; + value_3: string | null; + event_id: string; +} + +interface UsersTable { + pubkey: string; + username: string; + inserted_at: Date; +} + +const builder = new Kysely({ + dialect: new DenoSqliteDialect({ + database: new Sqlite('data/db.sqlite3'), + }), +}); + +export { builder }; diff --git a/src/deps.ts b/src/deps.ts index 12800d2a..b721ec9a 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -50,3 +50,4 @@ export * as secp from 'npm:@noble/secp256k1@^2.0.0'; export { LRUCache } from 'npm:lru-cache@^10.0.0'; export { DB as Sqlite } from 'https://deno.land/x/sqlite@v3.7.3/mod.ts'; export * as dotenv from 'https://deno.land/std@0.197.0/dotenv/mod.ts'; +export { DummyDriver, Kysely, SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from 'npm:kysely@^0.25.0'; From 465a3db501b221e6f456ba7b96c5c5a6a1eeec1d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 6 Aug 2023 20:22:12 -0500 Subject: [PATCH 04/34] kysely adapter pretty much working! --- deno.json | 4 ++-- lib/kysely-deno-sqlite.ts | 10 +++++++--- src/db.ts | 3 --- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/deno.json b/deno.json index b3a4d581..3dbda2dc 100644 --- a/deno.json +++ b/deno.json @@ -10,14 +10,14 @@ "@/": "./src/" }, "lint": { - "include": ["src/"], + "include": ["src/", "lib/"], "rules": { "tags": ["recommended"], "exclude": ["no-explicit-any"] } }, "fmt": { - "include": ["src/"], + "include": ["src/", "lib/"], "useTabs": false, "lineWidth": 120, "indentWidth": 2, diff --git a/lib/kysely-deno-sqlite.ts b/lib/kysely-deno-sqlite.ts index 4a0467ef..e7a40203 100644 --- a/lib/kysely-deno-sqlite.ts +++ b/lib/kysely-deno-sqlite.ts @@ -28,14 +28,18 @@ class DenoSqliteDatabase implements SqliteDatabase { } prepare(sql: string): SqliteStatement { + const query = this.#db.prepareQuery(sql); return { + // HACK: implement an actual driver to fix this. reader: true, all: (parameters: ReadonlyArray) => { - console.log(sql); - return this.#db.query(sql, parameters as any); + const result = query.allEntries(parameters as any); + query.finalize(); + return result; }, run: (parameters: ReadonlyArray) => { - this.#db.query(sql, parameters as any); + query.execute(parameters as any); + query.finalize(); return { changes: this.#db.changes, lastInsertRowid: this.#db.lastInsertRowId, diff --git a/src/db.ts b/src/db.ts index 08f83413..0e806550 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,4 +1,3 @@ -import { builder } from '@/db/builder.ts'; import { type Filter, Sqlite } from '@/deps.ts'; import { SignedEvent } from '@/event.ts'; @@ -111,6 +110,4 @@ const db = new DittoDB( new Sqlite('data/db.sqlite3'), ); -console.log(await builder.selectFrom('events').selectAll().limit(1).execute()) - export { db }; From ed896278e06a0b867cf87df2edc144fabbfbaa77 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 6 Aug 2023 21:27:34 -0500 Subject: [PATCH 05/34] fix lint --- src/db.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/db.ts b/src/db.ts index 0e806550..511612c4 100644 --- a/src/db.ts +++ b/src/db.ts @@ -102,7 +102,8 @@ class DittoDB { }); } - getFilter(filter: Filter) { + getFilter(_filter: Filter) { + // TODO } } From 7c2f290775de5152b4fb47b96c710d324f153a0c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 6 Aug 2023 21:57:26 -0500 Subject: [PATCH 06/34] Refactor kysely lib into separate files --- lib/kysely-deno-sqlite.ts | 92 ------------------- lib/kysely-deno-sqlite/deps.ts | 17 ++++ lib/kysely-deno-sqlite/mod.ts | 4 + .../src/deno-sqlite-dialect-config.ts | 7 ++ .../src/deno-sqlite-dialect.ts | 41 +++++++++ .../src/deno-sqlite-driver.ts | 51 ++++++++++ src/db/builder.ts | 2 +- 7 files changed, 121 insertions(+), 93 deletions(-) delete mode 100644 lib/kysely-deno-sqlite.ts create mode 100644 lib/kysely-deno-sqlite/deps.ts create mode 100644 lib/kysely-deno-sqlite/mod.ts create mode 100644 lib/kysely-deno-sqlite/src/deno-sqlite-dialect-config.ts create mode 100644 lib/kysely-deno-sqlite/src/deno-sqlite-dialect.ts create mode 100644 lib/kysely-deno-sqlite/src/deno-sqlite-driver.ts diff --git a/lib/kysely-deno-sqlite.ts b/lib/kysely-deno-sqlite.ts deleted file mode 100644 index e7a40203..00000000 --- a/lib/kysely-deno-sqlite.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { - type DatabaseIntrospector, - type Dialect, - type DialectAdapter, - type Driver, - Kysely, - type QueryCompiler, - SqliteAdapter, - type SqliteDatabase, - type SqliteDialectConfig, - SqliteDriver, - SqliteIntrospector, - SqliteQueryCompiler, - type SqliteStatement, -} from 'npm:kysely@^0.25.0'; - -import type { DB as DenoSqlite } from 'https://deno.land/x/sqlite@v3.7.3/mod.ts'; - -class DenoSqliteDatabase implements SqliteDatabase { - #db: DenoSqlite; - - constructor(db: DenoSqlite) { - this.#db = db; - } - - close(): void { - this.#db.close(); - } - - prepare(sql: string): SqliteStatement { - const query = this.#db.prepareQuery(sql); - return { - // HACK: implement an actual driver to fix this. - reader: true, - all: (parameters: ReadonlyArray) => { - const result = query.allEntries(parameters as any); - query.finalize(); - return result; - }, - run: (parameters: ReadonlyArray) => { - query.execute(parameters as any); - query.finalize(); - return { - changes: this.#db.changes, - lastInsertRowid: this.#db.lastInsertRowId, - }; - }, - }; - } -} - -interface DenoSqliteDialectConfig extends Omit { - database: DenoSqlite | (() => Promise); -} - -class DenoSqliteDriver extends SqliteDriver { - constructor(config: DenoSqliteDialectConfig) { - super({ - ...config, - database: async () => - new DenoSqliteDatabase( - typeof config.database === 'function' ? await config.database() : config.database, - ), - }); - } -} - -class DenoSqliteDialect implements Dialect { - readonly #config: DenoSqliteDialectConfig; - - constructor(config: DenoSqliteDialectConfig) { - this.#config = Object.freeze({ ...config }); - } - - createDriver(): Driver { - return new DenoSqliteDriver(this.#config); - } - - createQueryCompiler(): QueryCompiler { - return new SqliteQueryCompiler(); - } - - createAdapter(): DialectAdapter { - return new SqliteAdapter(); - } - - createIntrospector(db: Kysely): DatabaseIntrospector { - return new SqliteIntrospector(db); - } -} - -export { DenoSqliteDatabase, DenoSqliteDialect, type DenoSqliteDialectConfig, DenoSqliteDriver }; diff --git a/lib/kysely-deno-sqlite/deps.ts b/lib/kysely-deno-sqlite/deps.ts new file mode 100644 index 00000000..ff95c114 --- /dev/null +++ b/lib/kysely-deno-sqlite/deps.ts @@ -0,0 +1,17 @@ +export { + type DatabaseIntrospector, + type Dialect, + type DialectAdapter, + type Driver, + Kysely, + type QueryCompiler, + SqliteAdapter, + type SqliteDatabase, + type SqliteDialectConfig, + SqliteDriver, + SqliteIntrospector, + SqliteQueryCompiler, + type SqliteStatement, +} from 'npm:kysely@^0.25.0'; + +export type { DB as DenoSqlite } from 'https://deno.land/x/sqlite@v3.7.3/mod.ts'; diff --git a/lib/kysely-deno-sqlite/mod.ts b/lib/kysely-deno-sqlite/mod.ts new file mode 100644 index 00000000..a0e19980 --- /dev/null +++ b/lib/kysely-deno-sqlite/mod.ts @@ -0,0 +1,4 @@ +export { DenoSqliteDialect } from './src/deno-sqlite-dialect.ts'; +export { DenoSqliteDriver } from './src/deno-sqlite-driver.ts'; + +export type { DenoSqliteDialectConfig } from './src/deno-sqlite-dialect-config.ts'; diff --git a/lib/kysely-deno-sqlite/src/deno-sqlite-dialect-config.ts b/lib/kysely-deno-sqlite/src/deno-sqlite-dialect-config.ts new file mode 100644 index 00000000..12bba773 --- /dev/null +++ b/lib/kysely-deno-sqlite/src/deno-sqlite-dialect-config.ts @@ -0,0 +1,7 @@ +import { type DenoSqlite, type SqliteDialectConfig } from '../deps.ts'; + +interface DenoSqliteDialectConfig extends Omit { + database: DenoSqlite | (() => Promise); +} + +export type { DenoSqliteDialectConfig }; diff --git a/lib/kysely-deno-sqlite/src/deno-sqlite-dialect.ts b/lib/kysely-deno-sqlite/src/deno-sqlite-dialect.ts new file mode 100644 index 00000000..f3aa5175 --- /dev/null +++ b/lib/kysely-deno-sqlite/src/deno-sqlite-dialect.ts @@ -0,0 +1,41 @@ +import { + type DatabaseIntrospector, + type Dialect, + type DialectAdapter, + type Driver, + Kysely, + type QueryCompiler, + SqliteAdapter, + SqliteIntrospector, + SqliteQueryCompiler, +} from '../deps.ts'; + +import { DenoSqliteDriver } from './deno-sqlite-driver.ts'; + +import type { DenoSqliteDialectConfig } from './deno-sqlite-dialect-config.ts'; + +class DenoSqliteDialect implements Dialect { + readonly #config: DenoSqliteDialectConfig; + + constructor(config: DenoSqliteDialectConfig) { + this.#config = Object.freeze({ ...config }); + } + + createDriver(): Driver { + return new DenoSqliteDriver(this.#config); + } + + createQueryCompiler(): QueryCompiler { + return new SqliteQueryCompiler(); + } + + createAdapter(): DialectAdapter { + return new SqliteAdapter(); + } + + createIntrospector(db: Kysely): DatabaseIntrospector { + return new SqliteIntrospector(db); + } +} + +export { DenoSqliteDialect }; diff --git a/lib/kysely-deno-sqlite/src/deno-sqlite-driver.ts b/lib/kysely-deno-sqlite/src/deno-sqlite-driver.ts new file mode 100644 index 00000000..c68809ec --- /dev/null +++ b/lib/kysely-deno-sqlite/src/deno-sqlite-driver.ts @@ -0,0 +1,51 @@ +import { type DenoSqlite, type SqliteDatabase, SqliteDriver, type SqliteStatement } from '../deps.ts'; + +import type { DenoSqliteDialectConfig } from './deno-sqlite-dialect-config.ts'; + +class DenoSqliteDriver extends SqliteDriver { + constructor(config: DenoSqliteDialectConfig) { + super({ + ...config, + database: async () => + new DenoSqliteDatabase( + typeof config.database === 'function' ? await config.database() : config.database, + ), + }); + } +} + +/** HACK: This is an adapter class. */ +class DenoSqliteDatabase implements SqliteDatabase { + #db: DenoSqlite; + + constructor(db: DenoSqlite) { + this.#db = db; + } + + close(): void { + this.#db.close(); + } + + prepare(sql: string): SqliteStatement { + const query = this.#db.prepareQuery(sql); + return { + // HACK: implement an actual driver to fix this. + reader: true, + all: (parameters: ReadonlyArray) => { + const result = query.allEntries(parameters as any); + query.finalize(); + return result; + }, + run: (parameters: ReadonlyArray) => { + query.execute(parameters as any); + query.finalize(); + return { + changes: this.#db.changes, + lastInsertRowid: this.#db.lastInsertRowId, + }; + }, + }; + } +} + +export { DenoSqliteDriver }; diff --git a/src/db/builder.ts b/src/db/builder.ts index dec5debe..8a029fb4 100644 --- a/src/db/builder.ts +++ b/src/db/builder.ts @@ -1,5 +1,5 @@ import { Kysely, Sqlite } from '@/deps.ts'; -import { DenoSqliteDialect } from '../../lib/kysely-deno-sqlite.ts'; +import { DenoSqliteDialect } from '../../lib/kysely-deno-sqlite/mod.ts'; interface Tables { events: EventsTable; From 781ca741ddc6865fae7d1147062ca8e98c86fa51 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 6 Aug 2023 22:21:46 -0500 Subject: [PATCH 07/34] Implement the DenoSqliteDriver correctly --- lib/kysely-deno-sqlite/deps.ts | 6 +- .../src/deno-sqlite-driver.ts | 123 +++++++++++++----- 2 files changed, 93 insertions(+), 36 deletions(-) diff --git a/lib/kysely-deno-sqlite/deps.ts b/lib/kysely-deno-sqlite/deps.ts index ff95c114..71a9fa88 100644 --- a/lib/kysely-deno-sqlite/deps.ts +++ b/lib/kysely-deno-sqlite/deps.ts @@ -1,17 +1,17 @@ export { + CompiledQuery, + type DatabaseConnection, type DatabaseIntrospector, type Dialect, type DialectAdapter, type Driver, Kysely, type QueryCompiler, + type QueryResult, SqliteAdapter, - type SqliteDatabase, type SqliteDialectConfig, - SqliteDriver, SqliteIntrospector, SqliteQueryCompiler, - type SqliteStatement, } from 'npm:kysely@^0.25.0'; export type { DB as DenoSqlite } from 'https://deno.land/x/sqlite@v3.7.3/mod.ts'; diff --git a/lib/kysely-deno-sqlite/src/deno-sqlite-driver.ts b/lib/kysely-deno-sqlite/src/deno-sqlite-driver.ts index c68809ec..b0d46559 100644 --- a/lib/kysely-deno-sqlite/src/deno-sqlite-driver.ts +++ b/lib/kysely-deno-sqlite/src/deno-sqlite-driver.ts @@ -1,50 +1,107 @@ -import { type DenoSqlite, type SqliteDatabase, SqliteDriver, type SqliteStatement } from '../deps.ts'; +import { CompiledQuery, type DatabaseConnection, type DenoSqlite, type Driver, type QueryResult } from '../deps.ts'; import type { DenoSqliteDialectConfig } from './deno-sqlite-dialect-config.ts'; -class DenoSqliteDriver extends SqliteDriver { +class DenoSqliteDriver implements Driver { + readonly #config: DenoSqliteDialectConfig; + readonly #connectionMutex = new ConnectionMutex(); + + #db?: DenoSqlite; + #connection?: DatabaseConnection; + constructor(config: DenoSqliteDialectConfig) { - super({ - ...config, - database: async () => - new DenoSqliteDatabase( - typeof config.database === 'function' ? await config.database() : config.database, - ), - }); + this.#config = Object.freeze({ ...config }); + } + + async init(): Promise { + this.#db = typeof this.#config.database === 'function' ? await this.#config.database() : this.#config.database; + + this.#connection = new DenoSqliteConnection(this.#db); + + if (this.#config.onCreateConnection) { + await this.#config.onCreateConnection(this.#connection); + } + } + + async acquireConnection(): Promise { + // SQLite only has one single connection. We use a mutex here to wait + // until the single connection has been released. + await this.#connectionMutex.lock(); + return this.#connection!; + } + + async beginTransaction(connection: DatabaseConnection): Promise { + await connection.executeQuery(CompiledQuery.raw('begin')); + } + + async commitTransaction(connection: DatabaseConnection): Promise { + await connection.executeQuery(CompiledQuery.raw('commit')); + } + + async rollbackTransaction(connection: DatabaseConnection): Promise { + await connection.executeQuery(CompiledQuery.raw('rollback')); + } + + // deno-lint-ignore require-await + async releaseConnection(): Promise { + this.#connectionMutex.unlock(); + } + + // deno-lint-ignore require-await + async destroy(): Promise { + this.#db?.close(); } } -/** HACK: This is an adapter class. */ -class DenoSqliteDatabase implements SqliteDatabase { - #db: DenoSqlite; +class DenoSqliteConnection implements DatabaseConnection { + readonly #db: DenoSqlite; constructor(db: DenoSqlite) { this.#db = db; } - close(): void { - this.#db.close(); + executeQuery({ sql, parameters }: CompiledQuery): Promise> { + const query = this.#db.prepareQuery(sql); + + const rows = query.allEntries(parameters as any); + const { changes, lastInsertRowId } = this.#db; + + query.finalize(); + + return Promise.resolve({ + rows: rows as O[], + numAffectedRows: BigInt(changes), + insertId: BigInt(lastInsertRowId), + }); } - prepare(sql: string): SqliteStatement { - const query = this.#db.prepareQuery(sql); - return { - // HACK: implement an actual driver to fix this. - reader: true, - all: (parameters: ReadonlyArray) => { - const result = query.allEntries(parameters as any); - query.finalize(); - return result; - }, - run: (parameters: ReadonlyArray) => { - query.execute(parameters as any); - query.finalize(); - return { - changes: this.#db.changes, - lastInsertRowid: this.#db.lastInsertRowId, - }; - }, - }; + // deno-lint-ignore require-yield + async *streamQuery(): AsyncIterableIterator> { + throw new Error('Sqlite driver doesn\'t support streaming'); + } +} + +class ConnectionMutex { + #promise?: Promise; + #resolve?: () => void; + + async lock(): Promise { + while (this.#promise) { + await this.#promise; + } + + this.#promise = new Promise((resolve) => { + this.#resolve = resolve; + }); + } + + unlock(): void { + const resolve = this.#resolve; + + this.#promise = undefined; + this.#resolve = undefined; + + resolve?.(); } } From a4681e72816cdd3c85f185cc52170a1db10d0509 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 6 Aug 2023 22:29:56 -0500 Subject: [PATCH 08/34] Do it the normal way instead of the dumb way --- lib/kysely-deno-sqlite/src/deno-sqlite-driver.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/kysely-deno-sqlite/src/deno-sqlite-driver.ts b/lib/kysely-deno-sqlite/src/deno-sqlite-driver.ts index b0d46559..adf49ab7 100644 --- a/lib/kysely-deno-sqlite/src/deno-sqlite-driver.ts +++ b/lib/kysely-deno-sqlite/src/deno-sqlite-driver.ts @@ -61,13 +61,11 @@ class DenoSqliteConnection implements DatabaseConnection { } executeQuery({ sql, parameters }: CompiledQuery): Promise> { - const query = this.#db.prepareQuery(sql); + // @ts-expect-error `parameters` types are incompatible, but they should match in reality. + const rows = this.#db.queryEntries(sql, parameters); - const rows = query.allEntries(parameters as any); const { changes, lastInsertRowId } = this.#db; - query.finalize(); - return Promise.resolve({ rows: rows as O[], numAffectedRows: BigInt(changes), From ecc9db86dd4dd7c3d19be30535836ef7a4da2e45 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 6 Aug 2023 23:30:46 -0500 Subject: [PATCH 09/34] Import kysely-deno-sqlite as a dep --- lib/kysely-deno-sqlite/deps.ts | 17 --- lib/kysely-deno-sqlite/mod.ts | 4 - .../src/deno-sqlite-dialect-config.ts | 7 -- .../src/deno-sqlite-dialect.ts | 41 ------- .../src/deno-sqlite-driver.ts | 106 ------------------ src/db/builder.ts | 3 +- src/deps.ts | 3 +- 7 files changed, 3 insertions(+), 178 deletions(-) delete mode 100644 lib/kysely-deno-sqlite/deps.ts delete mode 100644 lib/kysely-deno-sqlite/mod.ts delete mode 100644 lib/kysely-deno-sqlite/src/deno-sqlite-dialect-config.ts delete mode 100644 lib/kysely-deno-sqlite/src/deno-sqlite-dialect.ts delete mode 100644 lib/kysely-deno-sqlite/src/deno-sqlite-driver.ts diff --git a/lib/kysely-deno-sqlite/deps.ts b/lib/kysely-deno-sqlite/deps.ts deleted file mode 100644 index 71a9fa88..00000000 --- a/lib/kysely-deno-sqlite/deps.ts +++ /dev/null @@ -1,17 +0,0 @@ -export { - CompiledQuery, - type DatabaseConnection, - type DatabaseIntrospector, - type Dialect, - type DialectAdapter, - type Driver, - Kysely, - type QueryCompiler, - type QueryResult, - SqliteAdapter, - type SqliteDialectConfig, - SqliteIntrospector, - SqliteQueryCompiler, -} from 'npm:kysely@^0.25.0'; - -export type { DB as DenoSqlite } from 'https://deno.land/x/sqlite@v3.7.3/mod.ts'; diff --git a/lib/kysely-deno-sqlite/mod.ts b/lib/kysely-deno-sqlite/mod.ts deleted file mode 100644 index a0e19980..00000000 --- a/lib/kysely-deno-sqlite/mod.ts +++ /dev/null @@ -1,4 +0,0 @@ -export { DenoSqliteDialect } from './src/deno-sqlite-dialect.ts'; -export { DenoSqliteDriver } from './src/deno-sqlite-driver.ts'; - -export type { DenoSqliteDialectConfig } from './src/deno-sqlite-dialect-config.ts'; diff --git a/lib/kysely-deno-sqlite/src/deno-sqlite-dialect-config.ts b/lib/kysely-deno-sqlite/src/deno-sqlite-dialect-config.ts deleted file mode 100644 index 12bba773..00000000 --- a/lib/kysely-deno-sqlite/src/deno-sqlite-dialect-config.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { type DenoSqlite, type SqliteDialectConfig } from '../deps.ts'; - -interface DenoSqliteDialectConfig extends Omit { - database: DenoSqlite | (() => Promise); -} - -export type { DenoSqliteDialectConfig }; diff --git a/lib/kysely-deno-sqlite/src/deno-sqlite-dialect.ts b/lib/kysely-deno-sqlite/src/deno-sqlite-dialect.ts deleted file mode 100644 index f3aa5175..00000000 --- a/lib/kysely-deno-sqlite/src/deno-sqlite-dialect.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { - type DatabaseIntrospector, - type Dialect, - type DialectAdapter, - type Driver, - Kysely, - type QueryCompiler, - SqliteAdapter, - SqliteIntrospector, - SqliteQueryCompiler, -} from '../deps.ts'; - -import { DenoSqliteDriver } from './deno-sqlite-driver.ts'; - -import type { DenoSqliteDialectConfig } from './deno-sqlite-dialect-config.ts'; - -class DenoSqliteDialect implements Dialect { - readonly #config: DenoSqliteDialectConfig; - - constructor(config: DenoSqliteDialectConfig) { - this.#config = Object.freeze({ ...config }); - } - - createDriver(): Driver { - return new DenoSqliteDriver(this.#config); - } - - createQueryCompiler(): QueryCompiler { - return new SqliteQueryCompiler(); - } - - createAdapter(): DialectAdapter { - return new SqliteAdapter(); - } - - createIntrospector(db: Kysely): DatabaseIntrospector { - return new SqliteIntrospector(db); - } -} - -export { DenoSqliteDialect }; diff --git a/lib/kysely-deno-sqlite/src/deno-sqlite-driver.ts b/lib/kysely-deno-sqlite/src/deno-sqlite-driver.ts deleted file mode 100644 index adf49ab7..00000000 --- a/lib/kysely-deno-sqlite/src/deno-sqlite-driver.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { CompiledQuery, type DatabaseConnection, type DenoSqlite, type Driver, type QueryResult } from '../deps.ts'; - -import type { DenoSqliteDialectConfig } from './deno-sqlite-dialect-config.ts'; - -class DenoSqliteDriver implements Driver { - readonly #config: DenoSqliteDialectConfig; - readonly #connectionMutex = new ConnectionMutex(); - - #db?: DenoSqlite; - #connection?: DatabaseConnection; - - constructor(config: DenoSqliteDialectConfig) { - this.#config = Object.freeze({ ...config }); - } - - async init(): Promise { - this.#db = typeof this.#config.database === 'function' ? await this.#config.database() : this.#config.database; - - this.#connection = new DenoSqliteConnection(this.#db); - - if (this.#config.onCreateConnection) { - await this.#config.onCreateConnection(this.#connection); - } - } - - async acquireConnection(): Promise { - // SQLite only has one single connection. We use a mutex here to wait - // until the single connection has been released. - await this.#connectionMutex.lock(); - return this.#connection!; - } - - async beginTransaction(connection: DatabaseConnection): Promise { - await connection.executeQuery(CompiledQuery.raw('begin')); - } - - async commitTransaction(connection: DatabaseConnection): Promise { - await connection.executeQuery(CompiledQuery.raw('commit')); - } - - async rollbackTransaction(connection: DatabaseConnection): Promise { - await connection.executeQuery(CompiledQuery.raw('rollback')); - } - - // deno-lint-ignore require-await - async releaseConnection(): Promise { - this.#connectionMutex.unlock(); - } - - // deno-lint-ignore require-await - async destroy(): Promise { - this.#db?.close(); - } -} - -class DenoSqliteConnection implements DatabaseConnection { - readonly #db: DenoSqlite; - - constructor(db: DenoSqlite) { - this.#db = db; - } - - executeQuery({ sql, parameters }: CompiledQuery): Promise> { - // @ts-expect-error `parameters` types are incompatible, but they should match in reality. - const rows = this.#db.queryEntries(sql, parameters); - - const { changes, lastInsertRowId } = this.#db; - - return Promise.resolve({ - rows: rows as O[], - numAffectedRows: BigInt(changes), - insertId: BigInt(lastInsertRowId), - }); - } - - // deno-lint-ignore require-yield - async *streamQuery(): AsyncIterableIterator> { - throw new Error('Sqlite driver doesn\'t support streaming'); - } -} - -class ConnectionMutex { - #promise?: Promise; - #resolve?: () => void; - - async lock(): Promise { - while (this.#promise) { - await this.#promise; - } - - this.#promise = new Promise((resolve) => { - this.#resolve = resolve; - }); - } - - unlock(): void { - const resolve = this.#resolve; - - this.#promise = undefined; - this.#resolve = undefined; - - resolve?.(); - } -} - -export { DenoSqliteDriver }; diff --git a/src/db/builder.ts b/src/db/builder.ts index 8a029fb4..0dd2c81a 100644 --- a/src/db/builder.ts +++ b/src/db/builder.ts @@ -1,5 +1,4 @@ -import { Kysely, Sqlite } from '@/deps.ts'; -import { DenoSqliteDialect } from '../../lib/kysely-deno-sqlite/mod.ts'; +import { DenoSqliteDialect, Kysely, Sqlite } from '@/deps.ts'; interface Tables { events: EventsTable; diff --git a/src/deps.ts b/src/deps.ts index b721ec9a..f5a311dc 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -50,4 +50,5 @@ export * as secp from 'npm:@noble/secp256k1@^2.0.0'; export { LRUCache } from 'npm:lru-cache@^10.0.0'; export { DB as Sqlite } from 'https://deno.land/x/sqlite@v3.7.3/mod.ts'; export * as dotenv from 'https://deno.land/std@0.197.0/dotenv/mod.ts'; -export { DummyDriver, Kysely, SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from 'npm:kysely@^0.25.0'; +export { Kysely } from 'npm:kysely@^0.25.0'; +export { DenoSqliteDialect } from 'https://gitlab.com/soapbox-pub/kysely-deno-sqlite/-/raw/76748303a45fac64a889cd2b9265c6c9b8ef2e8b/mod.ts'; From 3cb5f91d3b073a4c42b92c1e0d623e0f39bcff2f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 7 Aug 2023 00:50:12 -0500 Subject: [PATCH 10/34] Refactor db.ts to use kysely statements --- src/controllers/activitypub/actor.ts | 4 +- src/controllers/well-known/nostr.ts | 6 +- src/controllers/well-known/webfinger.ts | 5 +- src/db.ts | 164 +++++++++--------------- src/db/events.ts | 39 ++++++ src/db/users.ts | 27 ++++ src/deps.ts | 2 +- src/loopback.ts | 4 +- 8 files changed, 141 insertions(+), 110 deletions(-) create mode 100644 src/db/events.ts create mode 100644 src/db/users.ts diff --git a/src/controllers/activitypub/actor.ts b/src/controllers/activitypub/actor.ts index 0051c95e..d3547ac7 100644 --- a/src/controllers/activitypub/actor.ts +++ b/src/controllers/activitypub/actor.ts @@ -1,5 +1,5 @@ import { getAuthor } from '@/client.ts'; -import { db } from '@/db.ts'; +import { findUser } from '@/db/users.ts'; import { toActor } from '@/transformers/nostr-to-activitypub.ts'; import { activityJson } from '@/utils.ts'; @@ -8,7 +8,7 @@ import type { AppContext, AppController } from '@/app.ts'; const actorController: AppController = async (c) => { const username = c.req.param('username'); - const user = db.getUserByUsername(username); + const user = await findUser({ username }); if (!user) return notFound(c); const event = await getAuthor(user.pubkey); diff --git a/src/controllers/well-known/nostr.ts b/src/controllers/well-known/nostr.ts index 1e6a6d28..0d646fc7 100644 --- a/src/controllers/well-known/nostr.ts +++ b/src/controllers/well-known/nostr.ts @@ -1,5 +1,5 @@ import { Conf } from '@/config.ts'; -import { db } from '@/db.ts'; +import { findUser } from '@/db/users.ts'; import { z } from '@/deps.ts'; import type { AppController } from '@/app.ts'; @@ -10,9 +10,9 @@ const nameSchema = z.string().min(1).regex(/^\w+$/); * Serves NIP-05's nostr.json. * https://github.com/nostr-protocol/nips/blob/master/05.md */ -const nostrController: AppController = (c) => { +const nostrController: AppController = async (c) => { const name = nameSchema.safeParse(c.req.query('name')); - const user = name.success ? db.getUserByUsername(name.data) : null; + const user = name.success ? await findUser({ username: name.data }) : null; if (!user) return c.json({ names: {}, relays: {} }); diff --git a/src/controllers/well-known/webfinger.ts b/src/controllers/well-known/webfinger.ts index 0f65ef66..531acbc9 100644 --- a/src/controllers/well-known/webfinger.ts +++ b/src/controllers/well-known/webfinger.ts @@ -4,6 +4,7 @@ import { nip19, z } from '@/deps.ts'; import type { AppContext, AppController } from '@/app.ts'; import type { Webfinger } from '@/schemas/webfinger.ts'; +import { findUser } from '@/db/users.ts'; const webfingerQuerySchema = z.object({ resource: z.string().url(), @@ -36,14 +37,14 @@ const acctSchema = z.custom((value) => value instanceof URL) path: ['resource', 'acct'], }); -function handleAcct(c: AppContext, resource: URL): Response { +async function handleAcct(c: AppContext, resource: URL): Promise { const result = acctSchema.safeParse(resource); if (!result.success) { return c.json({ error: 'Invalid acct URI', schema: result.error }, 400); } const [username, host] = result.data; - const user = db.getUserByUsername(username); + const user = await findUser({ username }); if (!user) { return c.json({ error: 'Not found' }, 404); diff --git a/src/db.ts b/src/db.ts index 511612c4..240b871c 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,114 +1,78 @@ -import { type Filter, Sqlite } from '@/deps.ts'; -import { SignedEvent } from '@/event.ts'; +import { DenoSqliteDialect, Kysely, Sqlite } from '@/deps.ts'; -interface User { +interface Tables { + events: EventRow; + tags: TagRow; + users: UserRow; +} + +interface EventRow { + id: string; + kind: number; + pubkey: string; + content: string; + created_at: number; + tags: string; + sig: string; +} + +interface TagRow { + tag: string; + value_1: string | null; + value_2: string | null; + value_3: string | null; + event_id: string; +} + +interface UserRow { pubkey: string; username: string; inserted_at: Date; } -class DittoDB { - #db: Sqlite; +const sqlite = new Sqlite('data/db.sqlite3'); - constructor(db: Sqlite) { - this.#db = db; +// TODO: move this into a proper migration +sqlite.execute(` + CREATE TABLE IF NOT EXISTS events ( + id TEXT PRIMARY KEY, + kind INTEGER NOT NULL, + pubkey TEXT NOT NULL, + content TEXT NOT NULL, + created_at INTEGER NOT NULL, + tags TEXT NOT NULL, + sig TEXT NOT NULL + ); - this.#db.execute(` - CREATE TABLE IF NOT EXISTS events ( - id TEXT PRIMARY KEY, - kind INTEGER NOT NULL, - pubkey TEXT NOT NULL, - content TEXT NOT NULL, - created_at INTEGER NOT NULL, - tags TEXT NOT NULL, - sig TEXT NOT NULL - ); - - CREATE INDEX IF NOT EXISTS idx_events_kind ON events(kind); - CREATE INDEX IF NOT EXISTS idx_events_pubkey ON events(pubkey); - - CREATE TABLE IF NOT EXISTS tags ( - tag TEXT NOT NULL, - value_1 TEXT, - value_2 TEXT, - value_3 TEXT, - event_id TEXT NOT NULL, - FOREIGN KEY(event_id) REFERENCES events(id) ON DELETE CASCADE - ); - - CREATE INDEX IF NOT EXISTS idx_tags_tag ON tags(tag); - CREATE INDEX IF NOT EXISTS idx_tags_value_1 ON tags(value_1); - CREATE INDEX IF NOT EXISTS idx_tags_event_id ON tags(event_id); - - CREATE TABLE IF NOT EXISTS users ( - pubkey TEXT PRIMARY KEY, - username TEXT NOT NULL, - inserted_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL - ); + CREATE INDEX IF NOT EXISTS idx_events_kind ON events(kind); + CREATE INDEX IF NOT EXISTS idx_events_pubkey ON events(pubkey); - CREATE UNIQUE INDEX IF NOT EXISTS idx_users_username ON users(username); - `); - } + CREATE TABLE IF NOT EXISTS tags ( + tag TEXT NOT NULL, + value_1 TEXT, + value_2 TEXT, + value_3 TEXT, + event_id TEXT NOT NULL, + FOREIGN KEY(event_id) REFERENCES events(id) ON DELETE CASCADE + ); - insertUser(user: Pick): void { - this.#db.query( - 'INSERT INTO users(pubkey, username) VALUES (?, ?)', - [user.pubkey, user.username], - ); - } + CREATE INDEX IF NOT EXISTS idx_tags_tag ON tags(tag); + CREATE INDEX IF NOT EXISTS idx_tags_value_1 ON tags(value_1); + CREATE INDEX IF NOT EXISTS idx_tags_event_id ON tags(event_id); - getUserByUsername(username: string): User | null { - const result = this.#db.query<[string, string, Date]>( - 'SELECT pubkey, username, inserted_at FROM users WHERE username = ?', - [username], - )[0]; - if (!result) return null; - return { - pubkey: result[0], - username: result[1], - inserted_at: result[2], - }; - } + CREATE TABLE IF NOT EXISTS users ( + pubkey TEXT PRIMARY KEY, + username TEXT NOT NULL, + inserted_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL + ); - insertEvent(event: SignedEvent): void { - this.#db.transaction(() => { - this.#db.query( - ` - INSERT INTO events(id, kind, pubkey, content, created_at, tags, sig) - VALUES (?, ?, ?, ?, ?, ?, ?) - `, - [ - event.id, - event.kind, - event.pubkey, - event.content, - event.created_at, - JSON.stringify(event.tags), - event.sig, - ], - ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_users_username ON users(username); +`); - for (const [tag, value1, value2, value3] of event.tags) { - if (['p', 'e', 'q', 'd', 't', 'proxy'].includes(tag)) { - this.#db.query( - ` - INSERT INTO tags(event_id, tag, value_1, value_2, value_3) - VALUES (?, ?, ?, ?, ?) - `, - [event.id, tag, value1 || null, value2 || null, value3 || null], - ); - } - } - }); - } +const db = new Kysely({ + dialect: new DenoSqliteDialect({ + database: sqlite, + }), +}); - getFilter(_filter: Filter) { - // TODO - } -} - -const db = new DittoDB( - new Sqlite('data/db.sqlite3'), -); - -export { db }; +export { db, type EventRow, type TagRow, type UserRow }; diff --git a/src/db/events.ts b/src/db/events.ts new file mode 100644 index 00000000..e9fdf027 --- /dev/null +++ b/src/db/events.ts @@ -0,0 +1,39 @@ +import { type Filter, type Insertable } from '@/deps.ts'; +import { type SignedEvent } from '@/event.ts'; + +import { db, type TagRow } from '../db.ts'; + +function insertEvent(event: SignedEvent): Promise { + return db.transaction().execute(async (trx) => { + await trx.insertInto('events') + .values({ + ...event, + tags: JSON.stringify(event.tags), + }) + .executeTakeFirst(); + + const tags = event.tags.reduce[]>((results, tag) => { + if (['p', 'e', 'q', 'd', 't', 'proxy'].includes(tag[0])) { + results.push({ + event_id: event.id, + tag: tag[0], + value_1: tag[1] || null, + value_2: tag[2] || null, + value_3: tag[3] || null, + }); + } + + return results; + }, []); + + await trx.insertInto('tags') + .values(tags) + .execute(); + }); +} + +function getFilter(_filter: Filter) { + // TODO +} + +export { getFilter, insertEvent }; diff --git a/src/db/users.ts b/src/db/users.ts new file mode 100644 index 00000000..632eca7f --- /dev/null +++ b/src/db/users.ts @@ -0,0 +1,27 @@ +import { type Insertable } from '@/deps.ts'; + +import { db, type UserRow } from '../db.ts'; + +/** Adds a user to the database. */ +function insertUser(user: Insertable) { + return db.insertInto('users').values(user).execute(); +} + +/** + * Finds a single user based on one or more properties. + * + * ```ts + * await findUser({ username: 'alex' }); + * ``` + */ +function findUser(user: Partial>) { + let query = db.selectFrom('users').selectAll(); + + for (const [key, value] of Object.entries(user)) { + query = query.where(key as keyof UserRow, '=', value); + } + + return query.executeTakeFirst(); +} + +export { findUser, insertUser }; diff --git a/src/deps.ts b/src/deps.ts index f5a311dc..d83e6748 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -50,5 +50,5 @@ export * as secp from 'npm:@noble/secp256k1@^2.0.0'; export { LRUCache } from 'npm:lru-cache@^10.0.0'; export { DB as Sqlite } from 'https://deno.land/x/sqlite@v3.7.3/mod.ts'; export * as dotenv from 'https://deno.land/std@0.197.0/dotenv/mod.ts'; -export { Kysely } from 'npm:kysely@^0.25.0'; +export { type Insertable, Kysely, type NullableInsertKeys } from 'npm:kysely@^0.25.0'; export { DenoSqliteDialect } from 'https://gitlab.com/soapbox-pub/kysely-deno-sqlite/-/raw/76748303a45fac64a889cd2b9265c6c9b8ef2e8b/mod.ts'; diff --git a/src/loopback.ts b/src/loopback.ts index af69d918..15629131 100644 --- a/src/loopback.ts +++ b/src/loopback.ts @@ -1,5 +1,5 @@ import { Conf } from '@/config.ts'; -import { db } from '@/db.ts'; +import { insertEvent } from '@/db/events.ts'; import { RelayPool } from '@/deps.ts'; import { trends } from '@/trends.ts'; import { nostrDate, nostrNow } from '@/utils.ts'; @@ -22,7 +22,7 @@ relay.subscribe( /** Handle events through the loopback pipeline. */ function handleEvent(event: SignedEvent): void { console.info('loopback event:', event.id); - db.insertEvent(event); + insertEvent(event); trackHashtags(event); } From 4cbdda401ae623df2edc3c76f51dde093dd4d8b6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 7 Aug 2023 00:50:31 -0500 Subject: [PATCH 11/34] Remove unused builder.ts --- src/db/builder.ts | 39 --------------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 src/db/builder.ts diff --git a/src/db/builder.ts b/src/db/builder.ts deleted file mode 100644 index 0dd2c81a..00000000 --- a/src/db/builder.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { DenoSqliteDialect, Kysely, Sqlite } from '@/deps.ts'; - -interface Tables { - events: EventsTable; - tags: TagsTable; - users: UsersTable; -} - -interface EventsTable { - id: string; - kind: number; - pubkey: string; - content: string; - created_at: number; - tags: string; - sig: string; -} - -interface TagsTable { - tag: string; - value_1: string | null; - value_2: string | null; - value_3: string | null; - event_id: string; -} - -interface UsersTable { - pubkey: string; - username: string; - inserted_at: Date; -} - -const builder = new Kysely({ - dialect: new DenoSqliteDialect({ - database: new Sqlite('data/db.sqlite3'), - }), -}); - -export { builder }; From eb78a213f6a2eca7aa2532792c44c7610c22acfd Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 7 Aug 2023 00:51:11 -0500 Subject: [PATCH 12/34] deno lint --- src/controllers/well-known/webfinger.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/controllers/well-known/webfinger.ts b/src/controllers/well-known/webfinger.ts index 531acbc9..b3a14aff 100644 --- a/src/controllers/well-known/webfinger.ts +++ b/src/controllers/well-known/webfinger.ts @@ -1,5 +1,4 @@ import { Conf } from '@/config.ts'; -import { db } from '@/db.ts'; import { nip19, z } from '@/deps.ts'; import type { AppContext, AppController } from '@/app.ts'; From 97a250c1f794aa5df0ddb2c4c1ad600d7489952e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 7 Aug 2023 01:45:02 -0500 Subject: [PATCH 13/34] Add kysely migrations --- src/db.ts | 58 +++++++--------------- src/db/events.ts | 8 ++-- src/db/migrations/000_create_events.ts | 66 ++++++++++++++++++++++++++ src/deps.ts | 9 +++- 4 files changed, 96 insertions(+), 45 deletions(-) create mode 100644 src/db/migrations/000_create_events.ts diff --git a/src/db.ts b/src/db.ts index 240b871c..25365894 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,4 +1,7 @@ -import { DenoSqliteDialect, Kysely, Sqlite } from '@/deps.ts'; +import fs from 'node:fs/promises'; +import path from 'node:path'; + +import { DenoSqliteDialect, FileMigrationProvider, Kysely, Migrator, Sqlite } from '@/deps.ts'; interface Tables { events: EventRow; @@ -30,49 +33,22 @@ interface UserRow { inserted_at: Date; } -const sqlite = new Sqlite('data/db.sqlite3'); - -// TODO: move this into a proper migration -sqlite.execute(` - CREATE TABLE IF NOT EXISTS events ( - id TEXT PRIMARY KEY, - kind INTEGER NOT NULL, - pubkey TEXT NOT NULL, - content TEXT NOT NULL, - created_at INTEGER NOT NULL, - tags TEXT NOT NULL, - sig TEXT NOT NULL - ); - - CREATE INDEX IF NOT EXISTS idx_events_kind ON events(kind); - CREATE INDEX IF NOT EXISTS idx_events_pubkey ON events(pubkey); - - CREATE TABLE IF NOT EXISTS tags ( - tag TEXT NOT NULL, - value_1 TEXT, - value_2 TEXT, - value_3 TEXT, - event_id TEXT NOT NULL, - FOREIGN KEY(event_id) REFERENCES events(id) ON DELETE CASCADE - ); - - CREATE INDEX IF NOT EXISTS idx_tags_tag ON tags(tag); - CREATE INDEX IF NOT EXISTS idx_tags_value_1 ON tags(value_1); - CREATE INDEX IF NOT EXISTS idx_tags_event_id ON tags(event_id); - - CREATE TABLE IF NOT EXISTS users ( - pubkey TEXT PRIMARY KEY, - username TEXT NOT NULL, - inserted_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL - ); - - CREATE UNIQUE INDEX IF NOT EXISTS idx_users_username ON users(username); -`); - const db = new Kysely({ dialect: new DenoSqliteDialect({ - database: sqlite, + database: new Sqlite('data/db.sqlite3'), }), }); +const migrator = new Migrator({ + db, + provider: new FileMigrationProvider({ + fs, + path, + migrationFolder: new URL(import.meta.resolve('./db/migrations')).pathname, + }), +}); + +console.log('Running migrations...'); +await migrator.migrateToLatest(); + export { db, type EventRow, type TagRow, type UserRow }; diff --git a/src/db/events.ts b/src/db/events.ts index e9fdf027..560df7f4 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -26,9 +26,11 @@ function insertEvent(event: SignedEvent): Promise { return results; }, []); - await trx.insertInto('tags') - .values(tags) - .execute(); + await Promise.all(tags.map((tag) => { + return trx.insertInto('tags') + .values(tag) + .execute(); + })); }); } diff --git a/src/db/migrations/000_create_events.ts b/src/db/migrations/000_create_events.ts new file mode 100644 index 00000000..970edcd8 --- /dev/null +++ b/src/db/migrations/000_create_events.ts @@ -0,0 +1,66 @@ +import { Kysely, sql } from '@/deps.ts'; + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('events') + .addColumn('id', 'text', (col) => col.primaryKey()) + .addColumn('kind', 'integer', (col) => col.notNull()) + .addColumn('pubkey', 'text', (col) => col.notNull()) + .addColumn('content', 'text', (col) => col.notNull()) + .addColumn('created_at', 'integer', (col) => col.notNull()) + .addColumn('tags', 'text', (col) => col.notNull()) + .addColumn('sig', 'text', (col) => col.notNull()) + .execute(); + + await db.schema + .createTable('tags') + .addColumn('tag', 'text', (col) => col.notNull()) + .addColumn('value_1', 'text') + .addColumn('value_2', 'text') + .addColumn('value_3', 'text') + .addColumn('event_id', 'text', (col) => col.notNull().references('events.id')) + .execute(); + + await db.schema + .createTable('users') + .addColumn('pubkey', 'text', (col) => col.primaryKey()) + .addColumn('username', 'text', (col) => col.notNull().unique()) + .addColumn('inserted_at', 'datetime', (col) => col.notNull().defaultTo(sql`CURRENT_TIMESTAMP`)) + .execute(); + + await db.schema + .createIndex('idx_events_kind') + .on('events') + .column('kind') + .execute(); + + await db.schema + .createIndex('idx_events_pubkey') + .on('events') + .column('pubkey') + .execute(); + + await db.schema + .createIndex('idx_tags_tag') + .on('tags') + .column('tag') + .execute(); + + await db.schema + .createIndex('idx_tags_value_1') + .on('tags') + .column('value_1') + .execute(); + + await db.schema + .createIndex('idx_tags_event_id') + .on('tags') + .column('event_id') + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable('events').execute(); + await db.schema.dropTable('tags').execute(); + await db.schema.dropTable('users').execute(); +} diff --git a/src/deps.ts b/src/deps.ts index d83e6748..12e2baf6 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -50,5 +50,12 @@ export * as secp from 'npm:@noble/secp256k1@^2.0.0'; export { LRUCache } from 'npm:lru-cache@^10.0.0'; export { DB as Sqlite } from 'https://deno.land/x/sqlite@v3.7.3/mod.ts'; export * as dotenv from 'https://deno.land/std@0.197.0/dotenv/mod.ts'; -export { type Insertable, Kysely, type NullableInsertKeys } from 'npm:kysely@^0.25.0'; +export { + FileMigrationProvider, + type Insertable, + Kysely, + Migrator, + type NullableInsertKeys, + sql, +} from 'npm:kysely@^0.25.0'; export { DenoSqliteDialect } from 'https://gitlab.com/soapbox-pub/kysely-deno-sqlite/-/raw/76748303a45fac64a889cd2b9265c6c9b8ef2e8b/mod.ts'; From 176df2a2bc45b1f3f0814af67a32b887fca0358b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 7 Aug 2023 02:13:49 -0500 Subject: [PATCH 14/34] deno.json: remove "lib/" --- deno.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deno.json b/deno.json index 3dbda2dc..b3a4d581 100644 --- a/deno.json +++ b/deno.json @@ -10,14 +10,14 @@ "@/": "./src/" }, "lint": { - "include": ["src/", "lib/"], + "include": ["src/"], "rules": { "tags": ["recommended"], "exclude": ["no-explicit-any"] } }, "fmt": { - "include": ["src/", "lib/"], + "include": ["src/"], "useTabs": false, "lineWidth": 120, "indentWidth": 2, From 295b16e943db3901b6ab13d7b3bce98b0b21cedb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 8 Aug 2023 15:35:37 -0500 Subject: [PATCH 15/34] Add a very nice getFilter function --- src/db/events.ts | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/src/db/events.ts b/src/db/events.ts index 560df7f4..9e1edc34 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -34,8 +34,37 @@ function insertEvent(event: SignedEvent): Promise { }); } -function getFilter(_filter: Filter) { - // TODO +async function getFilter(filter: Filter): Promise[]> { + let query = db.selectFrom('events').selectAll(); + + for (const key of Object.keys(filter)) { + switch (key as keyof Filter) { + case 'ids': + query = query.where('id', 'in', filter.ids!); + break; + case 'kinds': + query = query.where('kind', 'in', filter.kinds!); + break; + case 'authors': + query = query.where('pubkey', 'in', filter.authors!); + break; + case 'since': + query = query.where('created_at', '>=', filter.since!); + break; + case 'until': + query = query.where('created_at', '<=', filter.until!); + break; + case 'limit': + query = query.limit(filter.limit!); + break; + } + } + + const events = await query.execute(); + + return events.map((event) => ( + { ...event, tags: JSON.parse(event.tags) } as SignedEvent + )); } export { getFilter, insertEvent }; From cd68da5b93a4a55a3fdb13907441069a4763ea5f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 8 Aug 2023 19:31:14 -0500 Subject: [PATCH 16/34] Minor refactoring --- src/db.ts | 3 ++- src/db/events.ts | 2 +- src/loopback.ts | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/db.ts b/src/db.ts index 25365894..4f382ebc 100644 --- a/src/db.ts +++ b/src/db.ts @@ -49,6 +49,7 @@ const migrator = new Migrator({ }); console.log('Running migrations...'); -await migrator.migrateToLatest(); +const results = await migrator.migrateToLatest(); +console.log('Migrations finished:', results); export { db, type EventRow, type TagRow, type UserRow }; diff --git a/src/db/events.ts b/src/db/events.ts index 9e1edc34..c82465a8 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -35,7 +35,7 @@ function insertEvent(event: SignedEvent): Promise { } async function getFilter(filter: Filter): Promise[]> { - let query = db.selectFrom('events').selectAll(); + let query = db.selectFrom('events').selectAll().orderBy('created_at', 'desc'); for (const key of Object.keys(filter)) { switch (key as keyof Filter) { diff --git a/src/loopback.ts b/src/loopback.ts index 15629131..7cd8fa2d 100644 --- a/src/loopback.ts +++ b/src/loopback.ts @@ -22,7 +22,7 @@ relay.subscribe( /** Handle events through the loopback pipeline. */ function handleEvent(event: SignedEvent): void { console.info('loopback event:', event.id); - insertEvent(event); + insertEvent(event).catch(console.warn); trackHashtags(event); } From 08756c340023ec9874a3d08c54896682810e7777 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 8 Aug 2023 19:36:58 -0500 Subject: [PATCH 17/34] deno.json: add deno task debug --- deno.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deno.json b/deno.json index b3a4d581..1463637a 100644 --- a/deno.json +++ b/deno.json @@ -2,7 +2,9 @@ "$schema": "https://deno.land/x/deno@v1.32.3/cli/schemas/config-file.v1.json", "lock": false, "tasks": { - "start": "deno run --allow-read --allow-write=data --allow-env --allow-net --unstable --watch src/server.ts", + "start": "deno run --allow-read --allow-write=data --allow-env --allow-net --unstable src/server.ts", + "dev": "deno run --allow-read --allow-write=data --allow-env --allow-net --unstable --watch src/server.ts", + "debug": "deno run --allow-read --allow-write=data --allow-env --allow-net --unstable --inspect src/server.ts", "test": "deno test --allow-read --allow-write=data --allow-env --unstable src", "check": "deno check --unstable src/server.ts" }, From 6c96240602faf9dede7c244266740fb13a6a9ce8 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 8 Aug 2023 22:35:43 -0500 Subject: [PATCH 18/34] Remove foreign key constraint from tags.event_id --- src/db/migrations/000_create_events.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db/migrations/000_create_events.ts b/src/db/migrations/000_create_events.ts index 970edcd8..84220718 100644 --- a/src/db/migrations/000_create_events.ts +++ b/src/db/migrations/000_create_events.ts @@ -18,7 +18,7 @@ export async function up(db: Kysely): Promise { .addColumn('value_1', 'text') .addColumn('value_2', 'text') .addColumn('value_3', 'text') - .addColumn('event_id', 'text', (col) => col.notNull().references('events.id')) + .addColumn('event_id', 'text', (col) => col.notNull()) .execute(); await db.schema From b408838ae0a3a4cb2ee0ef8f13aeb7fa2cb4ccf9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 8 Aug 2023 23:40:08 -0500 Subject: [PATCH 19/34] Support querying by tags --- src/db/events.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/db/events.ts b/src/db/events.ts index c82465a8..a0ddc650 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -58,6 +58,15 @@ async function getFilter(filter: Filter): Promise< query = query.limit(filter.limit!); break; } + + if (key.startsWith('#')) { + const tag = key.replace(/^#/, ''); + const value = filter[key as `#${string}`] as string[]; + query = query + .leftJoin('tags', 'tags.event_id', 'events.id') + .where('tags.tag', '=', tag) + .where('tags.value_1', 'in', value); + } } const events = await query.execute(); From 0bfd9d83c159b748ef34a47df1b4c3dfe7c9c3b1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 9 Aug 2023 00:22:20 -0500 Subject: [PATCH 20/34] Add getFilters function --- src/db/events.ts | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/db/events.ts b/src/db/events.ts index a0ddc650..4805d351 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -34,7 +34,7 @@ function insertEvent(event: SignedEvent): Promise { }); } -async function getFilter(filter: Filter): Promise[]> { +function getFilterQuery(filter: Filter) { let query = db.selectFrom('events').selectAll().orderBy('created_at', 'desc'); for (const key of Object.keys(filter)) { @@ -69,11 +69,25 @@ async function getFilter(filter: Filter): Promise< } } - const events = await query.execute(); + return query; +} + +async function getFilters(filters: [Filter]): Promise[]>; +async function getFilters(filters: Filter[]): Promise; +async function getFilters(filters: Filter[]) { + const queries = filters + .map(getFilterQuery) + .map((query) => query.execute()); + + const events = (await Promise.all(queries)).flat(); return events.map((event) => ( - { ...event, tags: JSON.parse(event.tags) } as SignedEvent + { ...event, tags: JSON.parse(event.tags) } )); } -export { getFilter, insertEvent }; +function getFilter(filter: Filter): Promise[]> { + return getFilters([filter]); +} + +export { getFilter, getFilters, insertEvent }; From 9671a77bee2abeea3d8b73a2a3543e1fea7b5cfc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 9 Aug 2023 01:18:43 -0500 Subject: [PATCH 21/34] Add isFollowed function --- src/db/events.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/db/events.ts b/src/db/events.ts index 4805d351..32d8aebd 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -1,7 +1,7 @@ import { type Filter, type Insertable } from '@/deps.ts'; import { type SignedEvent } from '@/event.ts'; -import { db, type TagRow } from '../db.ts'; +import { db, type TagRow } from '@/db.ts'; function insertEvent(event: SignedEvent): Promise { return db.transaction().execute(async (trx) => { @@ -90,4 +90,12 @@ function getFilter(filter: Filter): Promise([filter]); } -export { getFilter, getFilters, insertEvent }; +async function isFollowed({ pubkey }: SignedEvent): Promise { + const event = await getFilterQuery({ kinds: [3], '#p': [pubkey], limit: 1 }) + .innerJoin('users', 'users.pubkey', 'events.pubkey') + .executeTakeFirst(); + + return !!event; +} + +export { getFilter, getFilters, insertEvent, isFollowed }; From 55c957d10be0dca74903983fd97b21e79d5e8287 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 9 Aug 2023 01:28:38 -0500 Subject: [PATCH 22/34] Fix type error --- src/db/events.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db/events.ts b/src/db/events.ts index 32d8aebd..5f38cc09 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -62,7 +62,7 @@ function getFilterQuery(filter: Filter) { if (key.startsWith('#')) { const tag = key.replace(/^#/, ''); const value = filter[key as `#${string}`] as string[]; - query = query + return query .leftJoin('tags', 'tags.event_id', 'events.id') .where('tags.tag', '=', tag) .where('tags.value_1', 'in', value); From ad112ff9a6efad4547a82caa1a87d30cd3daca11 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 9 Aug 2023 01:53:50 -0500 Subject: [PATCH 23/34] Use explicit return type for getFilterQuery --- src/db.ts | 6 +++--- src/db/events.ts | 6 +++--- src/deps.ts | 1 + 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/db.ts b/src/db.ts index 4f382ebc..8f682d18 100644 --- a/src/db.ts +++ b/src/db.ts @@ -3,7 +3,7 @@ import path from 'node:path'; import { DenoSqliteDialect, FileMigrationProvider, Kysely, Migrator, Sqlite } from '@/deps.ts'; -interface Tables { +interface DittoDB { events: EventRow; tags: TagRow; users: UserRow; @@ -33,7 +33,7 @@ interface UserRow { inserted_at: Date; } -const db = new Kysely({ +const db = new Kysely({ dialect: new DenoSqliteDialect({ database: new Sqlite('data/db.sqlite3'), }), @@ -52,4 +52,4 @@ console.log('Running migrations...'); const results = await migrator.migrateToLatest(); console.log('Migrations finished:', results); -export { db, type EventRow, type TagRow, type UserRow }; +export { db, type DittoDB, type EventRow, type TagRow, type UserRow }; diff --git a/src/db/events.ts b/src/db/events.ts index 5f38cc09..f5b9f693 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -1,7 +1,7 @@ -import { type Filter, type Insertable } from '@/deps.ts'; +import { type Filter, type Insertable, type SelectQueryBuilder } from '@/deps.ts'; import { type SignedEvent } from '@/event.ts'; -import { db, type TagRow } from '@/db.ts'; +import { db, type DittoDB, type EventRow, type TagRow } from '@/db.ts'; function insertEvent(event: SignedEvent): Promise { return db.transaction().execute(async (trx) => { @@ -34,7 +34,7 @@ function insertEvent(event: SignedEvent): Promise { }); } -function getFilterQuery(filter: Filter) { +function getFilterQuery(filter: Filter): SelectQueryBuilder { let query = db.selectFrom('events').selectAll().orderBy('created_at', 'desc'); for (const key of Object.keys(filter)) { diff --git a/src/deps.ts b/src/deps.ts index 12e2baf6..cbb45376 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -56,6 +56,7 @@ export { Kysely, Migrator, type NullableInsertKeys, + type SelectQueryBuilder, sql, } from 'npm:kysely@^0.25.0'; export { DenoSqliteDialect } from 'https://gitlab.com/soapbox-pub/kysely-deno-sqlite/-/raw/76748303a45fac64a889cd2b9265c6c9b8ef2e8b/mod.ts'; From a1c8d3352f3dc09ddfdcce1e34ea6c80d512ec40 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 9 Aug 2023 02:01:12 -0500 Subject: [PATCH 24/34] Simplify --- src/db/events.ts | 6 +++--- src/deps.ts | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/db/events.ts b/src/db/events.ts index f5b9f693..c7a5fff9 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -1,7 +1,7 @@ -import { type Filter, type Insertable, type SelectQueryBuilder } from '@/deps.ts'; +import { type Filter, type Insertable } from '@/deps.ts'; import { type SignedEvent } from '@/event.ts'; -import { db, type DittoDB, type EventRow, type TagRow } from '@/db.ts'; +import { db, type TagRow } from '@/db.ts'; function insertEvent(event: SignedEvent): Promise { return db.transaction().execute(async (trx) => { @@ -34,7 +34,7 @@ function insertEvent(event: SignedEvent): Promise { }); } -function getFilterQuery(filter: Filter): SelectQueryBuilder { +function getFilterQuery(filter: Filter) { let query = db.selectFrom('events').selectAll().orderBy('created_at', 'desc'); for (const key of Object.keys(filter)) { diff --git a/src/deps.ts b/src/deps.ts index cbb45376..12e2baf6 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -56,7 +56,6 @@ export { Kysely, Migrator, type NullableInsertKeys, - type SelectQueryBuilder, sql, } from 'npm:kysely@^0.25.0'; export { DenoSqliteDialect } from 'https://gitlab.com/soapbox-pub/kysely-deno-sqlite/-/raw/76748303a45fac64a889cd2b9265c6c9b8ef2e8b/mod.ts'; From 2d2157293c597931d2be08d7d8b35c83ce4ce76f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 9 Aug 2023 02:03:18 -0500 Subject: [PATCH 25/34] Try this? --- src/db/events.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db/events.ts b/src/db/events.ts index c7a5fff9..50f1379f 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -65,7 +65,7 @@ function getFilterQuery(filter: Filter) { return query .leftJoin('tags', 'tags.event_id', 'events.id') .where('tags.tag', '=', tag) - .where('tags.value_1', 'in', value); + .where('tags.value_1', 'in', value) as typeof query; } } From 3b3947ea61a4eb486e6096b86e0852553a8e4a85 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 9 Aug 2023 14:06:57 -0500 Subject: [PATCH 26/34] Only track events which are locally followed --- src/db/events.ts | 5 +++-- src/loopback.ts | 10 +++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/db/events.ts b/src/db/events.ts index 50f1379f..96f4277a 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -90,7 +90,8 @@ function getFilter(filter: Filter): Promise([filter]); } -async function isFollowed({ pubkey }: SignedEvent): Promise { +/** Returns whether the pubkey is followed by a local user. */ +async function isLocallyFollowed(pubkey: string): Promise { const event = await getFilterQuery({ kinds: [3], '#p': [pubkey], limit: 1 }) .innerJoin('users', 'users.pubkey', 'events.pubkey') .executeTakeFirst(); @@ -98,4 +99,4 @@ async function isFollowed({ pubkey }: SignedEvent): Promise { return !!event; } -export { getFilter, getFilters, insertEvent, isFollowed }; +export { getFilter, getFilters, insertEvent, isLocallyFollowed }; diff --git a/src/loopback.ts b/src/loopback.ts index 7cd8fa2d..4cb45e1b 100644 --- a/src/loopback.ts +++ b/src/loopback.ts @@ -1,5 +1,5 @@ import { Conf } from '@/config.ts'; -import { insertEvent } from '@/db/events.ts'; +import { insertEvent, isLocallyFollowed } from '@/db/events.ts'; import { RelayPool } from '@/deps.ts'; import { trends } from '@/trends.ts'; import { nostrDate, nostrNow } from '@/utils.ts'; @@ -20,10 +20,14 @@ relay.subscribe( ); /** Handle events through the loopback pipeline. */ -function handleEvent(event: SignedEvent): void { +async function handleEvent(event: SignedEvent): Promise { console.info('loopback event:', event.id); - insertEvent(event).catch(console.warn); + trackHashtags(event); + + if (await isLocallyFollowed(event.pubkey)) { + insertEvent(event).catch(console.warn); + } } /** Track whenever a hashtag is used, for processing trending tags. */ From f127aa74061096df1aa8ba5cc755db0afa864b75 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 9 Aug 2023 14:24:49 -0500 Subject: [PATCH 27/34] Also track events from local users --- src/loopback.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/loopback.ts b/src/loopback.ts index 4cb45e1b..e07c73ea 100644 --- a/src/loopback.ts +++ b/src/loopback.ts @@ -1,5 +1,6 @@ import { Conf } from '@/config.ts'; import { insertEvent, isLocallyFollowed } from '@/db/events.ts'; +import { findUser } from '@/db/users.ts'; import { RelayPool } from '@/deps.ts'; import { trends } from '@/trends.ts'; import { nostrDate, nostrNow } from '@/utils.ts'; @@ -25,7 +26,7 @@ async function handleEvent(event: SignedEvent): Promise { trackHashtags(event); - if (await isLocallyFollowed(event.pubkey)) { + if (await findUser({ pubkey: event.pubkey }) || await isLocallyFollowed(event.pubkey)) { insertEvent(event).catch(console.warn); } } From 8ebd85b7604e5cfe59442b3eb98dc1c7799ba6a1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 9 Aug 2023 20:34:21 -0500 Subject: [PATCH 28/34] Improve tag indexing logic --- src/db/events.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/db/events.ts b/src/db/events.ts index 96f4277a..956517aa 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -3,6 +3,17 @@ import { type SignedEvent } from '@/event.ts'; import { db, type TagRow } from '@/db.ts'; +type TagCondition = ({ event, count }: { event: SignedEvent; count: number }) => boolean; + +/** Conditions for when to index certain tags. */ +const tagConditions: Record = { + 't': ({ count }) => count < 5, + 'p': ({ event }) => event.kind === 3, + 'd': ({ event, count }) => 30000 <= event.kind && event.kind < 40000 && count === 0, + 'q': ({ event, count }) => event.kind === 1 && count === 0, + 'proxy': ({ count }) => count === 0, +}; + function insertEvent(event: SignedEvent): Promise { return db.transaction().execute(async (trx) => { await trx.insertInto('events') @@ -12,11 +23,15 @@ function insertEvent(event: SignedEvent): Promise { }) .executeTakeFirst(); + const tagCounts: Record = {}; const tags = event.tags.reduce[]>((results, tag) => { - if (['p', 'e', 'q', 'd', 't', 'proxy'].includes(tag[0])) { + const tagName = tag[0]; + tagCounts[tagName] = (tagCounts[tagName] || 0) + 1; + + if (tagConditions[tagName]?.({ event, count: tagCounts[tagName] - 1 })) { results.push({ event_id: event.id, - tag: tag[0], + tag: tagName, value_1: tag[1] || null, value_2: tag[2] || null, value_3: tag[3] || null, From 14eb3cb43b38062a5e19e76b6cb41855f8eef725 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 10 Aug 2023 13:37:56 -0500 Subject: [PATCH 29/34] Let DB_PATH be configurable --- deno.json | 2 +- src/config.ts | 3 +++ src/db.ts | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/deno.json b/deno.json index 1463637a..f57d2de0 100644 --- a/deno.json +++ b/deno.json @@ -5,7 +5,7 @@ "start": "deno run --allow-read --allow-write=data --allow-env --allow-net --unstable src/server.ts", "dev": "deno run --allow-read --allow-write=data --allow-env --allow-net --unstable --watch src/server.ts", "debug": "deno run --allow-read --allow-write=data --allow-env --allow-net --unstable --inspect src/server.ts", - "test": "deno test --allow-read --allow-write=data --allow-env --unstable src", + "test": "DB_PATH=\":memory:\" deno test --allow-read --allow-write=data --allow-env --unstable src", "check": "deno check --unstable src/server.ts" }, "imports": { diff --git a/src/config.ts b/src/config.ts index 07e899d2..461d986b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -45,6 +45,9 @@ const Conf = { get localDomain() { return Deno.env.get('LOCAL_DOMAIN') || 'http://localhost:8000'; }, + get dbPath() { + return Deno.env.get('DB_PATH') || 'data/db.sqlite3'; + }, get postCharLimit() { return Number(Deno.env.get('POST_CHAR_LIMIT') || 5000); }, diff --git a/src/db.ts b/src/db.ts index 8f682d18..d03bf2f1 100644 --- a/src/db.ts +++ b/src/db.ts @@ -2,6 +2,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { DenoSqliteDialect, FileMigrationProvider, Kysely, Migrator, Sqlite } from '@/deps.ts'; +import { Conf } from '@/config.ts'; interface DittoDB { events: EventRow; @@ -35,7 +36,7 @@ interface UserRow { const db = new Kysely({ dialect: new DenoSqliteDialect({ - database: new Sqlite('data/db.sqlite3'), + database: new Sqlite(Conf.dbPath), }), }); From d150268a1d24c0de9e624b764187acca25e2cafc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 10 Aug 2023 14:08:09 -0500 Subject: [PATCH 30/34] Upgrade Deno to v1.36.1 --- .gitlab-ci.yml | 2 +- .tool-versions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5afd11d6..a80d9932 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:1.36.0 +image: denoland/deno:1.36.1 default: interruptible: true diff --git a/.tool-versions b/.tool-versions index d2861dc8..72d0a340 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -deno 1.36.0 +deno 1.36.1 From 5f37a68b9fee5309fe93141b3b9eab4517030229 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 10 Aug 2023 14:08:47 -0500 Subject: [PATCH 31/34] Bump std to v0.198.0 --- src/deps-test.ts | 2 +- src/deps.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/deps-test.ts b/src/deps-test.ts index e57b4adc..1448854f 100644 --- a/src/deps-test.ts +++ b/src/deps-test.ts @@ -1 +1 @@ -export { assert, assertEquals, assertThrows } from 'https://deno.land/std@0.177.0/testing/asserts.ts'; +export { assert, assertEquals, assertThrows } from 'https://deno.land/std@0.198.0/assert/mod.ts'; diff --git a/src/deps.ts b/src/deps.ts index 12e2baf6..7ee37ea5 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -49,7 +49,7 @@ export { generateSeededRsa } from 'https://gitlab.com/soapbox-pub/seeded-rsa/-/r export * as secp from 'npm:@noble/secp256k1@^2.0.0'; export { LRUCache } from 'npm:lru-cache@^10.0.0'; export { DB as Sqlite } from 'https://deno.land/x/sqlite@v3.7.3/mod.ts'; -export * as dotenv from 'https://deno.land/std@0.197.0/dotenv/mod.ts'; +export * as dotenv from 'https://deno.land/std@0.198.0/dotenv/mod.ts'; export { FileMigrationProvider, type Insertable, From 63772022cf7079c99a9ad75a544c0b1aaf6061d0 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 10 Aug 2023 14:56:14 -0500 Subject: [PATCH 32/34] deno fmt --- src/controllers/api/statuses.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 88c3ab98..50ea0320 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -95,8 +95,8 @@ const contextController: AppController = async (c) => { const descendantEvents = await getDescendants(event.id); return c.json({ - ancestors: (await Promise.all((ancestorEvents).map(toStatus))).filter(Boolean), - descendants: (await Promise.all((descendantEvents).map(toStatus))).filter(Boolean), + ancestors: (await Promise.all(ancestorEvents.map(toStatus))).filter(Boolean), + descendants: (await Promise.all(descendantEvents.map(toStatus))).filter(Boolean), }); } From 48343c80358999e8b8f274aab184fb37df3bba00 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 10 Aug 2023 14:56:27 -0500 Subject: [PATCH 33/34] Add events test --- deno.json | 3 ++- fixtures/events/55920b75.json | 15 +++++++++++++++ src/db/events.test.ts | 17 +++++++++++++++++ src/db/events.ts | 5 ++++- 4 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 fixtures/events/55920b75.json create mode 100644 src/db/events.test.ts diff --git a/deno.json b/deno.json index f57d2de0..5233fa9e 100644 --- a/deno.json +++ b/deno.json @@ -9,7 +9,8 @@ "check": "deno check --unstable src/server.ts" }, "imports": { - "@/": "./src/" + "@/": "./src/", + "~/": "./" }, "lint": { "include": ["src/"], diff --git a/fixtures/events/55920b75.json b/fixtures/events/55920b75.json new file mode 100644 index 00000000..f902786c --- /dev/null +++ b/fixtures/events/55920b75.json @@ -0,0 +1,15 @@ +{ + "kind": 1, + "content": "I'm vegan btw", + "tags": [ + [ + "proxy", + "https://gleasonator.com/objects/8f6fac53-4f66-4c6e-ac7d-92e5e78c3e79", + "activitypub" + ] + ], + "pubkey": "79c2cae114ea28a981e7559b4fe7854a473521a8d22a66bbab9fa248eb820ff6", + "created_at": 1691091365, + "id": "55920b758b9c7b17854b6e3d44e6a02a83d1cb49e1227e75a30426dea94d4cb2", + "sig": "a72f12c08f18e85d98fb92ae89e2fe63e48b8864c5e10fbdd5335f3c9f936397a6b0a7350efe251f8168b1601d7012d4a6d0ee6eec958067cf22a14f5a5ea579" +} \ No newline at end of file diff --git a/src/db/events.test.ts b/src/db/events.test.ts new file mode 100644 index 00000000..26fe7bff --- /dev/null +++ b/src/db/events.test.ts @@ -0,0 +1,17 @@ +import event55920b75 from '~/fixtures/events/55920b75.json' assert { type: 'json' }; +import { assertEquals } from '@/deps-test.ts'; + +import { getFilter, insertEvent } from './events.ts'; + +Deno.test('insert and filter events', async () => { + await insertEvent(event55920b75); + + assertEquals(await getFilter({ kinds: [1] }), [event55920b75]); + assertEquals(await getFilter({ kinds: [3] }), []); + assertEquals(await getFilter({ since: 1691091000 }), [event55920b75]); + assertEquals(await getFilter({ until: 1691091000 }), []); + assertEquals( + await getFilter({ '#proxy': ['https://gleasonator.com/objects/8f6fac53-4f66-4c6e-ac7d-92e5e78c3e79'] }), + [event55920b75], + ); +}); diff --git a/src/db/events.ts b/src/db/events.ts index 956517aa..473f30ae 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -50,7 +50,10 @@ function insertEvent(event: SignedEvent): Promise { } function getFilterQuery(filter: Filter) { - let query = db.selectFrom('events').selectAll().orderBy('created_at', 'desc'); + let query = db + .selectFrom('events') + .select(['id', 'kind', 'pubkey', 'content', 'tags', 'created_at', 'sig']) + .orderBy('created_at', 'desc'); for (const key of Object.keys(filter)) { switch (key as keyof Filter) { From 9cca801c800d4d37a5a1e41ff97d69ff6fdf7d22 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 11 Aug 2023 12:14:39 -0500 Subject: [PATCH 34/34] Order tag conditionals alphabetically, improve logic --- src/db/events.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/db/events.ts b/src/db/events.ts index 473f30ae..641149cf 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -7,11 +7,12 @@ type TagCondition = ({ event, count }: { event: SignedEvent; count: number }) => /** Conditions for when to index certain tags. */ const tagConditions: Record = { - 't': ({ count }) => count < 5, - 'p': ({ event }) => event.kind === 3, 'd': ({ event, count }) => 30000 <= event.kind && event.kind < 40000 && count === 0, - 'q': ({ event, count }) => event.kind === 1 && count === 0, + 'e': ({ count }) => count < 15, + 'p': ({ event, count }) => event.kind === 3 || count < 15, 'proxy': ({ count }) => count === 0, + 'q': ({ event, count }) => event.kind === 1 && count === 0, + 't': ({ count }) => count < 5, }; function insertEvent(event: SignedEvent): Promise {