From 149f8f6f04b5388bfff39e2742eeed594f855d48 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Oct 2023 22:30:07 -0500 Subject: [PATCH 1/6] Enable WAL mode on the database --- src/db/migrations/008_wal.ts | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 src/db/migrations/008_wal.ts diff --git a/src/db/migrations/008_wal.ts b/src/db/migrations/008_wal.ts new file mode 100644 index 00000000..7f96226e --- /dev/null +++ b/src/db/migrations/008_wal.ts @@ -0,0 +1,9 @@ +import { Kysely, sql } from '@/deps.ts'; + +export async function up(db: Kysely): Promise { + await sql`PRAGMA journal_mode = WAL`.execute(db); +} + +export async function down(db: Kysely): Promise { + await sql`PRAGMA journal_mode = DELETE`.execute(db); +} From d63de0ad0bd410268220277f729fd0b86c2c9be9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Oct 2023 23:03:56 -0500 Subject: [PATCH 2/6] Set SQLite PRAGMAs on start --- src/config.ts | 16 ++++++++++++++++ src/db.ts | 19 +++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/config.ts b/src/config.ts index cce1ce2c..246bf5a0 100644 --- a/src/config.ts +++ b/src/config.ts @@ -144,9 +144,25 @@ const Conf = { local(path: string): string { return mergePaths(Conf.localDomain, path); }, + /** URL to send Sentry errors to. */ get sentryDsn() { return Deno.env.get('SENTRY_DSN'); }, + /** SQLite settings. */ + sqlite: { + /** + * Number of bytes to use for memory-mapped IO. + * https://www.sqlite.org/pragma.html#pragma_mmap_size + */ + get mmapSize(): number { + const value = Deno.env.get('SQLITE_MMAP_SIZE'); + if (value) { + return Number(value); + } else { + return 1024 * 1024 * 1024; + } + }, + }, }; const optionalBooleanSchema = z diff --git a/src/db.ts b/src/db.ts index 43d7f048..58576e59 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,7 +1,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; -import { DenoSqlite3, DenoSqliteDialect, FileMigrationProvider, Kysely, Migrator } from '@/deps.ts'; +import { DenoSqlite3, DenoSqliteDialect, FileMigrationProvider, Kysely, Migrator, sql } from '@/deps.ts'; import { Conf } from '@/config.ts'; interface DittoDB { @@ -55,12 +55,27 @@ interface UnattachedMediaRow { uploaded_at: Date; } +const sqlite = new DenoSqlite3(Conf.dbPath); + +setPragmas(sqlite, { + synchronous: 'normal', + temp_store: 'memory', + mmap_size: Conf.sqlite.mmapSize, +}); + const db = new Kysely({ dialect: new DenoSqliteDialect({ - database: new DenoSqlite3(Conf.dbPath), + database: sqlite, }), }); +function setPragmas(db: DenoSqlite3, pragmas: Record) { + for (const [pragma, value] of Object.entries(pragmas)) { + db.prepare(`PRAGMA ${pragma} = ${value}`).run(); + console.log(`PRAGMA ${pragma} = ${db.prepare(`PRAGMA ${pragma}`).value()}`); + } +} + const migrator = new Migrator({ db, provider: new FileMigrationProvider({ From f35d38d83b1c7edaf99371ba2840c12a143803a4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Oct 2023 23:34:59 -0500 Subject: [PATCH 3/6] Refactor pragmas --- src/db.ts | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/db.ts b/src/db.ts index 58576e59..3d6b4dbf 100644 --- a/src/db.ts +++ b/src/db.ts @@ -55,24 +55,26 @@ interface UnattachedMediaRow { uploaded_at: Date; } -const sqlite = new DenoSqlite3(Conf.dbPath); - -setPragmas(sqlite, { - synchronous: 'normal', - temp_store: 'memory', - mmap_size: Conf.sqlite.mmapSize, -}); - const db = new Kysely({ dialect: new DenoSqliteDialect({ - database: sqlite, + database: new DenoSqlite3(Conf.dbPath), }), }); -function setPragmas(db: DenoSqlite3, pragmas: Record) { - for (const [pragma, value] of Object.entries(pragmas)) { - db.prepare(`PRAGMA ${pragma} = ${value}`).run(); - console.log(`PRAGMA ${pragma} = ${db.prepare(`PRAGMA ${pragma}`).value()}`); +await Promise.all([ + setPragma(db, 'synchronous', 'normal'), + setPragma(db, 'temp_store', 'memory'), + setPragma(db, 'mmap_size', Conf.sqlite.mmapSize), +]); + +/** Set the PRAGMA and then read back its value to confirm. */ +async function setPragma(db: Kysely, pragma: string, value: string | number) { + await sql.raw(`PRAGMA ${pragma} = ${value}`).execute(db); + const result = (await sql.raw(`PRAGMA ${pragma}`).execute(db)).rows[0] as object; + + for (const [key, value] of Object.entries(result)) { + console.log(`PRAGMA ${key} = ${value};`); + break; } } From 9686469c28885de20605cfaf8e29b7cffedf06a4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Oct 2023 23:44:28 -0500 Subject: [PATCH 4/6] Move PRAGMA utils to a separate file, log out PRAGMA values on start --- src/db.ts | 17 +++++++---------- src/pragma.ts | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 10 deletions(-) create mode 100644 src/pragma.ts diff --git a/src/db.ts b/src/db.ts index 3d6b4dbf..627679b8 100644 --- a/src/db.ts +++ b/src/db.ts @@ -3,6 +3,7 @@ import path from 'node:path'; import { DenoSqlite3, DenoSqliteDialect, FileMigrationProvider, Kysely, Migrator, sql } from '@/deps.ts'; import { Conf } from '@/config.ts'; +import { getPragma, setPragma } from '@/pragma.ts'; interface DittoDB { events: EventRow; @@ -61,22 +62,18 @@ const db = new Kysely({ }), }); +// Set PRAGMA values. await Promise.all([ setPragma(db, 'synchronous', 'normal'), setPragma(db, 'temp_store', 'memory'), setPragma(db, 'mmap_size', Conf.sqlite.mmapSize), ]); -/** Set the PRAGMA and then read back its value to confirm. */ -async function setPragma(db: Kysely, pragma: string, value: string | number) { - await sql.raw(`PRAGMA ${pragma} = ${value}`).execute(db); - const result = (await sql.raw(`PRAGMA ${pragma}`).execute(db)).rows[0] as object; - - for (const [key, value] of Object.entries(result)) { - console.log(`PRAGMA ${key} = ${value};`); - break; - } -} +// Log out PRAGMA values for debugging. +['journal_mode', 'synchronous', 'temp_store', 'mmap_size'].forEach(async (pragma) => { + const value = await getPragma(db, pragma); + console.log(`PRAGMA ${pragma} = ${value};`); +}); const migrator = new Migrator({ db, diff --git a/src/pragma.ts b/src/pragma.ts new file mode 100644 index 00000000..09a06102 --- /dev/null +++ b/src/pragma.ts @@ -0,0 +1,15 @@ +import { type Kysely, sql } from '@/deps.ts'; + +/** Set the PRAGMA and then read back its value to confirm. */ +function setPragma(db: Kysely, pragma: string, value: string | number) { + return sql.raw(`PRAGMA ${pragma} = ${value}`).execute(db); +} + +/** Get value of PRAGMA from the database. */ +async function getPragma(db: Kysely, pragma: string) { + const result = await sql.raw(`PRAGMA ${pragma}`).execute(db); + const row = result.rows[0] as Record; + return row[pragma]; +} + +export { getPragma, setPragma }; From 079cdcf7d720e957caf0cbf661fe62c94bb48fae Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Oct 2023 23:48:01 -0500 Subject: [PATCH 5/6] deno lint --- src/db.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db.ts b/src/db.ts index 627679b8..99339a83 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,7 +1,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; -import { DenoSqlite3, DenoSqliteDialect, FileMigrationProvider, Kysely, Migrator, sql } from '@/deps.ts'; +import { DenoSqlite3, DenoSqliteDialect, FileMigrationProvider, Kysely, Migrator } from '@/deps.ts'; import { Conf } from '@/config.ts'; import { getPragma, setPragma } from '@/pragma.ts'; From 1b8a2d764ce5ec17f44e87824605db3c492a49a2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Oct 2023 23:50:11 -0500 Subject: [PATCH 6/6] Fix tests --- src/pragma.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pragma.ts b/src/pragma.ts index 09a06102..f5aa8e4c 100644 --- a/src/pragma.ts +++ b/src/pragma.ts @@ -8,8 +8,8 @@ function setPragma(db: Kysely, pragma: string, value: string | number) { /** Get value of PRAGMA from the database. */ async function getPragma(db: Kysely, pragma: string) { const result = await sql.raw(`PRAGMA ${pragma}`).execute(db); - const row = result.rows[0] as Record; - return row[pragma]; + const row = result.rows[0] as Record | undefined; + return row?.[pragma]; } export { getPragma, setPragma };