From aefc8c71639d7b2ec97d28f46718d34042fa63dc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Feb 2025 22:47:11 -0600 Subject: [PATCH 01/15] Log verb with ditto.relay.message --- packages/ditto/controllers/nostr/relay.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ditto/controllers/nostr/relay.ts b/packages/ditto/controllers/nostr/relay.ts index 1b66415d..c64049cd 100644 --- a/packages/ditto/controllers/nostr/relay.ts +++ b/packages/ditto/controllers/nostr/relay.ts @@ -65,7 +65,7 @@ function connectStream(socket: WebSocket, ip: string | undefined, conf: DittoCon const result = n.json().pipe(n.clientMsg()).safeParse(e.data); if (result.success) { - logi({ level: 'trace', ns: 'ditto.relay.message', data: result.data as JsonValue, ip }); + logi({ level: 'trace', ns: 'ditto.relay.message', verb: result.data[0], data: result.data as JsonValue, ip }); relayMessagesCounter.inc({ verb: result.data[0] }); handleMsg(result.data); } else { From 7622230c19e937adcbb27c162899c0d8270f2c26 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Feb 2025 22:51:49 -0600 Subject: [PATCH 02/15] logi: ditto.relay.message -> ditto.relay.msg --- packages/ditto/controllers/nostr/relay.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ditto/controllers/nostr/relay.ts b/packages/ditto/controllers/nostr/relay.ts index c64049cd..941a2c20 100644 --- a/packages/ditto/controllers/nostr/relay.ts +++ b/packages/ditto/controllers/nostr/relay.ts @@ -65,7 +65,7 @@ function connectStream(socket: WebSocket, ip: string | undefined, conf: DittoCon const result = n.json().pipe(n.clientMsg()).safeParse(e.data); if (result.success) { - logi({ level: 'trace', ns: 'ditto.relay.message', verb: result.data[0], data: result.data as JsonValue, ip }); + logi({ level: 'trace', ns: 'ditto.relay.msg', verb: result.data[0], data: result.data as JsonValue, ip }); relayMessagesCounter.inc({ verb: result.data[0] }); handleMsg(result.data); } else { From 222bf84eab1b21c689592a59f65f906d4493547e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Feb 2025 23:05:41 -0600 Subject: [PATCH 03/15] More msg tweaks --- packages/ditto/controllers/nostr/relay.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/ditto/controllers/nostr/relay.ts b/packages/ditto/controllers/nostr/relay.ts index 941a2c20..6cdb0da7 100644 --- a/packages/ditto/controllers/nostr/relay.ts +++ b/packages/ditto/controllers/nostr/relay.ts @@ -64,9 +64,14 @@ function connectStream(socket: WebSocket, ip: string | undefined, conf: DittoCon } const result = n.json().pipe(n.clientMsg()).safeParse(e.data); + if (result.success) { - logi({ level: 'trace', ns: 'ditto.relay.msg', verb: result.data[0], data: result.data as JsonValue, ip }); - relayMessagesCounter.inc({ verb: result.data[0] }); + const msg = result.data; + const verb = msg[0]; + + logi({ level: 'trace', ns: 'ditto.relay.msg', verb, msg: msg as JsonValue, ip }); + relayMessagesCounter.inc({ verb }); + handleMsg(result.data); } else { relayMessagesCounter.inc(); From a9744ff4ad70757916907248dfd8f06c01d8c1d7 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 15 Feb 2025 23:23:04 -0600 Subject: [PATCH 04/15] Add streak.expires to the API --- packages/ditto/entities/MastodonAccount.ts | 1 + packages/ditto/views/mastodon/accounts.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/ditto/entities/MastodonAccount.ts b/packages/ditto/entities/MastodonAccount.ts index eedaaa29..4ea6665b 100644 --- a/packages/ditto/entities/MastodonAccount.ts +++ b/packages/ditto/entities/MastodonAccount.ts @@ -49,6 +49,7 @@ export interface MastodonAccount { days: number; start: string | null; end: string | null; + expires: string | null; }; }; domain?: string; diff --git a/packages/ditto/views/mastodon/accounts.ts b/packages/ditto/views/mastodon/accounts.ts index c2cf41ca..d541e633 100644 --- a/packages/ditto/views/mastodon/accounts.ts +++ b/packages/ditto/views/mastodon/accounts.ts @@ -121,6 +121,7 @@ function renderAccount(event: Omit, opts: ToAccountOpt days: streakDays, start: streakStart ? nostrDate(streakStart).toISOString() : null, end: streakEnd ? nostrDate(streakEnd).toISOString() : null, + expires: streakEnd ? nostrDate(streakEnd + streakWindow).toISOString() : null, }, }, domain: parsed05?.domain, From c59bb421c61cbfe4c3a87222507dc1c6daa8347e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Feb 2025 00:14:41 -0600 Subject: [PATCH 05/15] Add @ditto/db package --- deno.json | 1 + packages/{ditto => }/db/DittoDB.ts | 14 ++++++++------ packages/{ditto => }/db/DittoDatabase.ts | 4 ++-- packages/{ditto => }/db/DittoTables.ts | 5 ++--- packages/{ditto => }/db/KyselyLogger.ts | 8 ++++---- packages/{ditto => }/db/adapters/DittoPglite.ts | 9 +++++---- packages/{ditto => }/db/adapters/DittoPostgres.ts | 11 ++++++----- packages/db/deno.json | 6 ++++++ .../{ditto => }/db/migrations/000_create_events.ts | 2 +- .../{ditto => }/db/migrations/001_add_relays.ts | 2 +- .../{ditto => }/db/migrations/002_events_fts.ts | 2 +- .../{ditto => }/db/migrations/003_events_admin.ts | 2 +- .../db/migrations/004_add_user_indexes.ts | 2 +- .../{ditto => }/db/migrations/005_rework_tags.ts | 2 +- packages/{ditto => }/db/migrations/006_pragma.ts | 2 +- .../db/migrations/007_unattached_media.ts | 2 +- packages/{ditto => }/db/migrations/008_wal.ts | 2 +- .../{ditto => }/db/migrations/009_add_stats.ts | 2 +- .../{ditto => }/db/migrations/010_drop_users.ts | 2 +- .../db/migrations/011_kind_author_index.ts | 2 +- .../db/migrations/012_tags_composite_index.ts | 2 +- .../{ditto => }/db/migrations/013_soft_deletion.ts | 2 +- .../db/migrations/014_stats_indexes.ts.ts | 2 +- .../db/migrations/015_add_pubkey_domains.ts | 2 +- .../db/migrations/016_pubkey_domains_updated_at.ts | 2 +- .../{ditto => }/db/migrations/017_rm_relays.ts | 2 +- .../migrations/018_events_created_at_kind_index.ts | 2 +- .../db/migrations/019_ndatabase_schema.ts | 2 +- .../db/migrations/020_drop_deleted_at.ts | 2 +- packages/{ditto => }/db/migrations/020_pgfts.ts | 2 +- .../{ditto => }/db/migrations/021_pgfts_index.ts | 2 +- .../db/migrations/022_event_stats_reactions.ts | 2 +- .../db/migrations/023_add_nip46_tokens.ts | 2 +- .../db/migrations/024_event_stats_quotes_count.ts | 2 +- .../db/migrations/025_event_stats_add_zap_count.ts | 2 +- .../db/migrations/026_tags_name_index.ts | 2 +- .../db/migrations/027_add_zap_events.ts | 2 +- .../{ditto => }/db/migrations/028_stable_sort.ts | 2 +- .../{ditto => }/db/migrations/029_tag_queries.ts | 2 +- .../db/migrations/030_pg_events_jsonb.ts | 2 +- .../db/migrations/031_rm_unattached_media.ts | 2 +- .../db/migrations/032_add_author_search.ts | 2 +- .../{ditto => }/db/migrations/033_add_language.ts | 2 +- .../034_move_author_search_to_author_stats.ts | 2 +- .../migrations/035_author_stats_followers_index.ts | 2 +- packages/{ditto => }/db/migrations/036_stats64.ts | 2 +- .../{ditto => }/db/migrations/037_auth_tokens.ts | 2 +- .../db/migrations/038_push_subscriptions.ts | 2 +- .../{ditto => }/db/migrations/039_pg_notify.ts | 2 +- .../db/migrations/040_add_bunker_pubkey.ts | 2 +- .../db/migrations/041_pg_notify_id_only.ts | 2 +- .../db/migrations/042_add_search_ext.ts | 2 +- .../{ditto => }/db/migrations/043_rm_language.ts | 2 +- .../db/migrations/044_search_ext_drop_default.ts | 2 +- packages/{ditto => }/db/migrations/045_streaks.ts | 2 +- .../db/migrations/046_author_stats_nip05.ts | 2 +- .../db/migrations/047_add_domain_favicons.ts | 2 +- .../db/migrations/048_rm_pubkey_domains.ts | 2 +- .../db/migrations/049_author_stats_sorted.ts | 2 +- .../db/migrations/050_notify_only_insert.ts | 2 +- .../db/migrations/051_notify_replaceable.ts | 2 +- packages/db/mod.ts | 4 ++++ packages/{ditto => db}/utils/worker.test.ts | 2 +- packages/{ditto => db}/utils/worker.ts | 0 packages/ditto/app.ts | 2 +- packages/ditto/pipeline.ts | 2 +- packages/ditto/storages.ts | 3 +-- packages/ditto/storages/EventsDB.ts | 2 +- packages/ditto/storages/hydrate.ts | 2 +- packages/ditto/test.ts | 2 +- packages/ditto/trends.ts | 2 +- packages/ditto/utils/favicon.ts | 2 +- packages/ditto/utils/search.ts | 3 +-- packages/ditto/utils/stats.ts | 2 +- packages/ditto/workers/policy.worker.ts | 2 +- 75 files changed, 103 insertions(+), 91 deletions(-) rename packages/{ditto => }/db/DittoDB.ts (83%) rename packages/{ditto => }/db/DittoDatabase.ts (77%) rename packages/{ditto => }/db/DittoTables.ts (95%) rename packages/{ditto => }/db/KyselyLogger.ts (80%) rename packages/{ditto => }/db/adapters/DittoPglite.ts (80%) rename packages/{ditto => }/db/adapters/DittoPostgres.ts (86%) create mode 100644 packages/db/deno.json rename packages/{ditto => }/db/migrations/000_create_events.ts (97%) rename packages/{ditto => }/db/migrations/001_add_relays.ts (91%) rename packages/{ditto => }/db/migrations/002_events_fts.ts (85%) rename packages/{ditto => }/db/migrations/003_events_admin.ts (84%) rename packages/{ditto => }/db/migrations/004_add_user_indexes.ts (87%) rename packages/{ditto => }/db/migrations/005_rework_tags.ts (97%) rename packages/{ditto => }/db/migrations/006_pragma.ts (78%) rename packages/{ditto => }/db/migrations/007_unattached_media.ts (95%) rename packages/{ditto => }/db/migrations/008_wal.ts (78%) rename packages/{ditto => }/db/migrations/009_add_stats.ts (96%) rename packages/{ditto => }/db/migrations/010_drop_users.ts (83%) rename packages/{ditto => }/db/migrations/011_kind_author_index.ts (90%) rename packages/{ditto => }/db/migrations/012_tags_composite_index.ts (94%) rename packages/{ditto => }/db/migrations/013_soft_deletion.ts (88%) rename packages/{ditto => }/db/migrations/014_stats_indexes.ts.ts (93%) rename packages/{ditto => }/db/migrations/015_add_pubkey_domains.ts (93%) rename packages/{ditto => }/db/migrations/016_pubkey_domains_updated_at.ts (90%) rename packages/{ditto => }/db/migrations/017_rm_relays.ts (91%) rename packages/{ditto => }/db/migrations/018_events_created_at_kind_index.ts (90%) rename packages/{ditto => }/db/migrations/019_ndatabase_schema.ts (93%) rename packages/{ditto => }/db/migrations/020_drop_deleted_at.ts (91%) rename packages/{ditto => }/db/migrations/020_pgfts.ts (91%) rename packages/{ditto => }/db/migrations/021_pgfts_index.ts (90%) rename packages/{ditto => }/db/migrations/022_event_stats_reactions.ts (90%) rename packages/{ditto => }/db/migrations/023_add_nip46_tokens.ts (94%) rename packages/{ditto => }/db/migrations/024_event_stats_quotes_count.ts (90%) rename packages/{ditto => }/db/migrations/025_event_stats_add_zap_count.ts (90%) rename packages/{ditto => }/db/migrations/026_tags_name_index.ts (89%) rename packages/{ditto => }/db/migrations/027_add_zap_events.ts (96%) rename packages/{ditto => }/db/migrations/028_stable_sort.ts (96%) rename packages/{ditto => }/db/migrations/029_tag_queries.ts (98%) rename packages/{ditto => }/db/migrations/030_pg_events_jsonb.ts (99%) rename packages/{ditto => }/db/migrations/031_rm_unattached_media.ts (95%) rename packages/{ditto => }/db/migrations/032_add_author_search.ts (93%) rename packages/{ditto => }/db/migrations/033_add_language.ts (93%) rename packages/{ditto => }/db/migrations/034_move_author_search_to_author_stats.ts (96%) rename packages/{ditto => }/db/migrations/035_author_stats_followers_index.ts (93%) rename packages/{ditto => }/db/migrations/036_stats64.ts (94%) rename packages/{ditto => }/db/migrations/037_auth_tokens.ts (98%) rename packages/{ditto => }/db/migrations/038_push_subscriptions.ts (96%) rename packages/{ditto => }/db/migrations/039_pg_notify.ts (95%) rename packages/{ditto => }/db/migrations/040_add_bunker_pubkey.ts (94%) rename packages/{ditto => }/db/migrations/041_pg_notify_id_only.ts (94%) rename packages/{ditto => }/db/migrations/042_add_search_ext.ts (95%) rename packages/{ditto => }/db/migrations/043_rm_language.ts (92%) rename packages/{ditto => }/db/migrations/044_search_ext_drop_default.ts (90%) rename packages/{ditto => }/db/migrations/045_streaks.ts (91%) rename packages/{ditto => }/db/migrations/046_author_stats_nip05.ts (97%) rename packages/{ditto => }/db/migrations/047_add_domain_favicons.ts (92%) rename packages/{ditto => }/db/migrations/048_rm_pubkey_domains.ts (94%) rename packages/{ditto => }/db/migrations/049_author_stats_sorted.ts (94%) rename packages/{ditto => }/db/migrations/050_notify_only_insert.ts (93%) rename packages/{ditto => }/db/migrations/051_notify_replaceable.ts (96%) create mode 100644 packages/db/mod.ts rename packages/{ditto => db}/utils/worker.test.ts (93%) rename packages/{ditto => db}/utils/worker.ts (100%) diff --git a/deno.json b/deno.json index f7296fa0..f5acf018 100644 --- a/deno.json +++ b/deno.json @@ -2,6 +2,7 @@ "workspace": [ "./packages/api", "./packages/conf", + "./packages/db", "./packages/ditto" ], "tasks": { diff --git a/packages/ditto/db/DittoDB.ts b/packages/db/DittoDB.ts similarity index 83% rename from packages/ditto/db/DittoDB.ts rename to packages/db/DittoDB.ts index 8d242237..e18242cd 100644 --- a/packages/ditto/db/DittoDB.ts +++ b/packages/db/DittoDB.ts @@ -2,15 +2,17 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { logi } from '@soapbox/logi'; -import { JsonValue } from '@std/json'; -import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; +import { FileMigrationProvider, type Kysely, Migrator } from 'kysely'; -import { DittoPglite } from '@/db/adapters/DittoPglite.ts'; -import { DittoPostgres } from '@/db/adapters/DittoPostgres.ts'; -import { DittoDatabase, DittoDatabaseOpts } from '@/db/DittoDatabase.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; import { errorJson } from '@/utils/log.ts'; +import { DittoPglite } from './adapters/DittoPglite.ts'; +import { DittoPostgres } from './adapters/DittoPostgres.ts'; + +import type { JsonValue } from '@std/json'; +import type { DittoDatabase, DittoDatabaseOpts } from './DittoDatabase.ts'; +import type { DittoTables } from './DittoTables.ts'; + export class DittoDB { /** Open a new database connection. */ static create(databaseUrl: string, opts?: DittoDatabaseOpts): DittoDatabase { diff --git a/packages/ditto/db/DittoDatabase.ts b/packages/db/DittoDatabase.ts similarity index 77% rename from packages/ditto/db/DittoDatabase.ts rename to packages/db/DittoDatabase.ts index 3979cd12..e43356a0 100644 --- a/packages/ditto/db/DittoDatabase.ts +++ b/packages/db/DittoDatabase.ts @@ -1,6 +1,6 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; -import { DittoTables } from '@/db/DittoTables.ts'; +import type { DittoTables } from './DittoTables.ts'; export interface DittoDatabase { readonly kysely: Kysely; diff --git a/packages/ditto/db/DittoTables.ts b/packages/db/DittoTables.ts similarity index 95% rename from packages/ditto/db/DittoTables.ts rename to packages/db/DittoTables.ts index 5a7e4c73..92226a84 100644 --- a/packages/ditto/db/DittoTables.ts +++ b/packages/db/DittoTables.ts @@ -1,6 +1,5 @@ -import { Generated } from 'kysely'; - -import { NPostgresSchema } from '@nostrify/db'; +import type { NPostgresSchema } from '@nostrify/db'; +import type { Generated } from 'kysely'; export interface DittoTables extends NPostgresSchema { auth_tokens: AuthTokenRow; diff --git a/packages/ditto/db/KyselyLogger.ts b/packages/db/KyselyLogger.ts similarity index 80% rename from packages/ditto/db/KyselyLogger.ts rename to packages/db/KyselyLogger.ts index 45c10cc3..4ddd1305 100644 --- a/packages/ditto/db/KyselyLogger.ts +++ b/packages/db/KyselyLogger.ts @@ -1,8 +1,8 @@ -import { logi, LogiValue } from '@soapbox/logi'; -import { Logger } from 'kysely'; +import { logi, type LogiValue } from '@soapbox/logi'; import { dbQueriesCounter, dbQueryDurationHistogram } from '@/metrics.ts'; -import { errorJson } from '@/utils/log.ts'; + +import type { Logger } from 'kysely'; /** Log the SQL for queries. */ export const KyselyLogger: Logger = (event) => { @@ -24,7 +24,7 @@ export const KyselyLogger: Logger = (event) => { ns: 'ditto.sql', sql, parameters: parameters as LogiValue, - error: errorJson(event.error), + error: event.error instanceof Error ? event.error : null, duration, }); } diff --git a/packages/ditto/db/adapters/DittoPglite.ts b/packages/db/adapters/DittoPglite.ts similarity index 80% rename from packages/ditto/db/adapters/DittoPglite.ts rename to packages/db/adapters/DittoPglite.ts index df616458..2e7ca3fc 100644 --- a/packages/ditto/db/adapters/DittoPglite.ts +++ b/packages/db/adapters/DittoPglite.ts @@ -3,10 +3,11 @@ import { pg_trgm } from '@electric-sql/pglite/contrib/pg_trgm'; import { PgliteDialect } from '@soapbox/kysely-pglite'; import { Kysely } from 'kysely'; -import { DittoDatabase, DittoDatabaseOpts } from '@/db/DittoDatabase.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; -import { KyselyLogger } from '@/db/KyselyLogger.ts'; -import { isWorker } from '@/utils/worker.ts'; +import { KyselyLogger } from '../KyselyLogger.ts'; +import { isWorker } from '../utils/worker.ts'; + +import type { DittoDatabase, DittoDatabaseOpts } from '../DittoDatabase.ts'; +import type { DittoTables } from '../DittoTables.ts'; export class DittoPglite { static create(databaseUrl: string, opts?: DittoDatabaseOpts): DittoDatabase { diff --git a/packages/ditto/db/adapters/DittoPostgres.ts b/packages/db/adapters/DittoPostgres.ts similarity index 86% rename from packages/ditto/db/adapters/DittoPostgres.ts rename to packages/db/adapters/DittoPostgres.ts index 180e4a7a..9ab8156f 100644 --- a/packages/ditto/db/adapters/DittoPostgres.ts +++ b/packages/db/adapters/DittoPostgres.ts @@ -1,5 +1,5 @@ import { - BinaryOperationNode, + type BinaryOperationNode, FunctionNode, Kysely, OperatorNode, @@ -9,12 +9,13 @@ import { PrimitiveValueListNode, ValueNode, } from 'kysely'; -import { PostgresJSDialectConfig, PostgresJSDriver } from 'kysely-postgres-js'; +import { type PostgresJSDialectConfig, PostgresJSDriver } from 'kysely-postgres-js'; import postgres from 'postgres'; -import { DittoDatabase, DittoDatabaseOpts } from '@/db/DittoDatabase.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; -import { KyselyLogger } from '@/db/KyselyLogger.ts'; +import { KyselyLogger } from '../KyselyLogger.ts'; + +import type { DittoDatabase, DittoDatabaseOpts } from '../DittoDatabase.ts'; +import type { DittoTables } from '../DittoTables.ts'; export class DittoPostgres { static create(databaseUrl: string, opts?: DittoDatabaseOpts): DittoDatabase { diff --git a/packages/db/deno.json b/packages/db/deno.json new file mode 100644 index 00000000..51570d2f --- /dev/null +++ b/packages/db/deno.json @@ -0,0 +1,6 @@ +{ + "name": "@ditto/db", + "exports": { + ".": "./mod.ts" + } +} diff --git a/packages/ditto/db/migrations/000_create_events.ts b/packages/db/migrations/000_create_events.ts similarity index 97% rename from packages/ditto/db/migrations/000_create_events.ts rename to packages/db/migrations/000_create_events.ts index 9cfffc6c..e5f27f5d 100644 --- a/packages/ditto/db/migrations/000_create_events.ts +++ b/packages/db/migrations/000_create_events.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/001_add_relays.ts b/packages/db/migrations/001_add_relays.ts similarity index 91% rename from packages/ditto/db/migrations/001_add_relays.ts rename to packages/db/migrations/001_add_relays.ts index c1685e34..4d286fcb 100644 --- a/packages/ditto/db/migrations/001_add_relays.ts +++ b/packages/db/migrations/001_add_relays.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/002_events_fts.ts b/packages/db/migrations/002_events_fts.ts similarity index 85% rename from packages/ditto/db/migrations/002_events_fts.ts rename to packages/db/migrations/002_events_fts.ts index 392d3c0a..d88d0e7f 100644 --- a/packages/ditto/db/migrations/002_events_fts.ts +++ b/packages/db/migrations/002_events_fts.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(_db: Kysely): Promise { // This migration used to create an FTS table for SQLite, but SQLite support was removed. diff --git a/packages/ditto/db/migrations/003_events_admin.ts b/packages/db/migrations/003_events_admin.ts similarity index 84% rename from packages/ditto/db/migrations/003_events_admin.ts rename to packages/db/migrations/003_events_admin.ts index 46dbb37b..9d555c2d 100644 --- a/packages/ditto/db/migrations/003_events_admin.ts +++ b/packages/db/migrations/003_events_admin.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(_db: Kysely): Promise { } diff --git a/packages/ditto/db/migrations/004_add_user_indexes.ts b/packages/db/migrations/004_add_user_indexes.ts similarity index 87% rename from packages/ditto/db/migrations/004_add_user_indexes.ts rename to packages/db/migrations/004_add_user_indexes.ts index 1759644e..77fe1d31 100644 --- a/packages/ditto/db/migrations/004_add_user_indexes.ts +++ b/packages/db/migrations/004_add_user_indexes.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(_db: Kysely): Promise { } diff --git a/packages/ditto/db/migrations/005_rework_tags.ts b/packages/db/migrations/005_rework_tags.ts similarity index 97% rename from packages/ditto/db/migrations/005_rework_tags.ts rename to packages/db/migrations/005_rework_tags.ts index 29d83962..9d0cfcd2 100644 --- a/packages/ditto/db/migrations/005_rework_tags.ts +++ b/packages/db/migrations/005_rework_tags.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from 'kysely'; +import { type Kysely, sql } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/006_pragma.ts b/packages/db/migrations/006_pragma.ts similarity index 78% rename from packages/ditto/db/migrations/006_pragma.ts rename to packages/db/migrations/006_pragma.ts index 41c6883f..d56df6db 100644 --- a/packages/ditto/db/migrations/006_pragma.ts +++ b/packages/db/migrations/006_pragma.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(_db: Kysely): Promise { } diff --git a/packages/ditto/db/migrations/007_unattached_media.ts b/packages/db/migrations/007_unattached_media.ts similarity index 95% rename from packages/ditto/db/migrations/007_unattached_media.ts rename to packages/db/migrations/007_unattached_media.ts index eb738ecb..1a7a2b18 100644 --- a/packages/ditto/db/migrations/007_unattached_media.ts +++ b/packages/db/migrations/007_unattached_media.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/008_wal.ts b/packages/db/migrations/008_wal.ts similarity index 78% rename from packages/ditto/db/migrations/008_wal.ts rename to packages/db/migrations/008_wal.ts index 41c6883f..d56df6db 100644 --- a/packages/ditto/db/migrations/008_wal.ts +++ b/packages/db/migrations/008_wal.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(_db: Kysely): Promise { } diff --git a/packages/ditto/db/migrations/009_add_stats.ts b/packages/db/migrations/009_add_stats.ts similarity index 96% rename from packages/ditto/db/migrations/009_add_stats.ts rename to packages/db/migrations/009_add_stats.ts index 3865847e..a25ee09f 100644 --- a/packages/ditto/db/migrations/009_add_stats.ts +++ b/packages/db/migrations/009_add_stats.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/010_drop_users.ts b/packages/db/migrations/010_drop_users.ts similarity index 83% rename from packages/ditto/db/migrations/010_drop_users.ts rename to packages/db/migrations/010_drop_users.ts index e936fa00..3175cb04 100644 --- a/packages/ditto/db/migrations/010_drop_users.ts +++ b/packages/db/migrations/010_drop_users.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.dropTable('users').ifExists().execute(); diff --git a/packages/ditto/db/migrations/011_kind_author_index.ts b/packages/db/migrations/011_kind_author_index.ts similarity index 90% rename from packages/ditto/db/migrations/011_kind_author_index.ts rename to packages/db/migrations/011_kind_author_index.ts index 844c105c..03da79ab 100644 --- a/packages/ditto/db/migrations/011_kind_author_index.ts +++ b/packages/db/migrations/011_kind_author_index.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/012_tags_composite_index.ts b/packages/db/migrations/012_tags_composite_index.ts similarity index 94% rename from packages/ditto/db/migrations/012_tags_composite_index.ts rename to packages/db/migrations/012_tags_composite_index.ts index 3894ed27..9cca3cc1 100644 --- a/packages/ditto/db/migrations/012_tags_composite_index.ts +++ b/packages/db/migrations/012_tags_composite_index.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.dropIndex('idx_tags_tag').execute(); diff --git a/packages/ditto/db/migrations/013_soft_deletion.ts b/packages/db/migrations/013_soft_deletion.ts similarity index 88% rename from packages/ditto/db/migrations/013_soft_deletion.ts rename to packages/db/migrations/013_soft_deletion.ts index 17fcf5ea..7b336635 100644 --- a/packages/ditto/db/migrations/013_soft_deletion.ts +++ b/packages/db/migrations/013_soft_deletion.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.alterTable('events').addColumn('deleted_at', 'integer').execute(); diff --git a/packages/ditto/db/migrations/014_stats_indexes.ts.ts b/packages/db/migrations/014_stats_indexes.ts.ts similarity index 93% rename from packages/ditto/db/migrations/014_stats_indexes.ts.ts rename to packages/db/migrations/014_stats_indexes.ts.ts index db52b89a..7f8db099 100644 --- a/packages/ditto/db/migrations/014_stats_indexes.ts.ts +++ b/packages/db/migrations/014_stats_indexes.ts.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.createIndex('idx_author_stats_pubkey').on('author_stats').column('pubkey').execute(); diff --git a/packages/ditto/db/migrations/015_add_pubkey_domains.ts b/packages/db/migrations/015_add_pubkey_domains.ts similarity index 93% rename from packages/ditto/db/migrations/015_add_pubkey_domains.ts rename to packages/db/migrations/015_add_pubkey_domains.ts index 91a480d5..625de519 100644 --- a/packages/ditto/db/migrations/015_add_pubkey_domains.ts +++ b/packages/db/migrations/015_add_pubkey_domains.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/016_pubkey_domains_updated_at.ts b/packages/db/migrations/016_pubkey_domains_updated_at.ts similarity index 90% rename from packages/ditto/db/migrations/016_pubkey_domains_updated_at.ts rename to packages/db/migrations/016_pubkey_domains_updated_at.ts index 26f45fb7..8343d036 100644 --- a/packages/ditto/db/migrations/016_pubkey_domains_updated_at.ts +++ b/packages/db/migrations/016_pubkey_domains_updated_at.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/017_rm_relays.ts b/packages/db/migrations/017_rm_relays.ts similarity index 91% rename from packages/ditto/db/migrations/017_rm_relays.ts rename to packages/db/migrations/017_rm_relays.ts index eeea4d06..ccf53e67 100644 --- a/packages/ditto/db/migrations/017_rm_relays.ts +++ b/packages/db/migrations/017_rm_relays.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.dropTable('relays').execute(); diff --git a/packages/ditto/db/migrations/018_events_created_at_kind_index.ts b/packages/db/migrations/018_events_created_at_kind_index.ts similarity index 90% rename from packages/ditto/db/migrations/018_events_created_at_kind_index.ts rename to packages/db/migrations/018_events_created_at_kind_index.ts index 17ffa856..d6a9dcc1 100644 --- a/packages/ditto/db/migrations/018_events_created_at_kind_index.ts +++ b/packages/db/migrations/018_events_created_at_kind_index.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/019_ndatabase_schema.ts b/packages/db/migrations/019_ndatabase_schema.ts similarity index 93% rename from packages/ditto/db/migrations/019_ndatabase_schema.ts rename to packages/db/migrations/019_ndatabase_schema.ts index a394ed71..736cd0bc 100644 --- a/packages/ditto/db/migrations/019_ndatabase_schema.ts +++ b/packages/db/migrations/019_ndatabase_schema.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.alterTable('events').renameTo('nostr_events').execute(); diff --git a/packages/ditto/db/migrations/020_drop_deleted_at.ts b/packages/db/migrations/020_drop_deleted_at.ts similarity index 91% rename from packages/ditto/db/migrations/020_drop_deleted_at.ts rename to packages/db/migrations/020_drop_deleted_at.ts index a55fe537..6ba81031 100644 --- a/packages/ditto/db/migrations/020_drop_deleted_at.ts +++ b/packages/db/migrations/020_drop_deleted_at.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; // deno-lint-ignore no-explicit-any export async function up(db: Kysely): Promise { diff --git a/packages/ditto/db/migrations/020_pgfts.ts b/packages/db/migrations/020_pgfts.ts similarity index 91% rename from packages/ditto/db/migrations/020_pgfts.ts rename to packages/db/migrations/020_pgfts.ts index 8b22a4e3..e69bd508 100644 --- a/packages/ditto/db/migrations/020_pgfts.ts +++ b/packages/db/migrations/020_pgfts.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from 'kysely'; +import { type Kysely, sql } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.createTable('nostr_pgfts') diff --git a/packages/ditto/db/migrations/021_pgfts_index.ts b/packages/db/migrations/021_pgfts_index.ts similarity index 90% rename from packages/ditto/db/migrations/021_pgfts_index.ts rename to packages/db/migrations/021_pgfts_index.ts index 497adaeb..38e80aed 100644 --- a/packages/ditto/db/migrations/021_pgfts_index.ts +++ b/packages/db/migrations/021_pgfts_index.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/022_event_stats_reactions.ts b/packages/db/migrations/022_event_stats_reactions.ts similarity index 90% rename from packages/ditto/db/migrations/022_event_stats_reactions.ts rename to packages/db/migrations/022_event_stats_reactions.ts index 25cb7d99..45c780b6 100644 --- a/packages/ditto/db/migrations/022_event_stats_reactions.ts +++ b/packages/db/migrations/022_event_stats_reactions.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/023_add_nip46_tokens.ts b/packages/db/migrations/023_add_nip46_tokens.ts similarity index 94% rename from packages/ditto/db/migrations/023_add_nip46_tokens.ts rename to packages/db/migrations/023_add_nip46_tokens.ts index 27ac05d6..45c1522c 100644 --- a/packages/ditto/db/migrations/023_add_nip46_tokens.ts +++ b/packages/db/migrations/023_add_nip46_tokens.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from 'kysely'; +import { type Kysely, sql } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/024_event_stats_quotes_count.ts b/packages/db/migrations/024_event_stats_quotes_count.ts similarity index 90% rename from packages/ditto/db/migrations/024_event_stats_quotes_count.ts rename to packages/db/migrations/024_event_stats_quotes_count.ts index e5cffb2b..b9808bd2 100644 --- a/packages/ditto/db/migrations/024_event_stats_quotes_count.ts +++ b/packages/db/migrations/024_event_stats_quotes_count.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/025_event_stats_add_zap_count.ts b/packages/db/migrations/025_event_stats_add_zap_count.ts similarity index 90% rename from packages/ditto/db/migrations/025_event_stats_add_zap_count.ts rename to packages/db/migrations/025_event_stats_add_zap_count.ts index da021f07..0507fd18 100644 --- a/packages/ditto/db/migrations/025_event_stats_add_zap_count.ts +++ b/packages/db/migrations/025_event_stats_add_zap_count.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/026_tags_name_index.ts b/packages/db/migrations/026_tags_name_index.ts similarity index 89% rename from packages/ditto/db/migrations/026_tags_name_index.ts rename to packages/db/migrations/026_tags_name_index.ts index 18c2519d..3703953b 100644 --- a/packages/ditto/db/migrations/026_tags_name_index.ts +++ b/packages/db/migrations/026_tags_name_index.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/027_add_zap_events.ts b/packages/db/migrations/027_add_zap_events.ts similarity index 96% rename from packages/ditto/db/migrations/027_add_zap_events.ts rename to packages/db/migrations/027_add_zap_events.ts index 6445105f..8ccb8158 100644 --- a/packages/ditto/db/migrations/027_add_zap_events.ts +++ b/packages/db/migrations/027_add_zap_events.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/028_stable_sort.ts b/packages/db/migrations/028_stable_sort.ts similarity index 96% rename from packages/ditto/db/migrations/028_stable_sort.ts rename to packages/db/migrations/028_stable_sort.ts index c27c6a5f..76d771f5 100644 --- a/packages/ditto/db/migrations/028_stable_sort.ts +++ b/packages/db/migrations/028_stable_sort.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/029_tag_queries.ts b/packages/db/migrations/029_tag_queries.ts similarity index 98% rename from packages/ditto/db/migrations/029_tag_queries.ts rename to packages/db/migrations/029_tag_queries.ts index a2ad209c..9a2fd2b3 100644 --- a/packages/ditto/db/migrations/029_tag_queries.ts +++ b/packages/db/migrations/029_tag_queries.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from 'kysely'; +import { type Kysely, sql } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/030_pg_events_jsonb.ts b/packages/db/migrations/030_pg_events_jsonb.ts similarity index 99% rename from packages/ditto/db/migrations/030_pg_events_jsonb.ts rename to packages/db/migrations/030_pg_events_jsonb.ts index 6b28bfc3..b6a6328b 100644 --- a/packages/ditto/db/migrations/030_pg_events_jsonb.ts +++ b/packages/db/migrations/030_pg_events_jsonb.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from 'kysely'; +import { type Kysely, sql } from 'kysely'; export async function up(db: Kysely): Promise { // Create new table and indexes. diff --git a/packages/ditto/db/migrations/031_rm_unattached_media.ts b/packages/db/migrations/031_rm_unattached_media.ts similarity index 95% rename from packages/ditto/db/migrations/031_rm_unattached_media.ts rename to packages/db/migrations/031_rm_unattached_media.ts index 34a5a735..48e9e97b 100644 --- a/packages/ditto/db/migrations/031_rm_unattached_media.ts +++ b/packages/db/migrations/031_rm_unattached_media.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.dropTable('unattached_media').execute(); diff --git a/packages/ditto/db/migrations/032_add_author_search.ts b/packages/db/migrations/032_add_author_search.ts similarity index 93% rename from packages/ditto/db/migrations/032_add_author_search.ts rename to packages/db/migrations/032_add_author_search.ts index 8160f82b..d5c45c1b 100644 --- a/packages/ditto/db/migrations/032_add_author_search.ts +++ b/packages/db/migrations/032_add_author_search.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from 'kysely'; +import { type Kysely, sql } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/033_add_language.ts b/packages/db/migrations/033_add_language.ts similarity index 93% rename from packages/ditto/db/migrations/033_add_language.ts rename to packages/db/migrations/033_add_language.ts index a12c9ed3..9b680ee9 100644 --- a/packages/ditto/db/migrations/033_add_language.ts +++ b/packages/db/migrations/033_add_language.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.alterTable('nostr_events').addColumn('language', 'char(2)').execute(); diff --git a/packages/ditto/db/migrations/034_move_author_search_to_author_stats.ts b/packages/db/migrations/034_move_author_search_to_author_stats.ts similarity index 96% rename from packages/ditto/db/migrations/034_move_author_search_to_author_stats.ts rename to packages/db/migrations/034_move_author_search_to_author_stats.ts index 819fac0a..8c57c639 100644 --- a/packages/ditto/db/migrations/034_move_author_search_to_author_stats.ts +++ b/packages/db/migrations/034_move_author_search_to_author_stats.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from 'kysely'; +import { type Kysely, sql } from 'kysely'; // deno-lint-ignore no-explicit-any export async function up(db: Kysely): Promise { diff --git a/packages/ditto/db/migrations/035_author_stats_followers_index.ts b/packages/db/migrations/035_author_stats_followers_index.ts similarity index 93% rename from packages/ditto/db/migrations/035_author_stats_followers_index.ts rename to packages/db/migrations/035_author_stats_followers_index.ts index 83472220..d6b12f87 100644 --- a/packages/ditto/db/migrations/035_author_stats_followers_index.ts +++ b/packages/db/migrations/035_author_stats_followers_index.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/036_stats64.ts b/packages/db/migrations/036_stats64.ts similarity index 94% rename from packages/ditto/db/migrations/036_stats64.ts rename to packages/db/migrations/036_stats64.ts index ca13f69a..f9c4eabc 100644 --- a/packages/ditto/db/migrations/036_stats64.ts +++ b/packages/db/migrations/036_stats64.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from 'kysely'; +import { type Kysely, sql } from 'kysely'; // deno-lint-ignore no-explicit-any export async function up(db: Kysely): Promise { diff --git a/packages/ditto/db/migrations/037_auth_tokens.ts b/packages/db/migrations/037_auth_tokens.ts similarity index 98% rename from packages/ditto/db/migrations/037_auth_tokens.ts rename to packages/db/migrations/037_auth_tokens.ts index 2f6d1890..591ff379 100644 --- a/packages/ditto/db/migrations/037_auth_tokens.ts +++ b/packages/db/migrations/037_auth_tokens.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from 'kysely'; +import { type Kysely, sql } from 'kysely'; import { Conf } from '@/config.ts'; import { aesEncrypt } from '@/utils/aes.ts'; diff --git a/packages/ditto/db/migrations/038_push_subscriptions.ts b/packages/db/migrations/038_push_subscriptions.ts similarity index 96% rename from packages/ditto/db/migrations/038_push_subscriptions.ts rename to packages/db/migrations/038_push_subscriptions.ts index d75418bd..b06e82d5 100644 --- a/packages/ditto/db/migrations/038_push_subscriptions.ts +++ b/packages/db/migrations/038_push_subscriptions.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from 'kysely'; +import { type Kysely, sql } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/039_pg_notify.ts b/packages/db/migrations/039_pg_notify.ts similarity index 95% rename from packages/ditto/db/migrations/039_pg_notify.ts rename to packages/db/migrations/039_pg_notify.ts index fb0a21ea..2a91b6cf 100644 --- a/packages/ditto/db/migrations/039_pg_notify.ts +++ b/packages/db/migrations/039_pg_notify.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from 'kysely'; +import { type Kysely, sql } from 'kysely'; export async function up(db: Kysely): Promise { await sql` diff --git a/packages/ditto/db/migrations/040_add_bunker_pubkey.ts b/packages/db/migrations/040_add_bunker_pubkey.ts similarity index 94% rename from packages/ditto/db/migrations/040_add_bunker_pubkey.ts rename to packages/db/migrations/040_add_bunker_pubkey.ts index 9f0dff2b..60b5b942 100644 --- a/packages/ditto/db/migrations/040_add_bunker_pubkey.ts +++ b/packages/db/migrations/040_add_bunker_pubkey.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; // deno-lint-ignore no-explicit-any export async function up(db: Kysely): Promise { diff --git a/packages/ditto/db/migrations/041_pg_notify_id_only.ts b/packages/db/migrations/041_pg_notify_id_only.ts similarity index 94% rename from packages/ditto/db/migrations/041_pg_notify_id_only.ts rename to packages/db/migrations/041_pg_notify_id_only.ts index 47668894..ca14802a 100644 --- a/packages/ditto/db/migrations/041_pg_notify_id_only.ts +++ b/packages/db/migrations/041_pg_notify_id_only.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from 'kysely'; +import { type Kysely, sql } from 'kysely'; export async function up(db: Kysely): Promise { await sql`DROP TRIGGER IF EXISTS nostr_event_trigger ON nostr_events`.execute(db); diff --git a/packages/ditto/db/migrations/042_add_search_ext.ts b/packages/db/migrations/042_add_search_ext.ts similarity index 95% rename from packages/ditto/db/migrations/042_add_search_ext.ts rename to packages/db/migrations/042_add_search_ext.ts index 754e1571..11e2c3c0 100644 --- a/packages/ditto/db/migrations/042_add_search_ext.ts +++ b/packages/db/migrations/042_add_search_ext.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from 'kysely'; +import { type Kysely, sql } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/043_rm_language.ts b/packages/db/migrations/043_rm_language.ts similarity index 92% rename from packages/ditto/db/migrations/043_rm_language.ts rename to packages/db/migrations/043_rm_language.ts index e61edc12..eb69aca0 100644 --- a/packages/ditto/db/migrations/043_rm_language.ts +++ b/packages/db/migrations/043_rm_language.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.alterTable('nostr_events').dropColumn('language').execute(); diff --git a/packages/ditto/db/migrations/044_search_ext_drop_default.ts b/packages/db/migrations/044_search_ext_drop_default.ts similarity index 90% rename from packages/ditto/db/migrations/044_search_ext_drop_default.ts rename to packages/db/migrations/044_search_ext_drop_default.ts index 6c8c053f..e714bd62 100644 --- a/packages/ditto/db/migrations/044_search_ext_drop_default.ts +++ b/packages/db/migrations/044_search_ext_drop_default.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.alterTable('nostr_events').alterColumn('search_ext', (col) => col.dropDefault()).execute(); diff --git a/packages/ditto/db/migrations/045_streaks.ts b/packages/db/migrations/045_streaks.ts similarity index 91% rename from packages/ditto/db/migrations/045_streaks.ts rename to packages/db/migrations/045_streaks.ts index 553ef96a..e08727a8 100644 --- a/packages/ditto/db/migrations/045_streaks.ts +++ b/packages/db/migrations/045_streaks.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/046_author_stats_nip05.ts b/packages/db/migrations/046_author_stats_nip05.ts similarity index 97% rename from packages/ditto/db/migrations/046_author_stats_nip05.ts rename to packages/db/migrations/046_author_stats_nip05.ts index 12c23773..9cb5299a 100644 --- a/packages/ditto/db/migrations/046_author_stats_nip05.ts +++ b/packages/db/migrations/046_author_stats_nip05.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from 'kysely'; +import { type Kysely, sql } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/047_add_domain_favicons.ts b/packages/db/migrations/047_add_domain_favicons.ts similarity index 92% rename from packages/ditto/db/migrations/047_add_domain_favicons.ts rename to packages/db/migrations/047_add_domain_favicons.ts index b8d7af77..1086d157 100644 --- a/packages/ditto/db/migrations/047_add_domain_favicons.ts +++ b/packages/db/migrations/047_add_domain_favicons.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from 'kysely'; +import { type Kysely, sql } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/packages/ditto/db/migrations/048_rm_pubkey_domains.ts b/packages/db/migrations/048_rm_pubkey_domains.ts similarity index 94% rename from packages/ditto/db/migrations/048_rm_pubkey_domains.ts rename to packages/db/migrations/048_rm_pubkey_domains.ts index 5f052df2..d88681fd 100644 --- a/packages/ditto/db/migrations/048_rm_pubkey_domains.ts +++ b/packages/db/migrations/048_rm_pubkey_domains.ts @@ -1,4 +1,4 @@ -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.dropTable('pubkey_domains').execute(); diff --git a/packages/ditto/db/migrations/049_author_stats_sorted.ts b/packages/db/migrations/049_author_stats_sorted.ts similarity index 94% rename from packages/ditto/db/migrations/049_author_stats_sorted.ts rename to packages/db/migrations/049_author_stats_sorted.ts index 99aae4bf..0f18864e 100644 --- a/packages/ditto/db/migrations/049_author_stats_sorted.ts +++ b/packages/db/migrations/049_author_stats_sorted.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from 'kysely'; +import { type Kysely, sql } from 'kysely'; // deno-lint-ignore no-explicit-any export async function up(db: Kysely): Promise { diff --git a/packages/ditto/db/migrations/050_notify_only_insert.ts b/packages/db/migrations/050_notify_only_insert.ts similarity index 93% rename from packages/ditto/db/migrations/050_notify_only_insert.ts rename to packages/db/migrations/050_notify_only_insert.ts index 1e6bd4cb..6cbf2dde 100644 --- a/packages/ditto/db/migrations/050_notify_only_insert.ts +++ b/packages/db/migrations/050_notify_only_insert.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from 'kysely'; +import { type Kysely, sql } from 'kysely'; export async function up(db: Kysely): Promise { await sql`DROP TRIGGER IF EXISTS nostr_event_trigger ON nostr_events`.execute(db); diff --git a/packages/ditto/db/migrations/051_notify_replaceable.ts b/packages/db/migrations/051_notify_replaceable.ts similarity index 96% rename from packages/ditto/db/migrations/051_notify_replaceable.ts rename to packages/db/migrations/051_notify_replaceable.ts index e8233078..b4c91787 100644 --- a/packages/ditto/db/migrations/051_notify_replaceable.ts +++ b/packages/db/migrations/051_notify_replaceable.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from 'kysely'; +import { type Kysely, sql } from 'kysely'; export async function up(db: Kysely): Promise { await sql` diff --git a/packages/db/mod.ts b/packages/db/mod.ts new file mode 100644 index 00000000..39521f20 --- /dev/null +++ b/packages/db/mod.ts @@ -0,0 +1,4 @@ +export { DittoDB } from './DittoDB.ts'; + +export type { DittoDatabase } from './DittoDatabase.ts'; +export type { DittoTables } from './DittoTables.ts'; diff --git a/packages/ditto/utils/worker.test.ts b/packages/db/utils/worker.test.ts similarity index 93% rename from packages/ditto/utils/worker.test.ts rename to packages/db/utils/worker.test.ts index 89845e2b..3e94a91a 100644 --- a/packages/ditto/utils/worker.test.ts +++ b/packages/db/utils/worker.test.ts @@ -1,6 +1,6 @@ import { assertEquals } from '@std/assert'; -import { isWorker } from '@/utils/worker.ts'; +import { isWorker } from './worker.ts'; Deno.test('isWorker from the main thread returns false', () => { assertEquals(isWorker(), false); diff --git a/packages/ditto/utils/worker.ts b/packages/db/utils/worker.ts similarity index 100% rename from packages/ditto/utils/worker.ts rename to packages/db/utils/worker.ts diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index 3f5abee4..6a54f66f 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -1,5 +1,6 @@ import { confMw } from '@ditto/api/middleware'; import { type DittoConf } from '@ditto/conf'; +import { DittoTables } from '@ditto/db'; import { type Context, Env as HonoEnv, Handler, Hono, Input as HonoInput, MiddlewareHandler } from '@hono/hono'; import { every } from '@hono/hono/combine'; import { cors } from '@hono/hono/cors'; @@ -9,7 +10,6 @@ import { Kysely } from 'kysely'; import '@/startup.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; import { Time } from '@/utils/time.ts'; import { diff --git a/packages/ditto/pipeline.ts b/packages/ditto/pipeline.ts index 4fcd43bf..0ad1969d 100644 --- a/packages/ditto/pipeline.ts +++ b/packages/ditto/pipeline.ts @@ -1,3 +1,4 @@ +import { DittoTables } from '@ditto/db'; import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import { Kysely, UpdateObject } from 'kysely'; @@ -6,7 +7,6 @@ import { z } from 'zod'; import { pipelineEncounters } from '@/caches/pipelineEncounters.ts'; import { Conf } from '@/config.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; import { DittoPush } from '@/DittoPush.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { pipelineEventsCounter, policyEventsCounter, webPushNotificationsCounter } from '@/metrics.ts'; diff --git a/packages/ditto/storages.ts b/packages/ditto/storages.ts index 0bccc534..55566f51 100644 --- a/packages/ditto/storages.ts +++ b/packages/ditto/storages.ts @@ -1,9 +1,8 @@ // deno-lint-ignore-file require-await +import { type DittoDatabase, DittoDB } from '@ditto/db'; import { logi } from '@soapbox/logi'; import { Conf } from '@/config.ts'; -import { DittoDatabase } from '@/db/DittoDatabase.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { internalSubscriptionsSizeGauge } from '@/metrics.ts'; import { wsUrlSchema } from '@/schema.ts'; import { AdminStore } from '@/storages/AdminStore.ts'; diff --git a/packages/ditto/storages/EventsDB.ts b/packages/ditto/storages/EventsDB.ts index 91757449..b8c75c59 100644 --- a/packages/ditto/storages/EventsDB.ts +++ b/packages/ditto/storages/EventsDB.ts @@ -1,5 +1,6 @@ // deno-lint-ignore-file require-await +import { DittoTables } from '@ditto/db'; import { NPostgres, NPostgresSchema } from '@nostrify/db'; import { NIP50, NKinds, NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; @@ -11,7 +12,6 @@ import { nip27 } from 'nostr-tools'; import tldts from 'tldts'; import { z } from 'zod'; -import { DittoTables } from '@/db/DittoTables.ts'; import { dbEventsCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; import { isNostrId } from '@/utils.ts'; diff --git a/packages/ditto/storages/hydrate.ts b/packages/ditto/storages/hydrate.ts index 36df74f6..0836bd76 100644 --- a/packages/ditto/storages/hydrate.ts +++ b/packages/ditto/storages/hydrate.ts @@ -1,10 +1,10 @@ +import { DittoTables } from '@ditto/db'; import { NStore } from '@nostrify/nostrify'; import { Kysely } from 'kysely'; import { matchFilter } from 'nostr-tools'; import { NSchema as n } from '@nostrify/nostrify'; import { z } from 'zod'; -import { DittoTables } from '@/db/DittoTables.ts'; import { Conf } from '@/config.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { fallbackAuthor } from '@/utils.ts'; diff --git a/packages/ditto/test.ts b/packages/ditto/test.ts index 95aa2872..bc9a6787 100644 --- a/packages/ditto/test.ts +++ b/packages/ditto/test.ts @@ -1,10 +1,10 @@ +import { DittoDB } from '@ditto/db'; import ISO6391, { LanguageCode } from 'iso-639-1'; import lande from 'lande'; import { NostrEvent } from '@nostrify/nostrify'; import { finalizeEvent, generateSecretKey } from 'nostr-tools'; import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; import { purifyEvent } from '@/utils/purify.ts'; import { sql } from 'kysely'; diff --git a/packages/ditto/trends.ts b/packages/ditto/trends.ts index e4da152d..8dfdb5ae 100644 --- a/packages/ditto/trends.ts +++ b/packages/ditto/trends.ts @@ -1,9 +1,9 @@ +import { DittoTables } from '@ditto/db'; import { NostrFilter } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import { Kysely, sql } from 'kysely'; import { Conf } from '@/config.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; import { handleEvent } from '@/pipeline.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Storages } from '@/storages.ts'; diff --git a/packages/ditto/utils/favicon.ts b/packages/ditto/utils/favicon.ts index f1ae0f95..c181a198 100644 --- a/packages/ditto/utils/favicon.ts +++ b/packages/ditto/utils/favicon.ts @@ -1,11 +1,11 @@ import { DOMParser } from '@b-fuze/deno-dom'; +import { DittoTables } from '@ditto/db'; import { logi } from '@soapbox/logi'; import { safeFetch } from '@soapbox/safe-fetch'; import { Kysely } from 'kysely'; import tldts from 'tldts'; import { Conf } from '@/config.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; import { cachedFaviconsSizeGauge } from '@/metrics.ts'; import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; diff --git a/packages/ditto/utils/search.ts b/packages/ditto/utils/search.ts index e41cd413..7c6584c6 100644 --- a/packages/ditto/utils/search.ts +++ b/packages/ditto/utils/search.ts @@ -1,7 +1,6 @@ +import { DittoTables } from '@ditto/db'; import { Kysely, sql } from 'kysely'; -import { DittoTables } from '@/db/DittoTables.ts'; - /** Get pubkeys whose name and NIP-05 is similar to 'q' */ export async function getPubkeysBySearch( kysely: Kysely, diff --git a/packages/ditto/utils/stats.ts b/packages/ditto/utils/stats.ts index 972541d3..01ec80d9 100644 --- a/packages/ditto/utils/stats.ts +++ b/packages/ditto/utils/stats.ts @@ -1,10 +1,10 @@ +import { DittoTables } from '@ditto/db'; import { NostrEvent, NSchema as n, NStore } from '@nostrify/nostrify'; import { Insertable, Kysely, UpdateObject } from 'kysely'; import { SetRequired } from 'type-fest'; import { z } from 'zod'; import { Conf } from '@/config.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; import { findQuoteTag, findReplyTag, getTagSet } from '@/utils/tags.ts'; interface UpdateStatsOpts { diff --git a/packages/ditto/workers/policy.worker.ts b/packages/ditto/workers/policy.worker.ts index 00540b03..85a98240 100644 --- a/packages/ditto/workers/policy.worker.ts +++ b/packages/ditto/workers/policy.worker.ts @@ -1,9 +1,9 @@ +import { DittoDB } from '@ditto/db'; import '@soapbox/safe-fetch/load'; import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify'; import { ReadOnlyPolicy } from '@nostrify/policies'; import * as Comlink from 'comlink'; -import { DittoDB } from '@/db/DittoDB.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; // @ts-ignore Don't try to access the env from this worker. From 773b5da461b1cb42438ec3855ceb282044be7dd0 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Feb 2025 00:21:41 -0600 Subject: [PATCH 06/15] Add db tests --- packages/db/DittoDB.test.ts | 6 ++++++ packages/db/adapters/DittoPglite.test.ts | 10 ++++++++++ 2 files changed, 16 insertions(+) create mode 100644 packages/db/DittoDB.test.ts create mode 100644 packages/db/adapters/DittoPglite.test.ts diff --git a/packages/db/DittoDB.test.ts b/packages/db/DittoDB.test.ts new file mode 100644 index 00000000..1a283319 --- /dev/null +++ b/packages/db/DittoDB.test.ts @@ -0,0 +1,6 @@ +import { DittoDB } from './DittoDB.ts'; + +Deno.test('DittoDB', async () => { + const db = DittoDB.create('memory://'); + await DittoDB.migrate(db.kysely); +}); diff --git a/packages/db/adapters/DittoPglite.test.ts b/packages/db/adapters/DittoPglite.test.ts new file mode 100644 index 00000000..c4a6d8c6 --- /dev/null +++ b/packages/db/adapters/DittoPglite.test.ts @@ -0,0 +1,10 @@ +import { assertEquals } from '@std/assert'; + +import { DittoPglite } from './DittoPglite.ts'; + +Deno.test('DittoPglite.create', () => { + const db = DittoPglite.create('memory://'); + + assertEquals(db.poolSize, 1); + assertEquals(db.availableConnections, 1); +}); From 739153afc9ef37cdd301213797fe4b1bd95ad687 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Feb 2025 00:30:00 -0600 Subject: [PATCH 07/15] Add @ditto/metrics package --- deno.json | 3 ++- packages/db/KyselyLogger.ts | 3 +-- packages/ditto/controllers/api/streaming.ts | 10 +++++----- packages/ditto/controllers/api/translate.ts | 2 +- packages/ditto/controllers/metrics.ts | 8 ++++---- packages/ditto/controllers/nostr/relay.ts | 2 +- packages/ditto/firehose.ts | 2 +- packages/ditto/middleware/metricsMiddleware.ts | 3 +-- packages/ditto/pipeline.ts | 2 +- packages/ditto/storages.ts | 2 +- packages/ditto/storages/EventsDB.ts | 2 +- packages/ditto/utils/favicon.ts | 2 +- packages/ditto/utils/lnurl.ts | 2 +- packages/ditto/utils/nip05.ts | 4 ++-- packages/ditto/utils/unfurl.ts | 2 +- packages/metrics/deno.json | 6 ++++++ packages/{ditto => metrics}/metrics.ts | 0 17 files changed, 30 insertions(+), 25 deletions(-) create mode 100644 packages/metrics/deno.json rename packages/{ditto => metrics}/metrics.ts (100%) diff --git a/deno.json b/deno.json index f5acf018..629e1813 100644 --- a/deno.json +++ b/deno.json @@ -3,7 +3,8 @@ "./packages/api", "./packages/conf", "./packages/db", - "./packages/ditto" + "./packages/ditto", + "./packages/metrics" ], "tasks": { "start": "deno run -A --env-file --deny-read=.env packages/ditto/server.ts", diff --git a/packages/db/KyselyLogger.ts b/packages/db/KyselyLogger.ts index 4ddd1305..333e4285 100644 --- a/packages/db/KyselyLogger.ts +++ b/packages/db/KyselyLogger.ts @@ -1,7 +1,6 @@ +import { dbQueriesCounter, dbQueryDurationHistogram } from '@ditto/metrics'; import { logi, type LogiValue } from '@soapbox/logi'; -import { dbQueriesCounter, dbQueryDurationHistogram } from '@/metrics.ts'; - import type { Logger } from 'kysely'; /** Log the SQL for queries. */ diff --git a/packages/ditto/controllers/api/streaming.ts b/packages/ditto/controllers/api/streaming.ts index 43bf92be..7f2f8b64 100644 --- a/packages/ditto/controllers/api/streaming.ts +++ b/packages/ditto/controllers/api/streaming.ts @@ -1,14 +1,14 @@ +import { + streamingClientMessagesCounter, + streamingConnectionsGauge, + streamingServerMessagesCounter, +} from '@ditto/metrics'; import TTLCache from '@isaacs/ttlcache'; import { NostrEvent, NostrFilter } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; -import { - streamingClientMessagesCounter, - streamingConnectionsGauge, - streamingServerMessagesCounter, -} from '@/metrics.ts'; import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; import { getFeedPubkeys } from '@/queries.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; diff --git a/packages/ditto/controllers/api/translate.ts b/packages/ditto/controllers/api/translate.ts index d763c713..7395ff2f 100644 --- a/packages/ditto/controllers/api/translate.ts +++ b/packages/ditto/controllers/api/translate.ts @@ -1,10 +1,10 @@ +import { cachedTranslationsSizeGauge } from '@ditto/metrics'; import { LanguageCode } from 'iso-639-1'; import { z } from 'zod'; import { AppController } from '@/app.ts'; import { translationCache } from '@/caches/translationCache.ts'; import { MastodonTranslation } from '@/entities/MastodonTranslation.ts'; -import { cachedTranslationsSizeGauge } from '@/metrics.ts'; import { getEvent } from '@/queries.ts'; import { localeSchema } from '@/schema.ts'; import { parseBody } from '@/utils/api.ts'; diff --git a/packages/ditto/controllers/metrics.ts b/packages/ditto/controllers/metrics.ts index d168243b..32a8783d 100644 --- a/packages/ditto/controllers/metrics.ts +++ b/packages/ditto/controllers/metrics.ts @@ -1,12 +1,12 @@ -import { register } from 'prom-client'; - -import { AppController } from '@/app.ts'; import { dbAvailableConnectionsGauge, dbPoolSizeGauge, relayPoolRelaysSizeGauge, relayPoolSubscriptionsSizeGauge, -} from '@/metrics.ts'; +} from '@ditto/metrics'; +import { register } from 'prom-client'; + +import { AppController } from '@/app.ts'; import { Storages } from '@/storages.ts'; /** Prometheus/OpenMetrics controller. */ diff --git a/packages/ditto/controllers/nostr/relay.ts b/packages/ditto/controllers/nostr/relay.ts index 6cdb0da7..92906d04 100644 --- a/packages/ditto/controllers/nostr/relay.ts +++ b/packages/ditto/controllers/nostr/relay.ts @@ -1,4 +1,5 @@ import { type DittoConf } from '@ditto/conf'; +import { relayConnectionsGauge, relayEventsCounter, relayMessagesCounter } from '@ditto/metrics'; import { logi } from '@soapbox/logi'; import { JsonValue } from '@std/json'; import { @@ -14,7 +15,6 @@ import { import { AppController } from '@/app.ts'; import { relayInfoController } from '@/controllers/nostr/relay-info.ts'; -import { relayConnectionsGauge, relayEventsCounter, relayMessagesCounter } from '@/metrics.ts'; import * as pipeline from '@/pipeline.ts'; import { RelayError } from '@/RelayError.ts'; import { Storages } from '@/storages.ts'; diff --git a/packages/ditto/firehose.ts b/packages/ditto/firehose.ts index f04752b2..e967e1f2 100644 --- a/packages/ditto/firehose.ts +++ b/packages/ditto/firehose.ts @@ -1,8 +1,8 @@ +import { firehoseEventsCounter } from '@ditto/metrics'; import { Semaphore } from '@core/asyncutil'; import { logi } from '@soapbox/logi'; import { Conf } from '@/config.ts'; -import { firehoseEventsCounter } from '@/metrics.ts'; import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; diff --git a/packages/ditto/middleware/metricsMiddleware.ts b/packages/ditto/middleware/metricsMiddleware.ts index 0b213b82..91f2c422 100644 --- a/packages/ditto/middleware/metricsMiddleware.ts +++ b/packages/ditto/middleware/metricsMiddleware.ts @@ -1,8 +1,7 @@ +import { httpRequestsCounter, httpResponseDurationHistogram, httpResponsesCounter } from '@ditto/metrics'; import { ScopedPerformance } from '@esroyo/scoped-performance'; import { MiddlewareHandler } from '@hono/hono'; -import { httpRequestsCounter, httpResponseDurationHistogram, httpResponsesCounter } from '@/metrics.ts'; - /** Prometheus metrics middleware that tracks HTTP requests by methods and responses by status code. */ export const metricsMiddleware: MiddlewareHandler = async (c, next) => { // Start a timer to measure the duration of the response. diff --git a/packages/ditto/pipeline.ts b/packages/ditto/pipeline.ts index 0ad1969d..d3168c0e 100644 --- a/packages/ditto/pipeline.ts +++ b/packages/ditto/pipeline.ts @@ -1,4 +1,5 @@ import { DittoTables } from '@ditto/db'; +import { pipelineEventsCounter, policyEventsCounter, webPushNotificationsCounter } from '@ditto/metrics'; import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import { Kysely, UpdateObject } from 'kysely'; @@ -9,7 +10,6 @@ import { pipelineEncounters } from '@/caches/pipelineEncounters.ts'; import { Conf } from '@/config.ts'; import { DittoPush } from '@/DittoPush.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { pipelineEventsCounter, policyEventsCounter, webPushNotificationsCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; diff --git a/packages/ditto/storages.ts b/packages/ditto/storages.ts index 55566f51..be61beb6 100644 --- a/packages/ditto/storages.ts +++ b/packages/ditto/storages.ts @@ -1,9 +1,9 @@ // deno-lint-ignore-file require-await import { type DittoDatabase, DittoDB } from '@ditto/db'; +import { internalSubscriptionsSizeGauge } from '@ditto/metrics'; import { logi } from '@soapbox/logi'; import { Conf } from '@/config.ts'; -import { internalSubscriptionsSizeGauge } from '@/metrics.ts'; import { wsUrlSchema } from '@/schema.ts'; import { AdminStore } from '@/storages/AdminStore.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; diff --git a/packages/ditto/storages/EventsDB.ts b/packages/ditto/storages/EventsDB.ts index b8c75c59..622f5811 100644 --- a/packages/ditto/storages/EventsDB.ts +++ b/packages/ditto/storages/EventsDB.ts @@ -2,6 +2,7 @@ import { DittoTables } from '@ditto/db'; import { NPostgres, NPostgresSchema } from '@nostrify/db'; +import { dbEventsCounter } from '@ditto/metrics'; import { NIP50, NKinds, NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import { JsonValue } from '@std/json'; @@ -12,7 +13,6 @@ import { nip27 } from 'nostr-tools'; import tldts from 'tldts'; import { z } from 'zod'; -import { dbEventsCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; import { isNostrId } from '@/utils.ts'; import { abortError } from '@/utils/abort.ts'; diff --git a/packages/ditto/utils/favicon.ts b/packages/ditto/utils/favicon.ts index c181a198..ed218cfa 100644 --- a/packages/ditto/utils/favicon.ts +++ b/packages/ditto/utils/favicon.ts @@ -1,12 +1,12 @@ import { DOMParser } from '@b-fuze/deno-dom'; import { DittoTables } from '@ditto/db'; +import { cachedFaviconsSizeGauge } from '@ditto/metrics'; import { logi } from '@soapbox/logi'; import { safeFetch } from '@soapbox/safe-fetch'; import { Kysely } from 'kysely'; import tldts from 'tldts'; import { Conf } from '@/config.ts'; -import { cachedFaviconsSizeGauge } from '@/metrics.ts'; import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; diff --git a/packages/ditto/utils/lnurl.ts b/packages/ditto/utils/lnurl.ts index 4fd44988..ad2fefa6 100644 --- a/packages/ditto/utils/lnurl.ts +++ b/packages/ditto/utils/lnurl.ts @@ -1,10 +1,10 @@ +import { cachedLnurlsSizeGauge } from '@ditto/metrics'; import { NostrEvent } from '@nostrify/nostrify'; import { LNURL, LNURLDetails } from '@nostrify/nostrify/ln'; import { logi } from '@soapbox/logi'; import { safeFetch } from '@soapbox/safe-fetch'; import { JsonValue } from '@std/json'; -import { cachedLnurlsSizeGauge } from '@/metrics.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; import { errorJson } from '@/utils/log.ts'; import { Time } from '@/utils/time.ts'; diff --git a/packages/ditto/utils/nip05.ts b/packages/ditto/utils/nip05.ts index 1cc991b9..798fabdf 100644 --- a/packages/ditto/utils/nip05.ts +++ b/packages/ditto/utils/nip05.ts @@ -1,11 +1,11 @@ -import { nip19 } from 'nostr-tools'; +import { cachedNip05sSizeGauge } from '@ditto/metrics'; import { NIP05, NStore } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import { safeFetch } from '@soapbox/safe-fetch'; +import { nip19 } from 'nostr-tools'; import tldts from 'tldts'; import { Conf } from '@/config.ts'; -import { cachedNip05sSizeGauge } from '@/metrics.ts'; import { Storages } from '@/storages.ts'; import { errorJson } from '@/utils/log.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; diff --git a/packages/ditto/utils/unfurl.ts b/packages/ditto/utils/unfurl.ts index f895b71f..e2d4f855 100644 --- a/packages/ditto/utils/unfurl.ts +++ b/packages/ditto/utils/unfurl.ts @@ -1,3 +1,4 @@ +import { cachedLinkPreviewSizeGauge } from '@ditto/metrics'; import TTLCache from '@isaacs/ttlcache'; import { logi } from '@soapbox/logi'; import { safeFetch } from '@soapbox/safe-fetch'; @@ -6,7 +7,6 @@ import { unfurl } from 'unfurl.js'; import { Conf } from '@/config.ts'; import { PreviewCard } from '@/entities/PreviewCard.ts'; -import { cachedLinkPreviewSizeGauge } from '@/metrics.ts'; import { errorJson } from '@/utils/log.ts'; async function unfurlCard(url: string, signal: AbortSignal): Promise { diff --git a/packages/metrics/deno.json b/packages/metrics/deno.json new file mode 100644 index 00000000..12524c18 --- /dev/null +++ b/packages/metrics/deno.json @@ -0,0 +1,6 @@ +{ + "name": "@ditto/metrics", + "exports": { + ".": "./metrics.ts" + } +} diff --git a/packages/ditto/metrics.ts b/packages/metrics/metrics.ts similarity index 100% rename from packages/ditto/metrics.ts rename to packages/metrics/metrics.ts From 4e0bb16b85ed258eff983dde291fccf995cc0a5d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Feb 2025 00:39:00 -0600 Subject: [PATCH 08/15] Fix db tests --- packages/db/adapters/DittoPglite.test.ts | 5 ++++- packages/db/utils/worker.test.ts | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/db/adapters/DittoPglite.test.ts b/packages/db/adapters/DittoPglite.test.ts index c4a6d8c6..449ba02c 100644 --- a/packages/db/adapters/DittoPglite.test.ts +++ b/packages/db/adapters/DittoPglite.test.ts @@ -2,9 +2,12 @@ import { assertEquals } from '@std/assert'; import { DittoPglite } from './DittoPglite.ts'; -Deno.test('DittoPglite.create', () => { +Deno.test('DittoPglite.create', async () => { const db = DittoPglite.create('memory://'); assertEquals(db.poolSize, 1); assertEquals(db.availableConnections, 1); + + await db.kysely.destroy(); + await new Promise((resolve) => setTimeout(resolve, 100)); }); diff --git a/packages/db/utils/worker.test.ts b/packages/db/utils/worker.test.ts index 3e94a91a..73a90b87 100644 --- a/packages/db/utils/worker.test.ts +++ b/packages/db/utils/worker.test.ts @@ -7,8 +7,10 @@ Deno.test('isWorker from the main thread returns false', () => { }); Deno.test('isWorker from a worker thread returns true', async () => { + const url = new URL('./worker.ts', import.meta.url); + const script = ` - import { isWorker } from '@/utils/worker.ts'; + import { isWorker } from '${url.href}'; postMessage(isWorker()); self.close(); `; From 425e0bf3f0ca1bd80de50ab7f1e5bcae11fcdd40 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Feb 2025 00:42:46 -0600 Subject: [PATCH 09/15] ci: deno lint --allow-import --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c27e7584..39af1489 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,7 @@ test: stage: test script: - deno fmt --check - - deno lint + - deno lint --allow-import - deno task check - deno task test --coverage=cov_profile - deno coverage cov_profile From e1c1967a66dcd7d9f9a191c28952462e84b9837d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Feb 2025 11:42:48 -0600 Subject: [PATCH 10/15] db: remove some external deps --- packages/db/DittoDB.ts | 4 +--- packages/db/migrations/037_auth_tokens.ts | 17 ----------------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/packages/db/DittoDB.ts b/packages/db/DittoDB.ts index e18242cd..f3442808 100644 --- a/packages/db/DittoDB.ts +++ b/packages/db/DittoDB.ts @@ -4,8 +4,6 @@ import path from 'node:path'; import { logi } from '@soapbox/logi'; import { FileMigrationProvider, type Kysely, Migrator } from 'kysely'; -import { errorJson } from '@/utils/log.ts'; - import { DittoPglite } from './adapters/DittoPglite.ts'; import { DittoPostgres } from './adapters/DittoPostgres.ts'; @@ -51,7 +49,7 @@ export class DittoDB { msg: 'Migration failed.', state: 'failed', results: results as unknown as JsonValue, - error: errorJson(error), + error: error instanceof Error ? error : null, }); Deno.exit(1); } else { diff --git a/packages/db/migrations/037_auth_tokens.ts b/packages/db/migrations/037_auth_tokens.ts index 591ff379..f7ac340c 100644 --- a/packages/db/migrations/037_auth_tokens.ts +++ b/packages/db/migrations/037_auth_tokens.ts @@ -1,9 +1,5 @@ import { type Kysely, sql } from 'kysely'; -import { Conf } from '@/config.ts'; -import { aesEncrypt } from '@/utils/aes.ts'; -import { getTokenHash } from '@/utils/auth.ts'; - interface DB { nip46_tokens: { api_token: `token1${string}`; @@ -32,19 +28,6 @@ export async function up(db: Kysely): Promise { .addColumn('created_at', 'timestamp', (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`)) .execute(); - // There are probably not that many tokens in the database yet, so this should be fine. - const tokens = await db.selectFrom('nip46_tokens').selectAll().execute(); - - for (const token of tokens) { - await db.insertInto('auth_tokens').values({ - token_hash: await getTokenHash(token.api_token), - pubkey: token.user_pubkey, - nip46_sk_enc: await aesEncrypt(Conf.seckey, token.server_seckey), - nip46_relays: JSON.parse(token.relays), - created_at: token.connected_at, - }).execute(); - } - await db.schema.dropTable('nip46_tokens').execute(); } From fbb5c63c33d4b0d44a2907f908e3c4c174d663ba Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Feb 2025 11:45:08 -0600 Subject: [PATCH 11/15] metrics: make 'prefix' a variable --- packages/metrics/metrics.ts | 56 +++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/packages/metrics/metrics.ts b/packages/metrics/metrics.ts index 7fe75a8f..0dd87b7f 100644 --- a/packages/metrics/metrics.ts +++ b/packages/metrics/metrics.ts @@ -1,149 +1,151 @@ import { Counter, Gauge, Histogram } from 'prom-client'; +const prefix = 'ditto'; + export const httpRequestsCounter = new Counter({ - name: 'ditto_http_requests_total', + name: `${prefix}_http_requests_total`, help: 'Total number of HTTP requests', labelNames: ['method'], }); export const httpResponsesCounter = new Counter({ - name: 'ditto_http_responses_total', + name: `${prefix}_http_responses_total`, help: 'Total number of HTTP responses', labelNames: ['method', 'path', 'status'], }); export const httpResponseDurationHistogram = new Histogram({ - name: 'ditto_http_response_duration_seconds', + name: `${prefix}_http_response_duration_seconds`, help: 'Histogram of HTTP response times in seconds', labelNames: ['method', 'path', 'status'], }); export const streamingConnectionsGauge = new Gauge({ - name: 'ditto_streaming_connections', + name: `${prefix}_streaming_connections`, help: 'Number of active connections to the streaming API', }); export const streamingServerMessagesCounter = new Counter({ - name: 'ditto_streaming_server_messages_total', + name: `${prefix}_streaming_server_messages_total`, help: 'Total number of messages sent from the streaming API', }); export const streamingClientMessagesCounter = new Counter({ - name: 'ditto_streaming_client_messages_total', + name: `${prefix}_streaming_client_messages_total`, help: 'Total number of messages received by the streaming API', }); export const fetchResponsesCounter = new Counter({ - name: 'ditto_fetch_responses_total', + name: `${prefix}_fetch_responses_total`, help: 'Total number of fetch requests', labelNames: ['method', 'status'], }); export const firehoseEventsCounter = new Counter({ - name: 'ditto_firehose_events_total', + name: `${prefix}_firehose_events_total`, help: 'Total number of Nostr events processed by the firehose', labelNames: ['kind'], }); export const pipelineEventsCounter = new Counter({ - name: 'ditto_pipeline_events_total', + name: `${prefix}_pipeline_events_total`, help: 'Total number of Nostr events processed by the pipeline', labelNames: ['kind'], }); export const policyEventsCounter = new Counter({ - name: 'ditto_policy_events_total', + name: `${prefix}_policy_events_total`, help: 'Total number of policy OK responses', labelNames: ['ok'], }); export const relayEventsCounter = new Counter({ - name: 'ditto_relay_events_total', + name: `${prefix}_relay_events_total`, help: 'Total number of EVENT messages processed by the relay', labelNames: ['kind'], }); export const relayMessagesCounter = new Counter({ - name: 'ditto_relay_messages_total', + name: `${prefix}_relay_messages_total`, help: 'Total number of Nostr messages processed by the relay', labelNames: ['verb'], }); export const relayConnectionsGauge = new Gauge({ - name: 'ditto_relay_connections', + name: `${prefix}_relay_connections`, help: 'Number of active connections to the relay', }); export const dbQueriesCounter = new Counter({ - name: 'ditto_db_queries_total', + name: `${prefix}_db_queries_total`, help: 'Total number of database queries', labelNames: ['kind'], }); export const dbEventsCounter = new Counter({ - name: 'ditto_db_events_total', + name: `${prefix}_db_events_total`, help: 'Total number of database inserts', labelNames: ['kind'], }); export const dbPoolSizeGauge = new Gauge({ - name: 'ditto_db_pool_size', + name: `${prefix}_db_pool_size`, help: 'Number of connections in the database pool', }); export const dbAvailableConnectionsGauge = new Gauge({ - name: 'ditto_db_available_connections', + name: `${prefix}_db_available_connections`, help: 'Number of available connections in the database pool', }); export const dbQueryDurationHistogram = new Histogram({ - name: 'ditto_db_query_duration_seconds', + name: `${prefix}_db_query_duration_seconds`, help: 'Duration of database queries', }); export const cachedFaviconsSizeGauge = new Gauge({ - name: 'ditto_cached_favicons_size', + name: `${prefix}_cached_favicons_size`, help: 'Number of domain favicons in cache', }); export const cachedLnurlsSizeGauge = new Gauge({ - name: 'ditto_cached_lnurls_size', + name: `${prefix}_cached_lnurls_size`, help: 'Number of LNURL details in cache', }); export const cachedNip05sSizeGauge = new Gauge({ - name: 'ditto_cached_nip05s_size', + name: `${prefix}_cached_nip05s_size`, help: 'Number of NIP-05 results in cache', }); export const cachedLinkPreviewSizeGauge = new Gauge({ - name: 'ditto_cached_link_previews_size', + name: `${prefix}_cached_link_previews_size`, help: 'Number of link previews in cache', }); export const cachedTranslationsSizeGauge = new Gauge({ - name: 'ditto_cached_translations_size', + name: `${prefix}_cached_translations_size`, help: 'Number of translated statuses in cache', }); export const internalSubscriptionsSizeGauge = new Gauge({ - name: 'ditto_internal_subscriptions_size', + name: `${prefix}_internal_subscriptions_size`, help: "Number of active subscriptions to Ditto's internal relay", }); export const relayPoolRelaysSizeGauge = new Gauge({ - name: 'ditto_relay_pool_relays_size', + name: `${prefix}_relay_pool_relays_size`, help: 'Number of relays in the relay pool', labelNames: ['ready_state'], }); export const relayPoolSubscriptionsSizeGauge = new Gauge({ - name: 'ditto_relay_pool_subscriptions_size', + name: `${prefix}_relay_pool_subscriptions_size`, help: 'Number of active subscriptions to the relay pool', }); export const webPushNotificationsCounter = new Counter({ - name: 'ditto_web_push_notifications_total', + name: `${prefix}_web_push_notifications_total`, help: 'Total number of Web Push notifications sent', labelNames: ['type'], }); From 785ba1e053254fdc00f65ac35ffe8c361cf8700e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Feb 2025 11:50:25 -0600 Subject: [PATCH 12/15] Remove base alias from packages (make scripts use relative paths for now) --- deno.json | 2 +- packages/ditto/deno.json | 4 ++-- scripts/admin-event.ts | 8 ++++---- scripts/admin-role.ts | 6 +++--- scripts/db-export.test.ts | 1 + scripts/db-export.ts | 3 ++- scripts/db-import.ts | 4 ++-- scripts/db-migrate.ts | 2 +- scripts/db-policy.ts | 4 ++-- scripts/db-populate-extensions.ts | 4 ++-- scripts/db-populate-nip05.ts | 6 +++--- scripts/db-populate-search.ts | 3 ++- scripts/db-streak-recompute.ts | 4 ++-- scripts/nostr-pull.ts | 2 +- scripts/setup-kind0.ts | 9 +++++---- scripts/setup.ts | 2 +- scripts/stats-recompute.ts | 4 ++-- scripts/trends.ts | 2 +- 18 files changed, 37 insertions(+), 33 deletions(-) diff --git a/deno.json b/deno.json index 629e1813..debd7fa9 100644 --- a/deno.json +++ b/deno.json @@ -1,4 +1,5 @@ { + "version": "1.1.0", "workspace": [ "./packages/api", "./packages/conf", @@ -43,7 +44,6 @@ "./public" ], "imports": { - "@/": "./packages/ditto/", "@b-fuze/deno-dom": "jsr:@b-fuze/deno-dom@^0.1.47", "@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "@cashu/cashu-ts": "npm:@cashu/cashu-ts@^2.2.0", diff --git a/packages/ditto/deno.json b/packages/ditto/deno.json index 31f80278..82d28139 100644 --- a/packages/ditto/deno.json +++ b/packages/ditto/deno.json @@ -1,9 +1,9 @@ { "name": "@ditto/ditto", - "version": "1.1.0", "exports": {}, "imports": { - "deno.json": "./deno.json" + "@/": "./", + "deno.json": "../../deno.json" }, "lint": { "rules": { diff --git a/scripts/admin-event.ts b/scripts/admin-event.ts index 00711993..70f8ed48 100644 --- a/scripts/admin-event.ts +++ b/scripts/admin-event.ts @@ -1,10 +1,10 @@ import { JsonParseStream } from '@std/json/json-parse-stream'; import { TextLineStream } from '@std/streams/text-line-stream'; -import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { Storages } from '@/storages.ts'; -import { type EventStub } from '@/utils/api.ts'; -import { nostrNow } from '@/utils.ts'; +import { AdminSigner } from '../packages/ditto/signers/AdminSigner.ts'; +import { Storages } from '../packages/ditto/storages.ts'; +import { type EventStub } from '../packages/ditto/utils/api.ts'; +import { nostrNow } from '../packages/ditto/utils.ts'; const signer = new AdminSigner(); const store = await Storages.db(); diff --git a/scripts/admin-role.ts b/scripts/admin-role.ts index d275329f..369440c9 100644 --- a/scripts/admin-role.ts +++ b/scripts/admin-role.ts @@ -1,9 +1,9 @@ import { NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { Storages } from '@/storages.ts'; -import { nostrNow } from '@/utils.ts'; +import { AdminSigner } from '../packages/ditto/signers/AdminSigner.ts'; +import { Storages } from '../packages/ditto/storages.ts'; +import { nostrNow } from '../packages/ditto/utils.ts'; const store = await Storages.db(); diff --git a/scripts/db-export.test.ts b/scripts/db-export.test.ts index 939537d5..3b180291 100644 --- a/scripts/db-export.test.ts +++ b/scripts/db-export.test.ts @@ -1,4 +1,5 @@ import { assertEquals, assertThrows } from '@std/assert'; + import { buildFilter } from './db-export.ts'; Deno.test('buildFilter should return an empty filter when no arguments are provided', () => { diff --git a/scripts/db-export.ts b/scripts/db-export.ts index e32e08ad..d36d4f3f 100644 --- a/scripts/db-export.ts +++ b/scripts/db-export.ts @@ -1,7 +1,8 @@ -import { Storages } from '@/storages.ts'; import { NostrFilter } from '@nostrify/nostrify'; import { Command, InvalidOptionArgumentError } from 'commander'; +import { Storages } from '../packages/ditto/storages.ts'; + interface ExportFilter { authors?: string[]; ids?: string[]; diff --git a/scripts/db-import.ts b/scripts/db-import.ts index ed884453..2f6c1595 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -3,8 +3,8 @@ import { NostrEvent } from '@nostrify/nostrify'; import { JsonParseStream } from '@std/json/json-parse-stream'; import { TextLineStream } from '@std/streams/text-line-stream'; -import { Conf } from '@/config.ts'; -import { Storages } from '@/storages.ts'; +import { Conf } from '../packages/ditto/config.ts'; +import { Storages } from '../packages/ditto/storages.ts'; const store = await Storages.db(); const sem = new Semaphore(Conf.pg.poolSize); diff --git a/scripts/db-migrate.ts b/scripts/db-migrate.ts index d3e93783..21b8db22 100644 --- a/scripts/db-migrate.ts +++ b/scripts/db-migrate.ts @@ -1,4 +1,4 @@ -import { Storages } from '@/storages.ts'; +import { Storages } from '../packages/ditto/storages.ts'; // This migrates kysely internally. const kysely = await Storages.kysely(); diff --git a/scripts/db-policy.ts b/scripts/db-policy.ts index 4be3c4ef..caab55af 100644 --- a/scripts/db-policy.ts +++ b/scripts/db-policy.ts @@ -1,5 +1,5 @@ -import { policyWorker } from '@/workers/policy.ts'; -import { Storages } from '@/storages.ts'; +import { policyWorker } from '../packages/ditto/workers/policy.ts'; +import { Storages } from '../packages/ditto/storages.ts'; const db = await Storages.db(); let count = 0; diff --git a/scripts/db-populate-extensions.ts b/scripts/db-populate-extensions.ts index ca6d1927..2b40bd3d 100644 --- a/scripts/db-populate-extensions.ts +++ b/scripts/db-populate-extensions.ts @@ -1,7 +1,7 @@ import { NostrEvent } from '@nostrify/nostrify'; -import { Storages } from '@/storages.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; +import { Storages } from '../packages/ditto/storages.ts'; +import { EventsDB } from '../packages/ditto/storages/EventsDB.ts'; const kysely = await Storages.kysely(); diff --git a/scripts/db-populate-nip05.ts b/scripts/db-populate-nip05.ts index df11f007..acfe70da 100644 --- a/scripts/db-populate-nip05.ts +++ b/scripts/db-populate-nip05.ts @@ -1,9 +1,9 @@ import { Semaphore } from '@core/asyncutil'; - -import { updateAuthorData } from '@/pipeline.ts'; -import { Storages } from '@/storages.ts'; import { NostrEvent } from '@nostrify/nostrify'; +import { updateAuthorData } from '../packages/ditto/pipeline.ts'; +import { Storages } from '../packages/ditto/storages.ts'; + const kysely = await Storages.kysely(); const sem = new Semaphore(5); diff --git a/scripts/db-populate-search.ts b/scripts/db-populate-search.ts index 81b84ee6..e73f79ac 100644 --- a/scripts/db-populate-search.ts +++ b/scripts/db-populate-search.ts @@ -1,5 +1,6 @@ import { NSchema as n } from '@nostrify/nostrify'; -import { Storages } from '@/storages.ts'; + +import { Storages } from '../packages/ditto/storages.ts'; const store = await Storages.db(); const kysely = await Storages.kysely(); diff --git a/scripts/db-streak-recompute.ts b/scripts/db-streak-recompute.ts index a05eb08b..e45d4f64 100644 --- a/scripts/db-streak-recompute.ts +++ b/scripts/db-streak-recompute.ts @@ -1,5 +1,5 @@ -import { Conf } from '@/config.ts'; -import { Storages } from '@/storages.ts'; +import { Conf } from '../packages/ditto/config.ts'; +import { Storages } from '../packages/ditto/storages.ts'; const kysely = await Storages.kysely(); const statsQuery = kysely.selectFrom('author_stats').select('pubkey'); diff --git a/scripts/nostr-pull.ts b/scripts/nostr-pull.ts index c7ad21d3..7c21cb80 100644 --- a/scripts/nostr-pull.ts +++ b/scripts/nostr-pull.ts @@ -6,7 +6,7 @@ import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { Storages } from '@/storages.ts'; +import { Storages } from '../packages/ditto/storages.ts'; const store = await Storages.db(); diff --git a/scripts/setup-kind0.ts b/scripts/setup-kind0.ts index 7aa62df3..ff7cbd1a 100644 --- a/scripts/setup-kind0.ts +++ b/scripts/setup-kind0.ts @@ -1,9 +1,10 @@ -import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Command } from 'commander'; import { NostrEvent } from 'nostr-tools'; -import { nostrNow } from '@/utils.ts'; -import { Conf } from '@/config.ts'; -import { Storages } from '@/storages.ts'; + +import { AdminSigner } from '../packages/ditto/signers/AdminSigner.ts'; +import { nostrNow } from '../packages/ditto/utils.ts'; +import { Conf } from '../packages/ditto/config.ts'; +import { Storages } from '../packages/ditto/storages.ts'; function die(code: number, ...args: unknown[]) { console.error(...args); diff --git a/scripts/setup.ts b/scripts/setup.ts index 3f3fc955..f4ccf368 100644 --- a/scripts/setup.ts +++ b/scripts/setup.ts @@ -4,7 +4,7 @@ import { exists } from '@std/fs/exists'; import { generateSecretKey, nip19 } from 'nostr-tools'; import question from 'question-deno'; -import { Conf } from '@/config.ts'; +import { Conf } from '../packages/ditto/config.ts'; console.log(''); console.log('Hello! Welcome to the Ditto setup tool. We will ask you a few questions to generate a .env file for you.'); diff --git a/scripts/stats-recompute.ts b/scripts/stats-recompute.ts index 77be13fe..942d0012 100644 --- a/scripts/stats-recompute.ts +++ b/scripts/stats-recompute.ts @@ -1,7 +1,7 @@ import { nip19 } from 'nostr-tools'; -import { Storages } from '@/storages.ts'; -import { refreshAuthorStats } from '@/utils/stats.ts'; +import { Storages } from '../packages/ditto/storages.ts'; +import { refreshAuthorStats } from '../packages/ditto/utils/stats.ts'; let pubkey: string; try { diff --git a/scripts/trends.ts b/scripts/trends.ts index 6600f7e2..bb9708ab 100644 --- a/scripts/trends.ts +++ b/scripts/trends.ts @@ -6,7 +6,7 @@ import { updateTrendingLinks, updateTrendingPubkeys, updateTrendingZappedEvents, -} from '@/trends.ts'; +} from '../packages/ditto/trends.ts'; const trendSchema = z.enum(['pubkeys', 'zapped_events', 'events', 'hashtags', 'links']); const trends = trendSchema.array().parse(Deno.args); From e100f72a9baaf8d774cf2aa8c448326c215c3f5b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Feb 2025 11:53:34 -0600 Subject: [PATCH 13/15] Add deno task lint --- .gitlab-ci.yml | 2 +- deno.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 39af1489..766a144d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,7 @@ test: stage: test script: - deno fmt --check - - deno lint --allow-import + - deno task lint - deno task check - deno task test --coverage=cov_profile - deno coverage cov_profile diff --git a/deno.json b/deno.json index debd7fa9..412f32a3 100644 --- a/deno.json +++ b/deno.json @@ -19,6 +19,7 @@ "debug": "deno run -A --env-file --deny-read=.env --inspect packages/ditto/server.ts", "test": "deno test -A --env-file=.env.test --deny-read=.env --junit-path=./deno-test.xml", "check": "deno check --allow-import .", + "lint": "deno lint --allow-import", "nsec": "deno run scripts/nsec.ts", "admin:event": "deno run -A --env-file --deny-read=.env scripts/admin-event.ts", "admin:role": "deno run -A --env-file --deny-read=.env scripts/admin-role.ts", From 0ace14ffbb606d54b79da6fcd7a15145897eb6fc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Feb 2025 11:56:41 -0600 Subject: [PATCH 14/15] metrics: add types to all exports --- packages/metrics/metrics.ts | 54 ++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/packages/metrics/metrics.ts b/packages/metrics/metrics.ts index 0dd87b7f..716582d4 100644 --- a/packages/metrics/metrics.ts +++ b/packages/metrics/metrics.ts @@ -2,149 +2,149 @@ import { Counter, Gauge, Histogram } from 'prom-client'; const prefix = 'ditto'; -export const httpRequestsCounter = new Counter({ +export const httpRequestsCounter: Counter<'method'> = new Counter({ name: `${prefix}_http_requests_total`, help: 'Total number of HTTP requests', labelNames: ['method'], }); -export const httpResponsesCounter = new Counter({ +export const httpResponsesCounter: Counter<'method' | 'path' | 'status'> = new Counter({ name: `${prefix}_http_responses_total`, help: 'Total number of HTTP responses', labelNames: ['method', 'path', 'status'], }); -export const httpResponseDurationHistogram = new Histogram({ +export const httpResponseDurationHistogram: Histogram<'method' | 'path' | 'status'> = new Histogram({ name: `${prefix}_http_response_duration_seconds`, help: 'Histogram of HTTP response times in seconds', labelNames: ['method', 'path', 'status'], }); -export const streamingConnectionsGauge = new Gauge({ +export const streamingConnectionsGauge: Gauge = new Gauge({ name: `${prefix}_streaming_connections`, help: 'Number of active connections to the streaming API', }); -export const streamingServerMessagesCounter = new Counter({ +export const streamingServerMessagesCounter: Counter = new Counter({ name: `${prefix}_streaming_server_messages_total`, help: 'Total number of messages sent from the streaming API', }); -export const streamingClientMessagesCounter = new Counter({ +export const streamingClientMessagesCounter: Counter = new Counter({ name: `${prefix}_streaming_client_messages_total`, help: 'Total number of messages received by the streaming API', }); -export const fetchResponsesCounter = new Counter({ +export const fetchResponsesCounter: Counter<'method' | 'status'> = new Counter({ name: `${prefix}_fetch_responses_total`, help: 'Total number of fetch requests', labelNames: ['method', 'status'], }); -export const firehoseEventsCounter = new Counter({ +export const firehoseEventsCounter: Counter<'kind'> = new Counter({ name: `${prefix}_firehose_events_total`, help: 'Total number of Nostr events processed by the firehose', labelNames: ['kind'], }); -export const pipelineEventsCounter = new Counter({ +export const pipelineEventsCounter: Counter<'kind'> = new Counter({ name: `${prefix}_pipeline_events_total`, help: 'Total number of Nostr events processed by the pipeline', labelNames: ['kind'], }); -export const policyEventsCounter = new Counter({ +export const policyEventsCounter: Counter<'ok'> = new Counter({ name: `${prefix}_policy_events_total`, help: 'Total number of policy OK responses', labelNames: ['ok'], }); -export const relayEventsCounter = new Counter({ +export const relayEventsCounter: Counter<'kind'> = new Counter({ name: `${prefix}_relay_events_total`, help: 'Total number of EVENT messages processed by the relay', labelNames: ['kind'], }); -export const relayMessagesCounter = new Counter({ +export const relayMessagesCounter: Counter<'verb'> = new Counter({ name: `${prefix}_relay_messages_total`, help: 'Total number of Nostr messages processed by the relay', labelNames: ['verb'], }); -export const relayConnectionsGauge = new Gauge({ +export const relayConnectionsGauge: Gauge = new Gauge({ name: `${prefix}_relay_connections`, help: 'Number of active connections to the relay', }); -export const dbQueriesCounter = new Counter({ +export const dbQueriesCounter: Counter<'kind'> = new Counter({ name: `${prefix}_db_queries_total`, help: 'Total number of database queries', labelNames: ['kind'], }); -export const dbEventsCounter = new Counter({ +export const dbEventsCounter: Counter<'kind'> = new Counter({ name: `${prefix}_db_events_total`, help: 'Total number of database inserts', labelNames: ['kind'], }); -export const dbPoolSizeGauge = new Gauge({ +export const dbPoolSizeGauge: Gauge = new Gauge({ name: `${prefix}_db_pool_size`, help: 'Number of connections in the database pool', }); -export const dbAvailableConnectionsGauge = new Gauge({ +export const dbAvailableConnectionsGauge: Gauge = new Gauge({ name: `${prefix}_db_available_connections`, help: 'Number of available connections in the database pool', }); -export const dbQueryDurationHistogram = new Histogram({ +export const dbQueryDurationHistogram: Histogram = new Histogram({ name: `${prefix}_db_query_duration_seconds`, help: 'Duration of database queries', }); -export const cachedFaviconsSizeGauge = new Gauge({ +export const cachedFaviconsSizeGauge: Gauge = new Gauge({ name: `${prefix}_cached_favicons_size`, help: 'Number of domain favicons in cache', }); -export const cachedLnurlsSizeGauge = new Gauge({ +export const cachedLnurlsSizeGauge: Gauge = new Gauge({ name: `${prefix}_cached_lnurls_size`, help: 'Number of LNURL details in cache', }); -export const cachedNip05sSizeGauge = new Gauge({ +export const cachedNip05sSizeGauge: Gauge = new Gauge({ name: `${prefix}_cached_nip05s_size`, help: 'Number of NIP-05 results in cache', }); -export const cachedLinkPreviewSizeGauge = new Gauge({ +export const cachedLinkPreviewSizeGauge: Gauge = new Gauge({ name: `${prefix}_cached_link_previews_size`, help: 'Number of link previews in cache', }); -export const cachedTranslationsSizeGauge = new Gauge({ +export const cachedTranslationsSizeGauge: Gauge = new Gauge({ name: `${prefix}_cached_translations_size`, help: 'Number of translated statuses in cache', }); -export const internalSubscriptionsSizeGauge = new Gauge({ +export const internalSubscriptionsSizeGauge: Gauge = new Gauge({ name: `${prefix}_internal_subscriptions_size`, help: "Number of active subscriptions to Ditto's internal relay", }); -export const relayPoolRelaysSizeGauge = new Gauge({ +export const relayPoolRelaysSizeGauge: Gauge<'ready_state'> = new Gauge({ name: `${prefix}_relay_pool_relays_size`, help: 'Number of relays in the relay pool', labelNames: ['ready_state'], }); -export const relayPoolSubscriptionsSizeGauge = new Gauge({ +export const relayPoolSubscriptionsSizeGauge: Gauge = new Gauge({ name: `${prefix}_relay_pool_subscriptions_size`, help: 'Number of active subscriptions to the relay pool', }); -export const webPushNotificationsCounter = new Counter({ +export const webPushNotificationsCounter: Counter<'type'> = new Counter({ name: `${prefix}_web_push_notifications_total`, help: 'Total number of Web Push notifications sent', labelNames: ['type'], From af0e688ca317f67545605339b0cf7f86544f3de9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Feb 2025 12:52:27 -0600 Subject: [PATCH 15/15] Fix path to datadir and custom policies --- packages/conf/DittoConf.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/conf/DittoConf.ts b/packages/conf/DittoConf.ts index b0f1256f..6d4b45d7 100644 --- a/packages/conf/DittoConf.ts +++ b/packages/conf/DittoConf.ts @@ -1,4 +1,7 @@ +import Module from 'node:module'; import os from 'node:os'; +import path from 'node:path'; + import ISO6391, { type LanguageCode } from 'iso-639-1'; import { getPublicKey, nip19 } from 'nostr-tools'; import { decodeBase64 } from '@std/encoding/base64'; @@ -346,12 +349,12 @@ export class DittoConf { /** Path to the custom policy module. Must be an absolute path, https:, npm:, or jsr: URI. */ get policy(): string { - return this.env.get('DITTO_POLICY') || new URL('../data/policy.ts', import.meta.url).pathname; + return this.env.get('DITTO_POLICY') || path.join(this.dataDir, 'policy.ts'); } /** Absolute path to the data directory used by Ditto. */ get dataDir(): string { - return this.env.get('DITTO_DATA_DIR') || new URL('../data', import.meta.url).pathname; + return this.env.get('DITTO_DATA_DIR') || path.join(cwd(), 'data'); } /** Absolute path of the Deno directory. */ @@ -462,3 +465,12 @@ export class DittoConf { return Number(this.env.get('STREAK_WINDOW') || 129600); } } + +/** + * HACK: get cwd without read permissions. + * https://github.com/denoland/deno/issues/27080#issuecomment-2504150155 + */ +function cwd() { + // @ts-ignore Internal method, but it does exist. + return Module._nodeModulePaths('a')[0].slice(0, -15); +}