From 96fe8920cdacb2b27c6840a5179696b4a13213e6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 12 Jul 2024 13:47:29 -0500 Subject: [PATCH 1/6] Make database timeouts configurable --- src/config.ts | 15 +++++++++++++++ src/controllers/api/accounts.ts | 2 +- src/controllers/api/notifications.ts | 2 +- src/controllers/api/timelines.ts | 2 +- src/controllers/nostr/relay.ts | 5 +++-- src/storages/EventsDB.ts | 8 ++++---- 6 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/config.ts b/src/config.ts index 1dd688b7..33c233b8 100644 --- a/src/config.ts +++ b/src/config.ts @@ -98,6 +98,21 @@ class Conf { } return undefined; }, + /** Database query timeout configurations. */ + timeouts: { + /** Default query timeout when another setting isn't more specific. */ + get default(): number { + return Number(Deno.env.get('DB_TIMEOUT_DEFAULT') || 3_000); + }, + /** Timeout used for queries made through the Nostr relay. */ + get relay(): number { + return Number(Deno.env.get('DB_TIMEOUT_RELAY') || 1_000); + }, + /** Timeout used for timelines such as home, notifications, hashtag, etc. */ + get timelines(): number { + return Number(Deno.env.get('DB_TIMEOUT_TIMELINES') || 10_000); + }, + }, }; /** Character limit to enforce for posts made through Mastodon API. */ static get postCharLimit(): number { diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index ad4802ca..460aa6dd 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -209,7 +209,7 @@ const accountStatusesController: AppController = async (c) => { filter['#t'] = [tagged]; } - const opts = { signal, limit, timeout: 10_000 }; + const opts = { signal, limit, timeout: Conf.db.timeouts.timelines }; const events = await store.query([filter], opts) .then((events) => hydrateEvents({ events, store, signal })) diff --git a/src/controllers/api/notifications.ts b/src/controllers/api/notifications.ts index fab7c816..6f6036f2 100644 --- a/src/controllers/api/notifications.ts +++ b/src/controllers/api/notifications.ts @@ -78,7 +78,7 @@ async function renderNotifications( const store = c.get('store'); const pubkey = await c.get('signer')?.getPublicKey()!; const { signal } = c.req.raw; - const opts = { signal, limit: params.limit, timeout: 15_000 }; + const opts = { signal, limit: params.limit, timeout: Conf.db.timeouts.timelines }; const events = await store .query(filters, opts) diff --git a/src/controllers/api/timelines.ts b/src/controllers/api/timelines.ts index 62f1cd2f..848ae63f 100644 --- a/src/controllers/api/timelines.ts +++ b/src/controllers/api/timelines.ts @@ -60,7 +60,7 @@ const suggestedTimelineController: AppController = async (c) => { async function renderStatuses(c: AppContext, filters: NostrFilter[]) { const { signal } = c.req.raw; const store = c.get('store'); - const opts = { signal, timeout: 10_000 }; + const opts = { signal, timeout: Conf.db.timeouts.timelines }; const events = await store .query(filters, opts) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index f124360e..02d2995e 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -10,6 +10,7 @@ import { } from '@nostrify/nostrify'; import { AppController } from '@/app.ts'; +import { Conf } from '@/config.ts'; import { relayInfoController } from '@/controllers/nostr/relay-info.ts'; import { relayConnectionsGauge, relayEventCounter, relayMessageCounter } from '@/metrics.ts'; import * as pipeline from '@/pipeline.ts'; @@ -95,7 +96,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) { const pubsub = await Storages.pubsub(); try { - for (const event of await store.query(filters, { limit: FILTER_LIMIT, timeout: 1000 })) { + for (const event of await store.query(filters, { limit: FILTER_LIMIT, timeout: Conf.db.timeouts.relay })) { send(['EVENT', subId, event]); } } catch (e) { @@ -150,7 +151,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) { /** Handle COUNT. Return the number of events matching the filters. */ async function handleCount([_, subId, ...filters]: NostrClientCOUNT): Promise { const store = await Storages.db(); - const { count } = await store.count(filters, { timeout: 100 }); + const { count } = await store.count(filters, { timeout: Conf.db.timeouts.relay }); send(['COUNT', subId, { count, approximate: false }]); } diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index d66a65b7..abf076c7 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -64,7 +64,7 @@ class EventsDB implements NStore { await this.deleteEventsAdmin(event); try { - await this.store.event(event, { ...opts, timeout: opts.timeout ?? 1000 }); + await this.store.event(event, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); } catch (e) { if (e.message === 'Cannot add a deleted event') { throw new RelayError('blocked', 'event deleted by user'); @@ -164,7 +164,7 @@ class EventsDB implements NStore { this.console.debug('REQ', JSON.stringify(filters)); - return this.store.query(filters, { ...opts, timeout: opts.timeout ?? 1000 }); + return this.store.query(filters, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); } /** Delete events based on filters from the database. */ @@ -172,7 +172,7 @@ class EventsDB implements NStore { if (!filters.length) return Promise.resolve(); this.console.debug('DELETE', JSON.stringify(filters)); - return this.store.remove(filters, { ...opts, timeout: opts.timeout ?? 3000 }); + return this.store.remove(filters, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); } /** Get number of events that would be returned by filters. */ @@ -185,7 +185,7 @@ class EventsDB implements NStore { this.console.debug('COUNT', JSON.stringify(filters)); - return this.store.count(filters, { ...opts, timeout: opts.timeout ?? 500 }); + return this.store.count(filters, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); } /** Return only the tags that should be indexed. */ From 39fdbc1acb42410f48bf918e37a716333453806d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 12 Jul 2024 14:01:21 -0500 Subject: [PATCH 2/6] Delete scavenger.test.ts --- src/utils/scavenger.test.ts | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/utils/scavenger.test.ts diff --git a/src/utils/scavenger.test.ts b/src/utils/scavenger.test.ts deleted file mode 100644 index e69de29b..00000000 From 46c3e52b98aa2c5ea4443d209aaa9afe6a15547e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 13 Jul 2024 15:49:19 -0500 Subject: [PATCH 3/6] Upgrade Deno to v1.45.2 --- .gitlab-ci.yml | 2 +- .tool-versions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dc1e8456..3ea52cd3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:1.45.0 +image: denoland/deno:1.45.2 default: interruptible: true diff --git a/.tool-versions b/.tool-versions index 512c172e..9e95ec86 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -deno 1.45.0 \ No newline at end of file +deno 1.45.2 \ No newline at end of file From a49e56f2a0d734aa0a09ccfd0068a8d484acc49f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 14 Jul 2024 10:10:26 -0500 Subject: [PATCH 4/6] Increase default database timeouts --- src/config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config.ts b/src/config.ts index 33c233b8..dfb84618 100644 --- a/src/config.ts +++ b/src/config.ts @@ -102,7 +102,7 @@ class Conf { timeouts: { /** Default query timeout when another setting isn't more specific. */ get default(): number { - return Number(Deno.env.get('DB_TIMEOUT_DEFAULT') || 3_000); + return Number(Deno.env.get('DB_TIMEOUT_DEFAULT') || 5_000); }, /** Timeout used for queries made through the Nostr relay. */ get relay(): number { @@ -110,7 +110,7 @@ class Conf { }, /** Timeout used for timelines such as home, notifications, hashtag, etc. */ get timelines(): number { - return Number(Deno.env.get('DB_TIMEOUT_TIMELINES') || 10_000); + return Number(Deno.env.get('DB_TIMEOUT_TIMELINES') || 15_000); }, }, }; From c2026a340a7e5da79ba492028a36405ab95e8127 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 14 Jul 2024 15:59:02 -0500 Subject: [PATCH 5/6] Update CHANGELOG --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 990c4f42..f07ecec0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Prometheus support (`/metrics` endpoint). +- Sort zaps by amount; add pagination. + +### Fixed + +- Added IP rate-limiting of HTTP requests and WebSocket messages. +- Added database query timeouts. +- Fixed nos2x compatibility. + ## [1.0.0] - 2024-06-14 - Initial release From 6fe034d55af1d2b236b7c3e90ad1b7c436639445 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 15 Jul 2024 08:11:25 -0500 Subject: [PATCH 6/6] Bump version to v1.1.0 --- CHANGELOG.md | 5 ++++- deno.json | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f07ecec0..36881b2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.1.0] - 2024-07-15 + ### Added - Prometheus support (`/metrics` endpoint). @@ -22,5 +24,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Initial release -[unreleased]: https://gitlab.com/soapbox-pub/ditto/-/compare/v1.0.0...HEAD +[unreleased]: https://gitlab.com/soapbox-pub/ditto/-/compare/v1.1.0...HEAD +[1.1.0]: https://gitlab.com/soapbox-pub/ditto/-/compare/v1.0.0...v1.1.0 [1.0.0]: https://gitlab.com/soapbox-pub/ditto/-/tags/v1.0.0 diff --git a/deno.json b/deno.json index 084b527b..9e9dde90 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "$schema": "https://deno.land/x/deno@v1.41.0/cli/schemas/config-file.v1.json", - "version": "1.0.0", + "version": "1.1.0", "tasks": { "start": "deno run -A src/server.ts", "dev": "deno run -A --watch src/server.ts",