From 739153afc9ef37cdd301213797fe4b1bd95ad687 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Feb 2025 00:30:00 -0600 Subject: [PATCH 1/6] Add @ditto/metrics package --- deno.json | 3 ++- packages/db/KyselyLogger.ts | 3 +-- packages/ditto/controllers/api/streaming.ts | 10 +++++----- packages/ditto/controllers/api/translate.ts | 2 +- packages/ditto/controllers/metrics.ts | 8 ++++---- packages/ditto/controllers/nostr/relay.ts | 2 +- packages/ditto/firehose.ts | 2 +- packages/ditto/middleware/metricsMiddleware.ts | 3 +-- packages/ditto/pipeline.ts | 2 +- packages/ditto/storages.ts | 2 +- packages/ditto/storages/EventsDB.ts | 2 +- packages/ditto/utils/favicon.ts | 2 +- packages/ditto/utils/lnurl.ts | 2 +- packages/ditto/utils/nip05.ts | 4 ++-- packages/ditto/utils/unfurl.ts | 2 +- packages/metrics/deno.json | 6 ++++++ packages/{ditto => metrics}/metrics.ts | 0 17 files changed, 30 insertions(+), 25 deletions(-) create mode 100644 packages/metrics/deno.json rename packages/{ditto => metrics}/metrics.ts (100%) diff --git a/deno.json b/deno.json index f5acf018..629e1813 100644 --- a/deno.json +++ b/deno.json @@ -3,7 +3,8 @@ "./packages/api", "./packages/conf", "./packages/db", - "./packages/ditto" + "./packages/ditto", + "./packages/metrics" ], "tasks": { "start": "deno run -A --env-file --deny-read=.env packages/ditto/server.ts", diff --git a/packages/db/KyselyLogger.ts b/packages/db/KyselyLogger.ts index 4ddd1305..333e4285 100644 --- a/packages/db/KyselyLogger.ts +++ b/packages/db/KyselyLogger.ts @@ -1,7 +1,6 @@ +import { dbQueriesCounter, dbQueryDurationHistogram } from '@ditto/metrics'; import { logi, type LogiValue } from '@soapbox/logi'; -import { dbQueriesCounter, dbQueryDurationHistogram } from '@/metrics.ts'; - import type { Logger } from 'kysely'; /** Log the SQL for queries. */ diff --git a/packages/ditto/controllers/api/streaming.ts b/packages/ditto/controllers/api/streaming.ts index 43bf92be..7f2f8b64 100644 --- a/packages/ditto/controllers/api/streaming.ts +++ b/packages/ditto/controllers/api/streaming.ts @@ -1,14 +1,14 @@ +import { + streamingClientMessagesCounter, + streamingConnectionsGauge, + streamingServerMessagesCounter, +} from '@ditto/metrics'; import TTLCache from '@isaacs/ttlcache'; import { NostrEvent, NostrFilter } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; -import { - streamingClientMessagesCounter, - streamingConnectionsGauge, - streamingServerMessagesCounter, -} from '@/metrics.ts'; import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; import { getFeedPubkeys } from '@/queries.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; diff --git a/packages/ditto/controllers/api/translate.ts b/packages/ditto/controllers/api/translate.ts index d763c713..7395ff2f 100644 --- a/packages/ditto/controllers/api/translate.ts +++ b/packages/ditto/controllers/api/translate.ts @@ -1,10 +1,10 @@ +import { cachedTranslationsSizeGauge } from '@ditto/metrics'; import { LanguageCode } from 'iso-639-1'; import { z } from 'zod'; import { AppController } from '@/app.ts'; import { translationCache } from '@/caches/translationCache.ts'; import { MastodonTranslation } from '@/entities/MastodonTranslation.ts'; -import { cachedTranslationsSizeGauge } from '@/metrics.ts'; import { getEvent } from '@/queries.ts'; import { localeSchema } from '@/schema.ts'; import { parseBody } from '@/utils/api.ts'; diff --git a/packages/ditto/controllers/metrics.ts b/packages/ditto/controllers/metrics.ts index d168243b..32a8783d 100644 --- a/packages/ditto/controllers/metrics.ts +++ b/packages/ditto/controllers/metrics.ts @@ -1,12 +1,12 @@ -import { register } from 'prom-client'; - -import { AppController } from '@/app.ts'; import { dbAvailableConnectionsGauge, dbPoolSizeGauge, relayPoolRelaysSizeGauge, relayPoolSubscriptionsSizeGauge, -} from '@/metrics.ts'; +} from '@ditto/metrics'; +import { register } from 'prom-client'; + +import { AppController } from '@/app.ts'; import { Storages } from '@/storages.ts'; /** Prometheus/OpenMetrics controller. */ diff --git a/packages/ditto/controllers/nostr/relay.ts b/packages/ditto/controllers/nostr/relay.ts index 6cdb0da7..92906d04 100644 --- a/packages/ditto/controllers/nostr/relay.ts +++ b/packages/ditto/controllers/nostr/relay.ts @@ -1,4 +1,5 @@ import { type DittoConf } from '@ditto/conf'; +import { relayConnectionsGauge, relayEventsCounter, relayMessagesCounter } from '@ditto/metrics'; import { logi } from '@soapbox/logi'; import { JsonValue } from '@std/json'; import { @@ -14,7 +15,6 @@ import { import { AppController } from '@/app.ts'; import { relayInfoController } from '@/controllers/nostr/relay-info.ts'; -import { relayConnectionsGauge, relayEventsCounter, relayMessagesCounter } from '@/metrics.ts'; import * as pipeline from '@/pipeline.ts'; import { RelayError } from '@/RelayError.ts'; import { Storages } from '@/storages.ts'; diff --git a/packages/ditto/firehose.ts b/packages/ditto/firehose.ts index f04752b2..e967e1f2 100644 --- a/packages/ditto/firehose.ts +++ b/packages/ditto/firehose.ts @@ -1,8 +1,8 @@ +import { firehoseEventsCounter } from '@ditto/metrics'; import { Semaphore } from '@core/asyncutil'; import { logi } from '@soapbox/logi'; import { Conf } from '@/config.ts'; -import { firehoseEventsCounter } from '@/metrics.ts'; import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; diff --git a/packages/ditto/middleware/metricsMiddleware.ts b/packages/ditto/middleware/metricsMiddleware.ts index 0b213b82..91f2c422 100644 --- a/packages/ditto/middleware/metricsMiddleware.ts +++ b/packages/ditto/middleware/metricsMiddleware.ts @@ -1,8 +1,7 @@ +import { httpRequestsCounter, httpResponseDurationHistogram, httpResponsesCounter } from '@ditto/metrics'; import { ScopedPerformance } from '@esroyo/scoped-performance'; import { MiddlewareHandler } from '@hono/hono'; -import { httpRequestsCounter, httpResponseDurationHistogram, httpResponsesCounter } from '@/metrics.ts'; - /** Prometheus metrics middleware that tracks HTTP requests by methods and responses by status code. */ export const metricsMiddleware: MiddlewareHandler = async (c, next) => { // Start a timer to measure the duration of the response. diff --git a/packages/ditto/pipeline.ts b/packages/ditto/pipeline.ts index 0ad1969d..d3168c0e 100644 --- a/packages/ditto/pipeline.ts +++ b/packages/ditto/pipeline.ts @@ -1,4 +1,5 @@ import { DittoTables } from '@ditto/db'; +import { pipelineEventsCounter, policyEventsCounter, webPushNotificationsCounter } from '@ditto/metrics'; import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import { Kysely, UpdateObject } from 'kysely'; @@ -9,7 +10,6 @@ import { pipelineEncounters } from '@/caches/pipelineEncounters.ts'; import { Conf } from '@/config.ts'; import { DittoPush } from '@/DittoPush.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { pipelineEventsCounter, policyEventsCounter, webPushNotificationsCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; diff --git a/packages/ditto/storages.ts b/packages/ditto/storages.ts index 55566f51..be61beb6 100644 --- a/packages/ditto/storages.ts +++ b/packages/ditto/storages.ts @@ -1,9 +1,9 @@ // deno-lint-ignore-file require-await import { type DittoDatabase, DittoDB } from '@ditto/db'; +import { internalSubscriptionsSizeGauge } from '@ditto/metrics'; import { logi } from '@soapbox/logi'; import { Conf } from '@/config.ts'; -import { internalSubscriptionsSizeGauge } from '@/metrics.ts'; import { wsUrlSchema } from '@/schema.ts'; import { AdminStore } from '@/storages/AdminStore.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; diff --git a/packages/ditto/storages/EventsDB.ts b/packages/ditto/storages/EventsDB.ts index b8c75c59..622f5811 100644 --- a/packages/ditto/storages/EventsDB.ts +++ b/packages/ditto/storages/EventsDB.ts @@ -2,6 +2,7 @@ import { DittoTables } from '@ditto/db'; import { NPostgres, NPostgresSchema } from '@nostrify/db'; +import { dbEventsCounter } from '@ditto/metrics'; import { NIP50, NKinds, NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import { JsonValue } from '@std/json'; @@ -12,7 +13,6 @@ import { nip27 } from 'nostr-tools'; import tldts from 'tldts'; import { z } from 'zod'; -import { dbEventsCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; import { isNostrId } from '@/utils.ts'; import { abortError } from '@/utils/abort.ts'; diff --git a/packages/ditto/utils/favicon.ts b/packages/ditto/utils/favicon.ts index c181a198..ed218cfa 100644 --- a/packages/ditto/utils/favicon.ts +++ b/packages/ditto/utils/favicon.ts @@ -1,12 +1,12 @@ import { DOMParser } from '@b-fuze/deno-dom'; import { DittoTables } from '@ditto/db'; +import { cachedFaviconsSizeGauge } from '@ditto/metrics'; import { logi } from '@soapbox/logi'; import { safeFetch } from '@soapbox/safe-fetch'; import { Kysely } from 'kysely'; import tldts from 'tldts'; import { Conf } from '@/config.ts'; -import { cachedFaviconsSizeGauge } from '@/metrics.ts'; import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; diff --git a/packages/ditto/utils/lnurl.ts b/packages/ditto/utils/lnurl.ts index 4fd44988..ad2fefa6 100644 --- a/packages/ditto/utils/lnurl.ts +++ b/packages/ditto/utils/lnurl.ts @@ -1,10 +1,10 @@ +import { cachedLnurlsSizeGauge } from '@ditto/metrics'; import { NostrEvent } from '@nostrify/nostrify'; import { LNURL, LNURLDetails } from '@nostrify/nostrify/ln'; import { logi } from '@soapbox/logi'; import { safeFetch } from '@soapbox/safe-fetch'; import { JsonValue } from '@std/json'; -import { cachedLnurlsSizeGauge } from '@/metrics.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; import { errorJson } from '@/utils/log.ts'; import { Time } from '@/utils/time.ts'; diff --git a/packages/ditto/utils/nip05.ts b/packages/ditto/utils/nip05.ts index 1cc991b9..798fabdf 100644 --- a/packages/ditto/utils/nip05.ts +++ b/packages/ditto/utils/nip05.ts @@ -1,11 +1,11 @@ -import { nip19 } from 'nostr-tools'; +import { cachedNip05sSizeGauge } from '@ditto/metrics'; import { NIP05, NStore } from '@nostrify/nostrify'; import { logi } from '@soapbox/logi'; import { safeFetch } from '@soapbox/safe-fetch'; +import { nip19 } from 'nostr-tools'; import tldts from 'tldts'; import { Conf } from '@/config.ts'; -import { cachedNip05sSizeGauge } from '@/metrics.ts'; import { Storages } from '@/storages.ts'; import { errorJson } from '@/utils/log.ts'; import { SimpleLRU } from '@/utils/SimpleLRU.ts'; diff --git a/packages/ditto/utils/unfurl.ts b/packages/ditto/utils/unfurl.ts index f895b71f..e2d4f855 100644 --- a/packages/ditto/utils/unfurl.ts +++ b/packages/ditto/utils/unfurl.ts @@ -1,3 +1,4 @@ +import { cachedLinkPreviewSizeGauge } from '@ditto/metrics'; import TTLCache from '@isaacs/ttlcache'; import { logi } from '@soapbox/logi'; import { safeFetch } from '@soapbox/safe-fetch'; @@ -6,7 +7,6 @@ import { unfurl } from 'unfurl.js'; import { Conf } from '@/config.ts'; import { PreviewCard } from '@/entities/PreviewCard.ts'; -import { cachedLinkPreviewSizeGauge } from '@/metrics.ts'; import { errorJson } from '@/utils/log.ts'; async function unfurlCard(url: string, signal: AbortSignal): Promise { diff --git a/packages/metrics/deno.json b/packages/metrics/deno.json new file mode 100644 index 00000000..12524c18 --- /dev/null +++ b/packages/metrics/deno.json @@ -0,0 +1,6 @@ +{ + "name": "@ditto/metrics", + "exports": { + ".": "./metrics.ts" + } +} diff --git a/packages/ditto/metrics.ts b/packages/metrics/metrics.ts similarity index 100% rename from packages/ditto/metrics.ts rename to packages/metrics/metrics.ts From e1c1967a66dcd7d9f9a191c28952462e84b9837d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Feb 2025 11:42:48 -0600 Subject: [PATCH 2/6] db: remove some external deps --- packages/db/DittoDB.ts | 4 +--- packages/db/migrations/037_auth_tokens.ts | 17 ----------------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/packages/db/DittoDB.ts b/packages/db/DittoDB.ts index e18242cd..f3442808 100644 --- a/packages/db/DittoDB.ts +++ b/packages/db/DittoDB.ts @@ -4,8 +4,6 @@ import path from 'node:path'; import { logi } from '@soapbox/logi'; import { FileMigrationProvider, type Kysely, Migrator } from 'kysely'; -import { errorJson } from '@/utils/log.ts'; - import { DittoPglite } from './adapters/DittoPglite.ts'; import { DittoPostgres } from './adapters/DittoPostgres.ts'; @@ -51,7 +49,7 @@ export class DittoDB { msg: 'Migration failed.', state: 'failed', results: results as unknown as JsonValue, - error: errorJson(error), + error: error instanceof Error ? error : null, }); Deno.exit(1); } else { diff --git a/packages/db/migrations/037_auth_tokens.ts b/packages/db/migrations/037_auth_tokens.ts index 591ff379..f7ac340c 100644 --- a/packages/db/migrations/037_auth_tokens.ts +++ b/packages/db/migrations/037_auth_tokens.ts @@ -1,9 +1,5 @@ import { type Kysely, sql } from 'kysely'; -import { Conf } from '@/config.ts'; -import { aesEncrypt } from '@/utils/aes.ts'; -import { getTokenHash } from '@/utils/auth.ts'; - interface DB { nip46_tokens: { api_token: `token1${string}`; @@ -32,19 +28,6 @@ export async function up(db: Kysely): Promise { .addColumn('created_at', 'timestamp', (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`)) .execute(); - // There are probably not that many tokens in the database yet, so this should be fine. - const tokens = await db.selectFrom('nip46_tokens').selectAll().execute(); - - for (const token of tokens) { - await db.insertInto('auth_tokens').values({ - token_hash: await getTokenHash(token.api_token), - pubkey: token.user_pubkey, - nip46_sk_enc: await aesEncrypt(Conf.seckey, token.server_seckey), - nip46_relays: JSON.parse(token.relays), - created_at: token.connected_at, - }).execute(); - } - await db.schema.dropTable('nip46_tokens').execute(); } From fbb5c63c33d4b0d44a2907f908e3c4c174d663ba Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Feb 2025 11:45:08 -0600 Subject: [PATCH 3/6] metrics: make 'prefix' a variable --- packages/metrics/metrics.ts | 56 +++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/packages/metrics/metrics.ts b/packages/metrics/metrics.ts index 7fe75a8f..0dd87b7f 100644 --- a/packages/metrics/metrics.ts +++ b/packages/metrics/metrics.ts @@ -1,149 +1,151 @@ import { Counter, Gauge, Histogram } from 'prom-client'; +const prefix = 'ditto'; + export const httpRequestsCounter = new Counter({ - name: 'ditto_http_requests_total', + name: `${prefix}_http_requests_total`, help: 'Total number of HTTP requests', labelNames: ['method'], }); export const httpResponsesCounter = new Counter({ - name: 'ditto_http_responses_total', + name: `${prefix}_http_responses_total`, help: 'Total number of HTTP responses', labelNames: ['method', 'path', 'status'], }); export const httpResponseDurationHistogram = new Histogram({ - name: 'ditto_http_response_duration_seconds', + name: `${prefix}_http_response_duration_seconds`, help: 'Histogram of HTTP response times in seconds', labelNames: ['method', 'path', 'status'], }); export const streamingConnectionsGauge = new Gauge({ - name: 'ditto_streaming_connections', + name: `${prefix}_streaming_connections`, help: 'Number of active connections to the streaming API', }); export const streamingServerMessagesCounter = new Counter({ - name: 'ditto_streaming_server_messages_total', + name: `${prefix}_streaming_server_messages_total`, help: 'Total number of messages sent from the streaming API', }); export const streamingClientMessagesCounter = new Counter({ - name: 'ditto_streaming_client_messages_total', + name: `${prefix}_streaming_client_messages_total`, help: 'Total number of messages received by the streaming API', }); export const fetchResponsesCounter = new Counter({ - name: 'ditto_fetch_responses_total', + name: `${prefix}_fetch_responses_total`, help: 'Total number of fetch requests', labelNames: ['method', 'status'], }); export const firehoseEventsCounter = new Counter({ - name: 'ditto_firehose_events_total', + name: `${prefix}_firehose_events_total`, help: 'Total number of Nostr events processed by the firehose', labelNames: ['kind'], }); export const pipelineEventsCounter = new Counter({ - name: 'ditto_pipeline_events_total', + name: `${prefix}_pipeline_events_total`, help: 'Total number of Nostr events processed by the pipeline', labelNames: ['kind'], }); export const policyEventsCounter = new Counter({ - name: 'ditto_policy_events_total', + name: `${prefix}_policy_events_total`, help: 'Total number of policy OK responses', labelNames: ['ok'], }); export const relayEventsCounter = new Counter({ - name: 'ditto_relay_events_total', + name: `${prefix}_relay_events_total`, help: 'Total number of EVENT messages processed by the relay', labelNames: ['kind'], }); export const relayMessagesCounter = new Counter({ - name: 'ditto_relay_messages_total', + name: `${prefix}_relay_messages_total`, help: 'Total number of Nostr messages processed by the relay', labelNames: ['verb'], }); export const relayConnectionsGauge = new Gauge({ - name: 'ditto_relay_connections', + name: `${prefix}_relay_connections`, help: 'Number of active connections to the relay', }); export const dbQueriesCounter = new Counter({ - name: 'ditto_db_queries_total', + name: `${prefix}_db_queries_total`, help: 'Total number of database queries', labelNames: ['kind'], }); export const dbEventsCounter = new Counter({ - name: 'ditto_db_events_total', + name: `${prefix}_db_events_total`, help: 'Total number of database inserts', labelNames: ['kind'], }); export const dbPoolSizeGauge = new Gauge({ - name: 'ditto_db_pool_size', + name: `${prefix}_db_pool_size`, help: 'Number of connections in the database pool', }); export const dbAvailableConnectionsGauge = new Gauge({ - name: 'ditto_db_available_connections', + name: `${prefix}_db_available_connections`, help: 'Number of available connections in the database pool', }); export const dbQueryDurationHistogram = new Histogram({ - name: 'ditto_db_query_duration_seconds', + name: `${prefix}_db_query_duration_seconds`, help: 'Duration of database queries', }); export const cachedFaviconsSizeGauge = new Gauge({ - name: 'ditto_cached_favicons_size', + name: `${prefix}_cached_favicons_size`, help: 'Number of domain favicons in cache', }); export const cachedLnurlsSizeGauge = new Gauge({ - name: 'ditto_cached_lnurls_size', + name: `${prefix}_cached_lnurls_size`, help: 'Number of LNURL details in cache', }); export const cachedNip05sSizeGauge = new Gauge({ - name: 'ditto_cached_nip05s_size', + name: `${prefix}_cached_nip05s_size`, help: 'Number of NIP-05 results in cache', }); export const cachedLinkPreviewSizeGauge = new Gauge({ - name: 'ditto_cached_link_previews_size', + name: `${prefix}_cached_link_previews_size`, help: 'Number of link previews in cache', }); export const cachedTranslationsSizeGauge = new Gauge({ - name: 'ditto_cached_translations_size', + name: `${prefix}_cached_translations_size`, help: 'Number of translated statuses in cache', }); export const internalSubscriptionsSizeGauge = new Gauge({ - name: 'ditto_internal_subscriptions_size', + name: `${prefix}_internal_subscriptions_size`, help: "Number of active subscriptions to Ditto's internal relay", }); export const relayPoolRelaysSizeGauge = new Gauge({ - name: 'ditto_relay_pool_relays_size', + name: `${prefix}_relay_pool_relays_size`, help: 'Number of relays in the relay pool', labelNames: ['ready_state'], }); export const relayPoolSubscriptionsSizeGauge = new Gauge({ - name: 'ditto_relay_pool_subscriptions_size', + name: `${prefix}_relay_pool_subscriptions_size`, help: 'Number of active subscriptions to the relay pool', }); export const webPushNotificationsCounter = new Counter({ - name: 'ditto_web_push_notifications_total', + name: `${prefix}_web_push_notifications_total`, help: 'Total number of Web Push notifications sent', labelNames: ['type'], }); From 785ba1e053254fdc00f65ac35ffe8c361cf8700e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Feb 2025 11:50:25 -0600 Subject: [PATCH 4/6] Remove base alias from packages (make scripts use relative paths for now) --- deno.json | 2 +- packages/ditto/deno.json | 4 ++-- scripts/admin-event.ts | 8 ++++---- scripts/admin-role.ts | 6 +++--- scripts/db-export.test.ts | 1 + scripts/db-export.ts | 3 ++- scripts/db-import.ts | 4 ++-- scripts/db-migrate.ts | 2 +- scripts/db-policy.ts | 4 ++-- scripts/db-populate-extensions.ts | 4 ++-- scripts/db-populate-nip05.ts | 6 +++--- scripts/db-populate-search.ts | 3 ++- scripts/db-streak-recompute.ts | 4 ++-- scripts/nostr-pull.ts | 2 +- scripts/setup-kind0.ts | 9 +++++---- scripts/setup.ts | 2 +- scripts/stats-recompute.ts | 4 ++-- scripts/trends.ts | 2 +- 18 files changed, 37 insertions(+), 33 deletions(-) diff --git a/deno.json b/deno.json index 629e1813..debd7fa9 100644 --- a/deno.json +++ b/deno.json @@ -1,4 +1,5 @@ { + "version": "1.1.0", "workspace": [ "./packages/api", "./packages/conf", @@ -43,7 +44,6 @@ "./public" ], "imports": { - "@/": "./packages/ditto/", "@b-fuze/deno-dom": "jsr:@b-fuze/deno-dom@^0.1.47", "@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "@cashu/cashu-ts": "npm:@cashu/cashu-ts@^2.2.0", diff --git a/packages/ditto/deno.json b/packages/ditto/deno.json index 31f80278..82d28139 100644 --- a/packages/ditto/deno.json +++ b/packages/ditto/deno.json @@ -1,9 +1,9 @@ { "name": "@ditto/ditto", - "version": "1.1.0", "exports": {}, "imports": { - "deno.json": "./deno.json" + "@/": "./", + "deno.json": "../../deno.json" }, "lint": { "rules": { diff --git a/scripts/admin-event.ts b/scripts/admin-event.ts index 00711993..70f8ed48 100644 --- a/scripts/admin-event.ts +++ b/scripts/admin-event.ts @@ -1,10 +1,10 @@ import { JsonParseStream } from '@std/json/json-parse-stream'; import { TextLineStream } from '@std/streams/text-line-stream'; -import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { Storages } from '@/storages.ts'; -import { type EventStub } from '@/utils/api.ts'; -import { nostrNow } from '@/utils.ts'; +import { AdminSigner } from '../packages/ditto/signers/AdminSigner.ts'; +import { Storages } from '../packages/ditto/storages.ts'; +import { type EventStub } from '../packages/ditto/utils/api.ts'; +import { nostrNow } from '../packages/ditto/utils.ts'; const signer = new AdminSigner(); const store = await Storages.db(); diff --git a/scripts/admin-role.ts b/scripts/admin-role.ts index d275329f..369440c9 100644 --- a/scripts/admin-role.ts +++ b/scripts/admin-role.ts @@ -1,9 +1,9 @@ import { NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { Storages } from '@/storages.ts'; -import { nostrNow } from '@/utils.ts'; +import { AdminSigner } from '../packages/ditto/signers/AdminSigner.ts'; +import { Storages } from '../packages/ditto/storages.ts'; +import { nostrNow } from '../packages/ditto/utils.ts'; const store = await Storages.db(); diff --git a/scripts/db-export.test.ts b/scripts/db-export.test.ts index 939537d5..3b180291 100644 --- a/scripts/db-export.test.ts +++ b/scripts/db-export.test.ts @@ -1,4 +1,5 @@ import { assertEquals, assertThrows } from '@std/assert'; + import { buildFilter } from './db-export.ts'; Deno.test('buildFilter should return an empty filter when no arguments are provided', () => { diff --git a/scripts/db-export.ts b/scripts/db-export.ts index e32e08ad..d36d4f3f 100644 --- a/scripts/db-export.ts +++ b/scripts/db-export.ts @@ -1,7 +1,8 @@ -import { Storages } from '@/storages.ts'; import { NostrFilter } from '@nostrify/nostrify'; import { Command, InvalidOptionArgumentError } from 'commander'; +import { Storages } from '../packages/ditto/storages.ts'; + interface ExportFilter { authors?: string[]; ids?: string[]; diff --git a/scripts/db-import.ts b/scripts/db-import.ts index ed884453..2f6c1595 100644 --- a/scripts/db-import.ts +++ b/scripts/db-import.ts @@ -3,8 +3,8 @@ import { NostrEvent } from '@nostrify/nostrify'; import { JsonParseStream } from '@std/json/json-parse-stream'; import { TextLineStream } from '@std/streams/text-line-stream'; -import { Conf } from '@/config.ts'; -import { Storages } from '@/storages.ts'; +import { Conf } from '../packages/ditto/config.ts'; +import { Storages } from '../packages/ditto/storages.ts'; const store = await Storages.db(); const sem = new Semaphore(Conf.pg.poolSize); diff --git a/scripts/db-migrate.ts b/scripts/db-migrate.ts index d3e93783..21b8db22 100644 --- a/scripts/db-migrate.ts +++ b/scripts/db-migrate.ts @@ -1,4 +1,4 @@ -import { Storages } from '@/storages.ts'; +import { Storages } from '../packages/ditto/storages.ts'; // This migrates kysely internally. const kysely = await Storages.kysely(); diff --git a/scripts/db-policy.ts b/scripts/db-policy.ts index 4be3c4ef..caab55af 100644 --- a/scripts/db-policy.ts +++ b/scripts/db-policy.ts @@ -1,5 +1,5 @@ -import { policyWorker } from '@/workers/policy.ts'; -import { Storages } from '@/storages.ts'; +import { policyWorker } from '../packages/ditto/workers/policy.ts'; +import { Storages } from '../packages/ditto/storages.ts'; const db = await Storages.db(); let count = 0; diff --git a/scripts/db-populate-extensions.ts b/scripts/db-populate-extensions.ts index ca6d1927..2b40bd3d 100644 --- a/scripts/db-populate-extensions.ts +++ b/scripts/db-populate-extensions.ts @@ -1,7 +1,7 @@ import { NostrEvent } from '@nostrify/nostrify'; -import { Storages } from '@/storages.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; +import { Storages } from '../packages/ditto/storages.ts'; +import { EventsDB } from '../packages/ditto/storages/EventsDB.ts'; const kysely = await Storages.kysely(); diff --git a/scripts/db-populate-nip05.ts b/scripts/db-populate-nip05.ts index df11f007..acfe70da 100644 --- a/scripts/db-populate-nip05.ts +++ b/scripts/db-populate-nip05.ts @@ -1,9 +1,9 @@ import { Semaphore } from '@core/asyncutil'; - -import { updateAuthorData } from '@/pipeline.ts'; -import { Storages } from '@/storages.ts'; import { NostrEvent } from '@nostrify/nostrify'; +import { updateAuthorData } from '../packages/ditto/pipeline.ts'; +import { Storages } from '../packages/ditto/storages.ts'; + const kysely = await Storages.kysely(); const sem = new Semaphore(5); diff --git a/scripts/db-populate-search.ts b/scripts/db-populate-search.ts index 81b84ee6..e73f79ac 100644 --- a/scripts/db-populate-search.ts +++ b/scripts/db-populate-search.ts @@ -1,5 +1,6 @@ import { NSchema as n } from '@nostrify/nostrify'; -import { Storages } from '@/storages.ts'; + +import { Storages } from '../packages/ditto/storages.ts'; const store = await Storages.db(); const kysely = await Storages.kysely(); diff --git a/scripts/db-streak-recompute.ts b/scripts/db-streak-recompute.ts index a05eb08b..e45d4f64 100644 --- a/scripts/db-streak-recompute.ts +++ b/scripts/db-streak-recompute.ts @@ -1,5 +1,5 @@ -import { Conf } from '@/config.ts'; -import { Storages } from '@/storages.ts'; +import { Conf } from '../packages/ditto/config.ts'; +import { Storages } from '../packages/ditto/storages.ts'; const kysely = await Storages.kysely(); const statsQuery = kysely.selectFrom('author_stats').select('pubkey'); diff --git a/scripts/nostr-pull.ts b/scripts/nostr-pull.ts index c7ad21d3..7c21cb80 100644 --- a/scripts/nostr-pull.ts +++ b/scripts/nostr-pull.ts @@ -6,7 +6,7 @@ import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { Storages } from '@/storages.ts'; +import { Storages } from '../packages/ditto/storages.ts'; const store = await Storages.db(); diff --git a/scripts/setup-kind0.ts b/scripts/setup-kind0.ts index 7aa62df3..ff7cbd1a 100644 --- a/scripts/setup-kind0.ts +++ b/scripts/setup-kind0.ts @@ -1,9 +1,10 @@ -import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Command } from 'commander'; import { NostrEvent } from 'nostr-tools'; -import { nostrNow } from '@/utils.ts'; -import { Conf } from '@/config.ts'; -import { Storages } from '@/storages.ts'; + +import { AdminSigner } from '../packages/ditto/signers/AdminSigner.ts'; +import { nostrNow } from '../packages/ditto/utils.ts'; +import { Conf } from '../packages/ditto/config.ts'; +import { Storages } from '../packages/ditto/storages.ts'; function die(code: number, ...args: unknown[]) { console.error(...args); diff --git a/scripts/setup.ts b/scripts/setup.ts index 3f3fc955..f4ccf368 100644 --- a/scripts/setup.ts +++ b/scripts/setup.ts @@ -4,7 +4,7 @@ import { exists } from '@std/fs/exists'; import { generateSecretKey, nip19 } from 'nostr-tools'; import question from 'question-deno'; -import { Conf } from '@/config.ts'; +import { Conf } from '../packages/ditto/config.ts'; console.log(''); console.log('Hello! Welcome to the Ditto setup tool. We will ask you a few questions to generate a .env file for you.'); diff --git a/scripts/stats-recompute.ts b/scripts/stats-recompute.ts index 77be13fe..942d0012 100644 --- a/scripts/stats-recompute.ts +++ b/scripts/stats-recompute.ts @@ -1,7 +1,7 @@ import { nip19 } from 'nostr-tools'; -import { Storages } from '@/storages.ts'; -import { refreshAuthorStats } from '@/utils/stats.ts'; +import { Storages } from '../packages/ditto/storages.ts'; +import { refreshAuthorStats } from '../packages/ditto/utils/stats.ts'; let pubkey: string; try { diff --git a/scripts/trends.ts b/scripts/trends.ts index 6600f7e2..bb9708ab 100644 --- a/scripts/trends.ts +++ b/scripts/trends.ts @@ -6,7 +6,7 @@ import { updateTrendingLinks, updateTrendingPubkeys, updateTrendingZappedEvents, -} from '@/trends.ts'; +} from '../packages/ditto/trends.ts'; const trendSchema = z.enum(['pubkeys', 'zapped_events', 'events', 'hashtags', 'links']); const trends = trendSchema.array().parse(Deno.args); From e100f72a9baaf8d774cf2aa8c448326c215c3f5b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Feb 2025 11:53:34 -0600 Subject: [PATCH 5/6] Add deno task lint --- .gitlab-ci.yml | 2 +- deno.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 39af1489..766a144d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,7 @@ test: stage: test script: - deno fmt --check - - deno lint --allow-import + - deno task lint - deno task check - deno task test --coverage=cov_profile - deno coverage cov_profile diff --git a/deno.json b/deno.json index debd7fa9..412f32a3 100644 --- a/deno.json +++ b/deno.json @@ -19,6 +19,7 @@ "debug": "deno run -A --env-file --deny-read=.env --inspect packages/ditto/server.ts", "test": "deno test -A --env-file=.env.test --deny-read=.env --junit-path=./deno-test.xml", "check": "deno check --allow-import .", + "lint": "deno lint --allow-import", "nsec": "deno run scripts/nsec.ts", "admin:event": "deno run -A --env-file --deny-read=.env scripts/admin-event.ts", "admin:role": "deno run -A --env-file --deny-read=.env scripts/admin-role.ts", From 0ace14ffbb606d54b79da6fcd7a15145897eb6fc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 Feb 2025 11:56:41 -0600 Subject: [PATCH 6/6] metrics: add types to all exports --- packages/metrics/metrics.ts | 54 ++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/packages/metrics/metrics.ts b/packages/metrics/metrics.ts index 0dd87b7f..716582d4 100644 --- a/packages/metrics/metrics.ts +++ b/packages/metrics/metrics.ts @@ -2,149 +2,149 @@ import { Counter, Gauge, Histogram } from 'prom-client'; const prefix = 'ditto'; -export const httpRequestsCounter = new Counter({ +export const httpRequestsCounter: Counter<'method'> = new Counter({ name: `${prefix}_http_requests_total`, help: 'Total number of HTTP requests', labelNames: ['method'], }); -export const httpResponsesCounter = new Counter({ +export const httpResponsesCounter: Counter<'method' | 'path' | 'status'> = new Counter({ name: `${prefix}_http_responses_total`, help: 'Total number of HTTP responses', labelNames: ['method', 'path', 'status'], }); -export const httpResponseDurationHistogram = new Histogram({ +export const httpResponseDurationHistogram: Histogram<'method' | 'path' | 'status'> = new Histogram({ name: `${prefix}_http_response_duration_seconds`, help: 'Histogram of HTTP response times in seconds', labelNames: ['method', 'path', 'status'], }); -export const streamingConnectionsGauge = new Gauge({ +export const streamingConnectionsGauge: Gauge = new Gauge({ name: `${prefix}_streaming_connections`, help: 'Number of active connections to the streaming API', }); -export const streamingServerMessagesCounter = new Counter({ +export const streamingServerMessagesCounter: Counter = new Counter({ name: `${prefix}_streaming_server_messages_total`, help: 'Total number of messages sent from the streaming API', }); -export const streamingClientMessagesCounter = new Counter({ +export const streamingClientMessagesCounter: Counter = new Counter({ name: `${prefix}_streaming_client_messages_total`, help: 'Total number of messages received by the streaming API', }); -export const fetchResponsesCounter = new Counter({ +export const fetchResponsesCounter: Counter<'method' | 'status'> = new Counter({ name: `${prefix}_fetch_responses_total`, help: 'Total number of fetch requests', labelNames: ['method', 'status'], }); -export const firehoseEventsCounter = new Counter({ +export const firehoseEventsCounter: Counter<'kind'> = new Counter({ name: `${prefix}_firehose_events_total`, help: 'Total number of Nostr events processed by the firehose', labelNames: ['kind'], }); -export const pipelineEventsCounter = new Counter({ +export const pipelineEventsCounter: Counter<'kind'> = new Counter({ name: `${prefix}_pipeline_events_total`, help: 'Total number of Nostr events processed by the pipeline', labelNames: ['kind'], }); -export const policyEventsCounter = new Counter({ +export const policyEventsCounter: Counter<'ok'> = new Counter({ name: `${prefix}_policy_events_total`, help: 'Total number of policy OK responses', labelNames: ['ok'], }); -export const relayEventsCounter = new Counter({ +export const relayEventsCounter: Counter<'kind'> = new Counter({ name: `${prefix}_relay_events_total`, help: 'Total number of EVENT messages processed by the relay', labelNames: ['kind'], }); -export const relayMessagesCounter = new Counter({ +export const relayMessagesCounter: Counter<'verb'> = new Counter({ name: `${prefix}_relay_messages_total`, help: 'Total number of Nostr messages processed by the relay', labelNames: ['verb'], }); -export const relayConnectionsGauge = new Gauge({ +export const relayConnectionsGauge: Gauge = new Gauge({ name: `${prefix}_relay_connections`, help: 'Number of active connections to the relay', }); -export const dbQueriesCounter = new Counter({ +export const dbQueriesCounter: Counter<'kind'> = new Counter({ name: `${prefix}_db_queries_total`, help: 'Total number of database queries', labelNames: ['kind'], }); -export const dbEventsCounter = new Counter({ +export const dbEventsCounter: Counter<'kind'> = new Counter({ name: `${prefix}_db_events_total`, help: 'Total number of database inserts', labelNames: ['kind'], }); -export const dbPoolSizeGauge = new Gauge({ +export const dbPoolSizeGauge: Gauge = new Gauge({ name: `${prefix}_db_pool_size`, help: 'Number of connections in the database pool', }); -export const dbAvailableConnectionsGauge = new Gauge({ +export const dbAvailableConnectionsGauge: Gauge = new Gauge({ name: `${prefix}_db_available_connections`, help: 'Number of available connections in the database pool', }); -export const dbQueryDurationHistogram = new Histogram({ +export const dbQueryDurationHistogram: Histogram = new Histogram({ name: `${prefix}_db_query_duration_seconds`, help: 'Duration of database queries', }); -export const cachedFaviconsSizeGauge = new Gauge({ +export const cachedFaviconsSizeGauge: Gauge = new Gauge({ name: `${prefix}_cached_favicons_size`, help: 'Number of domain favicons in cache', }); -export const cachedLnurlsSizeGauge = new Gauge({ +export const cachedLnurlsSizeGauge: Gauge = new Gauge({ name: `${prefix}_cached_lnurls_size`, help: 'Number of LNURL details in cache', }); -export const cachedNip05sSizeGauge = new Gauge({ +export const cachedNip05sSizeGauge: Gauge = new Gauge({ name: `${prefix}_cached_nip05s_size`, help: 'Number of NIP-05 results in cache', }); -export const cachedLinkPreviewSizeGauge = new Gauge({ +export const cachedLinkPreviewSizeGauge: Gauge = new Gauge({ name: `${prefix}_cached_link_previews_size`, help: 'Number of link previews in cache', }); -export const cachedTranslationsSizeGauge = new Gauge({ +export const cachedTranslationsSizeGauge: Gauge = new Gauge({ name: `${prefix}_cached_translations_size`, help: 'Number of translated statuses in cache', }); -export const internalSubscriptionsSizeGauge = new Gauge({ +export const internalSubscriptionsSizeGauge: Gauge = new Gauge({ name: `${prefix}_internal_subscriptions_size`, help: "Number of active subscriptions to Ditto's internal relay", }); -export const relayPoolRelaysSizeGauge = new Gauge({ +export const relayPoolRelaysSizeGauge: Gauge<'ready_state'> = new Gauge({ name: `${prefix}_relay_pool_relays_size`, help: 'Number of relays in the relay pool', labelNames: ['ready_state'], }); -export const relayPoolSubscriptionsSizeGauge = new Gauge({ +export const relayPoolSubscriptionsSizeGauge: Gauge = new Gauge({ name: `${prefix}_relay_pool_subscriptions_size`, help: 'Number of active subscriptions to the relay pool', }); -export const webPushNotificationsCounter = new Counter({ +export const webPushNotificationsCounter: Counter<'type'> = new Counter({ name: `${prefix}_web_push_notifications_total`, help: 'Total number of Web Push notifications sent', labelNames: ['type'],