Merge branch 'main' into zapped-by-pagination-sort-amount

deno.json

@@ -59,6 +59,7 @@
     "nostr-relaypool": "npm:nostr-relaypool2@0.6.34",
     "nostr-tools": "npm:nostr-tools@2.5.1",
     "nostr-wasm": "npm:nostr-wasm@^0.1.0",
+    "prom-client": "npm:prom-client@^15.1.2",
     "question-deno": "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/mod.ts",
     "tldts": "npm:tldts@^6.0.14",
     "tseep": "npm:tseep@^1.2.1",

deno.lock

@@ -7,6 +7,7 @@
       "jsr:@denosaurs/plug@1": "jsr:@denosaurs/plug@1.0.6",
       "jsr:@gleasonator/policy": "jsr:@gleasonator/policy@0.2.0",
       "jsr:@gleasonator/policy@0.2.0": "jsr:@gleasonator/policy@0.2.0",
+      "jsr:@gleasonator/policy@0.4.0": "jsr:@gleasonator/policy@0.4.0",
       "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.4.6",
       "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5",
       "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4",
@@ -61,6 +62,7 @@
       "npm:nostr-tools@^2.5.0": "npm:nostr-tools@2.5.1",
       "npm:nostr-tools@^2.7.0": "npm:nostr-tools@2.7.0",
       "npm:nostr-wasm@^0.1.0": "npm:nostr-wasm@0.1.0",
+      "npm:prom-client@^15.1.2": "npm:prom-client@15.1.2",
       "npm:tldts@^6.0.14": "npm:tldts@6.1.18",
       "npm:type-fest@^4.3.0": "npm:type-fest@4.18.2",
       "npm:unfurl.js@^6.4.0": "npm:unfurl.js@6.4.0",
@@ -96,6 +98,12 @@
           "jsr:@nostrify/nostrify@^0.22.1"
         ]
       },
+      "@gleasonator/policy@0.4.0": {
+        "integrity": "59c2f3ab1dc663e99a3e10b7eb69bf9fe581ce5d428fe56653e38f7f961da5ea",
+        "dependencies": [
+          "jsr:@nostrify/nostrify@^0.22.1"
+        ]
+      },
       "@hono/hono@4.4.6": {
         "integrity": "aa557ca9930787ee86b9ca1730691f1ce1c379174c2cb244d5934db2b6314453"
       },
@@ -275,6 +283,10 @@
         "integrity": "sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg==",
         "dependencies": {}
       },
+      "@opentelemetry/api@1.9.0": {
+        "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==",
+        "dependencies": {}
+      },
       "@scure/base@1.1.1": {
         "integrity": "sha512-ZxOhsSyxYwLJj3pLZCefNitxsj093tb2vq90mp2txoYeBqbcjDjqFhyM8eUjq/uFm6zJ+mUuqxlS2FkuSY1MTA==",
         "dependencies": {}
@@ -351,6 +363,10 @@
         "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
         "dependencies": {}
       },
+      "bintrees@1.0.2": {
+        "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==",
+        "dependencies": {}
+      },
       "braces@3.0.2": {
         "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
         "dependencies": {
@@ -852,6 +868,13 @@
         "integrity": "sha512-eG2dWTVw5bzqGRztnHExczNxt5VGsE6OwTeCG3fdUf9KBsZzO3R5OIIIzWR+iZA0NtZ+RDVdaoE2dK1cn6jH4g==",
         "dependencies": {}
       },
+      "prom-client@15.1.2": {
+        "integrity": "sha512-on3h1iXb04QFLLThrmVYg1SChBQ9N1c+nKAjebBjokBqipddH3uxmOUcEkTnzmJ8Jh/5TSUnUqS40i2QB2dJHQ==",
+        "dependencies": {
+          "@opentelemetry/api": "@opentelemetry/api@1.9.0",
+          "tdigest": "tdigest@0.1.2"
+        }
+      },
       "psl@1.9.0": {
         "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==",
         "dependencies": {}
@@ -955,6 +978,12 @@
         "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==",
         "dependencies": {}
       },
+      "tdigest@0.1.2": {
+        "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==",
+        "dependencies": {
+          "bintrees": "bintrees@1.0.2"
+        }
+      },
       "tldts-core@6.1.18": {
         "integrity": "sha512-e4wx32F/7dMBSZyKAx825Yte3U0PQtZZ0bkWxYQiwLteRVnQ5zM40fEbi0IyNtwQssgJAk3GCr7Q+w39hX0VKA==",
         "dependencies": {}
@@ -1418,6 +1447,7 @@
       "npm:nostr-relaypool2@0.6.34",
       "npm:nostr-tools@2.5.1",
       "npm:nostr-wasm@^0.1.0",
+      "npm:prom-client@^15.1.2",
       "npm:tldts@^6.0.14",
       "npm:tseep@^1.2.1",
       "npm:type-fest@^4.3.0",

installation/ditto.conf

@@ -50,6 +50,12 @@ server {
     root /opt/ditto/public;
   }
 
+  location /metrics {
+    allow 127.0.0.1;
+    deny all;
+    proxy_pass http://ditto;
+  }
+
   location ~ ^/(instance|sw\.js$|sw\.js\.map$) {
     root /opt/ditto/public;
     try_files $uri =404;

src/app.ts

@@ -108,6 +108,7 @@ import {
   trendingStatusesController,
   trendingTagsController,
 } from '@/controllers/api/trends.ts';
+import { metricsController } from '@/controllers/metrics.ts';
 import { indexController } from '@/controllers/site.ts';
 import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts';
 import { nostrController } from '@/controllers/well-known/nostr.ts';
@@ -168,6 +169,8 @@ app.use(
   storeMiddleware,
 );
 
+app.get('/metrics', metricsController);
+
 app.get('/.well-known/nodeinfo', nodeInfoController);
 app.get('/.well-known/nostr.json', nostrController);
 

src/config.ts

@@ -63,6 +63,14 @@ class Conf {
   static get localDomain(): string {
     return Deno.env.get('LOCAL_DOMAIN') || `http://localhost:${Conf.port}`;
   }
+  /** Link to an external nostr viewer. */
+  static get externalDomain(): string {
+    return Deno.env.get('NOSTR_EXTERNAL') || 'https://njump.me';
+  }
+  /** Get a link to a nip19-encoded entity in the configured external viewer. */
+  static external(path: string) {
+    return new URL(path, Conf.externalDomain).toString();
+  }
   /**
    * Heroku-style database URL. This is used in production to connect to the
    * database.

src/controllers/metrics.ts

@@ -0,0 +1,14 @@
+import { register } from 'prom-client';
+
+import { AppController } from '@/app.ts';
+
+/** Prometheus/OpenMetrics controller. */
+export const metricsController: AppController = async (c) => {
+  const metrics = await register.metrics();
+
+  const headers: HeadersInit = {
+    'Content-Type': register.contentType,
+  };
+
+  return c.text(metrics, 200, headers);
+};

src/controllers/nostr/relay.ts

@@ -10,6 +10,7 @@ import {
 
 import { AppController } from '@/app.ts';
 import { relayInfoController } from '@/controllers/nostr/relay-info.ts';
+import { relayCountCounter, relayEventCounter, relayMessageCounter, relayReqCounter } from '@/metrics.ts';
 import * as pipeline from '@/pipeline.ts';
 import { RelayError } from '@/RelayError.ts';
 import { Storages } from '@/storages.ts';
@@ -22,6 +23,7 @@ function connectStream(socket: WebSocket) {
   const controllers = new Map<string, AbortController>();
 
   socket.onmessage = (e) => {
+    relayMessageCounter.inc();
     const result = n.json().pipe(n.clientMsg()).safeParse(e.data);
     if (result.success) {
       handleMsg(result.data);
@@ -40,15 +42,18 @@ function connectStream(socket: WebSocket) {
   function handleMsg(msg: NostrClientMsg) {
     switch (msg[0]) {
       case 'REQ':
+        relayReqCounter.inc();
         handleReq(msg);
         return;
       case 'EVENT':
+        relayEventCounter.inc({ kind: msg[1].kind.toString() });
         handleEvent(msg);
         return;
       case 'CLOSE':
         handleClose(msg);
         return;
       case 'COUNT':
+        relayCountCounter.inc();
         handleCount(msg);
         return;
     }

src/entities/MastodonAccount.ts

@@ -40,6 +40,7 @@ export interface MastodonAccount {
   username: string;
   ditto: {
     accepts_zaps: boolean;
+    external_url: string;
   };
   pleroma: {
     deactivated: boolean;

src/entities/MastodonStatus.ts

@@ -39,4 +39,7 @@ export interface MastodonStatus {
     expires_at?: string;
     quotes_count: number;
   };
+  ditto: {
+    external_url: string;
+  };
 }

src/firehose.ts

@@ -1,5 +1,6 @@
 import { Stickynotes } from '@soapbox/stickynotes';
 
+import { firehoseEventCounter } from '@/metrics.ts';
 import { Storages } from '@/storages.ts';
 import { nostrNow } from '@/utils.ts';
 
@@ -12,13 +13,14 @@ const console = new Stickynotes('ditto:firehose');
  * side-effects based on them, such as trending hashtag tracking
  * and storing events for notifications and the home feed.
  */
-export async function startFirehose() {
+export async function startFirehose(): Promise<void> {
   const store = await Storages.client();
 
   for await (const msg of store.req([{ kinds: [0, 1, 3, 5, 6, 7, 9735, 10002], limit: 0, since: nostrNow() }])) {
     if (msg[0] === 'EVENT') {
       const event = msg[2];
       console.debug(`NostrEvent<${event.kind}> ${event.id}`);
+      firehoseEventCounter.inc({ kind: event.kind });
 
       pipeline
         .handleEvent(event, AbortSignal.timeout(5000))

src/metrics.ts

@@ -0,0 +1,58 @@
+import { Counter } from 'prom-client';
+
+export const httpRequestCounter = new Counter({
+  name: 'http_requests_total',
+  help: 'Total number of HTTP requests',
+  labelNames: ['method'],
+});
+
+export const fetchCounter = new Counter({
+  name: 'fetch_total',
+  help: 'Total number of fetch requests',
+  labelNames: ['method'],
+});
+
+export const firehoseEventCounter = new Counter({
+  name: 'firehose_events_total',
+  help: 'Total number of Nostr events processed by the firehose',
+  labelNames: ['kind'],
+});
+
+export const pipelineEventCounter = new Counter({
+  name: 'pipeline_events_total',
+  help: 'Total number of Nostr events processed by the pipeline',
+  labelNames: ['kind'],
+});
+
+export const relayReqCounter = new Counter({
+  name: 'relay_reqs_total',
+  help: 'Total number of REQ messages processed by the relay',
+});
+
+export const relayEventCounter = new Counter({
+  name: 'relay_events_total',
+  help: 'Total number of EVENT messages processed by the relay',
+  labelNames: ['kind'],
+});
+
+export const relayCountCounter = new Counter({
+  name: 'relay_counts_total',
+  help: 'Total number of COUNT messages processed by the relay',
+});
+
+export const relayMessageCounter = new Counter({
+  name: 'relay_messages_total',
+  help: 'Total number of Nostr messages processed by the relay',
+});
+
+export const dbQueryCounter = new Counter({
+  name: 'db_query_total',
+  help: 'Total number of database queries',
+  labelNames: ['kind'],
+});
+
+export const dbEventCounter = new Counter({
+  name: 'db_events_total',
+  help: 'Total number of database inserts',
+  labelNames: ['kind'],
+});

src/middleware/metricsMiddleware.ts

@@ -0,0 +1,10 @@
+import { MiddlewareHandler } from '@hono/hono';
+
+import { httpRequestCounter } from '@/metrics.ts';
+
+export const metricsMiddleware: MiddlewareHandler = async (c, next) => {
+  const { method } = c.req;
+  httpRequestCounter.inc({ method });
+
+  await next();
+};

src/pipeline.ts

@@ -7,6 +7,7 @@ import { Conf } from '@/config.ts';
 import { DittoDB } from '@/db/DittoDB.ts';
 import { deleteAttachedMedia } from '@/db/unattached-media.ts';
 import { DittoEvent } from '@/interfaces/DittoEvent.ts';
+import { pipelineEventCounter } from '@/metrics.ts';
 import { RelayError } from '@/RelayError.ts';
 import { AdminSigner } from '@/signers/AdminSigner.ts';
 import { hydrateEvents } from '@/storages/hydrate.ts';
@@ -37,6 +38,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise<void
   if (encounterEvent(event)) return;
   if (await existsInDB(event)) return;
   debug(`NostrEvent<${event.kind}> ${event.id}`);
+  pipelineEventCounter.inc({ kind: event.kind });
 
   if (event.kind !== 24133) {
     await policyFilter(event);

src/storages/EventsDB.ts

@@ -7,6 +7,7 @@ import { nip27 } from 'nostr-tools';
 
 import { Conf } from '@/config.ts';
 import { DittoTables } from '@/db/DittoTables.ts';
+import { dbEventCounter, dbQueryCounter } from '@/metrics.ts';
 import { RelayError } from '@/RelayError.ts';
 import { purifyEvent } from '@/storages/hydrate.ts';
 import { isNostrId, isURL } from '@/utils.ts';
@@ -53,6 +54,7 @@ class EventsDB implements NStore {
   async event(event: NostrEvent, _opts?: { signal?: AbortSignal }): Promise<void> {
     event = purifyEvent(event);
     this.console.debug('EVENT', JSON.stringify(event));
+    dbEventCounter.inc({ kind: event.kind });
 
     if (await this.isDeletedAdmin(event)) {
       throw new RelayError('blocked', 'event deleted by admin');
@@ -137,6 +139,7 @@ class EventsDB implements NStore {
   /** Get events for filters from the database. */
   async query(filters: NostrFilter[], opts: { signal?: AbortSignal; limit?: number } = {}): Promise<NostrEvent[]> {
     filters = await this.expandFilters(filters);
+    dbQueryCounter.inc();
 
     for (const filter of filters) {
       if (filter.since && filter.since >= 2_147_483_647) {

src/views/mastodon/accounts.ts

@@ -82,6 +82,7 @@ async function renderAccount(
     username: parsed05?.nickname || npub.substring(0, 8),
     ditto: {
       accepts_zaps: Boolean(getLnurl({ lud06, lud16 })),
+      external_url: Conf.external(npub),
     },
     pleroma: {
       deactivated: names.has('disabled'),

src/views/mastodon/statuses.ts

@@ -124,6 +124,9 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
     uri: Conf.local(`/${note}`),
     url: Conf.local(`/${note}`),
     zapped: Boolean(zapEvent),
+    ditto: {
+      external_url: Conf.external(note),
+    },
     pleroma: {
       emoji_reactions: reactions,
       expires_at: !isNaN(expiresAt.getTime()) ? expiresAt.toISOString() : undefined,

src/workers/fetch.ts

@@ -1,8 +1,9 @@
 import * as Comlink from 'comlink';
 
+import { FetchWorker } from './fetch.worker.ts';
 import './handlers/abortsignal.ts';
 
-import type { FetchWorker } from './fetch.worker.ts';
+import { fetchCounter } from '@/metrics.ts';
 
 const worker = new Worker(new URL('./fetch.worker.ts', import.meta.url), { type: 'module' });
 const client = Comlink.wrap<typeof FetchWorker>(worker);
@@ -24,6 +25,7 @@ const fetchWorker: typeof fetch = async (...args) => {
   await ready;
   const [url, init] = serializeFetchArgs(args);
   const { body, signal, ...rest } = init;
+  fetchCounter.inc({ method: init.method });
   const result = await client.fetch(url, { ...rest, body: await prepareBodyForWorker(body) }, signal);
   return new Response(...result);
 };

src/workers/fetch.worker.ts

@@ -2,6 +2,7 @@ import Debug from '@soapbox/stickynotes/debug';
 import * as Comlink from 'comlink';
 
 import './handlers/abortsignal.ts';
+import '@/sentry.ts';
 
 const debug = Debug('ditto:fetch.worker');
 

src/workers/verify.worker.ts

@@ -3,6 +3,7 @@ import * as Comlink from 'comlink';
 import { VerifiedEvent, verifyEvent } from 'nostr-tools';
 
 import '@/nostr-wasm.ts';
+import '@/sentry.ts';
 
 export const VerifyWorker = {
   verifyEvent(event: NostrEvent): event is VerifiedEvent {