Update Prometheus metrics to conform to best practices

This commit is contained in:
Alex Gleason 2024-09-07 08:52:02 -05:00
parent 3fa97c0533
commit 5454942a2c
No known key found for this signature in database
GPG key ID: 7211D1F99744FBB7
8 changed files with 42 additions and 48 deletions

View file

@ -2,12 +2,11 @@ import { register } from 'prom-client';
import { AppController } from '@/app.ts'; import { AppController } from '@/app.ts';
import { DittoDB } from '@/db/DittoDB.ts'; import { DittoDB } from '@/db/DittoDB.ts';
import { dbAvailableConnectionsGauge, dbPoolSizeGauge } from '@/metrics.ts'; import { dbAvailableConnectionsGauge } from '@/metrics.ts';
/** Prometheus/OpenMetrics controller. */ /** Prometheus/OpenMetrics controller. */
export const metricsController: AppController = async (c) => { export const metricsController: AppController = async (c) => {
// Update some metrics at request time. // Update some metrics at request time.
dbPoolSizeGauge.set(DittoDB.poolSize);
dbAvailableConnectionsGauge.set(DittoDB.availableConnections); dbAvailableConnectionsGauge.set(DittoDB.availableConnections);
const metrics = await register.metrics(); const metrics = await register.metrics();

View file

@ -12,7 +12,7 @@ import {
import { AppController } from '@/app.ts'; import { AppController } from '@/app.ts';
import { Conf } from '@/config.ts'; import { Conf } from '@/config.ts';
import { relayInfoController } from '@/controllers/nostr/relay-info.ts'; import { relayInfoController } from '@/controllers/nostr/relay-info.ts';
import { relayConnectionsGauge, relayEventCounter, relayMessageCounter } from '@/metrics.ts'; import { relayConnectionsGauge, relayEventsCounter, relayMessagesCounter } from '@/metrics.ts';
import * as pipeline from '@/pipeline.ts'; import * as pipeline from '@/pipeline.ts';
import { RelayError } from '@/RelayError.ts'; import { RelayError } from '@/RelayError.ts';
import { Storages } from '@/storages.ts'; import { Storages } from '@/storages.ts';
@ -54,10 +54,10 @@ function connectStream(socket: WebSocket, ip: string | undefined) {
const result = n.json().pipe(n.clientMsg()).safeParse(e.data); const result = n.json().pipe(n.clientMsg()).safeParse(e.data);
if (result.success) { if (result.success) {
relayMessageCounter.inc({ verb: result.data[0] }); relayMessagesCounter.inc({ verb: result.data[0] });
handleMsg(result.data); handleMsg(result.data);
} else { } else {
relayMessageCounter.inc(); relayMessagesCounter.inc();
send(['NOTICE', 'Invalid message.']); send(['NOTICE', 'Invalid message.']);
} }
}; };
@ -130,7 +130,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) {
/** Handle EVENT. Store the event. */ /** Handle EVENT. Store the event. */
async function handleEvent([_, event]: NostrClientEVENT): Promise<void> { async function handleEvent([_, event]: NostrClientEVENT): Promise<void> {
relayEventCounter.inc({ kind: event.kind.toString() }); relayEventsCounter.inc({ kind: event.kind.toString() });
try { try {
// This will store it (if eligible) and run other side-effects. // This will store it (if eligible) and run other side-effects.
await pipeline.handleEvent(event, AbortSignal.timeout(1000)); await pipeline.handleEvent(event, AbortSignal.timeout(1000));

View file

@ -1,6 +1,6 @@
import { Stickynotes } from '@soapbox/stickynotes'; import { Stickynotes } from '@soapbox/stickynotes';
import { Logger } from 'kysely'; import { Logger } from 'kysely';
import { dbQueryCounter, dbQueryTimeHistogram } from '@/metrics.ts'; import { dbQueriesCounter, dbQueryDurationHistogram } from '@/metrics.ts';
/** Log the SQL for queries. */ /** Log the SQL for queries. */
export const KyselyLogger: Logger = (event) => { export const KyselyLogger: Logger = (event) => {
@ -9,8 +9,8 @@ export const KyselyLogger: Logger = (event) => {
const { query, queryDurationMillis } = event; const { query, queryDurationMillis } = event;
const { sql, parameters } = query; const { sql, parameters } = query;
dbQueryCounter.inc(); dbQueriesCounter.inc();
dbQueryTimeHistogram.observe(queryDurationMillis); dbQueryDurationHistogram.observe(queryDurationMillis);
console.debug( console.debug(
sql, sql,

View file

@ -2,7 +2,7 @@ import { Semaphore } from '@lambdalisue/async';
import { Stickynotes } from '@soapbox/stickynotes'; import { Stickynotes } from '@soapbox/stickynotes';
import { Conf } from '@/config.ts'; import { Conf } from '@/config.ts';
import { firehoseEventCounter } from '@/metrics.ts'; import { firehoseEventsCounter } from '@/metrics.ts';
import { Storages } from '@/storages.ts'; import { Storages } from '@/storages.ts';
import { nostrNow } from '@/utils.ts'; import { nostrNow } from '@/utils.ts';
@ -23,7 +23,7 @@ export async function startFirehose(): Promise<void> {
if (msg[0] === 'EVENT') { if (msg[0] === 'EVENT') {
const event = msg[2]; const event = msg[2];
console.debug(`NostrEvent<${event.kind}> ${event.id}`); console.debug(`NostrEvent<${event.kind}> ${event.id}`);
firehoseEventCounter.inc({ kind: event.kind }); firehoseEventsCounter.inc({ kind: event.kind });
sem.lock(async () => { sem.lock(async () => {
try { try {

View file

@ -1,86 +1,81 @@
import { Counter, Gauge, Histogram } from 'prom-client'; import { Counter, Gauge, Histogram } from 'prom-client';
export const httpRequestCounter = new Counter({ export const httpRequestsCounter = new Counter({
name: 'http_requests_total', name: 'ditto_http_requests_total',
help: 'Total number of HTTP requests', help: 'Total number of HTTP requests',
labelNames: ['method'], labelNames: ['method'],
}); });
export const httpResponseCounter = new Counter({ export const httpResponsesCounter = new Counter({
name: 'http_responses_total', name: 'ditto_http_responses_total',
help: 'Total number of HTTP responses', help: 'Total number of HTTP responses',
labelNames: ['method', 'path', 'status'], labelNames: ['method', 'path', 'status'],
}); });
export const streamingConnectionsGauge = new Gauge({ export const streamingConnectionsGauge = new Gauge({
name: 'streaming_connections', name: 'ditto_streaming_connections',
help: 'Number of active connections to the streaming API', help: 'Number of active connections to the streaming API',
}); });
export const fetchCounter = new Counter({ export const fetchCounter = new Counter({
name: 'fetch_total', name: 'ditto_fetch_total',
help: 'Total number of fetch requests', help: 'Total number of fetch requests',
labelNames: ['method'], labelNames: ['method'],
}); });
export const firehoseEventCounter = new Counter({ export const firehoseEventsCounter = new Counter({
name: 'firehose_events_total', name: 'ditto_firehose_events_total',
help: 'Total number of Nostr events processed by the firehose', help: 'Total number of Nostr events processed by the firehose',
labelNames: ['kind'], labelNames: ['kind'],
}); });
export const pipelineEventCounter = new Counter({ export const pipelineEventsCounter = new Counter({
name: 'pipeline_events_total', name: 'ditto_pipeline_events_total',
help: 'Total number of Nostr events processed by the pipeline', help: 'Total number of Nostr events processed by the pipeline',
labelNames: ['kind'], labelNames: ['kind'],
}); });
export const policyEventCounter = new Counter({ export const policyEventsCounter = new Counter({
name: 'policy_events_total', name: 'ditto_policy_events_total',
help: 'Total number of policy OK responses', help: 'Total number of policy OK responses',
labelNames: ['ok'], labelNames: ['ok'],
}); });
export const relayEventCounter = new Counter({ export const relayEventsCounter = new Counter({
name: 'relay_events_total', name: 'ditto_relay_events_total',
help: 'Total number of EVENT messages processed by the relay', help: 'Total number of EVENT messages processed by the relay',
labelNames: ['kind'], labelNames: ['kind'],
}); });
export const relayMessageCounter = new Counter({ export const relayMessagesCounter = new Counter({
name: 'relay_messages_total', name: 'ditto_relay_messages_total',
help: 'Total number of Nostr messages processed by the relay', help: 'Total number of Nostr messages processed by the relay',
labelNames: ['verb'], labelNames: ['verb'],
}); });
export const relayConnectionsGauge = new Gauge({ export const relayConnectionsGauge = new Gauge({
name: 'relay_connections', name: 'ditto_relay_connections',
help: 'Number of active connections to the relay', help: 'Number of active connections to the relay',
}); });
export const dbQueryCounter = new Counter({ export const dbQueriesCounter = new Counter({
name: 'db_query_total', name: 'ditto_db_queries_total',
help: 'Total number of database queries', help: 'Total number of database queries',
labelNames: ['kind'], labelNames: ['kind'],
}); });
export const dbEventCounter = new Counter({ export const dbEventsCounter = new Counter({
name: 'db_events_total', name: 'ditto_db_events_total',
help: 'Total number of database inserts', help: 'Total number of database inserts',
labelNames: ['kind'], labelNames: ['kind'],
}); });
export const dbPoolSizeGauge = new Gauge({
name: 'db_pool_size',
help: 'Number of connections in the database pool',
});
export const dbAvailableConnectionsGauge = new Gauge({ export const dbAvailableConnectionsGauge = new Gauge({
name: 'db_available_connections', name: 'ditto_db_available_connections',
help: 'Number of available connections in the database pool', help: 'Number of available connections in the database pool',
}); });
export const dbQueryTimeHistogram = new Histogram({ export const dbQueryDurationHistogram = new Histogram({
name: 'db_query_duration_ms', name: 'ditto_db_query_duration_ms',
help: 'Duration of database queries', help: 'Duration of database queries',
}); });

View file

@ -1,12 +1,12 @@
import { MiddlewareHandler } from '@hono/hono'; import { MiddlewareHandler } from '@hono/hono';
import { httpRequestCounter, httpResponseCounter } from '@/metrics.ts'; import { httpRequestsCounter, httpResponsesCounter } from '@/metrics.ts';
/** Prometheus metrics middleware that tracks HTTP requests by methods and responses by status code. */ /** Prometheus metrics middleware that tracks HTTP requests by methods and responses by status code. */
export const metricsMiddleware: MiddlewareHandler = async (c, next) => { export const metricsMiddleware: MiddlewareHandler = async (c, next) => {
// HTTP Request. // HTTP Request.
const { method } = c.req; const { method } = c.req;
httpRequestCounter.inc({ method }); httpRequestsCounter.inc({ method });
// Wait for other handlers to run. // Wait for other handlers to run.
await next(); await next();
@ -16,5 +16,5 @@ export const metricsMiddleware: MiddlewareHandler = async (c, next) => {
// Get a parameterized path name like `/posts/:id` instead of `/posts/1234`. // Get a parameterized path name like `/posts/:id` instead of `/posts/1234`.
// Tries to find actual route names first before falling back on potential middleware handlers like `app.use('*')`. // Tries to find actual route names first before falling back on potential middleware handlers like `app.use('*')`.
const path = c.req.matchedRoutes.find((r) => r.method !== 'ALL')?.path ?? c.req.routePath; const path = c.req.matchedRoutes.find((r) => r.method !== 'ALL')?.path ?? c.req.routePath;
httpResponseCounter.inc({ method, status, path }); httpResponsesCounter.inc({ method, status, path });
}; };

View file

@ -8,7 +8,7 @@ import { Conf } from '@/config.ts';
import { DittoDB } from '@/db/DittoDB.ts'; import { DittoDB } from '@/db/DittoDB.ts';
import { deleteAttachedMedia } from '@/db/unattached-media.ts'; import { deleteAttachedMedia } from '@/db/unattached-media.ts';
import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts';
import { pipelineEventCounter, policyEventCounter } from '@/metrics.ts'; import { pipelineEventsCounter, policyEventsCounter } from '@/metrics.ts';
import { RelayError } from '@/RelayError.ts'; import { RelayError } from '@/RelayError.ts';
import { AdminSigner } from '@/signers/AdminSigner.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts';
import { hydrateEvents } from '@/storages/hydrate.ts'; import { hydrateEvents } from '@/storages/hydrate.ts';
@ -40,7 +40,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise<void
if (encounterEvent(event)) return; if (encounterEvent(event)) return;
if (await existsInDB(event)) return; if (await existsInDB(event)) return;
debug(`NostrEvent<${event.kind}> ${event.id}`); debug(`NostrEvent<${event.kind}> ${event.id}`);
pipelineEventCounter.inc({ kind: event.kind }); pipelineEventsCounter.inc({ kind: event.kind });
if (event.kind !== 24133) { if (event.kind !== 24133) {
await policyFilter(event); await policyFilter(event);
@ -71,7 +71,7 @@ async function policyFilter(event: NostrEvent): Promise<void> {
try { try {
const result = await policyWorker.call(event); const result = await policyWorker.call(event);
policyEventCounter.inc({ ok: String(result[2]) }); policyEventsCounter.inc({ ok: String(result[2]) });
debug(JSON.stringify(result)); debug(JSON.stringify(result));
RelayError.assert(result); RelayError.assert(result);
} catch (e) { } catch (e) {

View file

@ -17,7 +17,7 @@ import { nip27 } from 'nostr-tools';
import { Conf } from '@/config.ts'; import { Conf } from '@/config.ts';
import { DittoDatabase } from '@/db/DittoDB.ts'; import { DittoDatabase } from '@/db/DittoDB.ts';
import { dbEventCounter } from '@/metrics.ts'; import { dbEventsCounter } from '@/metrics.ts';
import { RelayError } from '@/RelayError.ts'; import { RelayError } from '@/RelayError.ts';
import { purifyEvent } from '@/storages/hydrate.ts'; import { purifyEvent } from '@/storages/hydrate.ts';
import { isNostrId, isURL } from '@/utils.ts'; import { isNostrId, isURL } from '@/utils.ts';
@ -73,7 +73,7 @@ class EventsDB implements NStore {
async event(event: NostrEvent, opts: { signal?: AbortSignal; timeout?: number } = {}): Promise<void> { async event(event: NostrEvent, opts: { signal?: AbortSignal; timeout?: number } = {}): Promise<void> {
event = purifyEvent(event); event = purifyEvent(event);
this.console.debug('EVENT', JSON.stringify(event)); this.console.debug('EVENT', JSON.stringify(event));
dbEventCounter.inc({ kind: event.kind }); dbEventsCounter.inc({ kind: event.kind });
if (await this.isDeletedAdmin(event)) { if (await this.isDeletedAdmin(event)) {
throw new RelayError('blocked', 'event deleted by admin'); throw new RelayError('blocked', 'event deleted by admin');