mirror of
https://gitlab.com/soapbox-pub/ditto.git
synced 2025-12-06 03:19:46 +00:00
Merge remote-tracking branch 'origin/main' into feat-search-mime-type
This commit is contained in:
commit
b4184631c3
60 changed files with 1412 additions and 314 deletions
|
|
@ -46,15 +46,15 @@
|
|||
"@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1",
|
||||
"@negrel/webpush": "jsr:@negrel/webpush@^0.3.0",
|
||||
"@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0",
|
||||
"@nostrify/db": "jsr:@nostrify/db@^0.36.1",
|
||||
"@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.37.0",
|
||||
"@nostrify/db": "jsr:@nostrify/db@^0.36.2",
|
||||
"@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.38.0",
|
||||
"@nostrify/policies": "jsr:@nostrify/policies@^0.36.1",
|
||||
"@nostrify/types": "jsr:@nostrify/types@^0.36.0",
|
||||
"@scure/base": "npm:@scure/base@^1.1.6",
|
||||
"@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs",
|
||||
"@soapbox/kysely-pglite": "jsr:@soapbox/kysely-pglite@^1.0.0",
|
||||
"@soapbox/logi": "jsr:@soapbox/logi@^0.3.0",
|
||||
"@soapbox/safe-fetch": "jsr:@soapbox/safe-fetch@^2.0.0",
|
||||
"@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0",
|
||||
"@std/assert": "jsr:@std/assert@^0.225.1",
|
||||
"@std/cli": "jsr:@std/cli@^0.223.0",
|
||||
"@std/crypto": "jsr:@std/crypto@^0.224.0",
|
||||
|
|
|
|||
55
deno.lock
generated
55
deno.lock
generated
|
|
@ -30,11 +30,12 @@
|
|||
"jsr:@lambdalisue/async@^2.1.1": "2.1.1",
|
||||
"jsr:@negrel/http-ece@0.6.0": "0.6.0",
|
||||
"jsr:@negrel/webpush@0.3": "0.3.0",
|
||||
"jsr:@nostrify/db@~0.36.1": "0.36.1",
|
||||
"jsr:@nostrify/db@~0.36.2": "0.36.2",
|
||||
"jsr:@nostrify/nostrify@0.31": "0.31.0",
|
||||
"jsr:@nostrify/nostrify@0.32": "0.32.0",
|
||||
"jsr:@nostrify/nostrify@0.36": "0.36.2",
|
||||
"jsr:@nostrify/nostrify@0.37": "0.37.0",
|
||||
"jsr:@nostrify/nostrify@0.38": "0.38.0",
|
||||
"jsr:@nostrify/nostrify@~0.22.1": "0.22.5",
|
||||
"jsr:@nostrify/nostrify@~0.22.4": "0.22.4",
|
||||
"jsr:@nostrify/nostrify@~0.22.5": "0.22.5",
|
||||
|
|
@ -49,8 +50,8 @@
|
|||
"jsr:@nostrify/types@0.36": "0.36.0",
|
||||
"jsr:@nostrify/types@~0.30.1": "0.30.1",
|
||||
"jsr:@soapbox/kysely-pglite@1": "1.0.0",
|
||||
"jsr:@soapbox/logi@0.3": "0.3.0",
|
||||
"jsr:@soapbox/safe-fetch@2": "2.0.0",
|
||||
"jsr:@soapbox/stickynotes@0.4": "0.4.0",
|
||||
"jsr:@std/assert@0.223": "0.223.0",
|
||||
"jsr:@std/assert@0.224": "0.224.0",
|
||||
"jsr:@std/assert@~0.213.1": "0.213.1",
|
||||
|
|
@ -115,6 +116,7 @@
|
|||
"npm:lru-cache@^10.2.0": "10.2.2",
|
||||
"npm:lru-cache@^10.2.2": "10.2.2",
|
||||
"npm:nostr-tools@2.5.1": "2.5.1",
|
||||
"npm:nostr-tools@^2.10.4": "2.10.4",
|
||||
"npm:nostr-tools@^2.5.0": "2.5.1",
|
||||
"npm:nostr-tools@^2.7.0": "2.7.0",
|
||||
"npm:nostr-wasm@0.1": "0.1.0",
|
||||
|
|
@ -347,13 +349,13 @@
|
|||
"jsr:@std/path@0.224.0"
|
||||
]
|
||||
},
|
||||
"@nostrify/db@0.36.1": {
|
||||
"integrity": "b65b89ca6fe98d9dbcc0402b5c9c07b8430c2c91f84ba4128ff2eeed70c3d49f",
|
||||
"@nostrify/db@0.36.2": {
|
||||
"integrity": "6bf079b44fcb3ff5a85eadf9a9d4eb677fc770f1c80ad966602aa3d9dd8c88e8",
|
||||
"dependencies": [
|
||||
"jsr:@nostrify/nostrify@0.36",
|
||||
"jsr:@nostrify/types@0.35",
|
||||
"jsr:@nostrify/nostrify@0.37",
|
||||
"jsr:@nostrify/types@0.36",
|
||||
"npm:kysely@~0.27.3",
|
||||
"npm:nostr-tools@^2.7.0"
|
||||
"npm:nostr-tools@^2.10.4"
|
||||
]
|
||||
},
|
||||
"@nostrify/nostrify@0.22.4": {
|
||||
|
|
@ -469,6 +471,21 @@
|
|||
"npm:zod"
|
||||
]
|
||||
},
|
||||
"@nostrify/nostrify@0.38.0": {
|
||||
"integrity": "9ec7920057ee3a4dcbaef7e706dedea622bfdfdf0f6aac11047443f88d953deb",
|
||||
"dependencies": [
|
||||
"jsr:@nostrify/types@0.36",
|
||||
"jsr:@std/crypto",
|
||||
"jsr:@std/encoding@~0.224.1",
|
||||
"npm:@scure/base",
|
||||
"npm:@scure/bip32",
|
||||
"npm:@scure/bip39",
|
||||
"npm:lru-cache@^10.2.0",
|
||||
"npm:nostr-tools@^2.10.4",
|
||||
"npm:websocket-ts",
|
||||
"npm:zod"
|
||||
]
|
||||
},
|
||||
"@nostrify/policies@0.33.0": {
|
||||
"integrity": "c946b06d0527298b4d7c9819d142a10f522ba09eee76c37525aa4acfc5d87aee",
|
||||
"dependencies": [
|
||||
|
|
@ -525,15 +542,15 @@
|
|||
"npm:kysely@~0.27.4"
|
||||
]
|
||||
},
|
||||
"@soapbox/logi@0.3.0": {
|
||||
"integrity": "5aa5121e82422b0a1b5ec81790f75407c16c788d10af629cecef9a35d1b4c290"
|
||||
},
|
||||
"@soapbox/safe-fetch@2.0.0": {
|
||||
"integrity": "f451d686501c76a0faa058fe9d2073676282a8a42c3b93c59159eb9191f11b5f",
|
||||
"dependencies": [
|
||||
"npm:tldts@^6.1.61"
|
||||
]
|
||||
},
|
||||
"@soapbox/stickynotes@0.4.0": {
|
||||
"integrity": "60bfe61ab3d7e04bf708273b1e2d391a59534bdf29e54160e98d7afd328ca1ec"
|
||||
},
|
||||
"@std/assert@0.213.1": {
|
||||
"integrity": "24c28178b30c8e0782c18e8e94ea72b16282207569cdd10ffb9d1d26f2edebfe"
|
||||
},
|
||||
|
|
@ -1361,6 +1378,18 @@
|
|||
"whatwg-url@5.0.0"
|
||||
]
|
||||
},
|
||||
"nostr-tools@2.10.4": {
|
||||
"integrity": "sha512-biU7sk+jxHgVASfobg2T5ttxOGGSt69wEVBC51sHHOEaKAAdzHBLV/I2l9Rf61UzClhliZwNouYhqIso4a3HYg==",
|
||||
"dependencies": [
|
||||
"@noble/ciphers",
|
||||
"@noble/curves@1.2.0",
|
||||
"@noble/hashes@1.3.1",
|
||||
"@scure/base@1.1.1",
|
||||
"@scure/bip32@1.3.1",
|
||||
"@scure/bip39@1.2.1",
|
||||
"nostr-wasm"
|
||||
]
|
||||
},
|
||||
"nostr-tools@2.5.1": {
|
||||
"integrity": "sha512-bpkhGGAhdiCN0irfV+xoH3YP5CQeOXyXzUq7SYeM6D56xwTXZCPEmBlUGqFVfQidvRsoVeVxeAiOXW2c2HxoRQ==",
|
||||
"dependencies": [
|
||||
|
|
@ -2335,13 +2364,13 @@
|
|||
"jsr:@hono/hono@^4.4.6",
|
||||
"jsr:@lambdalisue/async@^2.1.1",
|
||||
"jsr:@negrel/webpush@0.3",
|
||||
"jsr:@nostrify/db@~0.36.1",
|
||||
"jsr:@nostrify/nostrify@0.37",
|
||||
"jsr:@nostrify/db@~0.36.2",
|
||||
"jsr:@nostrify/nostrify@0.38",
|
||||
"jsr:@nostrify/policies@~0.36.1",
|
||||
"jsr:@nostrify/types@0.36",
|
||||
"jsr:@soapbox/kysely-pglite@1",
|
||||
"jsr:@soapbox/logi@0.3",
|
||||
"jsr:@soapbox/safe-fetch@2",
|
||||
"jsr:@soapbox/stickynotes@0.4",
|
||||
"jsr:@std/assert@~0.225.1",
|
||||
"jsr:@std/cli@0.223",
|
||||
"jsr:@std/crypto@0.224",
|
||||
|
|
|
|||
|
|
@ -15,8 +15,7 @@ example.com {
|
|||
|
||||
handle /packs/* {
|
||||
root * /opt/ditto/public
|
||||
header Cache-Control "public, max-age=31536000, immutable"
|
||||
header Strict-Transport-Security "max-age=31536000"
|
||||
header Cache-Control "max-age=31536000, public, immutable"
|
||||
file_server
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ After=network-online.target
|
|||
[Service]
|
||||
Type=simple
|
||||
User=ditto
|
||||
SyslogIdentifier=ditto
|
||||
WorkingDirectory=/opt/ditto
|
||||
ExecStart=/usr/local/bin/deno task start
|
||||
Restart=on-failure
|
||||
|
|
|
|||
226
log.json
Normal file
226
log.json
Normal file
File diff suppressed because one or more lines are too long
45
scripts/deparameterize.ts
Normal file
45
scripts/deparameterize.ts
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
const decoder = new TextDecoder();
|
||||
|
||||
for await (const chunk of Deno.stdin.readable) {
|
||||
const text = decoder.decode(chunk);
|
||||
|
||||
const { sql, parameters } = JSON.parse(text) as { sql: string; parameters: unknown[] };
|
||||
|
||||
let result = sql;
|
||||
|
||||
for (let i = 0; i < parameters.length; i++) {
|
||||
const param = parameters[i];
|
||||
|
||||
result = result.replace(`$${i + 1}`, serializeParameter(param));
|
||||
}
|
||||
|
||||
console.log(result + ';');
|
||||
}
|
||||
|
||||
function serializeParameter(param: unknown): string {
|
||||
if (param === null) {
|
||||
return 'null';
|
||||
}
|
||||
|
||||
if (typeof param === 'string') {
|
||||
return `'${param}'`;
|
||||
}
|
||||
|
||||
if (typeof param === 'number' || typeof param === 'boolean') {
|
||||
return param.toString();
|
||||
}
|
||||
|
||||
if (param instanceof Date) {
|
||||
return `'${param.toISOString()}'`;
|
||||
}
|
||||
|
||||
if (Array.isArray(param)) {
|
||||
return `'{${param.join(',')}}'`;
|
||||
}
|
||||
|
||||
if (typeof param === 'object') {
|
||||
return `'${JSON.stringify(param)}'`;
|
||||
}
|
||||
|
||||
return JSON.stringify(param);
|
||||
}
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
import { ApplicationServer, PushMessageOptions, PushSubscriber, PushSubscription } from '@negrel/webpush';
|
||||
import { logi } from '@soapbox/logi';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
|
|
@ -20,7 +21,11 @@ export class DittoPush {
|
|||
vapidKeys: keys,
|
||||
});
|
||||
} else {
|
||||
console.warn('VAPID keys are not set. Push notifications will be disabled.');
|
||||
logi({
|
||||
level: 'warn',
|
||||
ns: 'ditto.push',
|
||||
msg: 'VAPID keys are not set. Push notifications will be disabled.',
|
||||
});
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
|
|
|||
148
src/app.ts
148
src/app.ts
|
|
@ -1,9 +1,8 @@
|
|||
import { type Context, Env as HonoEnv, Handler, Hono, Input as HonoInput, MiddlewareHandler } from '@hono/hono';
|
||||
import { every } from '@hono/hono/combine';
|
||||
import { cors } from '@hono/hono/cors';
|
||||
import { serveStatic } from '@hono/hono/deno';
|
||||
import { logger } from '@hono/hono/logger';
|
||||
import { NostrEvent, NostrSigner, NStore, NUploader } from '@nostrify/nostrify';
|
||||
import Debug from '@soapbox/stickynotes/debug';
|
||||
import { Kysely } from 'kysely';
|
||||
|
||||
import '@/startup.ts';
|
||||
|
|
@ -44,6 +43,7 @@ import { captchaController, captchaVerifyController } from '@/controllers/api/ca
|
|||
import {
|
||||
adminRelaysController,
|
||||
adminSetRelaysController,
|
||||
createCashuWalletController,
|
||||
deleteZapSplitsController,
|
||||
getZapSplitsController,
|
||||
nameRequestController,
|
||||
|
|
@ -109,7 +109,11 @@ import {
|
|||
zappedByController,
|
||||
} from '@/controllers/api/statuses.ts';
|
||||
import { streamingController } from '@/controllers/api/streaming.ts';
|
||||
import { suggestionsV1Controller, suggestionsV2Controller } from '@/controllers/api/suggestions.ts';
|
||||
import {
|
||||
localSuggestionsController,
|
||||
suggestionsV1Controller,
|
||||
suggestionsV2Controller,
|
||||
} from '@/controllers/api/suggestions.ts';
|
||||
import {
|
||||
hashtagTimelineController,
|
||||
homeTimelineController,
|
||||
|
|
@ -125,12 +129,12 @@ import { translateController } from '@/controllers/api/translate.ts';
|
|||
import { errorHandler } from '@/controllers/error.ts';
|
||||
import { frontendController } from '@/controllers/frontend.ts';
|
||||
import { metricsController } from '@/controllers/metrics.ts';
|
||||
import { indexController } from '@/controllers/site.ts';
|
||||
import { manifestController } from '@/controllers/manifest.ts';
|
||||
import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts';
|
||||
import { nostrController } from '@/controllers/well-known/nostr.ts';
|
||||
import { DittoTranslator } from '@/interfaces/DittoTranslator.ts';
|
||||
import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts';
|
||||
import { cacheControlMiddleware } from '@/middleware/cacheControlMiddleware.ts';
|
||||
import { cspMiddleware } from '@/middleware/cspMiddleware.ts';
|
||||
import { metricsMiddleware } from '@/middleware/metricsMiddleware.ts';
|
||||
import { notActivitypubMiddleware } from '@/middleware/notActivitypubMiddleware.ts';
|
||||
|
|
@ -141,6 +145,7 @@ import { signerMiddleware } from '@/middleware/signerMiddleware.ts';
|
|||
import { storeMiddleware } from '@/middleware/storeMiddleware.ts';
|
||||
import { uploaderMiddleware } from '@/middleware/uploaderMiddleware.ts';
|
||||
import { translatorMiddleware } from '@/middleware/translatorMiddleware.ts';
|
||||
import { logiMiddleware } from '@/middleware/logiMiddleware.ts';
|
||||
|
||||
export interface AppEnv extends HonoEnv {
|
||||
Variables: {
|
||||
|
|
@ -165,26 +170,29 @@ export interface AppEnv extends HonoEnv {
|
|||
|
||||
type AppContext = Context<AppEnv>;
|
||||
type AppMiddleware = MiddlewareHandler<AppEnv>;
|
||||
type AppController = Handler<AppEnv, any, HonoInput, Response | Promise<Response>>;
|
||||
type AppController<P extends string = any> = Handler<AppEnv, P, HonoInput, Response | Promise<Response>>;
|
||||
|
||||
const app = new Hono<AppEnv>({ strict: false });
|
||||
|
||||
const debug = Debug('ditto:http');
|
||||
|
||||
/** User-provided files in the gitignored `public/` directory. */
|
||||
const publicFiles = serveStatic({ root: './public/' });
|
||||
/** Static files provided by the Ditto repo, checked into git. */
|
||||
const staticFiles = serveStatic({ root: './static/' });
|
||||
|
||||
app.use('*', rateLimitMiddleware(300, Time.minutes(5)));
|
||||
app.use('*', cacheControlMiddleware({ noStore: true }));
|
||||
|
||||
app.use('/api/*', metricsMiddleware, paginationMiddleware, logger(debug));
|
||||
app.use('/.well-known/*', metricsMiddleware, logger(debug));
|
||||
app.use('/nodeinfo/*', metricsMiddleware, logger(debug));
|
||||
app.use('/oauth/*', metricsMiddleware, logger(debug));
|
||||
const ratelimit = every(
|
||||
rateLimitMiddleware(30, Time.seconds(5), false),
|
||||
rateLimitMiddleware(300, Time.minutes(5), false),
|
||||
);
|
||||
|
||||
app.get('/api/v1/streaming', metricsMiddleware, streamingController);
|
||||
app.get('/relay', metricsMiddleware, relayController);
|
||||
app.use('/api/*', metricsMiddleware, ratelimit, paginationMiddleware, logiMiddleware);
|
||||
app.use('/.well-known/*', metricsMiddleware, ratelimit, logiMiddleware);
|
||||
app.use('/nodeinfo/*', metricsMiddleware, ratelimit, logiMiddleware);
|
||||
app.use('/oauth/*', metricsMiddleware, ratelimit, logiMiddleware);
|
||||
|
||||
app.get('/api/v1/streaming', metricsMiddleware, ratelimit, streamingController);
|
||||
app.get('/relay', metricsMiddleware, ratelimit, relayController);
|
||||
|
||||
app.use(
|
||||
'*',
|
||||
|
|
@ -198,15 +206,39 @@ app.use(
|
|||
|
||||
app.get('/metrics', metricsController);
|
||||
|
||||
app.get('/.well-known/nodeinfo', nodeInfoController);
|
||||
app.get(
|
||||
'/.well-known/nodeinfo',
|
||||
cacheControlMiddleware({ maxAge: 300, staleWhileRevalidate: 300, staleIfError: 21600, public: true }),
|
||||
nodeInfoController,
|
||||
);
|
||||
app.get('/.well-known/nostr.json', nostrController);
|
||||
|
||||
app.get('/nodeinfo/:version', nodeInfoSchemaController);
|
||||
app.get('/manifest.webmanifest', manifestController);
|
||||
app.get(
|
||||
'/nodeinfo/:version',
|
||||
cacheControlMiddleware({ maxAge: 300, staleWhileRevalidate: 300, staleIfError: 21600, public: true }),
|
||||
nodeInfoSchemaController,
|
||||
);
|
||||
app.get(
|
||||
'/manifest.webmanifest',
|
||||
cacheControlMiddleware({ maxAge: 5, staleWhileRevalidate: 5, staleIfError: 21600, public: true }),
|
||||
manifestController,
|
||||
);
|
||||
|
||||
app.get('/api/v1/instance', instanceV1Controller);
|
||||
app.get('/api/v2/instance', instanceV2Controller);
|
||||
app.get('/api/v1/instance/extended_description', instanceDescriptionController);
|
||||
app.get(
|
||||
'/api/v1/instance',
|
||||
cacheControlMiddleware({ maxAge: 5, staleWhileRevalidate: 5, staleIfError: 21600, public: true }),
|
||||
instanceV1Controller,
|
||||
);
|
||||
app.get(
|
||||
'/api/v2/instance',
|
||||
cacheControlMiddleware({ maxAge: 5, staleWhileRevalidate: 5, staleIfError: 21600, public: true }),
|
||||
instanceV2Controller,
|
||||
);
|
||||
app.get(
|
||||
'/api/v1/instance/extended_description',
|
||||
cacheControlMiddleware({ maxAge: 5, staleWhileRevalidate: 5, staleIfError: 21600, public: true }),
|
||||
instanceDescriptionController,
|
||||
);
|
||||
|
||||
app.get('/api/v1/apps/verify_credentials', appCredentialsController);
|
||||
app.post('/api/v1/apps', createAppController);
|
||||
|
|
@ -295,15 +327,32 @@ app.get('/api/v1/preferences', preferencesController);
|
|||
app.get('/api/v1/search', searchController);
|
||||
app.get('/api/v2/search', searchController);
|
||||
|
||||
app.get('/api/pleroma/frontend_configurations', frontendConfigController);
|
||||
app.get(
|
||||
'/api/pleroma/frontend_configurations',
|
||||
cacheControlMiddleware({ maxAge: 5, staleWhileRevalidate: 5, staleIfError: 21600, public: true }),
|
||||
frontendConfigController,
|
||||
);
|
||||
|
||||
app.get('/api/v1/trends/statuses', rateLimitMiddleware(8, Time.seconds(30)), trendingStatusesController);
|
||||
app.get('/api/v1/trends/links', trendingLinksController);
|
||||
app.get('/api/v1/trends/tags', trendingTagsController);
|
||||
app.get('/api/v1/trends', trendingTagsController);
|
||||
app.get(
|
||||
'/api/v1/trends/links',
|
||||
cacheControlMiddleware({ maxAge: 300, staleWhileRevalidate: 300, staleIfError: 21600, public: true }),
|
||||
trendingLinksController,
|
||||
);
|
||||
app.get(
|
||||
'/api/v1/trends/tags',
|
||||
cacheControlMiddleware({ maxAge: 300, staleWhileRevalidate: 300, staleIfError: 21600, public: true }),
|
||||
trendingTagsController,
|
||||
);
|
||||
app.get(
|
||||
'/api/v1/trends',
|
||||
cacheControlMiddleware({ maxAge: 300, staleWhileRevalidate: 300, staleIfError: 21600, public: true }),
|
||||
trendingTagsController,
|
||||
);
|
||||
|
||||
app.get('/api/v1/suggestions', suggestionsV1Controller);
|
||||
app.get('/api/v2/suggestions', suggestionsV2Controller);
|
||||
app.get('/api/v2/ditto/suggestions/local', localSuggestionsController);
|
||||
|
||||
app.get('/api/v1/notifications', rateLimitMiddleware(8, Time.seconds(30)), requireSigner, notificationsController);
|
||||
app.get('/api/v1/notifications/:id', requireSigner, notificationController);
|
||||
|
|
@ -344,7 +393,11 @@ app.post(
|
|||
captchaVerifyController,
|
||||
);
|
||||
|
||||
app.get('/api/v1/ditto/zap_splits', getZapSplitsController);
|
||||
app.get(
|
||||
'/api/v1/ditto/zap_splits',
|
||||
cacheControlMiddleware({ maxAge: 5, staleWhileRevalidate: 5, public: true }),
|
||||
getZapSplitsController,
|
||||
);
|
||||
app.get('/api/v1/ditto/:id{[0-9a-f]{64}}/zap_splits', statusZapSplitsController);
|
||||
|
||||
app.put('/api/v1/admin/ditto/zap_splits', requireRole('admin'), updateZapSplitsController);
|
||||
|
|
@ -353,6 +406,8 @@ app.delete('/api/v1/admin/ditto/zap_splits', requireRole('admin'), deleteZapSpli
|
|||
app.post('/api/v1/ditto/zap', requireSigner, zapController);
|
||||
app.get('/api/v1/ditto/statuses/:id{[0-9a-f]{64}}/zapped_by', zappedByController);
|
||||
|
||||
app.post('/api/v1/ditto/wallet/create', requireSigner, createCashuWalletController);
|
||||
|
||||
app.post('/api/v1/reports', requireSigner, reportController);
|
||||
app.get('/api/v1/admin/reports', requireSigner, requireRole('admin'), adminReportsController);
|
||||
app.get('/api/v1/admin/reports/:id{[0-9a-f]{64}}', requireSigner, requireRole('admin'), adminReportController);
|
||||
|
|
@ -408,22 +463,39 @@ app.get('/timeline/*', frontendController);
|
|||
|
||||
// Known static file routes
|
||||
app.get('/sw.js', publicFiles);
|
||||
app.get('/favicon.ico', publicFiles, staticFiles);
|
||||
app.get('/images/*', publicFiles, staticFiles);
|
||||
app.get('/instance/*', publicFiles);
|
||||
app.get(
|
||||
'/favicon.ico',
|
||||
cacheControlMiddleware({ maxAge: 5, staleWhileRevalidate: 5, staleIfError: 21600, public: true }),
|
||||
publicFiles,
|
||||
staticFiles,
|
||||
);
|
||||
app.get(
|
||||
'/images/*',
|
||||
cacheControlMiddleware({ maxAge: 5, staleWhileRevalidate: 5, staleIfError: 21600, public: true }),
|
||||
publicFiles,
|
||||
staticFiles,
|
||||
);
|
||||
app.get(
|
||||
'/instance/*',
|
||||
cacheControlMiddleware({ maxAge: 5, staleWhileRevalidate: 5, staleIfError: 21600, public: true }),
|
||||
publicFiles,
|
||||
);
|
||||
|
||||
// Packs contains immutable static files
|
||||
app.get('/packs/*', async (c, next) => {
|
||||
c.header('Cache-Control', 'public, max-age=31536000, immutable');
|
||||
c.header('Strict-Transport-Security', '"max-age=31536000" always');
|
||||
await next();
|
||||
}, publicFiles);
|
||||
app.get(
|
||||
'/packs/*',
|
||||
cacheControlMiddleware({
|
||||
maxAge: 31536000,
|
||||
staleWhileRevalidate: 86400,
|
||||
staleIfError: 21600,
|
||||
public: true,
|
||||
immutable: true,
|
||||
}),
|
||||
publicFiles,
|
||||
);
|
||||
|
||||
// Site index
|
||||
app.get('/', frontendController, indexController);
|
||||
|
||||
// Fallback
|
||||
app.get('*', publicFiles, staticFiles, frontendController);
|
||||
app.get('/', ratelimit, frontendController);
|
||||
app.get('*', publicFiles, staticFiles, ratelimit, frontendController);
|
||||
|
||||
app.onError(errorHandler);
|
||||
|
||||
|
|
|
|||
|
|
@ -36,6 +36,10 @@ class Conf {
|
|||
static get port(): number {
|
||||
return parseInt(Deno.env.get('PORT') || '4036');
|
||||
}
|
||||
/** IP addresses not affected by rate limiting. */
|
||||
static get ipWhitelist(): string[] {
|
||||
return Deno.env.get('IP_WHITELIST')?.split(',') || [];
|
||||
}
|
||||
/** Relay URL to the Ditto server's relay. */
|
||||
static get relay(): `wss://${string}` | `ws://${string}` {
|
||||
const { protocol, host } = Conf.url;
|
||||
|
|
@ -248,7 +252,7 @@ class Conf {
|
|||
}
|
||||
/** Number of events the firehose is allowed to process at one time before they have to wait in a queue. */
|
||||
static get firehoseConcurrency(): number {
|
||||
return Math.ceil(Number(Deno.env.get('FIREHOSE_CONCURRENCY') ?? (Conf.pg.poolSize * 0.25)));
|
||||
return Math.ceil(Number(Deno.env.get('FIREHOSE_CONCURRENCY') ?? 1));
|
||||
}
|
||||
/** Nostr event kinds of events to listen for on the firehose. */
|
||||
static get firehoseKinds(): number[] {
|
||||
|
|
@ -261,20 +265,12 @@ class Conf {
|
|||
* This would make Nostr events inserted directly into Postgres available to the streaming API and relay.
|
||||
*/
|
||||
static get notifyEnabled(): boolean {
|
||||
return optionalBooleanSchema.parse(Deno.env.get('NOTIFY_ENABLED')) ?? false;
|
||||
return optionalBooleanSchema.parse(Deno.env.get('NOTIFY_ENABLED')) ?? true;
|
||||
}
|
||||
/** Whether to enable Ditto cron jobs. */
|
||||
static get cronEnabled(): boolean {
|
||||
return optionalBooleanSchema.parse(Deno.env.get('CRON_ENABLED')) ?? true;
|
||||
}
|
||||
/** Crawler User-Agent regex to render link previews to. */
|
||||
static get crawlerRegex(): RegExp {
|
||||
return new RegExp(
|
||||
Deno.env.get('CRAWLER_REGEX') ||
|
||||
'googlebot|bingbot|yandex|baiduspider|twitterbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterestbot|slackbot|vkShare|W3C_Validator|whatsapp|mastodon|pleroma|Discordbot|AhrefsBot|SEMrushBot|MJ12bot|SeekportBot|Synapse|Matrix',
|
||||
'i',
|
||||
);
|
||||
}
|
||||
/** User-Agent to use when fetching link previews. Pretend to be Facebook by default. */
|
||||
static get fetchUserAgent(): string {
|
||||
return Deno.env.get('DITTO_FETCH_USER_AGENT') ?? 'facebookexternalhit';
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { NostrFilter } from '@nostrify/nostrify';
|
||||
import { logi } from '@soapbox/logi';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { type AppController } from '@/app.ts';
|
||||
|
|
@ -9,6 +10,7 @@ import { hydrateEvents } from '@/storages/hydrate.ts';
|
|||
import { createAdminEvent, paginated, parseBody, updateEventInfo, updateUser } from '@/utils/api.ts';
|
||||
import { renderNameRequest } from '@/views/ditto.ts';
|
||||
import { renderAdminAccount, renderAdminAccountFromPubkey } from '@/views/mastodon/admin-accounts.ts';
|
||||
import { errorJson } from '@/utils/log.ts';
|
||||
|
||||
const adminAccountQuerySchema = z.object({
|
||||
local: booleanParamSchema.optional(),
|
||||
|
|
@ -148,11 +150,15 @@ const adminActionController: AppController = async (c) => {
|
|||
if (data.type === 'suspend') {
|
||||
n.disabled = true;
|
||||
n.suspended = true;
|
||||
store.remove([{ authors: [authorId] }]).catch(console.warn);
|
||||
store.remove([{ authors: [authorId] }]).catch((e: unknown) => {
|
||||
logi({ level: 'error', ns: 'ditto.api.admin.account.action', type: data.type, error: errorJson(e) });
|
||||
});
|
||||
}
|
||||
if (data.type === 'revoke_name') {
|
||||
n.revoke_name = true;
|
||||
store.remove([{ kinds: [30360], authors: [Conf.pubkey], '#p': [authorId] }]).catch(console.warn);
|
||||
store.remove([{ kinds: [30360], authors: [Conf.pubkey], '#p': [authorId] }]).catch((e: unknown) => {
|
||||
logi({ level: 'error', ns: 'ditto.api.admin.account.action', type: data.type, error: errorJson(e) });
|
||||
});
|
||||
}
|
||||
|
||||
await updateUser(authorId, n, c);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify';
|
||||
import { bytesToString } from '@scure/base';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { AppController } from '@/app.ts';
|
||||
|
|
@ -19,6 +20,7 @@ import { accountFromPubkey } from '@/views/mastodon/accounts.ts';
|
|||
import { renderAccount } from '@/views/mastodon/accounts.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
import { updateListAdminEvent } from '@/utils/api.ts';
|
||||
import { generateSecretKey } from 'nostr-tools';
|
||||
|
||||
const markerSchema = z.enum(['read', 'write']);
|
||||
|
||||
|
|
@ -342,3 +344,63 @@ export const updateInstanceController: AppController = async (c) => {
|
|||
|
||||
return c.json(204);
|
||||
};
|
||||
|
||||
const createCashuWalletSchema = z.object({
|
||||
description: z.string(),
|
||||
relays: z.array(z.string().url()),
|
||||
mints: z.array(z.string().url()).nonempty(), // must contain at least one item
|
||||
name: z.string(),
|
||||
});
|
||||
|
||||
export const createCashuWalletController: AppController = async (c) => {
|
||||
const signer = c.get('signer')!;
|
||||
const store = c.get('store');
|
||||
const pubkey = await signer.getPublicKey();
|
||||
const body = await parseBody(c.req.raw);
|
||||
const { signal } = c.req.raw;
|
||||
const result = createCashuWalletSchema.safeParse(body);
|
||||
|
||||
if (!result.success) {
|
||||
return c.json({ error: 'Bad request', schema: result.error }, 400);
|
||||
}
|
||||
|
||||
const [event] = await store.query([{ authors: [pubkey], kinds: [37375] }], { signal });
|
||||
if (event) {
|
||||
return c.json({ error: 'You already have a wallet 😏', schema: result.error }, 400);
|
||||
}
|
||||
|
||||
const { description, relays, mints, name } = result.data;
|
||||
relays.push(Conf.relay);
|
||||
|
||||
const tags: string[][] = [];
|
||||
|
||||
tags.push(['d', Math.random().toString(36).substring(3)]);
|
||||
tags.push(['name', name]);
|
||||
tags.push(['description', description]);
|
||||
tags.push(['unit', 'sat']);
|
||||
|
||||
for (const mint of new Set(mints)) {
|
||||
tags.push(['mint', mint]);
|
||||
}
|
||||
|
||||
for (const relay of new Set(relays)) {
|
||||
tags.push(['relay', relay]);
|
||||
}
|
||||
|
||||
const sk = generateSecretKey();
|
||||
const privkey = bytesToString('hex', sk);
|
||||
|
||||
const contentTags = [
|
||||
['privkey', privkey],
|
||||
];
|
||||
const encryptedContentTags = await signer.nip44?.encrypt(pubkey, JSON.stringify(contentTags));
|
||||
|
||||
// Wallet
|
||||
await createEvent({
|
||||
kind: 37375,
|
||||
content: encryptedContentTags,
|
||||
tags,
|
||||
}, c);
|
||||
|
||||
return c.json(201);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,6 +1,13 @@
|
|||
import { type Context } from '@hono/hono';
|
||||
import { Handler } from '@hono/hono';
|
||||
|
||||
const emptyArrayController = (c: Context) => c.json([]);
|
||||
const notImplementedController = (c: Context) => Promise.resolve(c.json({ error: 'Not implemented' }, 404));
|
||||
const emptyArrayController: Handler = (c) => {
|
||||
c.header('Cache-Control', 'max-age=300, public, stale-while-revalidate=60');
|
||||
return c.json([]);
|
||||
};
|
||||
|
||||
const notImplementedController: Handler = (c) => {
|
||||
c.header('Cache-Control', 'max-age=300, public, stale-while-revalidate=60');
|
||||
return c.json({ error: 'Not implemented' }, 404);
|
||||
};
|
||||
|
||||
export { emptyArrayController, notImplementedController };
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ const instanceV2Controller: AppController = async (c) => {
|
|||
},
|
||||
statuses: {
|
||||
max_characters: Conf.postCharLimit,
|
||||
max_media_attachments: 4,
|
||||
max_media_attachments: 20,
|
||||
characters_reserved_per_url: 23,
|
||||
},
|
||||
media_attachments: {
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
import { logi } from '@soapbox/logi';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { AppController } from '@/app.ts';
|
||||
import { dittoUploads } from '@/DittoUploads.ts';
|
||||
import { fileSchema } from '@/schema.ts';
|
||||
import { parseBody } from '@/utils/api.ts';
|
||||
import { renderAttachment } from '@/views/mastodon/attachments.ts';
|
||||
import { errorJson } from '@/utils/log.ts';
|
||||
import { uploadFile } from '@/utils/upload.ts';
|
||||
import { dittoUploads } from '@/DittoUploads.ts';
|
||||
|
||||
const mediaBodySchema = z.object({
|
||||
file: fileSchema,
|
||||
|
|
@ -32,7 +34,7 @@ const mediaController: AppController = async (c) => {
|
|||
const media = await uploadFile(c, file, { pubkey, description }, signal);
|
||||
return c.json(renderAttachment(media));
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
logi({ level: 'error', ns: 'ditto.api.media', error: errorJson(e) });
|
||||
return c.json({ error: 'Failed to upload file.' }, 500);
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import TTLCache from '@isaacs/ttlcache';
|
||||
import { NostrEvent, NostrFilter } from '@nostrify/nostrify';
|
||||
import { Stickynotes } from '@soapbox/stickynotes';
|
||||
import { logi } from '@soapbox/logi';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { type AppController } from '@/app.ts';
|
||||
|
|
@ -15,13 +15,12 @@ import { getFeedPubkeys } from '@/queries.ts';
|
|||
import { hydrateEvents } from '@/storages/hydrate.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
import { getTokenHash } from '@/utils/auth.ts';
|
||||
import { errorJson } from '@/utils/log.ts';
|
||||
import { bech32ToPubkey, Time } from '@/utils.ts';
|
||||
import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts';
|
||||
import { renderNotification } from '@/views/mastodon/notifications.ts';
|
||||
import { HTTPException } from '@hono/hono/http-exception';
|
||||
|
||||
const console = new Stickynotes('ditto:streaming');
|
||||
|
||||
/**
|
||||
* Streaming timelines/categories.
|
||||
* https://docs.joinmastodon.org/methods/streaming/#streams
|
||||
|
|
@ -101,7 +100,6 @@ const streamingController: AppController = async (c) => {
|
|||
|
||||
function send(e: StreamingEvent) {
|
||||
if (socket.readyState === WebSocket.OPEN) {
|
||||
console.debug('send', e.event, e.payload);
|
||||
streamingServerMessagesCounter.inc();
|
||||
socket.send(JSON.stringify(e));
|
||||
}
|
||||
|
|
@ -130,7 +128,7 @@ const streamingController: AppController = async (c) => {
|
|||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.debug('streaming error:', e);
|
||||
logi({ level: 'error', ns: 'ditto.streaming', msg: 'Error in streaming', error: errorJson(e) });
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import { matchFilter } from 'nostr-tools';
|
|||
import { AppContext, AppController } from '@/app.ts';
|
||||
import { Conf } from '@/config.ts';
|
||||
import { hydrateEvents } from '@/storages/hydrate.ts';
|
||||
import { paginatedList } from '@/utils/api.ts';
|
||||
import { paginated, paginatedList } from '@/utils/api.ts';
|
||||
import { getTagSet } from '@/utils/tags.ts';
|
||||
import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts';
|
||||
|
||||
|
|
@ -87,3 +87,41 @@ async function renderV2Suggestions(c: AppContext, params: { offset: number; limi
|
|||
};
|
||||
}));
|
||||
}
|
||||
|
||||
export const localSuggestionsController: AppController = async (c) => {
|
||||
const signal = c.req.raw.signal;
|
||||
const params = c.get('pagination');
|
||||
const store = c.get('store');
|
||||
|
||||
const grants = await store.query(
|
||||
[{ kinds: [30360], authors: [Conf.pubkey], ...params }],
|
||||
{ signal },
|
||||
);
|
||||
|
||||
const pubkeys = new Set<string>();
|
||||
|
||||
for (const grant of grants) {
|
||||
const pubkey = grant.tags.find(([name]) => name === 'p')?.[1];
|
||||
if (pubkey) {
|
||||
pubkeys.add(pubkey);
|
||||
}
|
||||
}
|
||||
|
||||
const profiles = await store.query(
|
||||
[{ kinds: [0], authors: [...pubkeys], search: `domain:${Conf.url.host}`, ...params }],
|
||||
{ signal },
|
||||
)
|
||||
.then((events) => hydrateEvents({ store, events, signal }));
|
||||
|
||||
const suggestions = (await Promise.all([...pubkeys].map(async (pubkey) => {
|
||||
const profile = profiles.find((event) => event.pubkey === pubkey);
|
||||
if (!profile) return;
|
||||
|
||||
return {
|
||||
source: 'global',
|
||||
account: await renderAccount(profile),
|
||||
};
|
||||
}))).filter(Boolean);
|
||||
|
||||
return paginated(c, grants, suggestions);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { NostrEvent, NostrFilter, NStore } from '@nostrify/nostrify';
|
||||
import { logi } from '@soapbox/logi';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { AppController } from '@/app.ts';
|
||||
|
|
@ -9,10 +10,17 @@ import { Storages } from '@/storages.ts';
|
|||
import { generateDateRange, Time } from '@/utils/time.ts';
|
||||
import { unfurlCardCached } from '@/utils/unfurl.ts';
|
||||
import { paginated } from '@/utils/api.ts';
|
||||
import { errorJson } from '@/utils/log.ts';
|
||||
import { renderStatus } from '@/views/mastodon/statuses.ts';
|
||||
|
||||
let trendingHashtagsCache = getTrendingHashtags().catch((e) => {
|
||||
console.error(`Failed to get trending hashtags: ${e}`);
|
||||
let trendingHashtagsCache = getTrendingHashtags().catch((e: unknown) => {
|
||||
logi({
|
||||
level: 'error',
|
||||
ns: 'ditto.trends.api',
|
||||
type: 'tags',
|
||||
msg: 'Failed to get trending hashtags',
|
||||
error: errorJson(e),
|
||||
});
|
||||
return Promise.resolve([]);
|
||||
});
|
||||
|
||||
|
|
@ -21,7 +29,13 @@ Deno.cron('update trending hashtags cache', '35 * * * *', async () => {
|
|||
const trends = await getTrendingHashtags();
|
||||
trendingHashtagsCache = Promise.resolve(trends);
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
logi({
|
||||
level: 'error',
|
||||
ns: 'ditto.trends.api',
|
||||
type: 'tags',
|
||||
msg: 'Failed to get trending hashtags',
|
||||
error: errorJson(e),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
|
|
@ -57,8 +71,14 @@ async function getTrendingHashtags() {
|
|||
});
|
||||
}
|
||||
|
||||
let trendingLinksCache = getTrendingLinks().catch((e) => {
|
||||
console.error(`Failed to get trending links: ${e}`);
|
||||
let trendingLinksCache = getTrendingLinks().catch((e: unknown) => {
|
||||
logi({
|
||||
level: 'error',
|
||||
ns: 'ditto.trends.api',
|
||||
type: 'links',
|
||||
msg: 'Failed to get trending links',
|
||||
error: errorJson(e),
|
||||
});
|
||||
return Promise.resolve([]);
|
||||
});
|
||||
|
||||
|
|
@ -67,7 +87,13 @@ Deno.cron('update trending links cache', '50 * * * *', async () => {
|
|||
const trends = await getTrendingLinks();
|
||||
trendingLinksCache = Promise.resolve(trends);
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
logi({
|
||||
level: 'error',
|
||||
ns: 'ditto.trends.api',
|
||||
type: 'links',
|
||||
msg: 'Failed to get trending links',
|
||||
error: errorJson(e),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,12 @@
|
|||
import { ErrorHandler } from '@hono/hono';
|
||||
import { HTTPException } from '@hono/hono/http-exception';
|
||||
import { logi } from '@soapbox/logi';
|
||||
|
||||
import { errorJson } from '@/utils/log.ts';
|
||||
|
||||
export const errorHandler: ErrorHandler = (err, c) => {
|
||||
c.header('Cache-Control', 'no-store');
|
||||
|
||||
if (err instanceof HTTPException) {
|
||||
if (err.res) {
|
||||
return err.res;
|
||||
|
|
@ -14,7 +19,7 @@ export const errorHandler: ErrorHandler = (err, c) => {
|
|||
return c.json({ error: 'The server was unable to respond in a timely manner' }, 500);
|
||||
}
|
||||
|
||||
console.error(err);
|
||||
logi({ level: 'error', ns: 'ditto.http', msg: 'Unhandled error', error: errorJson(err) });
|
||||
|
||||
return c.json({ error: 'Something went wrong' }, 500);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,31 +1,25 @@
|
|||
import { logi } from '@soapbox/logi';
|
||||
|
||||
import { AppMiddleware } from '@/app.ts';
|
||||
import { Conf } from '@/config.ts';
|
||||
import { Stickynotes } from '@soapbox/stickynotes';
|
||||
import { Storages } from '@/storages.ts';
|
||||
import { getPathParams, MetadataEntities } from '@/utils/og-metadata.ts';
|
||||
import { getInstanceMetadata } from '@/utils/instance.ts';
|
||||
import { errorJson } from '@/utils/log.ts';
|
||||
import { lookupPubkey } from '@/utils/lookup.ts';
|
||||
import { renderMetadata } from '@/views/meta.ts';
|
||||
import { getAuthor, getEvent } from '@/queries.ts';
|
||||
import { renderStatus } from '@/views/mastodon/statuses.ts';
|
||||
import { renderAccount } from '@/views/mastodon/accounts.ts';
|
||||
|
||||
const console = new Stickynotes('ditto:frontend');
|
||||
|
||||
/** Placeholder to find & replace with metadata. */
|
||||
const META_PLACEHOLDER = '<!--server-generated-meta-->' as const;
|
||||
|
||||
export const frontendController: AppMiddleware = async (c, next) => {
|
||||
export const frontendController: AppMiddleware = async (c) => {
|
||||
c.header('Cache-Control', 'max-age=86400, s-maxage=30, public, stale-if-error=604800');
|
||||
|
||||
try {
|
||||
const content = await Deno.readTextFile(new URL('../../public/index.html', import.meta.url));
|
||||
|
||||
const ua = c.req.header('User-Agent');
|
||||
console.debug('ua', ua);
|
||||
|
||||
if (!Conf.crawlerRegex.test(ua ?? '')) {
|
||||
return c.html(content);
|
||||
}
|
||||
|
||||
if (content.includes(META_PLACEHOLDER)) {
|
||||
const params = getPathParams(c.req.path);
|
||||
try {
|
||||
|
|
@ -33,14 +27,13 @@ export const frontendController: AppMiddleware = async (c, next) => {
|
|||
const meta = renderMetadata(c.req.url, entities);
|
||||
return c.html(content.replace(META_PLACEHOLDER, meta));
|
||||
} catch (e) {
|
||||
console.log(`Error building meta tags: ${e}`);
|
||||
logi({ level: 'error', ns: 'ditto.frontend', msg: 'Error building meta tags', error: errorJson(e) });
|
||||
return c.html(content);
|
||||
}
|
||||
}
|
||||
return c.html(content);
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
await next();
|
||||
} catch {
|
||||
return c.notFound();
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { Stickynotes } from '@soapbox/stickynotes';
|
||||
import TTLCache from '@isaacs/ttlcache';
|
||||
import { logi } from '@soapbox/logi';
|
||||
import { JsonValue } from '@std/json';
|
||||
import {
|
||||
NKinds,
|
||||
NostrClientCLOSE,
|
||||
NostrClientCOUNT,
|
||||
NostrClientEVENT,
|
||||
|
|
@ -17,22 +18,34 @@ import { relayConnectionsGauge, relayEventsCounter, relayMessagesCounter } from
|
|||
import * as pipeline from '@/pipeline.ts';
|
||||
import { RelayError } from '@/RelayError.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
import { Time } from '@/utils/time.ts';
|
||||
import { errorJson } from '@/utils/log.ts';
|
||||
import { purifyEvent } from '@/utils/purify.ts';
|
||||
import { MemoryRateLimiter } from '@/utils/ratelimiter/MemoryRateLimiter.ts';
|
||||
import { MultiRateLimiter } from '@/utils/ratelimiter/MultiRateLimiter.ts';
|
||||
import { RateLimiter } from '@/utils/ratelimiter/types.ts';
|
||||
import { Time } from '@/utils/time.ts';
|
||||
|
||||
/** Limit of initial events returned for a subscription. */
|
||||
const FILTER_LIMIT = 100;
|
||||
|
||||
const LIMITER_WINDOW = Time.minutes(1);
|
||||
const LIMITER_LIMIT = 300;
|
||||
|
||||
const limiter = new TTLCache<string, number>();
|
||||
const limiters = {
|
||||
msg: new MemoryRateLimiter({ limit: 300, window: Time.minutes(1) }),
|
||||
req: new MultiRateLimiter([
|
||||
new MemoryRateLimiter({ limit: 15, window: Time.seconds(5) }),
|
||||
new MemoryRateLimiter({ limit: 300, window: Time.minutes(5) }),
|
||||
new MemoryRateLimiter({ limit: 1000, window: Time.hours(1) }),
|
||||
]),
|
||||
event: new MultiRateLimiter([
|
||||
new MemoryRateLimiter({ limit: 10, window: Time.seconds(10) }),
|
||||
new MemoryRateLimiter({ limit: 100, window: Time.hours(1) }),
|
||||
new MemoryRateLimiter({ limit: 500, window: Time.days(1) }),
|
||||
]),
|
||||
ephemeral: new MemoryRateLimiter({ limit: 30, window: Time.seconds(10) }),
|
||||
};
|
||||
|
||||
/** Connections for metrics purposes. */
|
||||
const connections = new Set<WebSocket>();
|
||||
|
||||
const console = new Stickynotes('ditto:relay');
|
||||
|
||||
/** Set up the Websocket connection. */
|
||||
function connectStream(socket: WebSocket, ip: string | undefined) {
|
||||
const controllers = new Map<string, AbortController>();
|
||||
|
|
@ -43,15 +56,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) {
|
|||
};
|
||||
|
||||
socket.onmessage = (e) => {
|
||||
if (ip) {
|
||||
const count = limiter.get(ip) ?? 0;
|
||||
limiter.set(ip, count + 1, { ttl: LIMITER_WINDOW });
|
||||
|
||||
if (count > LIMITER_LIMIT) {
|
||||
socket.close(1008, 'Rate limit exceeded');
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (rateLimited(limiters.msg)) return;
|
||||
|
||||
if (typeof e.data !== 'string') {
|
||||
socket.close(1003, 'Invalid message');
|
||||
|
|
@ -60,6 +65,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) {
|
|||
|
||||
const result = n.json().pipe(n.clientMsg()).safeParse(e.data);
|
||||
if (result.success) {
|
||||
logi({ level: 'trace', ns: 'ditto.relay.message', data: result.data as JsonValue });
|
||||
relayMessagesCounter.inc({ verb: result.data[0] });
|
||||
handleMsg(result.data);
|
||||
} else {
|
||||
|
|
@ -77,6 +83,19 @@ function connectStream(socket: WebSocket, ip: string | undefined) {
|
|||
}
|
||||
};
|
||||
|
||||
function rateLimited(limiter: Pick<RateLimiter, 'client'>): boolean {
|
||||
if (ip) {
|
||||
const client = limiter.client(ip);
|
||||
try {
|
||||
client.hit();
|
||||
} catch {
|
||||
socket.close(1008, 'Rate limit exceeded');
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Handle client message. */
|
||||
function handleMsg(msg: NostrClientMsg) {
|
||||
switch (msg[0]) {
|
||||
|
|
@ -97,6 +116,8 @@ function connectStream(socket: WebSocket, ip: string | undefined) {
|
|||
|
||||
/** Handle REQ. Start a subscription. */
|
||||
async function handleReq([_, subId, ...filters]: NostrClientREQ): Promise<void> {
|
||||
if (rateLimited(limiters.req)) return;
|
||||
|
||||
const controller = new AbortController();
|
||||
controllers.get(subId)?.abort();
|
||||
controllers.set(subId, controller);
|
||||
|
|
@ -128,7 +149,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) {
|
|||
send(['EVENT', subId, msg[2]]);
|
||||
}
|
||||
}
|
||||
} catch (_e) {
|
||||
} catch {
|
||||
controllers.delete(subId);
|
||||
}
|
||||
}
|
||||
|
|
@ -136,6 +157,10 @@ function connectStream(socket: WebSocket, ip: string | undefined) {
|
|||
/** Handle EVENT. Store the event. */
|
||||
async function handleEvent([_, event]: NostrClientEVENT): Promise<void> {
|
||||
relayEventsCounter.inc({ kind: event.kind.toString() });
|
||||
|
||||
const limiter = NKinds.ephemeral(event.kind) ? limiters.ephemeral : limiters.event;
|
||||
if (rateLimited(limiter)) return;
|
||||
|
||||
try {
|
||||
// This will store it (if eligible) and run other side-effects.
|
||||
await pipeline.handleEvent(purifyEvent(event), { source: 'relay', signal: AbortSignal.timeout(1000) });
|
||||
|
|
@ -145,7 +170,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) {
|
|||
send(['OK', event.id, false, e.message]);
|
||||
} else {
|
||||
send(['OK', event.id, false, 'error: something went wrong']);
|
||||
console.error(e);
|
||||
logi({ level: 'error', ns: 'ditto.relay', msg: 'Error in relay', error: errorJson(e) });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -161,6 +186,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) {
|
|||
|
||||
/** Handle COUNT. Return the number of events matching the filters. */
|
||||
async function handleCount([_, subId, ...filters]: NostrClientCOUNT): Promise<void> {
|
||||
if (rateLimited(limiters.req)) return;
|
||||
const store = await Storages.db();
|
||||
const { count } = await store.count(filters, { timeout: Conf.db.timeouts.relay });
|
||||
send(['COUNT', subId, { count, approximate: false }]);
|
||||
|
|
@ -186,10 +212,18 @@ const relayController: AppController = (c, next) => {
|
|||
return c.text('Please use a Nostr client to connect.', 400);
|
||||
}
|
||||
|
||||
const ip = c.req.header('x-real-ip');
|
||||
let ip = c.req.header('x-real-ip');
|
||||
|
||||
if (ip && Conf.ipWhitelist.includes(ip)) {
|
||||
ip = undefined;
|
||||
}
|
||||
|
||||
if (ip) {
|
||||
const count = limiter.get(ip) ?? 0;
|
||||
if (count > LIMITER_LIMIT) {
|
||||
const remaining = Object
|
||||
.values(limiters)
|
||||
.reduce((acc, limiter) => Math.min(acc, limiter.client(ip).remaining), Infinity);
|
||||
|
||||
if (remaining < 0) {
|
||||
return c.json({ error: 'Rate limit exceeded' }, 429);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,17 +0,0 @@
|
|||
import { Conf } from '@/config.ts';
|
||||
|
||||
import type { AppController } from '@/app.ts';
|
||||
|
||||
/** Landing page controller. */
|
||||
const indexController: AppController = (c) => {
|
||||
const { origin } = Conf.url;
|
||||
|
||||
return c.text(`Please connect with a Mastodon client:
|
||||
|
||||
${origin}
|
||||
|
||||
Ditto <https://gitlab.com/soapbox-pub/ditto>
|
||||
`);
|
||||
};
|
||||
|
||||
export { indexController };
|
||||
|
|
@ -1,36 +1,50 @@
|
|||
import { NostrJson } from '@nostrify/nostrify';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { AppController } from '@/app.ts';
|
||||
import { localNip05Lookup } from '@/utils/nip05.ts';
|
||||
|
||||
const nameSchema = z.string().min(1).regex(/^\w+$/);
|
||||
const nameSchema = z.string().min(1).regex(/^[\w.-]+$/);
|
||||
const emptyResult: NostrJson = { names: {}, relays: {} };
|
||||
|
||||
/**
|
||||
* Serves NIP-05's nostr.json.
|
||||
* https://github.com/nostr-protocol/nips/blob/master/05.md
|
||||
*/
|
||||
const nostrController: AppController = async (c) => {
|
||||
// If there are no query parameters, this will always return an empty result.
|
||||
if (!Object.entries(c.req.queries()).length) {
|
||||
c.header('Cache-Control', 'max-age=31536000, public, immutable, stale-while-revalidate=86400');
|
||||
return c.json(emptyResult);
|
||||
}
|
||||
|
||||
const store = c.get('store');
|
||||
|
||||
const result = nameSchema.safeParse(c.req.query('name'));
|
||||
const name = result.success ? result.data : undefined;
|
||||
|
||||
const pointer = name ? await localNip05Lookup(store, name) : undefined;
|
||||
|
||||
if (!name || !pointer) {
|
||||
return c.json({ names: {}, relays: {} });
|
||||
// Not found, cache for 5 minutes.
|
||||
c.header('Cache-Control', 'max-age=300, public, stale-while-revalidate=30');
|
||||
return c.json(emptyResult);
|
||||
}
|
||||
|
||||
const { pubkey, relays } = pointer;
|
||||
const { pubkey, relays = [] } = pointer;
|
||||
|
||||
return c.json({
|
||||
names: {
|
||||
[name]: pubkey,
|
||||
},
|
||||
relays: {
|
||||
[pubkey]: relays,
|
||||
},
|
||||
});
|
||||
// It's found, so cache for 6 hours.
|
||||
c.header('Cache-Control', 'max-age=21600, public, stale-while-revalidate=3600');
|
||||
|
||||
return c.json(
|
||||
{
|
||||
names: {
|
||||
[name]: pubkey,
|
||||
},
|
||||
relays: {
|
||||
[pubkey]: relays,
|
||||
},
|
||||
} satisfies NostrJson,
|
||||
);
|
||||
};
|
||||
|
||||
export { nostrController };
|
||||
|
|
|
|||
|
|
@ -1,12 +1,15 @@
|
|||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
import { logi } from '@soapbox/logi';
|
||||
import { JsonValue } from '@std/json';
|
||||
import { FileMigrationProvider, Kysely, Migrator } from 'kysely';
|
||||
|
||||
import { DittoPglite } from '@/db/adapters/DittoPglite.ts';
|
||||
import { DittoPostgres } from '@/db/adapters/DittoPostgres.ts';
|
||||
import { DittoDatabase, DittoDatabaseOpts } from '@/db/DittoDatabase.ts';
|
||||
import { DittoTables } from '@/db/DittoTables.ts';
|
||||
import { errorJson } from '@/utils/log.ts';
|
||||
|
||||
export class DittoDB {
|
||||
/** Open a new database connection. */
|
||||
|
|
@ -36,20 +39,30 @@ export class DittoDB {
|
|||
}),
|
||||
});
|
||||
|
||||
console.warn('Running migrations...');
|
||||
logi({ level: 'info', ns: 'ditto.db.migration', msg: 'Running migrations...', state: 'started' });
|
||||
const { results, error } = await migrator.migrateToLatest();
|
||||
|
||||
if (error) {
|
||||
console.error(error);
|
||||
logi({
|
||||
level: 'fatal',
|
||||
ns: 'ditto.db.migration',
|
||||
msg: 'Migration failed.',
|
||||
state: 'failed',
|
||||
results: results as unknown as JsonValue,
|
||||
error: errorJson(error),
|
||||
});
|
||||
Deno.exit(1);
|
||||
} else {
|
||||
if (!results?.length) {
|
||||
console.warn('Everything up-to-date.');
|
||||
logi({ level: 'info', ns: 'ditto.db.migration', msg: 'Everything up-to-date.', state: 'skipped' });
|
||||
} else {
|
||||
console.warn('Migrations finished!');
|
||||
for (const { migrationName, status } of results!) {
|
||||
console.warn(` - ${migrationName}: ${status}`);
|
||||
}
|
||||
logi({
|
||||
level: 'info',
|
||||
ns: 'ditto.db.migration',
|
||||
msg: 'Migrations finished!',
|
||||
state: 'migrated',
|
||||
results: results as unknown as JsonValue,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,22 +1,31 @@
|
|||
import { Stickynotes } from '@soapbox/stickynotes';
|
||||
import { logi, LogiValue } from '@soapbox/logi';
|
||||
import { Logger } from 'kysely';
|
||||
|
||||
import { dbQueriesCounter, dbQueryDurationHistogram } from '@/metrics.ts';
|
||||
import { errorJson } from '@/utils/log.ts';
|
||||
|
||||
/** Log the SQL for queries. */
|
||||
export const KyselyLogger: Logger = (event) => {
|
||||
const console = new Stickynotes('ditto:sql');
|
||||
|
||||
const { query, queryDurationMillis } = event;
|
||||
const { sql, parameters } = query;
|
||||
const { parameters, sql } = query;
|
||||
|
||||
const queryDurationSeconds = queryDurationMillis / 1000;
|
||||
const duration = queryDurationMillis / 1000;
|
||||
|
||||
dbQueriesCounter.inc();
|
||||
dbQueryDurationHistogram.observe(queryDurationSeconds);
|
||||
dbQueryDurationHistogram.observe(duration);
|
||||
|
||||
console.debug(
|
||||
sql,
|
||||
JSON.stringify(parameters),
|
||||
`\x1b[90m(${(queryDurationSeconds / 1000).toFixed(2)}s)\x1b[0m`,
|
||||
);
|
||||
if (event.level === 'query') {
|
||||
logi({ level: 'debug', ns: 'ditto.sql', sql, parameters: parameters as LogiValue, duration });
|
||||
}
|
||||
|
||||
if (event.level === 'error') {
|
||||
logi({
|
||||
level: 'error',
|
||||
ns: 'ditto.sql',
|
||||
sql,
|
||||
parameters: parameters as LogiValue,
|
||||
error: errorJson(event.error),
|
||||
duration,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { Semaphore } from '@lambdalisue/async';
|
||||
import { Stickynotes } from '@soapbox/stickynotes';
|
||||
import { logi } from '@soapbox/logi';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
import { firehoseEventsCounter } from '@/metrics.ts';
|
||||
|
|
@ -8,7 +8,6 @@ import { nostrNow } from '@/utils.ts';
|
|||
|
||||
import * as pipeline from '@/pipeline.ts';
|
||||
|
||||
const console = new Stickynotes('ditto:firehose');
|
||||
const sem = new Semaphore(Conf.firehoseConcurrency);
|
||||
|
||||
/**
|
||||
|
|
@ -22,14 +21,14 @@ export async function startFirehose(): Promise<void> {
|
|||
for await (const msg of store.req([{ kinds: Conf.firehoseKinds, limit: 0, since: nostrNow() }])) {
|
||||
if (msg[0] === 'EVENT') {
|
||||
const event = msg[2];
|
||||
console.debug(`NostrEvent<${event.kind}> ${event.id}`);
|
||||
logi({ level: 'debug', ns: 'ditto.event', source: 'firehose', id: event.id, kind: event.kind });
|
||||
firehoseEventsCounter.inc({ kind: event.kind });
|
||||
|
||||
sem.lock(async () => {
|
||||
try {
|
||||
await pipeline.handleEvent(event, { source: 'firehose', signal: AbortSignal.timeout(5000) });
|
||||
} catch (e) {
|
||||
console.warn(e);
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
|
|||
33
src/middleware/cacheControlMiddleware.test.ts
Normal file
33
src/middleware/cacheControlMiddleware.test.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
import { Hono } from '@hono/hono';
|
||||
import { assertEquals } from '@std/assert';
|
||||
|
||||
import { cacheControlMiddleware } from '@/middleware/cacheControlMiddleware.ts';
|
||||
|
||||
Deno.test('cacheControlMiddleware with multiple options', async () => {
|
||||
const app = new Hono();
|
||||
|
||||
app.use(cacheControlMiddleware({
|
||||
maxAge: 31536000,
|
||||
public: true,
|
||||
immutable: true,
|
||||
}));
|
||||
|
||||
app.get('/', (c) => c.text('OK'));
|
||||
|
||||
const response = await app.request('/');
|
||||
const cacheControl = response.headers.get('Cache-Control');
|
||||
|
||||
assertEquals(cacheControl, 'max-age=31536000, public, immutable');
|
||||
});
|
||||
|
||||
Deno.test('cacheControlMiddleware with no options does not add header', async () => {
|
||||
const app = new Hono();
|
||||
|
||||
app.use(cacheControlMiddleware({}));
|
||||
app.get('/', (c) => c.text('OK'));
|
||||
|
||||
const response = await app.request('/');
|
||||
const cacheControl = response.headers.get('Cache-Control');
|
||||
|
||||
assertEquals(cacheControl, null);
|
||||
});
|
||||
102
src/middleware/cacheControlMiddleware.ts
Normal file
102
src/middleware/cacheControlMiddleware.ts
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
import { MiddlewareHandler } from '@hono/hono';
|
||||
|
||||
/**
|
||||
* Options for the `cacheControlMiddleware` middleware.
|
||||
*
|
||||
* NOTE: All numerical values are in **seconds**.
|
||||
*
|
||||
* See the definitions of [fresh](https://developer.mozilla.org/en-US/docs/Web/HTTP/Caching#fresh_and_stale_based_on_age) and [stale](https://developer.mozilla.org/en-US/docs/Web/HTTP/Caching#fresh_and_stale_based_on_age).
|
||||
*/
|
||||
export interface CacheControlMiddlewareOpts {
|
||||
/** Indicates that the response remains fresh until _N_ seconds after the response is generated. */
|
||||
maxAge?: number;
|
||||
/** Indicates how long the response remains fresh in a shared cache. */
|
||||
sMaxAge?: number;
|
||||
/** Indicates that the response can be stored in caches, but the response must be validated with the origin server before each reuse, even when the cache is disconnected from the origin server. */
|
||||
noCache?: boolean;
|
||||
/** Indicates that the response can be stored in caches and can be reused while fresh. */
|
||||
mustRevalidate?: boolean;
|
||||
/** Equivalent of `must-revalidate`, but specifically for shared caches only. */
|
||||
proxyRevalidate?: boolean;
|
||||
/** Indicates that any caches of any kind (private or shared) should not store this response. */
|
||||
noStore?: boolean;
|
||||
/** Indicates that the response can be stored only in a private cache (e.g. local caches in browsers). */
|
||||
private?: boolean;
|
||||
/** Indicates that the response can be stored in a shared cache. */
|
||||
public?: boolean;
|
||||
/** Indicates that a cache should store the response only if it understands the requirements for caching based on status code. */
|
||||
mustUnderstand?: boolean;
|
||||
/** Indicates that any intermediary (regardless of whether it implements a cache) shouldn't transform the response contents. */
|
||||
noTransform?: boolean;
|
||||
/** Indicates that the response will not be updated while it's fresh. */
|
||||
immutable?: boolean;
|
||||
/** Indicates that the cache could reuse a stale response while it revalidates it to a cache. */
|
||||
staleWhileRevalidate?: number;
|
||||
/** indicates that the cache can reuse a stale response when an upstream server generates an error, or when the error is generated locally. */
|
||||
staleIfError?: number;
|
||||
}
|
||||
|
||||
/** Adds a [`Cache-Control`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control) header to the response. */
|
||||
export function cacheControlMiddleware(opts: CacheControlMiddlewareOpts): MiddlewareHandler {
|
||||
return async (c, next) => {
|
||||
const directives: string[] = [];
|
||||
|
||||
if (typeof opts.maxAge === 'number') {
|
||||
directives.push(`max-age=${opts.maxAge}`);
|
||||
}
|
||||
|
||||
if (typeof opts.sMaxAge === 'number') {
|
||||
directives.push(`s-maxage=${opts.sMaxAge}`);
|
||||
}
|
||||
|
||||
if (opts.noCache) {
|
||||
directives.push('no-cache');
|
||||
}
|
||||
|
||||
if (opts.mustRevalidate) {
|
||||
directives.push('must-revalidate');
|
||||
}
|
||||
|
||||
if (opts.proxyRevalidate) {
|
||||
directives.push('proxy-revalidate');
|
||||
}
|
||||
|
||||
if (opts.noStore) {
|
||||
directives.push('no-store');
|
||||
}
|
||||
|
||||
if (opts.private) {
|
||||
directives.push('private');
|
||||
}
|
||||
|
||||
if (opts.public) {
|
||||
directives.push('public');
|
||||
}
|
||||
|
||||
if (opts.mustUnderstand) {
|
||||
directives.push('must-understand');
|
||||
}
|
||||
|
||||
if (opts.noTransform) {
|
||||
directives.push('no-transform');
|
||||
}
|
||||
|
||||
if (opts.immutable) {
|
||||
directives.push('immutable');
|
||||
}
|
||||
|
||||
if (typeof opts.staleWhileRevalidate === 'number') {
|
||||
directives.push(`stale-while-revalidate=${opts.staleWhileRevalidate}`);
|
||||
}
|
||||
|
||||
if (typeof opts.staleIfError === 'number') {
|
||||
directives.push(`stale-if-error=${opts.staleIfError}`);
|
||||
}
|
||||
|
||||
if (directives.length) {
|
||||
c.header('Cache-Control', directives.join(', '));
|
||||
}
|
||||
|
||||
await next();
|
||||
};
|
||||
}
|
||||
19
src/middleware/logiMiddleware.ts
Normal file
19
src/middleware/logiMiddleware.ts
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import { MiddlewareHandler } from '@hono/hono';
|
||||
import { logi } from '@soapbox/logi';
|
||||
|
||||
export const logiMiddleware: MiddlewareHandler = async (c, next) => {
|
||||
const { method } = c.req;
|
||||
const { pathname } = new URL(c.req.url);
|
||||
|
||||
logi({ level: 'info', ns: 'ditto.http.request', method, pathname });
|
||||
|
||||
const start = new Date();
|
||||
|
||||
await next();
|
||||
|
||||
const end = new Date();
|
||||
const delta = (end.getTime() - start.getTime()) / 1000;
|
||||
const level = c.res.status >= 500 ? 'error' : 'info';
|
||||
|
||||
logi({ level, ns: 'ditto.http.response', method, pathname, status: c.res.status, delta });
|
||||
};
|
||||
|
|
@ -1,15 +1,25 @@
|
|||
import { MiddlewareHandler } from '@hono/hono';
|
||||
import { rateLimiter } from 'hono-rate-limiter';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
|
||||
/**
|
||||
* Rate limit middleware for Hono, based on [`hono-rate-limiter`](https://github.com/rhinobase/hono-rate-limiter).
|
||||
*/
|
||||
export function rateLimitMiddleware(limit: number, windowMs: number): MiddlewareHandler {
|
||||
export function rateLimitMiddleware(limit: number, windowMs: number, includeHeaders?: boolean): MiddlewareHandler {
|
||||
// @ts-ignore Mismatched hono versions.
|
||||
return rateLimiter({
|
||||
limit,
|
||||
windowMs,
|
||||
skip: (c) => !c.req.header('x-real-ip'),
|
||||
standardHeaders: includeHeaders,
|
||||
handler: (c) => {
|
||||
c.header('Cache-Control', 'no-store');
|
||||
return c.text('Too many requests, please try again later.', 429);
|
||||
},
|
||||
skip: (c) => {
|
||||
const ip = c.req.header('x-real-ip');
|
||||
return !ip || Conf.ipWhitelist.includes(ip);
|
||||
},
|
||||
keyGenerator: (c) => c.req.header('x-real-ip')!,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
import { MiddlewareHandler } from '@hono/hono';
|
||||
import { HTTPException } from '@hono/hono/http-exception';
|
||||
|
||||
import { AppMiddleware } from '@/app.ts';
|
||||
import { NostrSigner } from '@nostrify/nostrify';
|
||||
|
||||
/** Throw a 401 if a signer isn't set. */
|
||||
export const requireSigner: AppMiddleware = async (c, next) => {
|
||||
export const requireSigner: MiddlewareHandler<{ Variables: { signer: NostrSigner } }> = async (c, next) => {
|
||||
if (!c.get('signer')) {
|
||||
throw new HTTPException(401, { message: 'No pubkey provided' });
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,13 +1,12 @@
|
|||
import { Semaphore } from '@lambdalisue/async';
|
||||
import { Stickynotes } from '@soapbox/stickynotes';
|
||||
|
||||
import { pipelineEncounters } from '@/caches/pipelineEncounters.ts';
|
||||
import { Conf } from '@/config.ts';
|
||||
import * as pipeline from '@/pipeline.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
import { logi } from '@soapbox/logi';
|
||||
|
||||
const sem = new Semaphore(1);
|
||||
const console = new Stickynotes('ditto:notify');
|
||||
|
||||
export async function startNotify(): Promise<void> {
|
||||
const { listen } = await Storages.database();
|
||||
|
|
@ -15,10 +14,12 @@ export async function startNotify(): Promise<void> {
|
|||
|
||||
listen('nostr_event', (id) => {
|
||||
if (pipelineEncounters.has(id)) {
|
||||
console.debug(`Skip event ${id} because it was already in the pipeline`);
|
||||
logi({ level: 'debug', ns: 'ditto.notify', id, skipped: true });
|
||||
return;
|
||||
}
|
||||
|
||||
logi({ level: 'debug', ns: 'ditto.notify', id, skipped: false });
|
||||
|
||||
sem.lock(async () => {
|
||||
try {
|
||||
const signal = AbortSignal.timeout(Conf.db.timeouts.default);
|
||||
|
|
@ -26,10 +27,11 @@ export async function startNotify(): Promise<void> {
|
|||
const [event] = await store.query([{ ids: [id], limit: 1 }], { signal });
|
||||
|
||||
if (event) {
|
||||
logi({ level: 'debug', ns: 'ditto.event', source: 'notify', id: event.id, kind: event.kind });
|
||||
await pipeline.handleEvent(event, { source: 'notify', signal });
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(e);
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify';
|
||||
import { Stickynotes } from '@soapbox/stickynotes';
|
||||
import { logi } from '@soapbox/logi';
|
||||
import { Kysely, sql } from 'kysely';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { pipelineEncounters } from '@/caches/pipelineEncounters.ts';
|
||||
import { Conf } from '@/config.ts';
|
||||
import { DittoTables } from '@/db/DittoTables.ts';
|
||||
import { DittoPush } from '@/DittoPush.ts';
|
||||
|
|
@ -15,6 +16,7 @@ import { Storages } from '@/storages.ts';
|
|||
import { eventAge, parseNip05, Time } from '@/utils.ts';
|
||||
import { getAmount } from '@/utils/bolt11.ts';
|
||||
import { detectLanguage } from '@/utils/language.ts';
|
||||
import { errorJson } from '@/utils/log.ts';
|
||||
import { nip05Cache } from '@/utils/nip05.ts';
|
||||
import { purifyEvent } from '@/utils/purify.ts';
|
||||
import { updateStats } from '@/utils/stats.ts';
|
||||
|
|
@ -22,9 +24,6 @@ import { getTagSet } from '@/utils/tags.ts';
|
|||
import { renderWebPushNotification } from '@/views/mastodon/push.ts';
|
||||
import { policyWorker } from '@/workers/policy.ts';
|
||||
import { verifyEventWorker } from '@/workers/verify.ts';
|
||||
import { pipelineEncounters } from '@/caches/pipelineEncounters.ts';
|
||||
|
||||
const console = new Stickynotes('ditto:pipeline');
|
||||
|
||||
interface PipelineOpts {
|
||||
signal: AbortSignal;
|
||||
|
|
@ -69,7 +68,7 @@ async function handleEvent(event: DittoEvent, opts: PipelineOpts): Promise<void>
|
|||
pipelineEncounters.set(event.id, true);
|
||||
|
||||
// Log the event.
|
||||
console.info(`NostrEvent<${event.kind}> ${event.id}`);
|
||||
logi({ level: 'debug', ns: 'ditto.event', source: 'pipeline', id: event.id, kind: event.kind });
|
||||
pipelineEventsCounter.inc({ kind: event.kind });
|
||||
|
||||
// NIP-46 events get special treatment.
|
||||
|
|
@ -136,18 +135,17 @@ async function handleEvent(event: DittoEvent, opts: PipelineOpts): Promise<void>
|
|||
}
|
||||
|
||||
async function policyFilter(event: NostrEvent, signal: AbortSignal): Promise<void> {
|
||||
const console = new Stickynotes('ditto:policy');
|
||||
|
||||
try {
|
||||
const result = await policyWorker.call(event, signal);
|
||||
policyEventsCounter.inc({ ok: String(result[2]) });
|
||||
console.debug(JSON.stringify(result));
|
||||
const [, , ok, reason] = result;
|
||||
logi({ level: 'debug', ns: 'ditto.policy', id: event.id, kind: event.kind, ok, reason });
|
||||
policyEventsCounter.inc({ ok: String(ok) });
|
||||
RelayError.assert(result);
|
||||
} catch (e) {
|
||||
if (e instanceof RelayError) {
|
||||
throw e;
|
||||
} else {
|
||||
console.error(e);
|
||||
logi({ level: 'error', ns: 'ditto.policy', id: event.id, kind: event.kind, error: errorJson(e) });
|
||||
throw new RelayError('blocked', 'policy error');
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import { NostrEvent, NostrFilter, NStore } from '@nostrify/nostrify';
|
||||
import Debug from '@soapbox/stickynotes/debug';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
|
|
@ -9,8 +8,6 @@ import { hydrateEvents } from '@/storages/hydrate.ts';
|
|||
import { fallbackAuthor } from '@/utils.ts';
|
||||
import { findReplyTag, getTagSet } from '@/utils/tags.ts';
|
||||
|
||||
const debug = Debug('ditto:queries');
|
||||
|
||||
interface GetEventOpts {
|
||||
/** Signal to abort the request. */
|
||||
signal?: AbortSignal;
|
||||
|
|
@ -20,12 +17,14 @@ interface GetEventOpts {
|
|||
relations?: DittoRelation[];
|
||||
}
|
||||
|
||||
/** Get a Nostr event by its ID. */
|
||||
/**
|
||||
* Get a Nostr event by its ID.
|
||||
* @deprecated Use `store.query` directly.
|
||||
*/
|
||||
const getEvent = async (
|
||||
id: string,
|
||||
opts: GetEventOpts = {},
|
||||
): Promise<DittoEvent | undefined> => {
|
||||
debug(`getEvent: ${id}`);
|
||||
const store = await Storages.db();
|
||||
const { kind, signal = AbortSignal.timeout(1000) } = opts;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,15 @@
|
|||
import * as Sentry from '@sentry/deno';
|
||||
import { logi } from '@soapbox/logi';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
|
||||
// Sentry
|
||||
if (Conf.sentryDsn) {
|
||||
console.log('Sentry enabled');
|
||||
logi({ level: 'info', ns: 'ditto.sentry', msg: 'Sentry enabled.', enabled: true });
|
||||
Sentry.init({
|
||||
dsn: Conf.sentryDsn,
|
||||
tracesSampleRate: 1.0,
|
||||
});
|
||||
} else {
|
||||
logi({ level: 'info', ns: 'ditto.sentry', msg: 'Sentry not configured. Skipping.', enabled: false });
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,14 @@
|
|||
import { logi } from '@soapbox/logi';
|
||||
|
||||
import '@/precheck.ts';
|
||||
import '@/sentry.ts';
|
||||
import '@/nostr-wasm.ts';
|
||||
import app from '@/app.ts';
|
||||
import { Conf } from '@/config.ts';
|
||||
|
||||
Deno.serve({ port: Conf.port }, app.fetch);
|
||||
Deno.serve({
|
||||
port: Conf.port,
|
||||
onListen({ hostname, port }): void {
|
||||
logi({ level: 'info', ns: 'ditto.server', msg: `Listening on http://${hostname}:${port}`, hostname, port });
|
||||
},
|
||||
}, app.fetch);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
// Starts up applications required to run before the HTTP server is on.
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
import { cron } from '@/cron.ts';
|
||||
import { startFirehose } from '@/firehose.ts';
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
// deno-lint-ignore-file require-await
|
||||
import { logi } from '@soapbox/logi';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
import { DittoDatabase } from '@/db/DittoDatabase.ts';
|
||||
import { DittoDB } from '@/db/DittoDB.ts';
|
||||
|
|
@ -89,13 +91,21 @@ export class Storages {
|
|||
return acc;
|
||||
}, []);
|
||||
|
||||
console.log(`pool: connecting to ${activeRelays.length} relays.`);
|
||||
logi({
|
||||
level: 'info',
|
||||
ns: 'ditto.pool',
|
||||
msg: `connecting to ${activeRelays.length} relays`,
|
||||
relays: activeRelays,
|
||||
});
|
||||
|
||||
return new NPool({
|
||||
open(url) {
|
||||
return new NRelay1(url, {
|
||||
// Skip event verification (it's done in the pipeline).
|
||||
verifyEvent: () => true,
|
||||
log(log) {
|
||||
logi(log);
|
||||
},
|
||||
});
|
||||
},
|
||||
reqRouter: async (filters) => {
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { generateSecretKey } from 'nostr-tools';
|
|||
import { RelayError } from '@/RelayError.ts';
|
||||
import { eventFixture, genEvent } from '@/test.ts';
|
||||
import { Conf } from '@/config.ts';
|
||||
import { EventsDB } from '@/storages/EventsDB.ts';
|
||||
import { createTestDB } from '@/test.ts';
|
||||
|
||||
Deno.test('count filters', async () => {
|
||||
|
|
@ -244,3 +245,42 @@ Deno.test('NPostgres.query with search', async (t) => {
|
|||
assertEquals(await store.query([{ search: "this shouldn't match" }]), []);
|
||||
});
|
||||
});
|
||||
|
||||
Deno.test('EventsDB.indexTags indexes only the final `e` and `p` tag of kind 7 events', () => {
|
||||
const event = {
|
||||
kind: 7,
|
||||
id: 'a92549a442d306b32273aa9456ba48e3851a4e6203af3f567543298ab964b35b',
|
||||
pubkey: 'f288a224a61b7361aa9dc41a90aba8a2dff4544db0bc386728e638b21da1792c',
|
||||
created_at: 1737908284,
|
||||
tags: [
|
||||
['e', '2503cea56931fb25914866e12ffc739741539db4d6815220b9974ef0967fe3f9', '', 'root'],
|
||||
['p', 'fad5c18326fb26d9019f1b2aa503802f0253494701bf311d7588a1e65cb8046b'],
|
||||
['p', '26d6a946675e603f8de4bf6f9cef442037b70c7eee170ff06ed7673fc34c98f1'],
|
||||
['p', '04c960497af618ae18f5147b3e5c309ef3d8a6251768a1c0820e02c93768cc3b'],
|
||||
['p', '0114bb11dd8eb89bfb40669509b2a5a473d27126e27acae58257f2fd7cd95776'],
|
||||
['p', '9fce3aea32b35637838fb45b75be32595742e16bb3e4742cc82bb3d50f9087e6'],
|
||||
['p', '26bd32c67232bdf16d05e763ec67d883015eb99fd1269025224c20c6cfdb0158'],
|
||||
['p', 'eab0e756d32b80bcd464f3d844b8040303075a13eabc3599a762c9ac7ab91f4f'],
|
||||
['p', 'edcd20558f17d99327d841e4582f9b006331ac4010806efa020ef0d40078e6da'],
|
||||
['p', 'bd1e19980e2c91e6dc657e92c25762ca882eb9272d2579e221f037f93788de91'],
|
||||
['p', 'bf2376e17ba4ec269d10fcc996a4746b451152be9031fa48e74553dde5526bce'],
|
||||
['p', '3878d95db7b854c3a0d3b2d6b7bf9bf28b36162be64326f5521ba71cf3b45a69'],
|
||||
['p', 'ede3866ddfc40aa4e458952c11c67e827e3cbb8a6a4f0a934c009aa2ed2fb477'],
|
||||
['p', 'f288a224a61b7361aa9dc41a90aba8a2dff4544db0bc386728e638b21da1792c'],
|
||||
['p', '9ce71f1506ccf4b99f234af49bd6202be883a80f95a155c6e9a1c36fd7e780c7', '', 'mention'],
|
||||
['p', '932614571afcbad4d17a191ee281e39eebbb41b93fac8fd87829622aeb112f4d', '', 'mention'],
|
||||
['e', 'e3653ae41ffb510e5fc071555ecfbc94d2fc31e355d61d941e39a97ac6acb15b'],
|
||||
['p', '4e088f3087f6a7e7097ce5fe7fd884ec04ddc69ed6cdd37c55e200f7744b1792'],
|
||||
],
|
||||
content: '🤙',
|
||||
sig:
|
||||
'44639d039a7f7fb8772fcfa13d134d3cda684ec34b6a777ead589676f9e8d81b08a24234066dcde1aacfbe193224940fba7586e7197c159757d3caf8f2b57e1b',
|
||||
};
|
||||
|
||||
const tags = EventsDB.indexTags(event);
|
||||
|
||||
assertEquals(tags, [
|
||||
['e', 'e3653ae41ffb510e5fc071555ecfbc94d2fc31e355d61d941e39a97ac6acb15b'],
|
||||
['p', '4e088f3087f6a7e7097ce5fe7fd884ec04ddc69ed6cdd37c55e200f7744b1792'],
|
||||
]);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@
|
|||
import { LanguageCode } from 'iso-639-1';
|
||||
import { NPostgres, NPostgresSchema } from '@nostrify/db';
|
||||
import { NIP50, NKinds, NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify';
|
||||
import { Stickynotes } from '@soapbox/stickynotes';
|
||||
import { logi } from '@soapbox/logi';
|
||||
import { JsonValue } from '@std/json';
|
||||
import { Kysely, SelectQueryBuilder } from 'kysely';
|
||||
import { nip27 } from 'nostr-tools';
|
||||
|
||||
|
|
@ -16,11 +17,19 @@ import { purifyEvent } from '@/utils/purify.ts';
|
|||
import { DittoEvent } from '@/interfaces/DittoEvent.ts';
|
||||
|
||||
/** Function to decide whether or not to index a tag. */
|
||||
type TagCondition = ({ event, count, value }: {
|
||||
type TagCondition = (opts: TagConditionOpts) => boolean;
|
||||
|
||||
/** Options for the tag condition function. */
|
||||
interface TagConditionOpts {
|
||||
/** Nostr event whose tags are being indexed. */
|
||||
event: NostrEvent;
|
||||
/** Count of the current tag name so far. Each tag name has a separate counter starting at 0. */
|
||||
count: number;
|
||||
/** Overall tag index. */
|
||||
index: number;
|
||||
/** Current vag value. */
|
||||
value: string;
|
||||
}) => boolean;
|
||||
}
|
||||
|
||||
/** Options for the EventsDB store. */
|
||||
interface EventsDBOpts {
|
||||
|
|
@ -36,19 +45,17 @@ interface EventsDBOpts {
|
|||
|
||||
/** SQL database storage adapter for Nostr events. */
|
||||
class EventsDB extends NPostgres {
|
||||
private console = new Stickynotes('ditto:db:events');
|
||||
|
||||
/** Conditions for when to index certain tags. */
|
||||
static tagConditions: Record<string, TagCondition> = {
|
||||
'a': ({ count }) => count < 15,
|
||||
'd': ({ event, count }) => count === 0 && NKinds.parameterizedReplaceable(event.kind),
|
||||
'e': ({ event, count, value }) => ((event.kind === 10003) || count < 15) && isNostrId(value),
|
||||
'e': EventsDB.eTagCondition,
|
||||
'k': ({ count, value }) => count === 0 && Number.isInteger(Number(value)),
|
||||
'L': ({ event, count }) => event.kind === 1985 || count === 0,
|
||||
'l': ({ event, count }) => event.kind === 1985 || count === 0,
|
||||
'n': ({ count, value }) => count < 50 && value.length < 50,
|
||||
'P': ({ count, value }) => count === 0 && isNostrId(value),
|
||||
'p': ({ event, count, value }) => (count < 15 || event.kind === 3) && isNostrId(value),
|
||||
'p': EventsDB.pTagCondition,
|
||||
'proxy': ({ count, value }) => count === 0 && value.length < 256,
|
||||
'q': ({ event, count, value }) => count === 0 && event.kind === 1 && isNostrId(value),
|
||||
'r': ({ event, count }) => (event.kind === 1985 ? count < 20 : count < 3),
|
||||
|
|
@ -65,7 +72,7 @@ class EventsDB extends NPostgres {
|
|||
/** Insert an event (and its tags) into the database. */
|
||||
override async event(event: NostrEvent, opts: { signal?: AbortSignal; timeout?: number } = {}): Promise<void> {
|
||||
event = purifyEvent(event);
|
||||
this.console.debug('EVENT', JSON.stringify(event));
|
||||
logi({ level: 'debug', ns: 'ditto.event', source: 'db', id: event.id, kind: event.kind });
|
||||
dbEventsCounter.inc({ kind: event.kind });
|
||||
|
||||
if (await this.isDeletedAdmin(event)) {
|
||||
|
|
@ -223,7 +230,7 @@ class EventsDB extends NPostgres {
|
|||
|
||||
if (opts.signal?.aborted) return Promise.resolve([]);
|
||||
|
||||
this.console.debug('REQ', JSON.stringify(filters));
|
||||
logi({ level: 'debug', ns: 'ditto.req', source: 'db', filters: filters as JsonValue });
|
||||
|
||||
return super.query(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout });
|
||||
}
|
||||
|
|
@ -253,7 +260,7 @@ class EventsDB extends NPostgres {
|
|||
|
||||
/** Delete events based on filters from the database. */
|
||||
override async remove(filters: NostrFilter[], opts: { signal?: AbortSignal; timeout?: number } = {}): Promise<void> {
|
||||
this.console.debug('DELETE', JSON.stringify(filters));
|
||||
logi({ level: 'debug', ns: 'ditto.remove', source: 'db', filters: filters as JsonValue });
|
||||
return super.remove(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout });
|
||||
}
|
||||
|
||||
|
|
@ -264,11 +271,33 @@ class EventsDB extends NPostgres {
|
|||
): Promise<{ count: number; approximate: any }> {
|
||||
if (opts.signal?.aborted) return Promise.reject(abortError());
|
||||
|
||||
this.console.debug('COUNT', JSON.stringify(filters));
|
||||
logi({ level: 'debug', ns: 'ditto.count', source: 'db', filters: filters as JsonValue });
|
||||
|
||||
return super.count(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout });
|
||||
}
|
||||
|
||||
/** Rule for indexing `e` tags. */
|
||||
private static eTagCondition({ event, count, value, index }: TagConditionOpts): boolean {
|
||||
if (!isNostrId(value)) return false;
|
||||
|
||||
if (event.kind === 7) {
|
||||
return index === event.tags.findLastIndex(([name]) => name === 'e');
|
||||
}
|
||||
|
||||
return event.kind === 10003 || count < 15;
|
||||
}
|
||||
|
||||
/** Rule for indexing `p` tags. */
|
||||
private static pTagCondition({ event, count, value, index }: TagConditionOpts): boolean {
|
||||
if (!isNostrId(value)) return false;
|
||||
|
||||
if (event.kind === 7) {
|
||||
return index === event.tags.findLastIndex(([name]) => name === 'p');
|
||||
}
|
||||
|
||||
return count < 15 || event.kind === 3;
|
||||
}
|
||||
|
||||
/** Return only the tags that should be indexed. */
|
||||
static override indexTags(event: NostrEvent): string[][] {
|
||||
const tagCounts: Record<string, number> = {};
|
||||
|
|
@ -281,19 +310,20 @@ class EventsDB extends NPostgres {
|
|||
tagCounts[name] = getCount(name) + 1;
|
||||
}
|
||||
|
||||
function checkCondition(name: string, value: string, condition: TagCondition) {
|
||||
function checkCondition(name: string, value: string, condition: TagCondition, index: number): boolean {
|
||||
return condition({
|
||||
event,
|
||||
count: getCount(name),
|
||||
value,
|
||||
index,
|
||||
});
|
||||
}
|
||||
|
||||
return event.tags.reduce<string[][]>((results, tag) => {
|
||||
return event.tags.reduce<string[][]>((results, tag, index) => {
|
||||
const [name, value] = tag;
|
||||
const condition = EventsDB.tagConditions[name] as TagCondition | undefined;
|
||||
|
||||
if (value && condition && value.length < 200 && checkCondition(name, value, condition)) {
|
||||
if (value && condition && value.length < 200 && checkCondition(name, value, condition, index)) {
|
||||
results.push(tag);
|
||||
}
|
||||
|
||||
|
|
|
|||
23
src/storages/InternalRelay.test.ts
Normal file
23
src/storages/InternalRelay.test.ts
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
import { assertEquals } from '@std/assert';
|
||||
|
||||
import { eventFixture } from '@/test.ts';
|
||||
|
||||
import { InternalRelay } from './InternalRelay.ts';
|
||||
|
||||
Deno.test('InternalRelay', async () => {
|
||||
const relay = new InternalRelay();
|
||||
const event1 = await eventFixture('event-1');
|
||||
|
||||
const promise = new Promise((resolve) => setTimeout(() => resolve(relay.event(event1)), 0));
|
||||
|
||||
for await (const msg of relay.req([{}])) {
|
||||
if (msg[0] === 'EVENT') {
|
||||
assertEquals(relay.subs.size, 1);
|
||||
assertEquals(msg[2], event1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
await promise;
|
||||
assertEquals(relay.subs.size, 0); // cleanup
|
||||
});
|
||||
|
|
@ -24,7 +24,7 @@ interface InternalRelayOpts {
|
|||
* The pipeline should push events to it, then anything in the application can subscribe to it.
|
||||
*/
|
||||
export class InternalRelay implements NRelay {
|
||||
private subs = new Map<string, { filters: NostrFilter[]; machina: Machina<NostrEvent> }>();
|
||||
readonly subs = new Map<string, { filters: NostrFilter[]; machina: Machina<NostrEvent> }>();
|
||||
|
||||
constructor(private opts: InternalRelayOpts = {}) {}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { NostrEvent, NostrFilter, NRelay1, NStore } from '@nostrify/nostrify';
|
||||
import Debug from '@soapbox/stickynotes/debug';
|
||||
import { logi } from '@soapbox/logi';
|
||||
import { JsonValue } from '@std/json';
|
||||
|
||||
import { normalizeFilters } from '@/filter.ts';
|
||||
import { type DittoEvent } from '@/interfaces/DittoEvent.ts';
|
||||
|
|
@ -13,8 +14,6 @@ interface SearchStoreOpts {
|
|||
}
|
||||
|
||||
class SearchStore implements NStore {
|
||||
#debug = Debug('ditto:storages:search');
|
||||
|
||||
#fallback: NStore;
|
||||
#hydrator: NStore;
|
||||
#relay: NRelay1 | undefined;
|
||||
|
|
@ -38,11 +37,11 @@ class SearchStore implements NStore {
|
|||
if (opts?.signal?.aborted) return Promise.reject(abortError());
|
||||
if (!filters.length) return Promise.resolve([]);
|
||||
|
||||
this.#debug('REQ', JSON.stringify(filters));
|
||||
logi({ level: 'debug', ns: 'ditto.req', source: 'search', filters: filters as JsonValue });
|
||||
const query = filters[0]?.search;
|
||||
|
||||
if (this.#relay && this.#relay.socket.readyState === WebSocket.OPEN) {
|
||||
this.#debug(`Searching for "${query}" at ${this.#relay.socket.url}...`);
|
||||
logi({ level: 'debug', ns: 'ditto.search', query, source: 'relay', relay: this.#relay.socket.url });
|
||||
|
||||
const events = await this.#relay.query(filters, opts);
|
||||
|
||||
|
|
@ -52,7 +51,7 @@ class SearchStore implements NStore {
|
|||
signal: opts?.signal,
|
||||
});
|
||||
} else {
|
||||
this.#debug(`Searching for "${query}" locally...`);
|
||||
logi({ level: 'debug', ns: 'ditto.search', query, source: 'db' });
|
||||
return this.#fallback.query(filters, opts);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,9 +16,11 @@ Deno.test("getTrendingTagValues(): 'e' tag and WITHOUT language parameter", asyn
|
|||
const post1uses = numberOfAuthorsWhoLikedPost1 * post1multiplier;
|
||||
for (let i = 0; i < numberOfAuthorsWhoLikedPost1; i++) {
|
||||
const sk = generateSecretKey();
|
||||
events.push(
|
||||
genEvent({ kind: 7, content: '+', tags: Array(post1multiplier).fill([...['e', post1.id]]) }, sk),
|
||||
);
|
||||
for (let j = 0; j < post1multiplier; j++) {
|
||||
events.push(
|
||||
genEvent({ kind: 7, content: '+', tags: [['e', post1.id, `${j}`]] }, sk),
|
||||
);
|
||||
}
|
||||
}
|
||||
events.push(post1);
|
||||
|
||||
|
|
@ -29,9 +31,11 @@ Deno.test("getTrendingTagValues(): 'e' tag and WITHOUT language parameter", asyn
|
|||
const post2uses = numberOfAuthorsWhoLikedPost2 * post2multiplier;
|
||||
for (let i = 0; i < numberOfAuthorsWhoLikedPost2; i++) {
|
||||
const sk = generateSecretKey();
|
||||
events.push(
|
||||
genEvent({ kind: 7, content: '+', tags: Array(post2multiplier).fill([...['e', post2.id]]) }, sk),
|
||||
);
|
||||
for (let j = 0; j < post2multiplier; j++) {
|
||||
events.push(
|
||||
genEvent({ kind: 7, content: '+', tags: [['e', post2.id, `${j}`]] }, sk),
|
||||
);
|
||||
}
|
||||
}
|
||||
events.push(post2);
|
||||
|
||||
|
|
@ -62,9 +66,11 @@ Deno.test("getTrendingTagValues(): 'e' tag and WITH language parameter", async (
|
|||
const post1uses = numberOfAuthorsWhoLikedPost1 * post1multiplier;
|
||||
for (let i = 0; i < numberOfAuthorsWhoLikedPost1; i++) {
|
||||
const sk = generateSecretKey();
|
||||
events.push(
|
||||
genEvent({ kind: 7, content: '+', tags: Array(post1multiplier).fill([...['e', post1.id]]) }, sk),
|
||||
);
|
||||
for (let j = 0; j < post1multiplier; j++) {
|
||||
events.push(
|
||||
genEvent({ kind: 7, content: '+', tags: [['e', post1.id, `${j}`]] }, sk),
|
||||
);
|
||||
}
|
||||
}
|
||||
events.push(post1);
|
||||
|
||||
|
|
@ -74,9 +80,11 @@ Deno.test("getTrendingTagValues(): 'e' tag and WITH language parameter", async (
|
|||
const post2multiplier = 1;
|
||||
for (let i = 0; i < numberOfAuthorsWhoLikedPost2; i++) {
|
||||
const sk = generateSecretKey();
|
||||
events.push(
|
||||
genEvent({ kind: 7, content: '+', tags: Array(post2multiplier).fill([...['e', post2.id]]) }, sk),
|
||||
);
|
||||
for (let j = 0; j < post2multiplier; j++) {
|
||||
events.push(
|
||||
genEvent({ kind: 7, content: '+', tags: [['e', post2.id, `${j}`]] }, sk),
|
||||
);
|
||||
}
|
||||
}
|
||||
events.push(post2);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { NostrFilter } from '@nostrify/nostrify';
|
||||
import { Stickynotes } from '@soapbox/stickynotes';
|
||||
import { logi } from '@soapbox/logi';
|
||||
import { Kysely, sql } from 'kysely';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
|
|
@ -7,10 +7,9 @@ import { DittoTables } from '@/db/DittoTables.ts';
|
|||
import { handleEvent } from '@/pipeline.ts';
|
||||
import { AdminSigner } from '@/signers/AdminSigner.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
import { errorJson } from '@/utils/log.ts';
|
||||
import { Time } from '@/utils/time.ts';
|
||||
|
||||
const console = new Stickynotes('ditto:trends');
|
||||
|
||||
/** Get trending tag values for a given tag in the given time frame. */
|
||||
export async function getTrendingTagValues(
|
||||
/** Kysely instance to execute queries on. */
|
||||
|
|
@ -75,7 +74,9 @@ export async function updateTrendingTags(
|
|||
aliases?: string[],
|
||||
values?: string[],
|
||||
) {
|
||||
console.info(`Updating trending ${l}...`);
|
||||
const params = { l, tagName, kinds, limit, extra, aliases, values };
|
||||
logi({ level: 'info', ns: 'ditto.trends', msg: 'Updating trending', ...params });
|
||||
|
||||
const kysely = await Storages.kysely();
|
||||
const signal = AbortSignal.timeout(1000);
|
||||
|
||||
|
|
@ -92,9 +93,10 @@ export async function updateTrendingTags(
|
|||
limit,
|
||||
}, values);
|
||||
|
||||
console.log(trends);
|
||||
if (!trends.length) {
|
||||
console.info(`No trending ${l} found. Skipping.`);
|
||||
if (trends.length) {
|
||||
logi({ level: 'info', ns: 'ditto.trends', msg: 'Trends found', trends, ...params });
|
||||
} else {
|
||||
logi({ level: 'info', ns: 'ditto.trends', msg: 'No trends found. Skipping.', ...params });
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -112,9 +114,9 @@ export async function updateTrendingTags(
|
|||
});
|
||||
|
||||
await handleEvent(label, { source: 'internal', signal });
|
||||
console.info(`Trending ${l} updated.`);
|
||||
logi({ level: 'info', ns: 'ditto.trends', msg: 'Trends updated', ...params });
|
||||
} catch (e) {
|
||||
console.error(`Error updating trending ${l}: ${e instanceof Error ? e.message : e}`);
|
||||
logi({ level: 'error', ns: 'ditto.trends', msg: 'Error updating trends', ...params, error: errorJson(e) });
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { type Context } from '@hono/hono';
|
||||
import { HTTPException } from '@hono/hono/http-exception';
|
||||
import { NostrEvent, NostrFilter } from '@nostrify/nostrify';
|
||||
import Debug from '@soapbox/stickynotes/debug';
|
||||
import { logi } from '@soapbox/logi';
|
||||
import { EventTemplate } from 'nostr-tools';
|
||||
import * as TypeFest from 'type-fest';
|
||||
|
||||
|
|
@ -15,8 +15,6 @@ import { nostrNow } from '@/utils.ts';
|
|||
import { parseFormData } from '@/utils/formdata.ts';
|
||||
import { purifyEvent } from '@/utils/purify.ts';
|
||||
|
||||
const debug = Debug('ditto:api');
|
||||
|
||||
/** EventTemplate with defaults. */
|
||||
type EventStub = TypeFest.SetOptional<EventTemplate, 'content' | 'created_at' | 'tags'>;
|
||||
|
||||
|
|
@ -159,7 +157,7 @@ async function updateNames(k: number, d: string, n: Record<string, boolean>, c:
|
|||
|
||||
/** Push the event through the pipeline, rethrowing any RelayError. */
|
||||
async function publishEvent(event: NostrEvent, c: AppContext): Promise<NostrEvent> {
|
||||
debug('EVENT', event);
|
||||
logi({ level: 'info', ns: 'ditto.event', source: 'api', id: event.id, kind: event.kind });
|
||||
try {
|
||||
await pipeline.handleEvent(event, { source: 'api', signal: c.req.raw.signal });
|
||||
const client = await Storages.client();
|
||||
|
|
@ -209,7 +207,8 @@ function buildLinkHeader(url: string, events: NostrEvent[]): string | undefined
|
|||
return `<${next}>; rel="next", <${prev}>; rel="prev"`;
|
||||
}
|
||||
|
||||
type Entity = { id: string };
|
||||
// deno-lint-ignore ban-types
|
||||
type Entity = {};
|
||||
type HeaderRecord = Record<string, string | string[]>;
|
||||
|
||||
/** Return results with pagination headers. Assumes chronological sorting of events. */
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { DOMParser } from '@b-fuze/deno-dom';
|
||||
import Debug from '@soapbox/stickynotes/debug';
|
||||
import { logi } from '@soapbox/logi';
|
||||
import tldts from 'tldts';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
|
|
@ -7,18 +7,16 @@ import { cachedFaviconsSizeGauge } from '@/metrics.ts';
|
|||
import { SimpleLRU } from '@/utils/SimpleLRU.ts';
|
||||
import { fetchWorker } from '@/workers/fetch.ts';
|
||||
|
||||
const debug = Debug('ditto:favicon');
|
||||
|
||||
const faviconCache = new SimpleLRU<string, URL>(
|
||||
async (key, { signal }) => {
|
||||
debug(`Fetching favicon ${key}`);
|
||||
const tld = tldts.parse(key);
|
||||
async (domain, { signal }) => {
|
||||
logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'started' });
|
||||
const tld = tldts.parse(domain);
|
||||
|
||||
if (!tld.isIcann || tld.isIp || tld.isPrivate) {
|
||||
throw new Error(`Invalid favicon domain: ${key}`);
|
||||
throw new Error(`Invalid favicon domain: ${domain}`);
|
||||
}
|
||||
|
||||
const rootUrl = new URL('/', `https://${key}/`);
|
||||
const rootUrl = new URL('/', `https://${domain}/`);
|
||||
const response = await fetchWorker(rootUrl, { signal });
|
||||
const html = await response.text();
|
||||
|
||||
|
|
@ -28,15 +26,28 @@ const faviconCache = new SimpleLRU<string, URL>(
|
|||
if (link) {
|
||||
const href = link.getAttribute('href');
|
||||
if (href) {
|
||||
let url: URL | undefined;
|
||||
|
||||
try {
|
||||
return new URL(href);
|
||||
url = new URL(href);
|
||||
} catch {
|
||||
return new URL(href, rootUrl);
|
||||
try {
|
||||
url = new URL(href, rootUrl);
|
||||
} catch {
|
||||
// fall through
|
||||
}
|
||||
}
|
||||
|
||||
if (url) {
|
||||
logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'found', url });
|
||||
return url;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`Favicon not found: ${key}`);
|
||||
logi({ level: 'info', ns: 'ditto.favicon', domain, state: 'failed' });
|
||||
|
||||
throw new Error(`Favicon not found: ${domain}`);
|
||||
},
|
||||
{ ...Conf.caches.favicon, gauge: cachedFaviconsSizeGauge },
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1,23 +1,23 @@
|
|||
import { NostrEvent } from '@nostrify/nostrify';
|
||||
import { LNURL, LNURLDetails } from '@nostrify/nostrify/ln';
|
||||
import { Stickynotes } from '@soapbox/stickynotes';
|
||||
import { logi } from '@soapbox/logi';
|
||||
import { JsonValue } from '@std/json';
|
||||
|
||||
import { cachedLnurlsSizeGauge } from '@/metrics.ts';
|
||||
import { SimpleLRU } from '@/utils/SimpleLRU.ts';
|
||||
import { errorJson } from '@/utils/log.ts';
|
||||
import { Time } from '@/utils/time.ts';
|
||||
import { fetchWorker } from '@/workers/fetch.ts';
|
||||
import { NostrEvent } from '@nostrify/nostrify';
|
||||
|
||||
const console = new Stickynotes('ditto:lnurl');
|
||||
|
||||
const lnurlCache = new SimpleLRU<string, LNURLDetails>(
|
||||
async (lnurl, { signal }) => {
|
||||
console.debug(`Lookup ${lnurl}`);
|
||||
logi({ level: 'info', ns: 'ditto.lnurl', lnurl, state: 'started' });
|
||||
try {
|
||||
const result = await LNURL.lookup(lnurl, { fetch: fetchWorker, signal });
|
||||
console.debug(`Found: ${lnurl}`);
|
||||
return result;
|
||||
const details = await LNURL.lookup(lnurl, { fetch: fetchWorker, signal });
|
||||
logi({ level: 'info', ns: 'ditto.lnurl', lnurl, state: 'found', details: details as unknown as JsonValue });
|
||||
return details;
|
||||
} catch (e) {
|
||||
console.debug(`Not found: ${lnurl}`);
|
||||
logi({ level: 'info', ns: 'ditto.lnurl', lnurl, state: 'failed', error: errorJson(e) });
|
||||
throw e;
|
||||
}
|
||||
},
|
||||
|
|
|
|||
8
src/utils/log.ts
Normal file
8
src/utils/log.ts
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
/** Serialize an error into JSON for JSON logging. */
|
||||
export function errorJson(error: unknown): Error | null {
|
||||
if (error instanceof Error) {
|
||||
return error;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -6,7 +6,6 @@ import tldts from 'tldts';
|
|||
import { getAuthor } from '@/queries.ts';
|
||||
import { bech32ToPubkey } from '@/utils.ts';
|
||||
import { nip05Cache } from '@/utils/nip05.ts';
|
||||
import { Stickynotes } from '@soapbox/stickynotes';
|
||||
|
||||
/** Resolve a bech32 or NIP-05 identifier to an account. */
|
||||
export async function lookupAccount(
|
||||
|
|
@ -22,8 +21,6 @@ export async function lookupAccount(
|
|||
|
||||
/** Resolve a bech32 or NIP-05 identifier to a pubkey. */
|
||||
export async function lookupPubkey(value: string, signal?: AbortSignal): Promise<string | undefined> {
|
||||
const console = new Stickynotes('ditto:lookup');
|
||||
|
||||
if (n.bech32().safeParse(value).success) {
|
||||
return bech32ToPubkey(value);
|
||||
}
|
||||
|
|
@ -31,8 +28,7 @@ export async function lookupPubkey(value: string, signal?: AbortSignal): Promise
|
|||
try {
|
||||
const { pubkey } = await nip05Cache.fetch(value, { signal });
|
||||
return pubkey;
|
||||
} catch (e) {
|
||||
console.debug(e);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,45 +1,45 @@
|
|||
import { nip19 } from 'nostr-tools';
|
||||
import { NIP05, NStore } from '@nostrify/nostrify';
|
||||
import Debug from '@soapbox/stickynotes/debug';
|
||||
import { logi } from '@soapbox/logi';
|
||||
import tldts from 'tldts';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
import { cachedNip05sSizeGauge } from '@/metrics.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
import { errorJson } from '@/utils/log.ts';
|
||||
import { SimpleLRU } from '@/utils/SimpleLRU.ts';
|
||||
import { Nip05, parseNip05 } from '@/utils.ts';
|
||||
import { fetchWorker } from '@/workers/fetch.ts';
|
||||
|
||||
const debug = Debug('ditto:nip05');
|
||||
|
||||
const nip05Cache = new SimpleLRU<string, nip19.ProfilePointer>(
|
||||
async (key, { signal }) => {
|
||||
debug(`Lookup ${key}`);
|
||||
const tld = tldts.parse(key);
|
||||
async (nip05, { signal }) => {
|
||||
const tld = tldts.parse(nip05);
|
||||
|
||||
if (!tld.isIcann || tld.isIp || tld.isPrivate) {
|
||||
throw new Error(`Invalid NIP-05: ${key}`);
|
||||
throw new Error(`Invalid NIP-05: ${nip05}`);
|
||||
}
|
||||
|
||||
const [name, domain] = key.split('@');
|
||||
const [name, domain] = nip05.split('@');
|
||||
|
||||
logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'started' });
|
||||
|
||||
try {
|
||||
if (domain === Conf.url.host) {
|
||||
const store = await Storages.db();
|
||||
const pointer = await localNip05Lookup(store, name);
|
||||
if (pointer) {
|
||||
debug(`Found: ${key} is ${pointer.pubkey}`);
|
||||
logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', pubkey: pointer.pubkey });
|
||||
return pointer;
|
||||
} else {
|
||||
throw new Error(`Not found: ${key}`);
|
||||
throw new Error(`Not found: ${nip05}`);
|
||||
}
|
||||
} else {
|
||||
const result = await NIP05.lookup(key, { fetch: fetchWorker, signal });
|
||||
debug(`Found: ${key} is ${result.pubkey}`);
|
||||
const result = await NIP05.lookup(nip05, { fetch: fetchWorker, signal });
|
||||
logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'found', pubkey: result.pubkey });
|
||||
return result;
|
||||
}
|
||||
} catch (e) {
|
||||
debug(`Not found: ${key}`);
|
||||
logi({ level: 'info', ns: 'ditto.nip05', nip05, state: 'failed', error: errorJson(e) });
|
||||
throw e;
|
||||
}
|
||||
},
|
||||
|
|
|
|||
31
src/utils/ratelimiter/MemoryRateLimiter.test.ts
Normal file
31
src/utils/ratelimiter/MemoryRateLimiter.test.ts
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
import { assertEquals, assertThrows } from '@std/assert';
|
||||
|
||||
import { MemoryRateLimiter } from './MemoryRateLimiter.ts';
|
||||
import { RateLimitError } from './RateLimitError.ts';
|
||||
|
||||
Deno.test('MemoryRateLimiter', async (t) => {
|
||||
const limit = 5;
|
||||
const window = 100;
|
||||
|
||||
using limiter = new MemoryRateLimiter({ limit, window });
|
||||
|
||||
await t.step('can hit up to limit', () => {
|
||||
for (let i = 0; i < limit; i++) {
|
||||
const client = limiter.client('test');
|
||||
assertEquals(client.hits, i);
|
||||
client.hit();
|
||||
}
|
||||
});
|
||||
|
||||
await t.step('throws when hit if limit exceeded', () => {
|
||||
assertThrows(() => limiter.client('test').hit(), RateLimitError);
|
||||
});
|
||||
|
||||
await t.step('can hit after window resets', async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, window + 1));
|
||||
|
||||
const client = limiter.client('test');
|
||||
assertEquals(client.hits, 0);
|
||||
client.hit();
|
||||
});
|
||||
});
|
||||
77
src/utils/ratelimiter/MemoryRateLimiter.ts
Normal file
77
src/utils/ratelimiter/MemoryRateLimiter.ts
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
import { RateLimitError } from './RateLimitError.ts';
|
||||
import { RateLimiter, RateLimiterClient } from './types.ts';
|
||||
|
||||
interface MemoryRateLimiterOpts {
|
||||
limit: number;
|
||||
window: number;
|
||||
}
|
||||
|
||||
export class MemoryRateLimiter implements RateLimiter {
|
||||
private iid: number;
|
||||
|
||||
private previous = new Map<string, RateLimiterClient>();
|
||||
private current = new Map<string, RateLimiterClient>();
|
||||
|
||||
constructor(private opts: MemoryRateLimiterOpts) {
|
||||
this.iid = setInterval(() => {
|
||||
this.previous = this.current;
|
||||
this.current = new Map();
|
||||
}, opts.window);
|
||||
}
|
||||
|
||||
get limit(): number {
|
||||
return this.opts.limit;
|
||||
}
|
||||
|
||||
get window(): number {
|
||||
return this.opts.window;
|
||||
}
|
||||
|
||||
client(key: string): RateLimiterClient {
|
||||
const curr = this.current.get(key);
|
||||
const prev = this.previous.get(key);
|
||||
|
||||
if (curr) {
|
||||
return curr;
|
||||
}
|
||||
|
||||
if (prev && prev.resetAt > new Date()) {
|
||||
this.current.set(key, prev);
|
||||
this.previous.delete(key);
|
||||
return prev;
|
||||
}
|
||||
|
||||
const next = new MemoryRateLimiterClient(this);
|
||||
this.current.set(key, next);
|
||||
return next;
|
||||
}
|
||||
|
||||
[Symbol.dispose](): void {
|
||||
clearInterval(this.iid);
|
||||
}
|
||||
}
|
||||
|
||||
class MemoryRateLimiterClient implements RateLimiterClient {
|
||||
private _hits: number = 0;
|
||||
readonly resetAt: Date;
|
||||
|
||||
constructor(private limiter: MemoryRateLimiter) {
|
||||
this.resetAt = new Date(Date.now() + limiter.window);
|
||||
}
|
||||
|
||||
get hits(): number {
|
||||
return this._hits;
|
||||
}
|
||||
|
||||
get remaining(): number {
|
||||
return this.limiter.limit - this.hits;
|
||||
}
|
||||
|
||||
hit(n: number = 1): void {
|
||||
this._hits += n;
|
||||
|
||||
if (this.remaining < 0) {
|
||||
throw new RateLimitError(this.limiter, this);
|
||||
}
|
||||
}
|
||||
}
|
||||
41
src/utils/ratelimiter/MultiRateLimiter.test.ts
Normal file
41
src/utils/ratelimiter/MultiRateLimiter.test.ts
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
import { assertEquals, assertThrows } from '@std/assert';
|
||||
|
||||
import { MemoryRateLimiter } from './MemoryRateLimiter.ts';
|
||||
import { MultiRateLimiter } from './MultiRateLimiter.ts';
|
||||
|
||||
Deno.test('MultiRateLimiter', async (t) => {
|
||||
using limiter1 = new MemoryRateLimiter({ limit: 5, window: 100 });
|
||||
using limiter2 = new MemoryRateLimiter({ limit: 8, window: 200 });
|
||||
|
||||
const limiter = new MultiRateLimiter([limiter1, limiter2]);
|
||||
|
||||
await t.step('can hit up to first limit', () => {
|
||||
for (let i = 0; i < limiter1.limit; i++) {
|
||||
const client = limiter.client('test');
|
||||
assertEquals(client.hits, i);
|
||||
client.hit();
|
||||
}
|
||||
});
|
||||
|
||||
await t.step('throws when hit if first limit exceeded', () => {
|
||||
assertThrows(() => limiter.client('test').hit(), Error);
|
||||
});
|
||||
|
||||
await t.step('can hit up to second limit after the first window resets', async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, limiter1.window + 1));
|
||||
|
||||
const limit = limiter2.limit - limiter1.limit - 1;
|
||||
|
||||
for (let i = 0; i < limit; i++) {
|
||||
const client = limiter.client('test');
|
||||
assertEquals(client.hits, i);
|
||||
client.hit();
|
||||
}
|
||||
});
|
||||
|
||||
await t.step('throws when hit if second limit exceeded', () => {
|
||||
assertEquals(limiter.client('test').limiter, limiter1);
|
||||
assertThrows(() => limiter.client('test').hit(), Error);
|
||||
assertEquals(limiter.client('test').limiter, limiter2);
|
||||
});
|
||||
});
|
||||
51
src/utils/ratelimiter/MultiRateLimiter.ts
Normal file
51
src/utils/ratelimiter/MultiRateLimiter.ts
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
import { RateLimiter, RateLimiterClient } from './types.ts';
|
||||
|
||||
export class MultiRateLimiter {
|
||||
constructor(private limiters: RateLimiter[]) {}
|
||||
|
||||
client(key: string): MultiRateLimiterClient {
|
||||
return new MultiRateLimiterClient(key, this.limiters);
|
||||
}
|
||||
}
|
||||
|
||||
class MultiRateLimiterClient implements RateLimiterClient {
|
||||
constructor(private key: string, private limiters: RateLimiter[]) {
|
||||
if (!limiters.length) {
|
||||
throw new Error('No limiters provided');
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns the _active_ limiter, which is either the first exceeded or the first. */
|
||||
get limiter(): RateLimiter {
|
||||
const exceeded = this.limiters.find((limiter) => limiter.client(this.key).remaining < 0);
|
||||
return exceeded ?? this.limiters[0];
|
||||
}
|
||||
|
||||
get hits(): number {
|
||||
return this.limiter.client(this.key).hits;
|
||||
}
|
||||
|
||||
get resetAt(): Date {
|
||||
return this.limiter.client(this.key).resetAt;
|
||||
}
|
||||
|
||||
get remaining(): number {
|
||||
return this.limiter.client(this.key).remaining;
|
||||
}
|
||||
|
||||
hit(n?: number): void {
|
||||
let error: unknown;
|
||||
|
||||
for (const limiter of this.limiters) {
|
||||
try {
|
||||
limiter.client(this.key).hit(n);
|
||||
} catch (e) {
|
||||
error ??= e;
|
||||
}
|
||||
}
|
||||
|
||||
if (error instanceof Error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
10
src/utils/ratelimiter/RateLimitError.ts
Normal file
10
src/utils/ratelimiter/RateLimitError.ts
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
import { RateLimiter, RateLimiterClient } from './types.ts';
|
||||
|
||||
export class RateLimitError extends Error {
|
||||
constructor(
|
||||
readonly limiter: RateLimiter,
|
||||
readonly client: RateLimiterClient,
|
||||
) {
|
||||
super('Rate limit exceeded');
|
||||
}
|
||||
}
|
||||
12
src/utils/ratelimiter/types.ts
Normal file
12
src/utils/ratelimiter/types.ts
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
export interface RateLimiter extends Disposable {
|
||||
readonly limit: number;
|
||||
readonly window: number;
|
||||
client(key: string): RateLimiterClient;
|
||||
}
|
||||
|
||||
export interface RateLimiterClient {
|
||||
readonly hits: number;
|
||||
readonly resetAt: Date;
|
||||
readonly remaining: number;
|
||||
hit(n?: number): void;
|
||||
}
|
||||
|
|
@ -1,17 +1,15 @@
|
|||
import TTLCache from '@isaacs/ttlcache';
|
||||
import Debug from '@soapbox/stickynotes/debug';
|
||||
import { logi } from '@soapbox/logi';
|
||||
import DOMPurify from 'isomorphic-dompurify';
|
||||
import { unfurl } from 'unfurl.js';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
import { PreviewCard } from '@/entities/PreviewCard.ts';
|
||||
import { cachedLinkPreviewSizeGauge } from '@/metrics.ts';
|
||||
import { errorJson } from '@/utils/log.ts';
|
||||
import { fetchWorker } from '@/workers/fetch.ts';
|
||||
|
||||
const debug = Debug('ditto:unfurl');
|
||||
|
||||
async function unfurlCard(url: string, signal: AbortSignal): Promise<PreviewCard | null> {
|
||||
debug(`Unfurling ${url}...`);
|
||||
try {
|
||||
const result = await unfurl(url, {
|
||||
fetch: (url) =>
|
||||
|
|
@ -26,7 +24,7 @@ async function unfurlCard(url: string, signal: AbortSignal): Promise<PreviewCard
|
|||
|
||||
const { oEmbed, title, description, canonical_url, open_graph } = result;
|
||||
|
||||
return {
|
||||
const card = {
|
||||
type: oEmbed?.type || 'link',
|
||||
url: canonical_url || url,
|
||||
title: oEmbed?.title || title || '',
|
||||
|
|
@ -46,9 +44,12 @@ async function unfurlCard(url: string, signal: AbortSignal): Promise<PreviewCard
|
|||
embed_url: '',
|
||||
blurhash: null,
|
||||
};
|
||||
|
||||
logi({ level: 'info', ns: 'ditto.unfurl', url, success: true });
|
||||
|
||||
return card;
|
||||
} catch (e) {
|
||||
debug(`Failed to unfurl ${url}`);
|
||||
debug(e);
|
||||
logi({ level: 'info', ns: 'ditto.unfurl', url, success: false, error: errorJson(e) });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { HTTPException } from '@hono/hono/http-exception';
|
||||
import { Stickynotes } from '@soapbox/stickynotes';
|
||||
import { logi } from '@soapbox/logi';
|
||||
import { crypto } from '@std/crypto';
|
||||
import { encodeHex } from '@std/encoding/hex';
|
||||
import { encode } from 'blurhash';
|
||||
|
|
@ -8,8 +8,7 @@ import sharp from 'sharp';
|
|||
import { AppContext } from '@/app.ts';
|
||||
import { Conf } from '@/config.ts';
|
||||
import { DittoUpload, dittoUploads } from '@/DittoUploads.ts';
|
||||
|
||||
const console = new Stickynotes('ditto:uploader');
|
||||
import { errorJson } from '@/utils/log.ts';
|
||||
|
||||
interface FileMeta {
|
||||
pubkey: string;
|
||||
|
|
@ -86,7 +85,7 @@ export async function uploadFile(
|
|||
tags.push(['blurhash', blurhash]);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`Error parsing image metadata: ${e}`);
|
||||
logi({ level: 'error', ns: 'ditto.upload.analyze', error: errorJson(e) });
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,22 +1,22 @@
|
|||
/// <reference lib="webworker" />
|
||||
|
||||
import { safeFetch } from '@soapbox/safe-fetch';
|
||||
import { Stickynotes } from '@soapbox/stickynotes';
|
||||
import { logi } from '@soapbox/logi';
|
||||
import * as Comlink from 'comlink';
|
||||
|
||||
import '@/workers/handlers/abortsignal.ts';
|
||||
import '@/sentry.ts';
|
||||
|
||||
const console = new Stickynotes('ditto:fetch.worker');
|
||||
|
||||
export const FetchWorker = {
|
||||
async fetch(
|
||||
url: string,
|
||||
init: Omit<RequestInit, 'signal'>,
|
||||
signal: AbortSignal | null | undefined,
|
||||
): Promise<[BodyInit, ResponseInit]> {
|
||||
console.debug(init.method, url);
|
||||
logi({ level: 'debug', ns: 'ditto.fetch', method: init.method ?? 'GET', url });
|
||||
|
||||
const response = await safeFetch(url, { ...init, signal });
|
||||
|
||||
return [
|
||||
await response.arrayBuffer(),
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify';
|
||||
import { Stickynotes } from '@soapbox/stickynotes';
|
||||
import { logi } from '@soapbox/logi';
|
||||
import * as Comlink from 'comlink';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
|
|
@ -7,8 +7,6 @@ import type { CustomPolicy } from '@/workers/policy.worker.ts';
|
|||
|
||||
import '@/workers/handlers/abortsignal.ts';
|
||||
|
||||
const console = new Stickynotes('ditto:policy');
|
||||
|
||||
class PolicyWorker implements NPolicy {
|
||||
private worker: Comlink.Remote<CustomPolicy>;
|
||||
private ready: Promise<void>;
|
||||
|
|
@ -55,16 +53,34 @@ class PolicyWorker implements NPolicy {
|
|||
pubkey: Conf.pubkey,
|
||||
});
|
||||
|
||||
console.warn(`Using custom policy: ${Conf.policy}`);
|
||||
logi({
|
||||
level: 'info',
|
||||
ns: 'ditto.system.policy',
|
||||
msg: 'Using custom policy',
|
||||
path: Conf.policy,
|
||||
enabled: true,
|
||||
});
|
||||
} catch (e) {
|
||||
if (e instanceof Error && e.message.includes('Module not found')) {
|
||||
console.warn('Custom policy not found <https://docs.soapbox.pub/ditto/policies/>');
|
||||
logi({
|
||||
level: 'info',
|
||||
ns: 'ditto.system.policy',
|
||||
msg: 'Custom policy not found <https://docs.soapbox.pub/ditto/policies/>',
|
||||
path: null,
|
||||
enabled: false,
|
||||
});
|
||||
this.enabled = false;
|
||||
return;
|
||||
}
|
||||
|
||||
if (e instanceof Error && e.message.includes('PGlite is not supported in worker threads')) {
|
||||
console.warn('Custom policies are not supported with PGlite. The policy is disabled.');
|
||||
logi({
|
||||
level: 'warn',
|
||||
ns: 'ditto.system.policy',
|
||||
msg: 'Custom policies are not supported with PGlite. The policy is disabled.',
|
||||
path: Conf.policy,
|
||||
enabled: false,
|
||||
});
|
||||
this.enabled = false;
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue