diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b97226a0..48c5b253 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:1.45.5 +image: denoland/deno:1.46.3 default: interruptible: true @@ -35,11 +35,10 @@ test: postgres: stage: test - script: deno task db:migrate && deno task test + script: sleep 1 && deno task test services: - postgres:16 variables: DITTO_NSEC: nsec1zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs4rm7hz - DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres + TEST_DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres POSTGRES_HOST_AUTH_METHOD: trust - ALLOW_TO_USE_DATABASE_URL: true diff --git a/.tool-versions b/.tool-versions index 73d0a9db..900b9e20 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -deno 1.45.5 \ No newline at end of file +deno 1.46.3 \ No newline at end of file diff --git a/ansible/playbook.yml b/ansible/playbook.yml new file mode 100644 index 00000000..a2d8226e --- /dev/null +++ b/ansible/playbook.yml @@ -0,0 +1,23 @@ +--- +- name: Update Ditto + hosts: all + become: true + tasks: + - name: Update Soapbox + shell: + cmd: deno task soapbox + chdir: /opt/ditto + become_user: ditto + + - name: Update ditto from the main branch + git: + repo: 'https://gitlab.com/soapbox-pub/ditto.git' + dest: '/opt/ditto' + version: main + become_user: ditto + + - name: Restart ditto service + systemd: + name: ditto + state: restarted + become_user: root diff --git a/deno.json b/deno.json index 392f6f34..71f778d9 100644 --- a/deno.json +++ b/deno.json @@ -1,5 +1,4 @@ { - "$schema": "https://deno.land/x/deno@v1.41.0/cli/schemas/config-file.v1.json", "version": "1.1.0", "tasks": { "start": "deno run -A src/server.ts", @@ -19,8 +18,10 @@ "admin:role": "deno run -A scripts/admin-role.ts", "setup": "deno run -A scripts/setup.ts", "stats:recompute": "deno run -A scripts/stats-recompute.ts", - "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip soapbox.zip && rm soapbox.zip", - "trends": "deno run -A scripts/trends.ts" + "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip -o soapbox.zip && rm soapbox.zip", + "trends": "deno run -A scripts/trends.ts", + "clean:deps": "deno cache --reload src/app.ts", + "db:populate-search": "deno run -A scripts/db-populate-search.ts" }, "unstable": ["cron", "ffi", "kv", "worker-options"], "exclude": ["./public"], @@ -28,16 +29,16 @@ "@/": "./src/", "@b-fuze/deno-dom": "jsr:@b-fuze/deno-dom@^0.1.47", "@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", - "@db/sqlite": "jsr:@db/sqlite@^0.11.1", + "@electric-sql/pglite": "npm:@soapbox.pub/pglite@^0.2.10", "@hono/hono": "jsr:@hono/hono@^4.4.6", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/db": "jsr:@nostrify/db@^0.31.2", + "@nostrify/db": "jsr:@nostrify/db@^0.32.2", "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.30.1", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", - "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", + "@soapbox/kysely-pglite": "jsr:@soapbox/kysely-pglite@^0.0.1", "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0", "@std/assert": "jsr:@std/assert@^0.225.1", "@std/cli": "jsr:@std/cli@^0.223.0", @@ -62,6 +63,7 @@ "isomorphic-dompurify": "npm:isomorphic-dompurify@^2.11.0", "kysely": "npm:kysely@^0.27.4", "kysely-postgres-js": "npm:kysely-postgres-js@2.0.0", + "lande": "npm:lande@^1.0.10", "light-bolt11-decoder": "npm:light-bolt11-decoder", "linkify-plugin-hashtag": "npm:linkify-plugin-hashtag@^4.1.1", "linkify-string": "npm:linkify-string@^4.1.1", @@ -71,7 +73,7 @@ "nostr-tools": "npm:nostr-tools@2.5.1", "nostr-wasm": "npm:nostr-wasm@^0.1.0", "path-to-regexp": "npm:path-to-regexp@^7.1.0", - "postgres": "https://raw.githubusercontent.com/xyzshantaram/postgres.js/8a9bbce88b3f6425ecaacd99a80372338b157a53/deno/mod.js", + "postgres": "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/mod.js", "prom-client": "npm:prom-client@^15.1.2", "question-deno": "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/mod.ts", "tldts": "npm:tldts@^6.0.14", diff --git a/deno.lock b/deno.lock index f83c7ed2..11789c33 100644 --- a/deno.lock +++ b/deno.lock @@ -2,63 +2,68 @@ "version": "3", "packages": { "specifiers": { - "jsr:@b-fuze/deno-dom@^0.1.47": "jsr:@b-fuze/deno-dom@0.1.47", + "jsr:@b-fuze/deno-dom@^0.1.47": "jsr:@b-fuze/deno-dom@0.1.48", "jsr:@bradenmacdonald/s3-lite-client@^0.7.4": "jsr:@bradenmacdonald/s3-lite-client@0.7.6", - "jsr:@db/sqlite@^0.11.1": "jsr:@db/sqlite@0.11.1", - "jsr:@denosaurs/plug@1": "jsr:@denosaurs/plug@1.0.6", "jsr:@denosaurs/plug@1.0.3": "jsr:@denosaurs/plug@1.0.3", "jsr:@gleasonator/policy": "jsr:@gleasonator/policy@0.2.0", "jsr:@gleasonator/policy@0.2.0": "jsr:@gleasonator/policy@0.2.0", "jsr:@gleasonator/policy@0.4.0": "jsr:@gleasonator/policy@0.4.0", "jsr:@gleasonator/policy@0.4.1": "jsr:@gleasonator/policy@0.4.1", "jsr:@gleasonator/policy@0.4.2": "jsr:@gleasonator/policy@0.4.2", - "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.9", + "jsr:@gleasonator/policy@0.5.0": "jsr:@gleasonator/policy@0.5.0", + "jsr:@gleasonator/policy@0.5.1": "jsr:@gleasonator/policy@0.5.1", + "jsr:@gleasonator/policy@0.5.2": "jsr:@gleasonator/policy@0.5.2", + "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.11", "jsr:@lambdalisue/async@^2.1.1": "jsr:@lambdalisue/async@2.1.1", - "jsr:@nostrify/db@^0.31.2": "jsr:@nostrify/db@0.31.2", + "jsr:@nostrify/db@^0.32.2": "jsr:@nostrify/db@0.32.2", "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.30.0": "jsr:@nostrify/nostrify@0.30.1", "jsr:@nostrify/nostrify@^0.30.1": "jsr:@nostrify/nostrify@0.30.1", - "jsr:@nostrify/types@^0.30.0": "jsr:@nostrify/types@0.30.0", - "jsr:@soapbox/kysely-deno-sqlite@^2.1.0": "jsr:@soapbox/kysely-deno-sqlite@2.2.0", + "jsr:@nostrify/nostrify@^0.31.0": "jsr:@nostrify/nostrify@0.31.0", + "jsr:@nostrify/policies@^0.33.0": "jsr:@nostrify/policies@0.33.0", + "jsr:@nostrify/policies@^0.33.1": "jsr:@nostrify/policies@0.33.1", + "jsr:@nostrify/types@^0.30.0": "jsr:@nostrify/types@0.30.1", + "jsr:@nostrify/types@^0.30.1": "jsr:@nostrify/types@0.30.1", + "jsr:@soapbox/kysely-pglite@^0.0.1": "jsr:@soapbox/kysely-pglite@0.0.1", "jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0", "jsr:@std/assert@^0.213.1": "jsr:@std/assert@0.213.1", - "jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0", - "jsr:@std/assert@^0.221.0": "jsr:@std/assert@0.221.0", + "jsr:@std/assert@^0.223.0": "jsr:@std/assert@0.223.0", "jsr:@std/assert@^0.224.0": "jsr:@std/assert@0.224.0", "jsr:@std/assert@^0.225.1": "jsr:@std/assert@0.225.3", + "jsr:@std/bytes@^0.223.0": "jsr:@std/bytes@0.223.0", "jsr:@std/bytes@^0.224.0": "jsr:@std/bytes@0.224.0", "jsr:@std/bytes@^1.0.0-rc.3": "jsr:@std/bytes@1.0.0", "jsr:@std/bytes@^1.0.1-rc.3": "jsr:@std/bytes@1.0.2", "jsr:@std/bytes@^1.0.2": "jsr:@std/bytes@1.0.2", "jsr:@std/bytes@^1.0.2-rc.3": "jsr:@std/bytes@1.0.2", + "jsr:@std/cli@^0.223.0": "jsr:@std/cli@0.223.0", "jsr:@std/crypto@^0.224.0": "jsr:@std/crypto@0.224.0", "jsr:@std/dotenv@^0.224.0": "jsr:@std/dotenv@0.224.2", "jsr:@std/encoding@0.213.1": "jsr:@std/encoding@0.213.1", - "jsr:@std/encoding@^0.221.0": "jsr:@std/encoding@0.221.0", "jsr:@std/encoding@^0.224.0": "jsr:@std/encoding@0.224.3", "jsr:@std/encoding@^0.224.1": "jsr:@std/encoding@0.224.3", "jsr:@std/fmt@0.213.1": "jsr:@std/fmt@0.213.1", - "jsr:@std/fmt@^0.221.0": "jsr:@std/fmt@0.221.0", "jsr:@std/fs@0.213.1": "jsr:@std/fs@0.213.1", - "jsr:@std/fs@^0.221.0": "jsr:@std/fs@0.221.0", "jsr:@std/fs@^0.229.3": "jsr:@std/fs@0.229.3", - "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.1", - "jsr:@std/io@^0.224": "jsr:@std/io@0.224.6", + "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.3", + "jsr:@std/io@^0.223.0": "jsr:@std/io@0.223.0", + "jsr:@std/io@^0.224": "jsr:@std/io@0.224.7", "jsr:@std/json@^0.223.0": "jsr:@std/json@0.223.0", "jsr:@std/media-types@^0.224.1": "jsr:@std/media-types@0.224.1", "jsr:@std/path@0.213.1": "jsr:@std/path@0.213.1", - "jsr:@std/path@0.217": "jsr:@std/path@0.217.0", "jsr:@std/path@0.217.0": "jsr:@std/path@0.217.0", + "jsr:@std/path@1.0.0-rc.1": "jsr:@std/path@1.0.0-rc.1", "jsr:@std/path@^0.213.1": "jsr:@std/path@0.213.1", - "jsr:@std/path@^0.221.0": "jsr:@std/path@0.221.0", "jsr:@std/streams@^0.223.0": "jsr:@std/streams@0.223.0", "npm:@isaacs/ttlcache@^1.4.1": "npm:@isaacs/ttlcache@1.4.1", "npm:@noble/hashes@^1.4.0": "npm:@noble/hashes@1.4.0", + "npm:@noble/secp256k1@^2.0.0": "npm:@noble/secp256k1@2.1.0", "npm:@scure/base@^1.1.6": "npm:@scure/base@1.1.6", "npm:@scure/bip32@^1.4.0": "npm:@scure/bip32@1.4.0", "npm:@scure/bip39@^1.3.0": "npm:@scure/bip39@1.3.0", + "npm:@soapbox.pub/pglite@^0.2.10": "npm:@soapbox.pub/pglite@0.2.10", "npm:@types/node": "npm:@types/node@18.16.19", "npm:comlink-async-generator": "npm:comlink-async-generator@0.0.1", "npm:comlink-async-generator@^0.0.1": "npm:comlink-async-generator@0.0.1", @@ -74,6 +79,7 @@ "npm:kysely@^0.27.2": "npm:kysely@0.27.4", "npm:kysely@^0.27.3": "npm:kysely@0.27.4", "npm:kysely@^0.27.4": "npm:kysely@0.27.4", + "npm:lande@^1.0.10": "npm:lande@1.0.10", "npm:light-bolt11-decoder": "npm:light-bolt11-decoder@3.1.1", "npm:linkify-plugin-hashtag@^4.1.1": "npm:linkify-plugin-hashtag@4.1.3_linkifyjs@4.1.3", "npm:linkify-string@^4.1.1": "npm:linkify-string@4.1.3_linkifyjs@4.1.3", @@ -90,6 +96,7 @@ "npm:postgres@3.4.4": "npm:postgres@3.4.4", "npm:prom-client@^15.1.2": "npm:prom-client@15.1.2", "npm:tldts@^6.0.14": "npm:tldts@6.1.18", + "npm:tseep@^1.2.1": "npm:tseep@1.2.1", "npm:type-fest@^4.3.0": "npm:type-fest@4.18.2", "npm:unfurl.js@^6.4.0": "npm:unfurl.js@6.4.0", "npm:websocket-ts@^2.1.5": "npm:websocket-ts@2.1.5", @@ -102,19 +109,15 @@ "jsr:@denosaurs/plug@1.0.3" ] }, + "@b-fuze/deno-dom@0.1.48": { + "integrity": "bf5b591aef2e9e9c59adfcbb93a9ecd45bab5b7c8263625beafa5c8f1662e7da" + }, "@bradenmacdonald/s3-lite-client@0.7.6": { "integrity": "2b5976dca95d207dc88e23f9807e3eecbc441b0cf547dcda5784afe6668404d1", "dependencies": [ "jsr:@std/io@^0.224" ] }, - "@db/sqlite@0.11.1": { - "integrity": "546434e7ed762db07e6ade0f963540dd5e06723b802937bf260ff855b21ef9c5", - "dependencies": [ - "jsr:@denosaurs/plug@1", - "jsr:@std/path@0.217" - ] - }, "@denosaurs/plug@1.0.3": { "integrity": "b010544e386bea0ff3a1d05e0c88f704ea28cbd4d753439c2f1ee021a85d4640", "dependencies": [ @@ -124,15 +127,6 @@ "jsr:@std/path@0.213.1" ] }, - "@denosaurs/plug@1.0.6": { - "integrity": "6cf5b9daba7799837b9ffbe89f3450510f588fafef8115ddab1ff0be9cb7c1a7", - "dependencies": [ - "jsr:@std/encoding@^0.221.0", - "jsr:@std/fmt@^0.221.0", - "jsr:@std/fs@^0.221.0", - "jsr:@std/path@^0.221.0" - ] - }, "@gleasonator/policy@0.2.0": { "integrity": "3fe58b853ab203b2b67e65b64391dbcf5c07bc1caaf46e97b2f8ed5b14f30fdf", "dependencies": [ @@ -157,6 +151,27 @@ "jsr:@nostrify/nostrify@^0.22.1" ] }, + "@gleasonator/policy@0.5.0": { + "integrity": "c2882eb3b4147dfe96b6ec2870b012b5a614f686770d1d4b2f778fdc44e8b1f5", + "dependencies": [ + "jsr:@nostrify/nostrify@^0.31.0", + "jsr:@nostrify/policies@^0.33.0" + ] + }, + "@gleasonator/policy@0.5.1": { + "integrity": "2d687c5166556ce13ac05c4542f61ef8a47d8b96b57f6e43d52035805f895551", + "dependencies": [ + "jsr:@nostrify/nostrify@^0.31.0", + "jsr:@nostrify/policies@^0.33.0" + ] + }, + "@gleasonator/policy@0.5.2": { + "integrity": "cdd3add87be3132eb05736bca640dfb3bbb1aa79928a44d3563cde20bab7c0d3", + "dependencies": [ + "jsr:@nostrify/nostrify@^0.31.0", + "jsr:@nostrify/policies@^0.33.1" + ] + }, "@hono/hono@4.4.6": { "integrity": "aa557ca9930787ee86b9ca1730691f1ce1c379174c2cb244d5934db2b6314453" }, @@ -166,6 +181,9 @@ "@hono/hono@4.5.1": { "integrity": "459748ed4d4146c6e4bdff0213ff1ac44749904066ae02e7550d6c7f28c9bc4c" }, + "@hono/hono@4.5.11": { + "integrity": "5bd6b1a3a503efb746fdcf0aae3ac536dd09229d372988bde5db0798ef64ae4f" + }, "@hono/hono@4.5.3": { "integrity": "429923b2b3c6586a1450862328d61a1346fee5841e8ae86c494250475057213c" }, @@ -181,11 +199,11 @@ "@lambdalisue/async@2.1.1": { "integrity": "1fc9bc6f4ed50215cd2f7217842b18cea80f81c25744f88f8c5eb4be5a1c9ab4" }, - "@nostrify/db@0.31.2": { - "integrity": "a906b64edbf84a6b482cd7c9f5df2d2237c4ec42589116097d99ceb41347b1f5", + "@nostrify/db@0.32.2": { + "integrity": "265fb41e9d5810b99f1003ce56c89e4b468e6d0c04e7b9d9e3126c4efd49c1c2", "dependencies": [ - "jsr:@nostrify/nostrify@^0.30.0", - "jsr:@nostrify/types@^0.30.0", + "jsr:@nostrify/nostrify@^0.31.0", + "jsr:@nostrify/types@^0.30.1", "npm:kysely@^0.27.3", "npm:nostr-tools@^2.7.0" ] @@ -249,13 +267,43 @@ "npm:zod@^3.23.8" ] }, + "@nostrify/nostrify@0.31.0": { + "integrity": "1c1b686bb9ca3ad8d19807e3b96ef3793a65d70fd0f433fe6ef8b3fdb9f45557", + "dependencies": [ + "jsr:@nostrify/types@^0.30.1", + "jsr:@std/encoding@^0.224.1", + "npm:@scure/bip32@^1.4.0", + "npm:@scure/bip39@^1.3.0", + "npm:lru-cache@^10.2.0", + "npm:nostr-tools@^2.7.0", + "npm:websocket-ts@^2.1.5", + "npm:zod@^3.23.8" + ] + }, + "@nostrify/policies@0.33.0": { + "integrity": "c946b06d0527298b4d7c9819d142a10f522ba09eee76c37525aa4acfc5d87aee", + "dependencies": [ + "jsr:@nostrify/types@^0.30.1", + "npm:nostr-tools@^2.7.0" + ] + }, + "@nostrify/policies@0.33.1": { + "integrity": "381e1f9406a6da22da03a254e46b1aa07d5491b9761961cda3a4aeb5bf3f5286", + "dependencies": [ + "jsr:@nostrify/types@^0.30.1", + "npm:nostr-tools@^2.7.0" + ] + }, "@nostrify/types@0.30.0": { "integrity": "1f38fa849cff930bd709edbf94ef9ac02f46afb8b851f86c8736517b354616da" }, - "@soapbox/kysely-deno-sqlite@2.2.0": { - "integrity": "668ec94600bc4b4d7bd618dd7ca65d4ef30ee61c46ffcb379b6f45203c08517a", + "@nostrify/types@0.30.1": { + "integrity": "245da176f6893a43250697db51ad32bfa29bf9b1cdc1ca218043d9abf6de5ae5" + }, + "@soapbox/kysely-pglite@0.0.1": { + "integrity": "7a4221aa780aad6fba9747c45c59dfb1c62017ba8cad9db5607f6e5822c058d5", "dependencies": [ - "npm:kysely@^0.27.2" + "npm:kysely@^0.27.4" ] }, "@soapbox/stickynotes@0.4.0": { @@ -264,11 +312,8 @@ "@std/assert@0.213.1": { "integrity": "24c28178b30c8e0782c18e8e94ea72b16282207569cdd10ffb9d1d26f2edebfe" }, - "@std/assert@0.217.0": { - "integrity": "c98e279362ca6982d5285c3b89517b757c1e3477ee9f14eb2fdf80a45aaa9642" - }, - "@std/assert@0.221.0": { - "integrity": "a5f1aa6e7909dbea271754fd4ab3f4e687aeff4873b4cef9a320af813adb489a" + "@std/assert@0.223.0": { + "integrity": "eb8d6d879d76e1cc431205bd346ed4d88dc051c6366365b1af47034b0670be24" }, "@std/assert@0.224.0": { "integrity": "8643233ec7aec38a940a8264a6e3eed9bfa44e7a71cc6b3c8874213ff401967f" @@ -279,6 +324,9 @@ "jsr:@std/internal@^1.0.0" ] }, + "@std/bytes@0.223.0": { + "integrity": "84b75052cd8680942c397c2631318772b295019098f40aac5c36cead4cba51a8" + }, "@std/bytes@0.224.0": { "integrity": "a2250e1d0eb7d1c5a426f21267ab9bdeac2447fa87a3d0d1a467d3f7a6058e49" }, @@ -288,6 +336,12 @@ "@std/bytes@1.0.2": { "integrity": "fbdee322bbd8c599a6af186a1603b3355e59a5fb1baa139f8f4c3c9a1b3e3d57" }, + "@std/cli@0.223.0": { + "integrity": "2feb7970f2028904c3edc22ea916ce9538113dfc170844f3eae03578c333c356", + "dependencies": [ + "jsr:@std/assert@^0.223.0" + ] + }, "@std/crypto@0.224.0": { "integrity": "154ef3ff08ef535562ef1a718718c5b2c5fc3808f0f9100daad69e829bfcdf2d", "dependencies": [ @@ -304,18 +358,12 @@ "@std/encoding@0.213.1": { "integrity": "fcbb6928713dde941a18ca5db88ca1544d0755ec8fb20fe61e2dc8144b390c62" }, - "@std/encoding@0.221.0": { - "integrity": "d1dd76ef0dc5d14088411e6dc1dede53bf8308c95d1537df1214c97137208e45" - }, "@std/encoding@0.224.3": { "integrity": "5e861b6d81be5359fad4155e591acf17c0207b595112d1840998bb9f476dbdaf" }, "@std/fmt@0.213.1": { "integrity": "a06d31777566d874b9c856c10244ac3e6b660bdec4c82506cd46be052a1082c3" }, - "@std/fmt@0.221.0": { - "integrity": "379fed69bdd9731110f26b9085aeb740606b20428ce6af31ef6bd45ef8efa62a" - }, "@std/fs@0.213.1": { "integrity": "fbcaf099f8a85c27ab0712b666262cda8fe6d02e9937bf9313ecaea39a22c501", "dependencies": [ @@ -323,15 +371,11 @@ "jsr:@std/path@^0.213.1" ] }, - "@std/fs@0.221.0": { - "integrity": "028044450299de8ed5a716ade4e6d524399f035513b85913794f4e81f07da286", - "dependencies": [ - "jsr:@std/assert@^0.221.0", - "jsr:@std/path@^0.221.0" - ] - }, "@std/fs@0.229.3": { - "integrity": "783bca21f24da92e04c3893c9e79653227ab016c48e96b3078377ebd5222e6eb" + "integrity": "783bca21f24da92e04c3893c9e79653227ab016c48e96b3078377ebd5222e6eb", + "dependencies": [ + "jsr:@std/path@1.0.0-rc.1" + ] }, "@std/internal@1.0.0": { "integrity": "ac6a6dfebf838582c4b4f61a6907374e27e05bedb6ce276e0f1608fe84e7cd9a" @@ -339,6 +383,16 @@ "@std/internal@1.0.1": { "integrity": "6f8c7544d06a11dd256c8d6ba54b11ed870aac6c5aeafff499892662c57673e6" }, + "@std/internal@1.0.3": { + "integrity": "208e9b94a3d5649bd880e9ca38b885ab7651ab5b5303a56ed25de4755fb7b11e" + }, + "@std/io@0.223.0": { + "integrity": "2d8c3c2ab3a515619b90da2c6ff5ea7b75a94383259ef4d02116b228393f84f1", + "dependencies": [ + "jsr:@std/assert@^0.223.0", + "jsr:@std/bytes@^0.223.0" + ] + }, "@std/io@0.224.0": { "integrity": "0aff885d21d829c050b8a08b1d71b54aed5841aecf227f8d77e99ec529a11e8e", "dependencies": [ @@ -369,8 +423,17 @@ "jsr:@std/bytes@^1.0.2" ] }, + "@std/io@0.224.7": { + "integrity": "a70848793c44a7c100926571a8c9be68ba85487bfcd4d0540d86deabe1123dc9", + "dependencies": [ + "jsr:@std/bytes@^1.0.2" + ] + }, "@std/json@0.223.0": { - "integrity": "9a4a255931dd0397924c6b10bb6a72fe3e28ddd876b981ada2e3b8dd0764163f" + "integrity": "9a4a255931dd0397924c6b10bb6a72fe3e28ddd876b981ada2e3b8dd0764163f", + "dependencies": [ + "jsr:@std/streams@^0.223.0" + ] }, "@std/media-types@0.224.1": { "integrity": "9e69a5daed37c5b5c6d3ce4731dc191f80e67f79bed392b0957d1d03b87f11e1" @@ -381,20 +444,16 @@ "jsr:@std/assert@^0.213.1" ] }, - "@std/path@0.217.0": { - "integrity": "1217cc25534bca9a2f672d7fe7c6f356e4027df400c0e85c0ef3e4343bc67d11", - "dependencies": [ - "jsr:@std/assert@^0.217.0" - ] - }, - "@std/path@0.221.0": { - "integrity": "0a36f6b17314ef653a3a1649740cc8db51b25a133ecfe838f20b79a56ebe0095", - "dependencies": [ - "jsr:@std/assert@^0.221.0" - ] + "@std/path@1.0.0-rc.1": { + "integrity": "b8c00ae2f19106a6bb7cbf1ab9be52aa70de1605daeb2dbdc4f87a7cbaf10ff6" }, "@std/streams@0.223.0": { - "integrity": "d6b28e498ced3960b04dc5d251f2dcfc1df244b5ec5a48dc23a8f9b490be3b99" + "integrity": "d6b28e498ced3960b04dc5d251f2dcfc1df244b5ec5a48dc23a8f9b490be3b99", + "dependencies": [ + "jsr:@std/assert@^0.223.0", + "jsr:@std/bytes@^0.223.0", + "jsr:@std/io@^0.223.0" + ] } }, "npm": { @@ -440,6 +499,10 @@ "integrity": "sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg==", "dependencies": {} }, + "@noble/secp256k1@2.1.0": { + "integrity": "sha512-XLEQQNdablO0XZOIniFQimiXsZDNwaYgL96dZwC54Q30imSbAOFf3NKtepc+cXyuZf5Q1HCgbqgZ2UFFuHVcEw==", + "dependencies": {} + }, "@opentelemetry/api@1.9.0": { "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", "dependencies": {} @@ -482,6 +545,10 @@ "@scure/base": "@scure/base@1.1.6" } }, + "@soapbox.pub/pglite@0.2.10": { + "integrity": "sha512-DEHejCr+R99RNdyOo34Nbl1FKLmpBCc0pMlPhH3yTyc/KH5HV7dPYbTGCgqRXPxODVkQhvaEuIF2266KsUlZcg==", + "dependencies": {} + }, "@types/dompurify@3.0.5": { "integrity": "sha512-1Wg0g3BtQF7sSb27fJQAKck1HECM6zV1EB66j8JH9i3LCjYabJa0FSdiSgsD5K/RbrsR0SiraKacLB+T8ZVYAg==", "dependencies": { @@ -846,6 +913,12 @@ "integrity": "sha512-dyNKv2KRvYOQPLCAOCjjQuCk4YFd33BvGdf/o5bC7FiW+BB6snA81Zt+2wT9QDFzKqxKa5rrOmvlK/anehCcgA==", "dependencies": {} }, + "lande@1.0.10": { + "integrity": "sha512-yT52DQh+UV2pEp08jOYrA4drDv0DbjpiRyZYgl25ak9G2cVR2AimzrqkYQWrD9a7Ud+qkAcaiDDoNH9DXfHPmw==", + "dependencies": { + "toygrad": "toygrad@2.6.0" + } + }, "light-bolt11-decoder@3.1.1": { "integrity": "sha512-sLg/KCwYkgsHWkefWd6KqpCHrLFWWaXTOX3cf6yD2hAzL0SLpX+lFcaFK2spkjbgzG6hhijKfORDc9WoUHwX0A==", "dependencies": { @@ -1195,6 +1268,10 @@ "url-parse": "url-parse@1.5.10" } }, + "toygrad@2.6.0": { + "integrity": "sha512-g4zBmlSbvzOE5FOILxYkAybTSxijKLkj1WoNqVGnbMcWDyj4wWQ+eYSr3ik7XOpIgMq/7eBcPRTJX3DM2E0YMg==", + "dependencies": {} + }, "tr46@0.0.3": { "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", "dependencies": {} @@ -1205,6 +1282,10 @@ "punycode": "punycode@2.3.1" } }, + "tseep@1.2.1": { + "integrity": "sha512-VFnsNcPGC4qFJ1nxbIPSjTmtRZOhlqLmtwRqtLVos8mbRHki8HO9cy9Z1e89EiWyxFmq6LBviI9TQjijxw/mEw==", + "dependencies": {} + }, "type-fest@3.13.1": { "integrity": "sha512-tLq3bSNx+xSpwvAJnzrK0Ep5CLNWjvFTOp71URMaAEWBfRb9nnJiBoUe0tF8bI4ZFO3omgBR6NvnbzVUT3Ly4g==", "dependencies": {} @@ -1863,6 +1944,18 @@ "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/main/mod.ts": "662438fd3909984bb8cbaf3fd44d2121e949d11301baf21d6c3f057ccf9887de", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/main/src/PostgreSQLDriver.ts": "590c2fa248cff38e6e0f623050983039b5fde61e9c7131593d2922fb1f0eb921", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/main/src/PostgreSQLDriverDatabaseConnection.ts": "2158de426860bfd4f8e73afff0289bd40a11e273c8d883d4fd6474db01a9c2a7", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/mod.js": "cb68f17d6d90df318934deccdb469d740be0888e7a597a9e7eea7100ce36a252", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/polyfills.js": "318eb01f2b4cc33a46c59f3ddc11f22a56d6b1db8b7719b2ad7decee63a5bd47", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/bytes.js": "f2de43bdc8fa5dc4b169f2c70d5d8b053a3dea8f85ef011d7b27dec69e14ebb7", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/connection.js": "63bb06ad07cf802d295b35788261c34e82a80cec30b0dffafe05ccd74af3716f", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/errors.js": "85cfbed9a5ab0db41ab8e97b806c881af29807dfe99bc656fdf1a18c1c13b6c6", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/index.js": "4e8b09c7d0ce6e9eea386f59337867266498d5bb60ccd567d0bea5da03f6094d", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/large.js": "f3e770cdb7cc695f7b50687b4c6c4b7252129515486ec8def98b7582ee7c54ef", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/query.js": "67c45a5151032aa46b587abc15381fe4efd97c696e5c1b53082b8161309c4ee2", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/queue.js": "709624843223ea842bf095f6934080f19f1a059a51cbbf82e9827f3bb1bf2ca7", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/result.js": "001ff5e0c8d634674f483d07fbcd620a797e3101f842d6c20ca3ace936260465", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/subscribe.js": "9e4d0c3e573a6048e77ee2f15abbd5bcd17da9ca85a78c914553472c6d6c169b", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/types.js": "471f4a6c35412aa202a7c177c0a7e5a7c3bd225f01bbde67c947894c1b8bf6ed", "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/KeyCombo.ts": "a370b2dca76faa416d00e45479c8ce344971b5b86b44b4d0b213245c4bd2f8a3", "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/checkbox.ts": "e337ee7396aaefe6cc8c6349a445542fe7f0760311773369c9012b3fa278d21e", "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/config.ts": "a94a022c757f63ee7c410e29b97d3bfab1811889fb4483f56395cf376a911d1b", @@ -1915,12 +2008,11 @@ "dependencies": [ "jsr:@b-fuze/deno-dom@^0.1.47", "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", - "jsr:@db/sqlite@^0.11.1", "jsr:@hono/hono@^4.4.6", "jsr:@lambdalisue/async@^2.1.1", - "jsr:@nostrify/db@^0.31.2", + "jsr:@nostrify/db@^0.32.2", "jsr:@nostrify/nostrify@^0.30.1", - "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", + "jsr:@soapbox/kysely-pglite@^0.0.1", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", "jsr:@std/cli@^0.223.0", @@ -1935,6 +2027,7 @@ "npm:@isaacs/ttlcache@^1.4.1", "npm:@noble/secp256k1@^2.0.0", "npm:@scure/base@^1.1.6", + "npm:@soapbox.pub/pglite@^0.2.10", "npm:comlink-async-generator@^0.0.1", "npm:comlink@^4.4.1", "npm:commander@12.1.0", @@ -1946,6 +2039,7 @@ "npm:isomorphic-dompurify@^2.11.0", "npm:kysely-postgres-js@2.0.0", "npm:kysely@^0.27.4", + "npm:lande@^1.0.10", "npm:light-bolt11-decoder", "npm:linkify-plugin-hashtag@^4.1.1", "npm:linkify-string@^4.1.1", diff --git a/docs/debugging.md b/docs/debugging.md index 6abc513d..879f36cd 100644 --- a/docs/debugging.md +++ b/docs/debugging.md @@ -16,12 +16,12 @@ ssh -L 9229:localhost:9229 @ Then, in Chromium, go to `chrome://inspect` and the Ditto server should be available. -## SQLite performance +## SQL performance -To track slow queries, first set `DEBUG=ditto:sqlite.worker` in the environment so only SQLite logs are shown. +To track slow queries, first set `DEBUG=ditto:sql` in the environment so only SQL logs are shown. Then, grep for any logs above 0.001s: ```sh journalctl -fu ditto | grep -v '(0.00s)' -``` \ No newline at end of file +``` diff --git a/grafana/Ditto-Dashboard.json b/grafana/Ditto-Dashboard.json new file mode 100644 index 00000000..4b632806 --- /dev/null +++ b/grafana/Ditto-Dashboard.json @@ -0,0 +1,4570 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + }, + { + "name": "DS_DITTO-PG", + "label": "ditto-pg", + "description": "", + "type": "datasource", + "pluginId": "grafana-postgresql-datasource", + "pluginName": "PostgreSQL" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.2.0" + }, + { + "type": "datasource", + "id": "grafana-postgresql-datasource", + "name": "PostgreSQL", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Ditto application performance", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 22, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${prometheus}" + }, + "description": "Responses sent back to users from the server. Organized by status code.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",path!=\"/relay\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",path!=\"/relay\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",path!=\"/relay\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",path!=\"/relay\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "HTTP Responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of idle database connections available to the server. Higher is better. At 0, the site stops working.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "red", + "value": 0 + }, + { + "color": "#EAB839", + "value": 25 + }, + { + "color": "green", + "value": 50 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 11 + }, + "id": 8, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "ditto_db_available_connections", + "instant": false, + "legendFormat": "Connections", + "range": true, + "refId": "A" + } + ], + "title": "Database Available Connections", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Usage of system resources.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 6, + "y": 11 + }, + "id": 20, + "options": { + "displayMode": "lcd", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "CPU", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", + "hide": false, + "instant": false, + "legendFormat": "RAM", + "range": true, + "refId": "B" + } + ], + "title": "System Usage", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of individual database calls.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 11 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_db_queries_total[$__rate_interval])", + "instant": false, + "legendFormat": "Queries", + "range": true, + "refId": "A" + } + ], + "title": "Database Queries", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of Nostr clients currently connected to the Nostr relay.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 6, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "ditto_relay_connections", + "instant": false, + "legendFormat": "Connections", + "range": true, + "refId": "A" + } + ], + "title": "Relay Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Nostr events that are accepted or rejected by the custom policy script.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Accept" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Reject" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 19 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_policy_events_total{ok=\"true\"}[$__rate_interval])", + "instant": false, + "legendFormat": "Accept", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_policy_events_total{ok=\"false\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "Reject", + "range": true, + "refId": "B" + } + ], + "title": "Policy Events", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${prometheus}" + }, + "description": "Sum duration of all queries in milliseconds.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 19 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_db_query_duration_ms_sum[$__rate_interval])", + "instant": false, + "legendFormat": "Time", + "range": true, + "refId": "A" + } + ], + "title": "Database Query Time", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 27, + "panels": [ + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${prometheus}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 28 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v1/timelines/home\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"GET\",path=\"/api/v1/timelines/home\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v1/timelines/home\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"GET\",path=\"/api/v1/timelines/home\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "GET /api/v1/timelines/home", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 28 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v1/notifications\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"GET\",path=\"/api/v1/notifications\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v1/notifications\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"GET\",path=\"/api/v1/notifications\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "GET /api/v1/notifications", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 28 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v1/accounts/verify_credentials\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"GET\",path=\"/api/v1/accounts/verify_credentials\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v1/accounts/verify_credentials\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"GET\",path=\"/api/v1/accounts/verify_credentials\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "GET /api/v1/accounts/verify_credentials", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 28 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"POST\",path=\"/oauth/token\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"POST\",path=\"/oauth/token\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"POST\",path=\"/oauth/token\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"POST\",path=\"/oauth/token\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "POST /oauth/token", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 35 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v1/instance\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"GET\",path=\"/api/v1/instance\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v1/instance\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"GET\",path=\"/api/v1/instance\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "GET /api/v1/instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 35 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v1/timelines/public\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"GET\",path=\"/api/v1/timelines/public\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v1/timelines/public\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"GET\",path=\"/api/v1/timelines/public\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "GET /api/v1/timelines/public", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 35 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v1/timelines/tag/:hashtag\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"GET\",path=\"/api/v1/timelines/tag/:hashtag\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v1/timelines/tag/:hashtag\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"GET\",path=\"/api/v1/timelines/tag/:hashtag\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "GET /api/v1/timelines/tag/:hashtag", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 35 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v2/search\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"GET\",path=\"/api/v2/search\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v2/search\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"GET\",path=\"/api/v2/search\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "GET /api/v2/search", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 42 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"POST\",path=\"/api/v1/statuses\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"POST\",path=\"/api/v1/statuses\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"POST\",path=\"/api/v1/statuses\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"POST\",path=\"/api/v1/statuses\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "POST /api/v1/statuses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 42 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/favourite\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/favourite\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/favourite\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/favourite\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "POST /api/v1/statuses/:id/favourite", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 42 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/reblog\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/reblog\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/reblog\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/reblog\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "POST /api/v1/statuses/:id/reblog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 42 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"POST\",path=\"/api/v1/ditto/zap\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"POST\",path=\"/api/v1/ditto/zap\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"POST\",path=\"/api/v1/ditto/zap\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"POST\",path=\"/api/v1/ditto/zap\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "POST /api/v1/ditto/zap", + "type": "timeseries" + } + ], + "title": "API", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 21, + "panels": [], + "title": "Database", + "type": "row" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_DITTO-PG}" + }, + "description": "SQL queries ranked by total time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "super-light-green", + "value": 100 + }, + { + "color": "light-orange", + "value": 500 + }, + { + "color": "red", + "value": 1000 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "query" + }, + "properties": [ + { + "id": "displayName", + "value": "Query" + }, + { + "id": "custom.inspect", + "value": true + }, + { + "id": "custom.minWidth", + "value": 500 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "total_time" + }, + "properties": [ + { + "id": "displayName", + "value": "Total Time" + }, + { + "id": "unit", + "value": "ms" + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "average_time" + }, + "properties": [ + { + "id": "displayName", + "value": "Average Time" + }, + { + "id": "unit", + "value": "ms" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "calls" + }, + "properties": [ + { + "id": "displayName", + "value": "Calls" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "total_percent" + }, + "properties": [ + { + "id": "displayName", + "value": "Total Percent" + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "max_time" + }, + "properties": [ + { + "id": "displayName", + "value": "Max Time" + }, + { + "id": "unit", + "value": "ms" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 15, + "x": 0, + "y": 29 + }, + "id": 13, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Total Percent" + } + ] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_DITTO-PG}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH query_stats AS (\n SELECT\n LEFT(query, 10000) AS query,\n (total_plan_time + total_exec_time) AS total_time,\n (mean_plan_time + mean_exec_time) AS average_time,\n (max_plan_time + max_exec_time) AS max_time,\n calls\n FROM\n pg_stat_statements\n WHERE\n calls > 0\n)\nSELECT\n query,\n total_time,\n average_time,\n max_time,\n calls,\n total_time / (SELECT SUM(total_time) FROM query_stats) AS total_percent\nFROM\n query_stats\nORDER BY\n total_time DESC\nLIMIT 100;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "pg_stat_statements" + } + ], + "title": "Query Performance", + "type": "table" + }, + { + "datasource": { + "name": "${postgres}", + "type": "grafana-postgresql-datasource", + "uid": "${postgres}" + }, + "description": "Index hit rate. Higher is better.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "table" + }, + "properties": [ + { + "id": "displayName", + "value": "Table" + }, + { + "id": "custom.filterable", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "index" + }, + "properties": [ + { + "id": "displayName", + "value": "Index" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "percentage_used" + }, + "properties": [ + { + "id": "displayName", + "value": "Percentage Used" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge", + "valueDisplayMode": "color" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-RdYlGr" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 9, + "x": 15, + "y": 29 + }, + "id": 14, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_DITTO-PG}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT \n relname AS table,\n indexrelname AS index,\n (idx_blks_hit * 100) / (CASE WHEN idx_blks_hit + idx_blks_read = 0 THEN 1 ELSE idx_blks_hit + idx_blks_read END) as percentage_used\nFROM \n pg_statio_user_indexes\nWHERE \n schemaname = 'public'\nORDER BY\n percentage_used DESC;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Index Uses", + "type": "table" + }, + { + "datasource": { + "name": "${postgres}", + "type": "grafana-postgresql-datasource", + "uid": "${postgres}" + }, + "description": "Size of database tables.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 7, + "x": 0, + "y": 40 + }, + "id": 16, + "options": { + "displayLabels": [ + "name" + ], + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_DITTO-PG}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n c.relname AS table,\n pg_table_size(c.oid) AS size\nFROM\n pg_class c\n LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace)\nWHERE\n n.nspname NOT IN ('pg_catalog', 'information_schema')\n AND n.nspname !~ '^pg_toast'\n AND c.relkind IN ('r', 'm')\nORDER BY\n pg_table_size(c.oid) DESC;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Table Size", + "transformations": [ + { + "id": "rowsToFields", + "options": {} + } + ], + "type": "piechart" + }, + { + "datasource": { + "name": "${postgres}", + "type": "grafana-postgresql-datasource", + "uid": "${postgres}" + }, + "description": "Size of all indexes in the database.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 7, + "y": 40 + }, + "id": 17, + "options": { + "displayLabels": [ + "name" + ], + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false, + "values": [] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_DITTO-PG}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n c.relname AS name,\n sum(c.relpages :: bigint * 8192) :: bigint AS size\nFROM\n pg_class c\n LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace)\nWHERE\n n.nspname NOT IN ('pg_catalog', 'information_schema')\n AND n.nspname !~ '^pg_toast'\n AND c.relkind = 'i'\nGROUP BY\n (n.nspname, c.relname)\nORDER BY\n sum(c.relpages) DESC;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Index Size", + "transformations": [ + { + "id": "rowsToFields", + "options": {} + } + ], + "type": "piechart" + }, + { + "datasource": { + "name": "${postgres}", + "type": "grafana-postgresql-datasource", + "uid": "${postgres}" + }, + "description": "Indexes that are not used or very rarely used.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "table" + }, + "properties": [ + { + "id": "displayName", + "value": "Table" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "index" + }, + "properties": [ + { + "id": "displayName", + "value": "Index" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "index_size" + }, + "properties": [ + { + "id": "displayName", + "value": "Size" + }, + { + "id": "unit", + "value": "decbytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "index_scans" + }, + "properties": [ + { + "id": "displayName", + "value": "Scans" + }, + { + "id": "unit", + "value": "short" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 9, + "x": 15, + "y": 40 + }, + "id": 18, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_DITTO-PG}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n relname AS table,\n indexrelname AS index,\n pg_relation_size(i.indexrelid) AS index_size,\n idx_scan as index_scans\nFROM\n pg_stat_user_indexes ui\n JOIN pg_index i ON ui.indexrelid = i.indexrelid\nWHERE\n NOT indisunique\n AND idx_scan < 50\n AND pg_relation_size(relid) > 5 * 8192\nORDER BY\n pg_relation_size(i.indexrelid) / nullif(idx_scan, 0) DESC NULLS FIRST,\n pg_relation_size(i.indexrelid) DESC;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Unused Indexes", + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 23, + "panels": [], + "title": "Nostr", + "type": "row" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${prometheus}" + }, + "description": "Nostr messages sent to the relay over WebSocket.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "{instance=\"localhost:4036\", job=\"ditto\"}" + }, + "properties": [ + { + "id": "displayName", + "value": "Other" + }, + { + "id": "color", + "value": { + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "REQ" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "EVENT" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CLOSE" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(ditto_relay_messages_total[$__rate_interval])", + "format": "time_series", + "instant": false, + "legendFormat": "{{verb}}", + "range": true, + "refId": "A" + } + ], + "title": "Relay Messages", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${prometheus}" + }, + "description": "Nostr events from all sources. Organized by kind.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Kind 0" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "NIP-01", + "url": "https://github.com/nostr-protocol/nips/blob/master/01.md" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Kind 1" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "NIP-01", + "url": "https://github.com/nostr-protocol/nips/blob/master/01.md" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_pipeline_events_total[$__rate_interval])", + "instant": false, + "legendFormat": "Kind {{kind}}", + "range": true, + "refId": "A" + } + ], + "title": "Pipeline Events", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${prometheus}" + }, + "description": "Rate of events being processed by different parts of the application.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 74 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(ditto_pipeline_events_total[$__rate_interval]))", + "instant": false, + "legendFormat": "Pipeline Events", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "sum(increase(ditto_relay_events_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Relay Events", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(ditto_firehose_events_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Firehose Events", + "range": true, + "refId": "C" + } + ], + "title": "Event Sources", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "cdtcs576ar7cwb" + }, + "filters": [], + "hide": 0, + "name": "Filters", + "skipUrlSync": false, + "type": "adhoc" + }, + { + "current": {}, + "description": "Prometheus datasource", + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "prometheus", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "description": "PostgreSQL datasource", + "hide": 0, + "includeAll": false, + "label": "Postgres", + "multi": false, + "name": "postgres", + "options": [], + "query": "grafana-postgresql-datasource", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Ditto", + "uid": "ddps3ap51fv28d", + "version": 7, + "weekStart": "" +} \ No newline at end of file diff --git a/scripts/admin-event.ts b/scripts/admin-event.ts index 71c957f4..00711993 100644 --- a/scripts/admin-event.ts +++ b/scripts/admin-event.ts @@ -1,16 +1,13 @@ import { JsonParseStream } from '@std/json/json-parse-stream'; import { TextLineStream } from '@std/streams/text-line-stream'; -import { DittoDB } from '@/db/DittoDB.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; +import { Storages } from '@/storages.ts'; import { type EventStub } from '@/utils/api.ts'; import { nostrNow } from '@/utils.ts'; const signer = new AdminSigner(); - -const db = await DittoDB.getInstance(); -const eventsDB = new EventsDB(db); +const store = await Storages.db(); const readable = Deno.stdin.readable .pipeThrough(new TextDecoderStream()) @@ -25,7 +22,7 @@ for await (const t of readable) { ...t as EventStub, }); - await eventsDB.event(event); + await store.event(event); } Deno.exit(0); diff --git a/scripts/admin-role.ts b/scripts/admin-role.ts index 3f3c53f2..d275329f 100644 --- a/scripts/admin-role.ts +++ b/scripts/admin-role.ts @@ -1,13 +1,11 @@ import { NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { DittoDB } from '@/db/DittoDB.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; +import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; -const db = await DittoDB.getInstance(); -const eventsDB = new EventsDB(db); +const store = await Storages.db(); const [pubkeyOrNpub, role] = Deno.args; const pubkey = pubkeyOrNpub.startsWith('npub1') ? nip19.decode(pubkeyOrNpub as `npub1${string}`).data : pubkeyOrNpub; @@ -25,7 +23,7 @@ if (!['admin', 'user'].includes(role)) { const signer = new AdminSigner(); const admin = await signer.getPublicKey(); -const [existing] = await eventsDB.query([{ +const [existing] = await store.query([{ kinds: [30382], authors: [admin], '#d': [pubkey], @@ -59,6 +57,6 @@ const event = await signer.signEvent({ created_at: nostrNow(), }); -await eventsDB.event(event); +await store.event(event); Deno.exit(0); diff --git a/scripts/db-migrate.ts b/scripts/db-migrate.ts index ab0b9747..d3e93783 100644 --- a/scripts/db-migrate.ts +++ b/scripts/db-migrate.ts @@ -1,14 +1,7 @@ -import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; -import { sleep } from '@/test.ts'; - -if (Deno.env.get('CI') && Conf.db.dialect === 'postgres') { - console.info('Waiting 1 second for postgres to start...'); - await sleep(1_000); -} +import { Storages } from '@/storages.ts'; // This migrates kysely internally. -const { kysely } = await DittoDB.getInstance(); +const kysely = await Storages.kysely(); // Close the connection before exiting. await kysely.destroy(); diff --git a/scripts/db-populate-search.ts b/scripts/db-populate-search.ts new file mode 100644 index 00000000..e34aaa75 --- /dev/null +++ b/scripts/db-populate-search.ts @@ -0,0 +1,32 @@ +import { NSchema as n } from '@nostrify/nostrify'; +import { Storages } from '@/storages.ts'; + +const store = await Storages.db(); +const kysely = await Storages.kysely(); + +for await (const msg of store.req([{ kinds: [0] }])) { + if (msg[0] === 'EVENT') { + const { pubkey, content } = msg[2]; + + const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(content); + const search = [name, nip05].filter(Boolean).join(' ').trim(); + + try { + await kysely.insertInto('author_search').values({ + pubkey, + search, + }).onConflict( + (oc) => + oc.column('pubkey') + .doUpdateSet((eb) => ({ search: eb.ref('excluded.search') })), + ) + .execute(); + } catch { + // do nothing + } + } else { + break; + } +} + +Deno.exit(); diff --git a/scripts/nostr-pull.ts b/scripts/nostr-pull.ts index 68766c8f..f7a3840c 100644 --- a/scripts/nostr-pull.ts +++ b/scripts/nostr-pull.ts @@ -6,11 +6,9 @@ import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { DittoDB } from '@/db/DittoDB.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; +import { Storages } from '@/storages.ts'; -const db = await DittoDB.getInstance(); -const eventsDB = new EventsDB(db); +const store = await Storages.db(); interface ImportEventsOpts { profilesOnly: boolean; @@ -21,7 +19,7 @@ const importUsers = async ( authors: string[], relays: string[], opts?: Partial, - doEvent: DoEvent = async (event: NostrEvent) => await eventsDB.event(event), + doEvent: DoEvent = async (event: NostrEvent) => await store.event(event), ) => { // Kind 0s + follow lists. const profiles: Record> = {}; @@ -29,6 +27,18 @@ const importUsers = async ( const notes = new Set(); const { profilesOnly = false } = opts || {}; + const put = async (event: NostrEvent) => { + try { + await doEvent(event); + } catch (error) { + if (error.message.includes('violates unique constraint')) { + console.warn(`Skipping existing event ${event.id}...`); + } else { + console.error(error); + } + } + }; + await Promise.all(relays.map(async (relay) => { if (!relay.startsWith('wss://')) console.error(`Invalid relay url ${relay}`); const conn = new NRelay1(relay); @@ -49,7 +59,7 @@ const importUsers = async ( if (kind === 1 && !notes.has(event.id)) { // add the event to eventsDB only if it has not been found already. notes.add(event.id); - await doEvent(event); + await put(event); return; } @@ -64,7 +74,7 @@ const importUsers = async ( for (const user in profiles) { const profile = profiles[user]; for (const kind in profile) { - await doEvent(profile[kind]); + await put(profile[kind]); } let name = user; diff --git a/scripts/setup.ts b/scripts/setup.ts index 9a6d6f34..32376692 100644 --- a/scripts/setup.ts +++ b/scripts/setup.ts @@ -45,16 +45,16 @@ const DATABASE_URL = Deno.env.get('DATABASE_URL'); if (DATABASE_URL) { vars.DATABASE_URL = await question('input', 'Database URL', DATABASE_URL); } else { - const database = await question('list', 'Which database do you want to use?', ['postgres', 'sqlite']); - if (database === 'sqlite') { - const path = await question('input', 'Path to SQLite database', 'data/db.sqlite3'); - vars.DATABASE_URL = `sqlite://${path}`; + const database = await question('list', 'Which database do you want to use?', ['postgres', 'pglite']); + if (database === 'pglite') { + const path = await question('input', 'Path to PGlite data directory', 'data/pgdata'); + vars.DATABASE_URL = `file://${path}`; } if (database === 'postgres') { const host = await question('input', 'Postgres host', 'localhost'); const port = await question('input', 'Postgres port', '5432'); const user = await question('input', 'Postgres user', 'ditto'); - const password = await question('input', 'Postgres password', 'ditto'); + const password = await question('password', 'Postgres password', true); const database = await question('input', 'Postgres database', 'ditto'); vars.DATABASE_URL = `postgres://${user}:${password}@${host}:${port}/${database}`; } diff --git a/scripts/stats-recompute.ts b/scripts/stats-recompute.ts index 107a3167..77be13fe 100644 --- a/scripts/stats-recompute.ts +++ b/scripts/stats-recompute.ts @@ -1,6 +1,5 @@ import { nip19 } from 'nostr-tools'; -import { DittoDB } from '@/db/DittoDB.ts'; import { Storages } from '@/storages.ts'; import { refreshAuthorStats } from '@/utils/stats.ts'; @@ -18,6 +17,6 @@ try { } const store = await Storages.db(); -const kysely = await DittoDB.getInstance(); +const kysely = await Storages.kysely(); await refreshAuthorStats({ pubkey, kysely, store }); diff --git a/src/DittoUploads.ts b/src/DittoUploads.ts new file mode 100644 index 00000000..0024b1bf --- /dev/null +++ b/src/DittoUploads.ts @@ -0,0 +1,16 @@ +import { LRUCache } from 'lru-cache'; + +import { Time } from '@/utils/time.ts'; + +export interface DittoUpload { + id: string; + pubkey: string; + url: string; + tags: string[][]; + uploadedAt: Date; +} + +export const dittoUploads = new LRUCache({ + max: 1000, + ttl: Time.minutes(15), +}); diff --git a/src/app.ts b/src/app.ts index 9828724d..43ad4b00 100644 --- a/src/app.ts +++ b/src/app.ts @@ -53,7 +53,7 @@ import { instanceV2Controller, } from '@/controllers/api/instance.ts'; import { markersController, updateMarkersController } from '@/controllers/api/markers.ts'; -import { mediaController } from '@/controllers/api/media.ts'; +import { mediaController, updateMediaController } from '@/controllers/api/media.ts'; import { mutesController } from '@/controllers/api/mutes.ts'; import { notificationsController } from '@/controllers/api/notifications.ts'; import { createTokenController, oauthAuthorizeController, oauthController } from '@/controllers/api/oauth.ts'; @@ -226,6 +226,10 @@ app.delete('/api/v1/statuses/:id{[0-9a-f]{64}}', requireSigner, deleteStatusCont app.get('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/quotes', quotesController); app.post('/api/v1/media', mediaController); +app.put( + '/api/v1/media/:id{[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}', + updateMediaController, +); app.post('/api/v2/media', mediaController); app.get('/api/v1/timelines/home', requireSigner, homeTimelineController); diff --git a/src/config.ts b/src/config.ts index 0cb71509..f007341f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,5 +1,4 @@ -import url from 'node:url'; - +import os from 'node:os'; import * as dotenv from '@std/dotenv'; import { getPublicKey, nip19 } from 'nostr-tools'; import { z } from 'zod'; @@ -36,21 +35,11 @@ class Conf { } return this._pubkey; } - /** Ditto admin secret key as a Web Crypto key. */ - static get cryptoKey(): Promise { - return crypto.subtle.importKey( - 'raw', - Conf.seckey, - { name: 'HMAC', hash: 'SHA-256' }, - false, - ['sign', 'verify'], - ); - } - + /** Port to use when serving the HTTP server. */ static get port(): number { return parseInt(Deno.env.get('PORT') || '4036'); } - + /** Relay URL to the Ditto server's relay. */ static get relay(): `wss://${string}` | `ws://${string}` { const { protocol, host } = Conf.url; return `${protocol === 'https:' ? 'wss:' : 'ws:'}//${host}/relay`; @@ -82,22 +71,17 @@ class Conf { * ``` */ static get databaseUrl(): string { - return Deno.env.get('DATABASE_URL') ?? 'sqlite://data/db.sqlite3'; + return Deno.env.get('DATABASE_URL') ?? 'file://data/pgdata'; + } + /** Database to use in tests. */ + static get testDatabaseUrl(): string { + return Deno.env.get('TEST_DATABASE_URL') ?? 'memory://'; + } + /** PGlite debug level. 0 disables logging. */ + static get pgliteDebug(): 0 | 1 | 2 | 3 | 4 | 5 { + return Number(Deno.env.get('PGLITE_DEBUG') || 0) as 0 | 1 | 2 | 3 | 4 | 5; } static db = { - get url(): url.UrlWithStringQuery { - return url.parse(Conf.databaseUrl); - }, - get dialect(): 'sqlite' | 'postgres' | undefined { - switch (Conf.db.url.protocol) { - case 'sqlite:': - return 'sqlite'; - case 'postgres:': - case 'postgresql:': - return 'postgres'; - } - return undefined; - }, /** Database query timeout configurations. */ timeouts: { /** Default query timeout when another setting isn't more specific. */ @@ -198,12 +182,6 @@ class Conf { 'system', ]; } - /** Proof-of-work configuration. */ - static pow = { - get registrations(): number { - return Number(Deno.env.get('DITTO_POW_REGISTRATIONS') ?? 20); - }, - }; /** Domain of the Ditto server as a `URL` object, for easily grabbing the `hostname`, etc. */ static get url(): URL { return new URL(Conf.localDomain); @@ -216,21 +194,6 @@ class Conf { static get sentryDsn(): string | undefined { return Deno.env.get('SENTRY_DSN'); } - /** SQLite settings. */ - static sqlite = { - /** - * Number of bytes to use for memory-mapped IO. - * https://www.sqlite.org/pragma.html#pragma_mmap_size - */ - get mmapSize(): number { - const value = Deno.env.get('SQLITE_MMAP_SIZE'); - if (value) { - return Number(value); - } else { - return 1024 * 1024 * 1024; - } - }, - }; /** Postgres settings. */ static pg = { /** Number of connections to use in the pool. */ @@ -258,10 +221,22 @@ class Conf { 'i', ); } + /** User-Agent to use when fetching link previews. Pretend to be Facebook by default. */ + static get fetchUserAgent(): string { + return Deno.env.get('DITTO_FETCH_USER_AGENT') ?? 'facebookexternalhit'; + } /** Path to the custom policy module. Must be an absolute path, https:, npm:, or jsr: URI. */ static get policy(): string { return Deno.env.get('DITTO_POLICY') || new URL('../data/policy.ts', import.meta.url).pathname; } + /** Absolute path to the data directory used by Ditto. */ + static get dataDir(): string { + return Deno.env.get('DITTO_DATA_DIR') || new URL('../data', import.meta.url).pathname; + } + /** Absolute path of the Deno directory. */ + static get denoDir(): string { + return Deno.env.get('DENO_DIR') || `${os.userInfo().homedir}/.cache/deno`; + } /** Whether zap splits should be enabled. */ static get zapSplitsEnabled(): boolean { return optionalBooleanSchema.parse(Deno.env.get('ZAP_SPLITS_ENABLED')) ?? false; diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 9c635565..c946b697 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -6,6 +6,7 @@ import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { getAuthor, getFollowedPubkeys } from '@/queries.ts'; import { booleanParamSchema, fileSchema } from '@/schema.ts'; +import { getPubkeysBySearch } from '@/controllers/api/search.ts'; import { Storages } from '@/storages.ts'; import { uploadFile } from '@/utils/upload.ts'; import { nostrNow } from '@/utils.ts'; @@ -115,6 +116,7 @@ const accountSearchQuerySchema = z.object({ const accountSearchController: AppController = async (c) => { const { signal } = c.req.raw; const { limit } = c.get('pagination'); + const kysely = await Storages.kysely(); const result = accountSearchQuerySchema.safeParse(c.req.query()); @@ -133,8 +135,17 @@ const accountSearchController: AppController = async (c) => { return c.json(pubkey ? [await accountFromPubkey(pubkey)] : []); } - const events = event ? [event] : await store.query([{ kinds: [0], search: query, limit }], { signal }); + const pubkeys = await getPubkeysBySearch(kysely, { q: query, limit }); + let events = event ? [event] : await store.query([{ kinds: [0], authors: pubkeys, limit }], { + signal, + }); + + if (!event) { + events = pubkeys + .map((pubkey) => events.find((event) => event.pubkey === pubkey)) + .filter((event) => !!event); + } const accounts = await hydrateEvents({ events, store, signal }).then( (events) => Promise.all( diff --git a/src/controllers/api/instance.ts b/src/controllers/api/instance.ts index faca9c9f..b350963d 100644 --- a/src/controllers/api/instance.ts +++ b/src/controllers/api/instance.ts @@ -63,9 +63,6 @@ const instanceV1Controller: AppController = async (c) => { nostr: { pubkey: Conf.pubkey, relay: `${wsProtocol}//${host}/relay`, - pow: { - registrations: Conf.pow.registrations, - }, }, rules: [], }); diff --git a/src/controllers/api/media.ts b/src/controllers/api/media.ts index 71b3e782..7dc398ca 100644 --- a/src/controllers/api/media.ts +++ b/src/controllers/api/media.ts @@ -5,6 +5,7 @@ import { fileSchema } from '@/schema.ts'; import { parseBody } from '@/utils/api.ts'; import { renderAttachment } from '@/views/mastodon/attachments.ts'; import { uploadFile } from '@/utils/upload.ts'; +import { dittoUploads } from '@/DittoUploads.ts'; const mediaBodySchema = z.object({ file: fileSchema, @@ -13,6 +14,10 @@ const mediaBodySchema = z.object({ focus: z.string().optional(), }); +const mediaUpdateSchema = z.object({ + description: z.string(), +}); + const mediaController: AppController = async (c) => { const pubkey = await c.get('signer')?.getPublicKey()!; const result = mediaBodySchema.safeParse(await parseBody(c.req.raw)); @@ -32,4 +37,27 @@ const mediaController: AppController = async (c) => { } }; -export { mediaController }; +const updateMediaController: AppController = async (c) => { + const result = mediaUpdateSchema.safeParse(await parseBody(c.req.raw)); + + if (!result.success) { + return c.json({ error: 'Bad request.', schema: result.error }, 422); + } + + const id = c.req.param('id'); + const { description } = result.data; + const upload = dittoUploads.get(id); + + if (!upload) { + return c.json({ error: 'File with specified ID not found.' }, 404); + } + + dittoUploads.set(id, { + ...upload, + tags: upload.tags.filter(([name]) => name !== 'alt').concat([['alt', description]]), + }); + + return c.json({ message: 'ok' }, 200); +}; + +export { mediaController, updateMediaController }; diff --git a/src/controllers/api/notifications.ts b/src/controllers/api/notifications.ts index 54e88edf..1b9c746a 100644 --- a/src/controllers/api/notifications.ts +++ b/src/controllers/api/notifications.ts @@ -4,9 +4,10 @@ import { z } from 'zod'; import { AppContext, AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { DittoPagination } from '@/interfaces/DittoPagination.ts'; +import { getAmount } from '@/utils/bolt11.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { paginated } from '@/utils/api.ts'; -import { renderNotification } from '@/views/mastodon/notifications.ts'; +import { renderNotification, RenderNotificationOpts } from '@/views/mastodon/notifications.ts'; /** Set of known notification types across backends. */ const notificationTypes = new Set([ @@ -23,6 +24,7 @@ const notificationTypes = new Set([ 'severed_relationships', 'pleroma:emoji_reaction', 'ditto:name_grant', + 'ditto:zap', ]); const notificationsSchema = z.object({ @@ -50,6 +52,9 @@ const notificationsController: AppController = async (c) => { if (types.has('favourite') || types.has('pleroma:emoji_reaction')) { kinds.add(7); } + if (types.has('ditto:zap')) { + kinds.add(9735); + } const filter: NostrFilter = { kinds: [...kinds], @@ -81,16 +86,55 @@ async function renderNotifications( const { signal } = c.req.raw; const opts = { signal, limit: params.limit, timeout: Conf.db.timeouts.timelines }; + const zapsRelatedFilter: NostrFilter[] = []; + const events = await store .query(filters, opts) - .then((events) => events.filter((event) => event.pubkey !== pubkey)) + .then((events) => + events.filter((event) => { + if (event.kind === 9735) { + const zappedEventId = event.tags.find(([name]) => name === 'e')?.[1]; + if (zappedEventId) zapsRelatedFilter.push({ kinds: [1], ids: [zappedEventId] }); + const zapSender = event.tags.find(([name]) => name === 'P')?.[1]; + if (zapSender) zapsRelatedFilter.push({ kinds: [0], authors: [zapSender] }); + } + + return event.pubkey !== pubkey; + }) + ) .then((events) => hydrateEvents({ events, store, signal })); if (!events.length) { return c.json([]); } - const notifications = (await Promise.all(events.map((event) => renderNotification(event, { viewerPubkey: pubkey })))) + const zapSendersAndPosts = await store + .query(zapsRelatedFilter, opts) + .then((events) => hydrateEvents({ events, store, signal })); + + const notifications = (await Promise.all(events.map((event) => { + const opts: RenderNotificationOpts = { viewerPubkey: pubkey }; + if (event.kind === 9735) { + const zapRequestString = event?.tags?.find(([name]) => name === 'description')?.[1]; + const zapRequest = n.json().pipe(n.event()).optional().catch(undefined).parse(zapRequestString); + // By getting the pubkey from the zap request we guarantee who is the sender + // some clients don't put the P tag in the zap receipt... + const zapSender = zapRequest?.pubkey; + const zappedPost = event.tags.find(([name]) => name === 'e')?.[1]; + + const amountSchema = z.coerce.number().int().nonnegative().catch(0); + // amount in millisats + const amount = amountSchema.parse(getAmount(event?.tags.find(([name]) => name === 'bolt11')?.[1])); + + opts['zap'] = { + zapSender: zapSendersAndPosts.find(({ pubkey, kind }) => kind === 0 && pubkey === zapSender) ?? zapSender, + zappedPost: zapSendersAndPosts.find(({ id }) => id === zappedPost), + amount, + message: zapRequest?.content, + }; + } + return renderNotification(event, opts); + }))) .filter((notification) => notification && types.has(notification.type)); if (!notifications.length) { diff --git a/src/controllers/api/oauth.ts b/src/controllers/api/oauth.ts index a736f5ca..94aaeecd 100644 --- a/src/controllers/api/oauth.ts +++ b/src/controllers/api/oauth.ts @@ -6,7 +6,6 @@ import { z } from 'zod'; import { AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { nostrNow } from '@/utils.ts'; import { parseBody } from '@/utils/api.ts'; import { Storages } from '@/storages.ts'; @@ -82,7 +81,7 @@ const createTokenController: AppController = async (c) => { async function getToken( { pubkey, secret, relays = [] }: { pubkey: string; secret?: string; relays?: string[] }, ): Promise<`token1${string}`> { - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); const token = generateToken(); const serverSeckey = generateSecretKey(); diff --git a/src/controllers/api/search.test.ts b/src/controllers/api/search.test.ts new file mode 100644 index 00000000..2c5e91bd --- /dev/null +++ b/src/controllers/api/search.test.ts @@ -0,0 +1,21 @@ +import { assertEquals } from '@std/assert'; + +import { createTestDB } from '@/test.ts'; +import { getPubkeysBySearch } from '@/controllers/api/search.ts'; + +Deno.test('fuzzy search works', async () => { + await using db = await createTestDB(); + + await db.kysely.insertInto('author_search').values({ + pubkey: '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', + search: 'patrickReiis patrickdosreis.com', + }).execute(); + + assertEquals(await getPubkeysBySearch(db.kysely, { q: 'pat rick', limit: 1 }), []); + assertEquals(await getPubkeysBySearch(db.kysely, { q: 'patrick dos reis', limit: 1 }), [ + '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', + ]); + assertEquals(await getPubkeysBySearch(db.kysely, { q: 'dosreis.com', limit: 1 }), [ + '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', + ]); +}); diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 9bddc336..01fb6665 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -1,8 +1,10 @@ import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; +import { Kysely, sql } from 'kysely'; import { z } from 'zod'; import { AppController } from '@/app.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; import { booleanParamSchema } from '@/schema.ts'; import { Storages } from '@/storages.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; @@ -47,9 +49,8 @@ const searchController: AppController = async (c) => { if (event) { events = [event]; - } else { - events = await searchEvents(result.data, signal); } + events.push(...(await searchEvents(result.data, signal))); const viewerPubkey = await c.get('signer')?.getPublicKey(); @@ -89,10 +90,33 @@ async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: filter.authors = [account_id]; } + const pubkeys: string[] = []; + if (type === 'accounts') { + const kysely = await Storages.kysely(); + + pubkeys.push(...(await getPubkeysBySearch(kysely, { q, limit }))); + + if (!filter?.authors) { + filter.authors = pubkeys; + } else { + filter.authors.push(...pubkeys); + } + + filter.search = undefined; + } + const store = await Storages.search(); - return store.query([filter], { signal }) + let events = await store.query([filter], { signal }) .then((events) => hydrateEvents({ events, store, signal })); + + if (type !== 'accounts') return events; + + events = pubkeys + .map((pubkey) => events.find((event) => event.pubkey === pubkey)) + .filter((event) => !!event); + + return events; } /** Get event kinds to search from `type` query param. */ @@ -170,4 +194,16 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort return []; } -export { searchController }; +/** Get pubkeys whose name and NIP-05 is similar to 'q' */ +async function getPubkeysBySearch(kysely: Kysely, { q, limit }: Pick) { + const pubkeys = (await sql<{ pubkey: string }>` + SELECT *, word_similarity(${q}, search) AS sml + FROM author_search + WHERE ${q} % search + ORDER BY sml DESC, search LIMIT ${limit} + `.execute(kysely)).rows.map(({ pubkey }) => pubkey); + + return pubkeys; +} + +export { getPubkeysBySearch, searchController }; diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 621e26a8..e0956e74 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -1,3 +1,4 @@ +import { HTTPException } from '@hono/hono/http-exception'; import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; import ISO6391 from 'iso-639-1'; import 'linkify-plugin-hashtag'; @@ -6,22 +7,22 @@ import { nip19 } from 'nostr-tools'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; -import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; -import { addTag, deleteTag } from '@/utils/tags.ts'; -import { asyncReplaceAll } from '@/utils/text.ts'; import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; +import { DittoUpload, dittoUploads } from '@/DittoUploads.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { getAncestors, getAuthor, getDescendants, getEvent } from '@/queries.ts'; -import { getUnattachedMediaByIds } from '@/db/unattached-media.ts'; +import { addTag, deleteTag } from '@/utils/tags.ts'; +import { asyncReplaceAll } from '@/utils/text.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; -import { renderEventAccounts } from '@/views.ts'; -import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts'; import { Storages } from '@/storages.ts'; -import { hydrateEvents, purifyEvent } from '@/storages/hydrate.ts'; +import { hydrateEvents } from '@/storages/hydrate.ts'; import { createEvent, paginated, paginatedList, parseBody, updateListEvent } from '@/utils/api.ts'; import { getInvoice, getLnurl } from '@/utils/lnurl.ts'; +import { purifyEvent } from '@/utils/purify.ts'; import { getZapSplits } from '@/utils/zap-split.ts'; +import { renderEventAccounts } from '@/views.ts'; +import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; +import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts'; const createStatusSchema = z.object({ in_reply_to_id: n.id().nullish(), @@ -49,7 +50,6 @@ const statusController: AppController = async (c) => { const id = c.req.param('id'); const event = await getEvent(id, { - kind: 1, signal: AbortSignal.timeout(1500), }); @@ -63,7 +63,6 @@ const statusController: AppController = async (c) => { const createStatusController: AppController = async (c) => { const body = await parseBody(c.req.raw); const result = createStatusSchema.safeParse(body); - const { kysely } = await DittoDB.getInstance(); const store = c.get('store'); if (!result.success) { @@ -112,10 +111,18 @@ const createStatusController: AppController = async (c) => { tags.push(['l', data.language, 'ISO-639-1']); } - const media = data.media_ids?.length ? await getUnattachedMediaByIds(kysely, data.media_ids) : []; + const media: DittoUpload[] = (data.media_ids ?? []).map((id) => { + const upload = dittoUploads.get(id); - const imeta: string[][] = media.map(({ data }) => { - const values: string[] = data.map((tag) => tag.join(' ')); + if (!upload) { + throw new HTTPException(422, { message: 'Uploaded attachment is no longer available.' }); + } + + return upload; + }); + + const imeta: string[][] = media.map(({ tags }) => { + const values: string[] = tags.map((tag) => tag.join(' ')); return ['imeta', ...values]; }); @@ -123,21 +130,25 @@ const createStatusController: AppController = async (c) => { const pubkeys = new Set(); - const content = await asyncReplaceAll(data.status ?? '', /@([\w@+._]+)/g, async (match, username) => { - const pubkey = await lookupPubkey(username); - if (!pubkey) return match; + const content = await asyncReplaceAll( + data.status ?? '', + /(? { + const pubkey = await lookupPubkey(username); + if (!pubkey) return match; - // Content addressing (default) - if (!data.to) { - pubkeys.add(pubkey); - } + // Content addressing (default) + if (!data.to) { + pubkeys.add(pubkey); + } - try { - return `nostr:${nip19.npubEncode(pubkey)}`; - } catch { - return match; - } - }); + try { + return `nostr:${nip19.npubEncode(pubkey)}`; + } catch { + return match; + } + }, + ); // Explicit addressing for (const to of data.to ?? []) { @@ -161,7 +172,7 @@ const createStatusController: AppController = async (c) => { } const mediaUrls: string[] = media - .map(({ data }) => data.find(([name]) => name === 'url')?.[1]) + .map(({ url }) => url) .filter((url): url is string => Boolean(url)); const quoteCompat = data.quote_id ? `\n\nnostr:${nip19.noteEncode(data.quote_id)}` : ''; @@ -567,7 +578,7 @@ const zappedByController: AppController = async (c) => { const id = c.req.param('id'); const params = c.get('listPagination'); const store = await Storages.db(); - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); const zaps = await kysely.selectFrom('event_zaps') .selectAll() diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index 9a5b5deb..cfa8c3c5 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -4,7 +4,6 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { streamingConnectionsGauge } from '@/metrics.ts'; import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; import { getFeedPubkeys } from '@/queries.ts'; @@ -38,6 +37,25 @@ const streamSchema = z.enum([ type Stream = z.infer; +/** https://docs.joinmastodon.org/methods/streaming/#events-11 */ +interface StreamingEvent { + /** https://docs.joinmastodon.org/methods/streaming/#events */ + event: + | 'update' + | 'delete' + | 'notification' + | 'filters_changed' + | 'conversation' + | 'announcement' + | 'announcement.reaction' + | 'announcement.delete' + | 'status.update' + | 'encrypted_message' + | 'notifications_merged'; + payload: string; + stream: Stream[]; +} + const LIMITER_WINDOW = Time.minutes(5); const LIMITER_LIMIT = 100; @@ -73,18 +91,14 @@ const streamingController: AppController = async (c) => { const policy = pubkey ? new MuteListPolicy(pubkey, await Storages.admin()) : undefined; - function send(name: string, payload: object) { + function send(e: StreamingEvent) { if (socket.readyState === WebSocket.OPEN) { - debug('send', name, JSON.stringify(payload)); - socket.send(JSON.stringify({ - event: name, - payload: JSON.stringify(payload), - stream: [stream], - })); + debug('send', e.event, e.payload); + socket.send(JSON.stringify(e)); } } - async function sub(type: string, filters: NostrFilter[], render: (event: NostrEvent) => Promise) { + async function sub(filters: NostrFilter[], render: (event: NostrEvent) => Promise) { try { for await (const msg of pubsub.req(filters, { signal: controller.signal })) { if (msg[0] === 'EVENT') { @@ -102,7 +116,7 @@ const streamingController: AppController = async (c) => { const result = await render(event); if (result) { - send(type, result); + send(result); } } } @@ -118,19 +132,37 @@ const streamingController: AppController = async (c) => { const topicFilter = await topicToFilter(stream, c.req.query(), pubkey); if (topicFilter) { - sub('update', [topicFilter], async (event) => { + sub([topicFilter], async (event) => { + let payload: object | undefined; + if (event.kind === 1) { - return await renderStatus(event, { viewerPubkey: pubkey }); + payload = await renderStatus(event, { viewerPubkey: pubkey }); } if (event.kind === 6) { - return await renderReblog(event, { viewerPubkey: pubkey }); + payload = await renderReblog(event, { viewerPubkey: pubkey }); + } + + if (payload) { + return { + event: 'update', + payload: JSON.stringify(payload), + stream: [stream], + }; } }); } if (['user', 'user:notification'].includes(stream) && pubkey) { - sub('notification', [{ '#p': [pubkey] }], async (event) => { - return await renderNotification(event, { viewerPubkey: pubkey }); + sub([{ '#p': [pubkey] }], async (event) => { + if (event.pubkey === pubkey) return; // skip own events + const payload = await renderNotification(event, { viewerPubkey: pubkey }); + if (payload) { + return { + event: 'notification', + payload: JSON.stringify(payload), + stream: [stream], + }; + } }); return; } @@ -189,7 +221,7 @@ async function topicToFilter( async function getTokenPubkey(token: string): Promise { if (token.startsWith('token1')) { - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); const { user_pubkey } = await kysely .selectFrom('nip46_tokens') diff --git a/src/controllers/metrics.ts b/src/controllers/metrics.ts index e85294c0..4ef378a0 100644 --- a/src/controllers/metrics.ts +++ b/src/controllers/metrics.ts @@ -1,14 +1,16 @@ import { register } from 'prom-client'; import { AppController } from '@/app.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { dbAvailableConnectionsGauge, dbPoolSizeGauge } from '@/metrics.ts'; +import { Storages } from '@/storages.ts'; /** Prometheus/OpenMetrics controller. */ export const metricsController: AppController = async (c) => { + const db = await Storages.database(); + // Update some metrics at request time. - dbPoolSizeGauge.set(DittoDB.poolSize); - dbAvailableConnectionsGauge.set(DittoDB.availableConnections); + dbPoolSizeGauge.set(db.poolSize); + dbAvailableConnectionsGauge.set(db.availableConnections); const metrics = await register.metrics(); diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 583fd153..9f47b382 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -12,7 +12,7 @@ import { import { AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { relayInfoController } from '@/controllers/nostr/relay-info.ts'; -import { relayConnectionsGauge, relayEventCounter, relayMessageCounter } from '@/metrics.ts'; +import { relayConnectionsGauge, relayEventsCounter, relayMessagesCounter } from '@/metrics.ts'; import * as pipeline from '@/pipeline.ts'; import { RelayError } from '@/RelayError.ts'; import { Storages } from '@/storages.ts'; @@ -54,10 +54,10 @@ function connectStream(socket: WebSocket, ip: string | undefined) { const result = n.json().pipe(n.clientMsg()).safeParse(e.data); if (result.success) { - relayMessageCounter.inc({ verb: result.data[0] }); + relayMessagesCounter.inc({ verb: result.data[0] }); handleMsg(result.data); } else { - relayMessageCounter.inc(); + relayMessagesCounter.inc(); send(['NOTICE', 'Invalid message.']); } }; @@ -130,7 +130,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) { /** Handle EVENT. Store the event. */ async function handleEvent([_, event]: NostrClientEVENT): Promise { - relayEventCounter.inc({ kind: event.kind.toString() }); + relayEventsCounter.inc({ kind: event.kind.toString() }); try { // This will store it (if eligible) and run other side-effects. await pipeline.handleEvent(event, AbortSignal.timeout(1000)); diff --git a/src/db/DittoDB.ts b/src/db/DittoDB.ts index 5ed5d15e..923a109d 100644 --- a/src/db/DittoDB.ts +++ b/src/db/DittoDB.ts @@ -1,69 +1,32 @@ import fs from 'node:fs/promises'; import path from 'node:path'; -import { NDatabaseSchema, NPostgresSchema } from '@nostrify/db'; import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; -import { Conf } from '@/config.ts'; +import { DittoPglite } from '@/db/adapters/DittoPglite.ts'; import { DittoPostgres } from '@/db/adapters/DittoPostgres.ts'; -import { DittoSQLite } from '@/db/adapters/DittoSQLite.ts'; +import { DittoDatabase, DittoDatabaseOpts } from '@/db/DittoDatabase.ts'; import { DittoTables } from '@/db/DittoTables.ts'; -export type DittoDatabase = { - dialect: 'sqlite'; - kysely: Kysely & Kysely; -} | { - dialect: 'postgres'; - kysely: Kysely & Kysely; -}; - export class DittoDB { - private static db: Promise | undefined; + /** Open a new database connection. */ + static create(databaseUrl: string, opts?: DittoDatabaseOpts): DittoDatabase { + const { protocol } = new URL(databaseUrl); - static getInstance(): Promise { - if (!this.db) { - this.db = this._getInstance(); - } - return this.db; - } - - static async _getInstance(): Promise { - const result = {} as DittoDatabase; - - switch (Conf.db.dialect) { - case 'sqlite': - result.dialect = 'sqlite'; - result.kysely = await DittoSQLite.getInstance(); - break; - case 'postgres': - result.dialect = 'postgres'; - result.kysely = await DittoPostgres.getInstance(); - break; + switch (protocol) { + case 'file:': + case 'memory:': + return DittoPglite.create(databaseUrl, opts); + case 'postgres:': + case 'postgresql:': + return DittoPostgres.create(databaseUrl, opts); default: throw new Error('Unsupported database URL.'); } - - await this.migrate(result.kysely); - - return result; - } - - static get poolSize(): number { - if (Conf.db.dialect === 'postgres') { - return DittoPostgres.poolSize; - } - return 1; - } - - static get availableConnections(): number { - if (Conf.db.dialect === 'postgres') { - return DittoPostgres.availableConnections; - } - return 1; } /** Migrate the database to the latest version. */ - static async migrate(kysely: DittoDatabase['kysely']) { + static async migrate(kysely: Kysely) { const migrator = new Migrator({ db: kysely, provider: new FileMigrationProvider({ diff --git a/src/db/DittoDatabase.ts b/src/db/DittoDatabase.ts new file mode 100644 index 00000000..ec9a103d --- /dev/null +++ b/src/db/DittoDatabase.ts @@ -0,0 +1,14 @@ +import { Kysely } from 'kysely'; + +import { DittoTables } from '@/db/DittoTables.ts'; + +export interface DittoDatabase { + readonly kysely: Kysely; + readonly poolSize: number; + readonly availableConnections: number; +} + +export interface DittoDatabaseOpts { + poolSize?: number; + debug?: 0 | 1 | 2 | 3 | 4 | 5; +} diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index 69356649..642db484 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -1,12 +1,21 @@ -export interface DittoTables { +import { Nullable } from 'kysely'; + +import { NPostgresSchema } from '@nostrify/db'; + +export interface DittoTables extends NPostgresSchema { + nostr_events: NostrEventsRow; nip46_tokens: NIP46TokenRow; - unattached_media: UnattachedMediaRow; author_stats: AuthorStatsRow; event_stats: EventStatsRow; pubkey_domains: PubkeyDomainRow; event_zaps: EventZapRow; + author_search: AuthorSearch; } +type NostrEventsRow = NPostgresSchema['nostr_events'] & { + language: Nullable; +}; + interface AuthorStatsRow { pubkey: string; followers_count: number; @@ -33,14 +42,6 @@ interface NIP46TokenRow { connected_at: Date; } -interface UnattachedMediaRow { - id: string; - pubkey: string; - url: string; - data: string; - uploaded_at: number; -} - interface PubkeyDomainRow { pubkey: string; domain: string; @@ -54,3 +55,8 @@ interface EventZapRow { amount_millisats: number; comment: string; } + +interface AuthorSearch { + pubkey: string; + search: string; +} diff --git a/src/db/KyselyLogger.ts b/src/db/KyselyLogger.ts index 3b5b4398..72167e21 100644 --- a/src/db/KyselyLogger.ts +++ b/src/db/KyselyLogger.ts @@ -1,6 +1,6 @@ import { Stickynotes } from '@soapbox/stickynotes'; import { Logger } from 'kysely'; -import { dbQueryCounter, dbQueryTimeHistogram } from '@/metrics.ts'; +import { dbQueriesCounter, dbQueryDurationHistogram } from '@/metrics.ts'; /** Log the SQL for queries. */ export const KyselyLogger: Logger = (event) => { @@ -9,8 +9,8 @@ export const KyselyLogger: Logger = (event) => { const { query, queryDurationMillis } = event; const { sql, parameters } = query; - dbQueryCounter.inc(); - dbQueryTimeHistogram.observe(queryDurationMillis); + dbQueriesCounter.inc(); + dbQueryDurationHistogram.observe(queryDurationMillis); console.debug( sql, diff --git a/src/db/adapters/DittoPglite.ts b/src/db/adapters/DittoPglite.ts new file mode 100644 index 00000000..2455fc37 --- /dev/null +++ b/src/db/adapters/DittoPglite.ts @@ -0,0 +1,28 @@ +import { PGlite } from '@electric-sql/pglite'; +import { pg_trgm } from '@electric-sql/pglite/contrib/pg_trgm'; +import { PgliteDialect } from '@soapbox/kysely-pglite'; +import { Kysely } from 'kysely'; + +import { DittoDatabase, DittoDatabaseOpts } from '@/db/DittoDatabase.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; +import { KyselyLogger } from '@/db/KyselyLogger.ts'; + +export class DittoPglite { + static create(databaseUrl: string, opts?: DittoDatabaseOpts): DittoDatabase { + const kysely = new Kysely({ + dialect: new PgliteDialect({ + database: new PGlite(databaseUrl, { + extensions: { pg_trgm }, + debug: opts?.debug, + }), + }), + log: KyselyLogger, + }); + + return { + kysely, + poolSize: 1, + availableConnections: 1, + }; + } +} diff --git a/src/db/adapters/DittoPostgres.ts b/src/db/adapters/DittoPostgres.ts index d1127117..f1a5bcc9 100644 --- a/src/db/adapters/DittoPostgres.ts +++ b/src/db/adapters/DittoPostgres.ts @@ -1,4 +1,3 @@ -import { NPostgresSchema } from '@nostrify/db'; import { BinaryOperationNode, FunctionNode, @@ -13,51 +12,43 @@ import { import { PostgresJSDialectConfig, PostgresJSDriver } from 'kysely-postgres-js'; import postgres from 'postgres'; -import { Conf } from '@/config.ts'; +import { DittoDatabase, DittoDatabaseOpts } from '@/db/DittoDatabase.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { KyselyLogger } from '@/db/KyselyLogger.ts'; export class DittoPostgres { - static db: Kysely & Kysely | undefined; - static postgres?: postgres.Sql; + static create(databaseUrl: string, opts?: DittoDatabaseOpts): DittoDatabase { + const pg = postgres(databaseUrl, { max: opts?.poolSize }); - // deno-lint-ignore require-await - static async getInstance(): Promise & Kysely> { - if (!this.postgres) { - this.postgres = postgres(Conf.databaseUrl, { max: Conf.pg.poolSize }); - } - - if (!this.db) { - this.db = new Kysely({ - dialect: { - createAdapter() { - return new PostgresAdapter(); - }, - createDriver() { - return new PostgresJSDriver({ - postgres: DittoPostgres.postgres as unknown as PostgresJSDialectConfig['postgres'], - }); - }, - createIntrospector(db) { - return new PostgresIntrospector(db); - }, - createQueryCompiler() { - return new DittoPostgresQueryCompiler(); - }, + const kysely = new Kysely({ + dialect: { + createAdapter() { + return new PostgresAdapter(); }, - log: KyselyLogger, - }) as Kysely & Kysely; - } + createDriver() { + return new PostgresJSDriver({ + postgres: pg as unknown as PostgresJSDialectConfig['postgres'], + }); + }, + createIntrospector(db) { + return new PostgresIntrospector(db); + }, + createQueryCompiler() { + return new DittoPostgresQueryCompiler(); + }, + }, + log: KyselyLogger, + }); - return this.db; - } - - static get poolSize() { - return this.postgres?.connections.open ?? 0; - } - - static get availableConnections(): number { - return this.postgres?.connections.idle ?? 0; + return { + kysely, + get poolSize() { + return pg.connections.open; + }, + get availableConnections() { + return pg.connections.idle; + }, + }; } } diff --git a/src/db/adapters/DittoSQLite.ts b/src/db/adapters/DittoSQLite.ts deleted file mode 100644 index e54292dd..00000000 --- a/src/db/adapters/DittoSQLite.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { NDatabaseSchema } from '@nostrify/db'; -import { PolySqliteDialect } from '@soapbox/kysely-deno-sqlite'; -import { Kysely, sql } from 'kysely'; - -import { Conf } from '@/config.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; -import { KyselyLogger } from '@/db/KyselyLogger.ts'; -import SqliteWorker from '@/workers/sqlite.ts'; - -export class DittoSQLite { - static db: Kysely & Kysely | undefined; - - static async getInstance(): Promise & Kysely> { - if (!this.db) { - const sqliteWorker = new SqliteWorker(); - await sqliteWorker.open(this.path); - - this.db = new Kysely({ - dialect: new PolySqliteDialect({ - database: sqliteWorker, - }), - log: KyselyLogger, - }) as Kysely & Kysely; - - // Set PRAGMA values. - await Promise.all([ - sql`PRAGMA synchronous = normal`.execute(this.db), - sql`PRAGMA temp_store = memory`.execute(this.db), - sql`PRAGMA foreign_keys = ON`.execute(this.db), - sql`PRAGMA auto_vacuum = FULL`.execute(this.db), - sql`PRAGMA journal_mode = WAL`.execute(this.db), - sql.raw(`PRAGMA mmap_size = ${Conf.sqlite.mmapSize}`).execute(this.db), - ]); - } - return this.db; - } - - /** Get the relative or absolute path based on the `DATABASE_URL`. */ - static get path() { - if (Conf.databaseUrl === 'sqlite://:memory:') { - return ':memory:'; - } - - const { host, pathname } = Conf.db.url; - - if (!pathname) return ''; - - // Get relative path. - if (host === '') { - return pathname; - } else if (host === '.') { - return pathname; - } else if (host) { - return host + pathname; - } - - return ''; - } -} diff --git a/src/db/migrations/002_events_fts.ts b/src/db/migrations/002_events_fts.ts index 56abab5f..45ad03e4 100644 --- a/src/db/migrations/002_events_fts.ts +++ b/src/db/migrations/002_events_fts.ts @@ -1,13 +1,8 @@ -import { Kysely, sql } from 'kysely'; +import { Kysely } from 'kysely'; -import { Conf } from '@/config.ts'; - -export async function up(db: Kysely): Promise { - if (Conf.db.dialect === 'sqlite') { - await sql`CREATE VIRTUAL TABLE events_fts USING fts5(id, content)`.execute(db); - } +export async function up(_db: Kysely): Promise { + // This migration used to create an FTS table for SQLite, but SQLite support was removed. } -export async function down(db: Kysely): Promise { - await db.schema.dropTable('events_fts').ifExists().execute(); +export async function down(_db: Kysely): Promise { } diff --git a/src/db/migrations/019_ndatabase_schema.ts b/src/db/migrations/019_ndatabase_schema.ts index 31b86cd3..79d8cbc9 100644 --- a/src/db/migrations/019_ndatabase_schema.ts +++ b/src/db/migrations/019_ndatabase_schema.ts @@ -1,25 +1,13 @@ -import { Kysely, sql } from 'kysely'; - -import { Conf } from '@/config.ts'; +import { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.alterTable('events').renameTo('nostr_events').execute(); await db.schema.alterTable('tags').renameTo('nostr_tags').execute(); await db.schema.alterTable('nostr_tags').renameColumn('tag', 'name').execute(); - - if (Conf.db.dialect === 'sqlite') { - await db.schema.dropTable('events_fts').execute(); - await sql`CREATE VIRTUAL TABLE nostr_fts5 USING fts5(event_id, content)`.execute(db); - } } export async function down(db: Kysely): Promise { await db.schema.alterTable('nostr_events').renameTo('events').execute(); await db.schema.alterTable('nostr_tags').renameTo('tags').execute(); await db.schema.alterTable('tags').renameColumn('name', 'tag').execute(); - - if (Conf.db.dialect === 'sqlite') { - await db.schema.dropTable('nostr_fts5').execute(); - await sql`CREATE VIRTUAL TABLE events_fts USING fts5(id, content)`.execute(db); - } } diff --git a/src/db/migrations/020_pgfts.ts b/src/db/migrations/020_pgfts.ts index 835de117..26e320ec 100644 --- a/src/db/migrations/020_pgfts.ts +++ b/src/db/migrations/020_pgfts.ts @@ -1,19 +1,13 @@ import { Kysely, sql } from 'kysely'; -import { Conf } from '@/config.ts'; - export async function up(db: Kysely): Promise { - if (Conf.db.dialect === 'postgres') { - await db.schema.createTable('nostr_pgfts') - .ifNotExists() - .addColumn('event_id', 'text', (c) => c.primaryKey().references('nostr_events.id').onDelete('cascade')) - .addColumn('search_vec', sql`tsvector`, (c) => c.notNull()) - .execute(); - } + await db.schema.createTable('nostr_pgfts') + .ifNotExists() + .addColumn('event_id', 'text', (c) => c.primaryKey().references('nostr_events.id').onDelete('cascade')) + .addColumn('search_vec', sql`tsvector`, (c) => c.notNull()) + .execute(); } export async function down(db: Kysely): Promise { - if (Conf.db.dialect === 'postgres') { - await db.schema.dropTable('nostr_pgfts').ifExists().execute(); - } + await db.schema.dropTable('nostr_pgfts').ifExists().execute(); } diff --git a/src/db/migrations/021_pgfts_index.ts b/src/db/migrations/021_pgfts_index.ts index 4b834995..7ad24546 100644 --- a/src/db/migrations/021_pgfts_index.ts +++ b/src/db/migrations/021_pgfts_index.ts @@ -1,21 +1,15 @@ import { Kysely } from 'kysely'; -import { Conf } from '@/config.ts'; - export async function up(db: Kysely): Promise { - if (Conf.db.dialect === 'postgres') { - await db.schema - .createIndex('nostr_pgfts_gin_search_vec') - .ifNotExists() - .on('nostr_pgfts') - .using('gin') - .column('search_vec') - .execute(); - } + await db.schema + .createIndex('nostr_pgfts_gin_search_vec') + .ifNotExists() + .on('nostr_pgfts') + .using('gin') + .column('search_vec') + .execute(); } export async function down(db: Kysely): Promise { - if (Conf.db.dialect === 'postgres') { - await db.schema.dropIndex('nostr_pgfts_gin_search_vec').ifExists().execute(); - } + await db.schema.dropIndex('nostr_pgfts_gin_search_vec').ifExists().execute(); } diff --git a/src/db/migrations/030_pg_events_jsonb.ts b/src/db/migrations/030_pg_events_jsonb.ts index 7bfc6c17..dcd6ad85 100644 --- a/src/db/migrations/030_pg_events_jsonb.ts +++ b/src/db/migrations/030_pg_events_jsonb.ts @@ -1,10 +1,6 @@ import { Kysely, sql } from 'kysely'; -import { Conf } from '@/config.ts'; - export async function up(db: Kysely): Promise { - if (Conf.db.dialect !== 'postgres') return; - // Create new table and indexes. await db.schema .createTable('nostr_events_new') diff --git a/src/db/migrations/031_rm_unattached_media.ts b/src/db/migrations/031_rm_unattached_media.ts new file mode 100644 index 00000000..febd85e1 --- /dev/null +++ b/src/db/migrations/031_rm_unattached_media.ts @@ -0,0 +1,34 @@ +import { Kysely } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema.dropTable('unattached_media').execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema + .createTable('unattached_media') + .addColumn('id', 'text', (c) => c.primaryKey()) + .addColumn('pubkey', 'text', (c) => c.notNull()) + .addColumn('url', 'text', (c) => c.notNull()) + .addColumn('data', 'text', (c) => c.notNull()) + .addColumn('uploaded_at', 'bigint', (c) => c.notNull()) + .execute(); + + await db.schema + .createIndex('unattached_media_id') + .on('unattached_media') + .column('id') + .execute(); + + await db.schema + .createIndex('unattached_media_pubkey') + .on('unattached_media') + .column('pubkey') + .execute(); + + await db.schema + .createIndex('unattached_media_url') + .on('unattached_media') + .column('url') + .execute(); +} diff --git a/src/db/migrations/032_add_author_search.ts b/src/db/migrations/032_add_author_search.ts new file mode 100644 index 00000000..4323c252 --- /dev/null +++ b/src/db/migrations/032_add_author_search.ts @@ -0,0 +1,18 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('author_search') + .addColumn('pubkey', 'char(64)', (col) => col.primaryKey()) + .addColumn('search', 'text', (col) => col.notNull()) + .ifNotExists() + .execute(); + + await sql`CREATE EXTENSION IF NOT EXISTS pg_trgm`.execute(db); + await sql`CREATE INDEX author_search_search_idx ON author_search USING GIN (search gin_trgm_ops)`.execute(db); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropIndex('author_search_search_idx').ifExists().execute(); + await db.schema.dropTable('author_search').execute(); +} diff --git a/src/db/migrations/033_add_language.ts b/src/db/migrations/033_add_language.ts new file mode 100644 index 00000000..77bfc37e --- /dev/null +++ b/src/db/migrations/033_add_language.ts @@ -0,0 +1,15 @@ +import { Kysely } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema.alterTable('nostr_events').addColumn('language', 'char(2)').execute(); + + await db.schema.createIndex('nostr_events_language_created_idx') + .on('nostr_events') + .columns(['language', 'created_at desc', 'id asc', 'kind']) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.alterTable('nostr_events').dropColumn('language').execute(); + await db.schema.dropIndex('nostr_events_language_created_idx').execute(); +} diff --git a/src/db/unattached-media.ts b/src/db/unattached-media.ts deleted file mode 100644 index fac7a1d9..00000000 --- a/src/db/unattached-media.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { Kysely } from 'kysely'; - -import { DittoDB } from '@/db/DittoDB.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; - -interface UnattachedMedia { - id: string; - pubkey: string; - url: string; - /** NIP-94 tags. */ - data: string[][]; - uploaded_at: number; -} - -/** Add unattached media into the database. */ -async function insertUnattachedMedia(media: UnattachedMedia) { - const { kysely } = await DittoDB.getInstance(); - await kysely.insertInto('unattached_media') - .values({ ...media, data: JSON.stringify(media.data) }) - .execute(); - - return media; -} - -/** Select query for unattached media. */ -function selectUnattachedMediaQuery(kysely: Kysely) { - return kysely.selectFrom('unattached_media') - .select([ - 'unattached_media.id', - 'unattached_media.pubkey', - 'unattached_media.url', - 'unattached_media.data', - 'unattached_media.uploaded_at', - ]); -} - -/** Delete unattached media by URL. */ -async function deleteUnattachedMediaByUrl(url: string) { - const { kysely } = await DittoDB.getInstance(); - return kysely.deleteFrom('unattached_media') - .where('url', '=', url) - .execute(); -} - -/** Get unattached media by IDs. */ -async function getUnattachedMediaByIds(kysely: Kysely, ids: string[]): Promise { - if (!ids.length) return []; - - const results = await selectUnattachedMediaQuery(kysely) - .where('id', 'in', ids) - .execute(); - - return results.map((row) => ({ - ...row, - data: JSON.parse(row.data), - })); -} - -/** Delete rows as an event with media is being created. */ -async function deleteAttachedMedia(pubkey: string, urls: string[]): Promise { - if (!urls.length) return; - const { kysely } = await DittoDB.getInstance(); - await kysely.deleteFrom('unattached_media') - .where('pubkey', '=', pubkey) - .where('url', 'in', urls) - .execute(); -} - -export { - deleteAttachedMedia, - deleteUnattachedMediaByUrl, - getUnattachedMediaByIds, - insertUnattachedMedia, - type UnattachedMedia, -}; diff --git a/src/firehose.ts b/src/firehose.ts index 86d19f74..85e3dc89 100644 --- a/src/firehose.ts +++ b/src/firehose.ts @@ -2,7 +2,7 @@ import { Semaphore } from '@lambdalisue/async'; import { Stickynotes } from '@soapbox/stickynotes'; import { Conf } from '@/config.ts'; -import { firehoseEventCounter } from '@/metrics.ts'; +import { firehoseEventsCounter } from '@/metrics.ts'; import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; @@ -23,7 +23,7 @@ export async function startFirehose(): Promise { if (msg[0] === 'EVENT') { const event = msg[2]; console.debug(`NostrEvent<${event.kind}> ${event.id}`); - firehoseEventCounter.inc({ kind: event.kind }); + firehoseEventsCounter.inc({ kind: event.kind }); sem.lock(async () => { try { diff --git a/src/metrics.ts b/src/metrics.ts index c1fb8238..ac1db2ee 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -1,86 +1,86 @@ import { Counter, Gauge, Histogram } from 'prom-client'; -export const httpRequestCounter = new Counter({ - name: 'http_requests_total', +export const httpRequestsCounter = new Counter({ + name: 'ditto_http_requests_total', help: 'Total number of HTTP requests', labelNames: ['method'], }); -export const httpResponseCounter = new Counter({ - name: 'http_responses_total', +export const httpResponsesCounter = new Counter({ + name: 'ditto_http_responses_total', help: 'Total number of HTTP responses', labelNames: ['method', 'path', 'status'], }); export const streamingConnectionsGauge = new Gauge({ - name: 'streaming_connections', + name: 'ditto_streaming_connections', help: 'Number of active connections to the streaming API', }); export const fetchCounter = new Counter({ - name: 'fetch_total', + name: 'ditto_fetch_total', help: 'Total number of fetch requests', labelNames: ['method'], }); -export const firehoseEventCounter = new Counter({ - name: 'firehose_events_total', +export const firehoseEventsCounter = new Counter({ + name: 'ditto_firehose_events_total', help: 'Total number of Nostr events processed by the firehose', labelNames: ['kind'], }); -export const pipelineEventCounter = new Counter({ - name: 'pipeline_events_total', +export const pipelineEventsCounter = new Counter({ + name: 'ditto_pipeline_events_total', help: 'Total number of Nostr events processed by the pipeline', labelNames: ['kind'], }); -export const policyEventCounter = new Counter({ - name: 'policy_events_total', +export const policyEventsCounter = new Counter({ + name: 'ditto_policy_events_total', help: 'Total number of policy OK responses', labelNames: ['ok'], }); -export const relayEventCounter = new Counter({ - name: 'relay_events_total', +export const relayEventsCounter = new Counter({ + name: 'ditto_relay_events_total', help: 'Total number of EVENT messages processed by the relay', labelNames: ['kind'], }); -export const relayMessageCounter = new Counter({ - name: 'relay_messages_total', +export const relayMessagesCounter = new Counter({ + name: 'ditto_relay_messages_total', help: 'Total number of Nostr messages processed by the relay', labelNames: ['verb'], }); export const relayConnectionsGauge = new Gauge({ - name: 'relay_connections', + name: 'ditto_relay_connections', help: 'Number of active connections to the relay', }); -export const dbQueryCounter = new Counter({ - name: 'db_query_total', +export const dbQueriesCounter = new Counter({ + name: 'ditto_db_queries_total', help: 'Total number of database queries', labelNames: ['kind'], }); -export const dbEventCounter = new Counter({ - name: 'db_events_total', +export const dbEventsCounter = new Counter({ + name: 'ditto_db_events_total', help: 'Total number of database inserts', labelNames: ['kind'], }); export const dbPoolSizeGauge = new Gauge({ - name: 'db_pool_size', + name: 'ditto_db_pool_size', help: 'Number of connections in the database pool', }); export const dbAvailableConnectionsGauge = new Gauge({ - name: 'db_available_connections', + name: 'ditto_db_available_connections', help: 'Number of available connections in the database pool', }); -export const dbQueryTimeHistogram = new Histogram({ - name: 'db_query_duration_ms', +export const dbQueryDurationHistogram = new Histogram({ + name: 'ditto_db_query_duration_ms', help: 'Duration of database queries', }); diff --git a/src/middleware/metricsMiddleware.ts b/src/middleware/metricsMiddleware.ts index 6cf0e6de..e8a30972 100644 --- a/src/middleware/metricsMiddleware.ts +++ b/src/middleware/metricsMiddleware.ts @@ -1,12 +1,12 @@ import { MiddlewareHandler } from '@hono/hono'; -import { httpRequestCounter, httpResponseCounter } from '@/metrics.ts'; +import { httpRequestsCounter, httpResponsesCounter } from '@/metrics.ts'; /** Prometheus metrics middleware that tracks HTTP requests by methods and responses by status code. */ export const metricsMiddleware: MiddlewareHandler = async (c, next) => { // HTTP Request. const { method } = c.req; - httpRequestCounter.inc({ method }); + httpRequestsCounter.inc({ method }); // Wait for other handlers to run. await next(); @@ -16,5 +16,5 @@ export const metricsMiddleware: MiddlewareHandler = async (c, next) => { // Get a parameterized path name like `/posts/:id` instead of `/posts/1234`. // Tries to find actual route names first before falling back on potential middleware handlers like `app.use('*')`. const path = c.req.matchedRoutes.find((r) => r.method !== 'ALL')?.path ?? c.req.routePath; - httpResponseCounter.inc({ method, status, path }); + httpResponsesCounter.inc({ method, status, path }); }; diff --git a/src/middleware/signerMiddleware.ts b/src/middleware/signerMiddleware.ts index 60826db9..344e14ef 100644 --- a/src/middleware/signerMiddleware.ts +++ b/src/middleware/signerMiddleware.ts @@ -5,7 +5,7 @@ import { nip19 } from 'nostr-tools'; import { AppMiddleware } from '@/app.ts'; import { ConnectSigner } from '@/signers/ConnectSigner.ts'; import { ReadOnlySigner } from '@/signers/ReadOnlySigner.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; +import { Storages } from '@/storages.ts'; /** We only accept "Bearer" type. */ const BEARER_REGEX = new RegExp(`^Bearer (${nip19.BECH32_REGEX.source})$`); @@ -20,7 +20,7 @@ export const signerMiddleware: AppMiddleware = async (c, next) => { if (bech32.startsWith('token1')) { try { - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); const { user_pubkey, server_seckey, relays } = await kysely .selectFrom('nip46_tokens') diff --git a/src/pipeline.test.ts b/src/pipeline.test.ts index 2af2b8c3..76b1fe51 100644 --- a/src/pipeline.test.ts +++ b/src/pipeline.test.ts @@ -1,7 +1,7 @@ import { assertEquals } from '@std/assert'; import { generateSecretKey } from 'nostr-tools'; -import { createTestDB, genEvent, getTestDB } from '@/test.ts'; +import { createTestDB, genEvent } from '@/test.ts'; import { handleZaps } from '@/pipeline.ts'; Deno.test('store one zap receipt in nostr_events; convert it into event_zaps table format and store it', async () => { @@ -58,7 +58,7 @@ Deno.test('store one zap receipt in nostr_events; convert it into event_zaps tab // If no error happens = ok Deno.test('zap receipt does not have a "description" tag', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const kysely = db.kysely; const sk = generateSecretKey(); @@ -71,7 +71,7 @@ Deno.test('zap receipt does not have a "description" tag', async () => { }); Deno.test('zap receipt does not have a zap request stringified value in the "description" tag', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const kysely = db.kysely; const sk = generateSecretKey(); @@ -84,7 +84,7 @@ Deno.test('zap receipt does not have a zap request stringified value in the "des }); Deno.test('zap receipt does not have a "bolt11" tag', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const kysely = db.kysely; const sk = generateSecretKey(); @@ -103,7 +103,7 @@ Deno.test('zap receipt does not have a "bolt11" tag', async () => { }); Deno.test('zap request inside zap receipt does not have an "e" tag', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const kysely = db.kysely; const sk = generateSecretKey(); diff --git a/src/pipeline.ts b/src/pipeline.ts index 2fb18649..8ca7ae5f 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,14 +1,15 @@ import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; +import ISO6391 from 'iso-639-1'; import { Kysely, sql } from 'kysely'; +import lande from 'lande'; import { LRUCache } from 'lru-cache'; import { z } from 'zod'; import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; -import { deleteAttachedMedia } from '@/db/unattached-media.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { pipelineEventCounter, policyEventCounter } from '@/metrics.ts'; +import { pipelineEventsCounter, policyEventsCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; @@ -16,11 +17,11 @@ import { Storages } from '@/storages.ts'; import { eventAge, parseNip05, Time } from '@/utils.ts'; import { policyWorker } from '@/workers/policy.ts'; import { verifyEventWorker } from '@/workers/verify.ts'; +import { getAmount } from '@/utils/bolt11.ts'; import { nip05Cache } from '@/utils/nip05.ts'; +import { purifyEvent } from '@/utils/purify.ts'; import { updateStats } from '@/utils/stats.ts'; import { getTagSet } from '@/utils/tags.ts'; -import { getAmount } from '@/utils/bolt11.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; const debug = Debug('ditto:pipeline'); @@ -40,7 +41,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise ${event.id}`); - pipelineEventCounter.inc({ kind: event.kind }); + pipelineEventsCounter.inc({ kind: event.kind }); if (event.kind !== 24133) { await policyFilter(event); @@ -54,14 +55,14 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { try { const result = await policyWorker.call(event); - policyEventCounter.inc({ ok: String(result[2]) }); + policyEventsCounter.inc({ ok: String(result[2]) }); debug(JSON.stringify(result)); RelayError.assert(result); } catch (e) { @@ -106,7 +107,7 @@ async function existsInDB(event: DittoEvent): Promise { async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { await hydrateEvents({ events: [event], store: await Storages.db(), signal }); - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); const domain = await kysely .selectFrom('pubkey_domains') .select('domain') @@ -120,10 +121,11 @@ async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { if (NKinds.ephemeral(event.kind)) return; const store = await Storages.db(); - const { kysely } = await DittoDB.getInstance(); - await updateStats({ event, store, kysely }).catch(debug); - await store.event(event, { signal }); + await store.transaction(async (store, kysely) => { + await updateStats({ event, store, kysely }); + await store.event(event, { signal }); + }); } /** Parse kind 0 metadata and track indexes in the database. */ @@ -134,41 +136,64 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise undefined) : undefined; - // Fetch nip05. - const result = await nip05Cache.fetch(nip05, { signal }).catch(() => undefined); - if (!result) return; - - // Ensure pubkey matches event. - const { pubkey } = result; - if (pubkey !== event.pubkey) return; - - // Track pubkey domain. + // Populate author_search. try { - const { kysely } = await DittoDB.getInstance(); - const { domain } = parseNip05(nip05); + const search = result?.pubkey === event.pubkey ? [name, nip05].filter(Boolean).join(' ').trim() : name ?? ''; - await sql` - INSERT INTO pubkey_domains (pubkey, domain, last_updated_at) - VALUES (${pubkey}, ${domain}, ${event.created_at}) - ON CONFLICT(pubkey) DO UPDATE SET - domain = excluded.domain, - last_updated_at = excluded.last_updated_at - WHERE excluded.last_updated_at > pubkey_domains.last_updated_at - `.execute(kysely); - } catch (_e) { + if (search) { + await kysely.insertInto('author_search') + .values({ pubkey: event.pubkey, search }) + .onConflict((oc) => oc.column('pubkey').doUpdateSet({ search })) + .execute(); + } + } catch { // do nothing } + + if (nip05 && result && result.pubkey === event.pubkey) { + // Track pubkey domain. + try { + const { domain } = parseNip05(nip05); + + await sql` + INSERT INTO pubkey_domains (pubkey, domain, last_updated_at) + VALUES (${event.pubkey}, ${domain}, ${event.created_at}) + ON CONFLICT(pubkey) DO UPDATE SET + domain = excluded.domain, + last_updated_at = excluded.last_updated_at + WHERE excluded.last_updated_at > pubkey_domains.last_updated_at + `.execute(kysely); + } catch (_e) { + // do nothing + } + } } -/** Delete unattached media entries that are attached to the event. */ -function processMedia({ tags, pubkey, user }: DittoEvent) { - if (user) { - const urls = getTagSet(tags, 'media'); - return deleteAttachedMedia(pubkey, [...urls]); +/** Update the event in the database and set its language. */ +async function setLanguage(event: NostrEvent): Promise { + const [topResult] = lande(event.content); + + if (topResult) { + const [iso6393, confidence] = topResult; + const locale = new Intl.Locale(iso6393); + + if (confidence >= 0.95 && ISO6391.validate(locale.language)) { + const kysely = await Storages.kysely(); + try { + await kysely.updateTable('nostr_events') + .set('language', locale.language) + .where('id', '=', event.id) + .execute(); + } catch { + // do nothing + } + } } } diff --git a/src/queries.ts b/src/queries.ts index 9bce58ca..9ee86a36 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -74,7 +74,7 @@ async function getAncestors(store: NStore, event: NostrEvent, result: NostrEvent const inReplyTo = replyTag ? replyTag[1] : undefined; if (inReplyTo) { - const [parentEvent] = await store.query([{ kinds: [1], ids: [inReplyTo], until: event.created_at, limit: 1 }]); + const [parentEvent] = await store.query([{ ids: [inReplyTo], until: event.created_at, limit: 1 }]); if (parentEvent) { result.push(parentEvent); diff --git a/src/schemas/pagination.ts b/src/schemas/pagination.ts index d7a6cf68..89e3c5f6 100644 --- a/src/schemas/pagination.ts +++ b/src/schemas/pagination.ts @@ -2,7 +2,10 @@ import { z } from 'zod'; /** Schema to parse pagination query params. */ export const paginationSchema = z.object({ - max_id: z.string().optional().catch(undefined), + max_id: z.string().transform((val) => { + if (!val.includes('-')) return val; + return val.split('-')[1]; + }).optional().catch(undefined), min_id: z.string().optional().catch(undefined), since: z.coerce.number().nonnegative().optional().catch(undefined), until: z.coerce.number().nonnegative().optional().catch(undefined), diff --git a/src/storages.ts b/src/storages.ts index c49b62cf..073b6135 100644 --- a/src/storages.ts +++ b/src/storages.ts @@ -1,5 +1,6 @@ // deno-lint-ignore-file require-await import { Conf } from '@/config.ts'; +import { DittoDatabase } from '@/db/DittoDatabase.ts'; import { DittoDB } from '@/db/DittoDB.ts'; import { AdminStore } from '@/storages/AdminStore.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; @@ -11,17 +12,37 @@ import { seedZapSplits } from '@/utils/zap-split.ts'; export class Storages { private static _db: Promise | undefined; + private static _database: Promise | undefined; private static _admin: Promise | undefined; private static _client: Promise | undefined; private static _pubsub: Promise | undefined; private static _search: Promise | undefined; - /** SQLite database to store events this Ditto server cares about. */ + public static async database(): Promise { + if (!this._database) { + this._database = (async () => { + const db = DittoDB.create(Conf.databaseUrl, { + poolSize: Conf.pg.poolSize, + debug: Conf.pgliteDebug, + }); + await DittoDB.migrate(db.kysely); + return db; + })(); + } + return this._database; + } + + public static async kysely(): Promise { + const { kysely } = await this.database(); + return kysely; + } + + /** SQL database to store events this Ditto server cares about. */ public static async db(): Promise { if (!this._db) { this._db = (async () => { - const db = await DittoDB.getInstance(); - const store = new EventsDB(db); + const kysely = await this.kysely(); + const store = new EventsDB({ kysely, pubkey: Conf.pubkey, timeout: Conf.db.timeouts.default }); await seedZapSplits(store); return store; })(); diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 7a5f7b93..b24032aa 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -54,6 +54,23 @@ Deno.test('query events with domain search filter', async () => { assertEquals(await store.query([{ kinds: [1], search: 'domain:example.com' }]), []); }); +Deno.test('query events with language search filter', async () => { + await using db = await createTestDB(); + const { store, kysely } = db; + + const en = genEvent({ kind: 1, content: 'hello world!' }); + const es = genEvent({ kind: 1, content: 'hola mundo!' }); + + await store.event(en); + await store.event(es); + + await kysely.updateTable('nostr_events').set('language', 'en').where('id', '=', en.id).execute(); + await kysely.updateTable('nostr_events').set('language', 'es').where('id', '=', es.id).execute(); + + assertEquals(await store.query([{ search: 'language:en' }]), [en]); + assertEquals(await store.query([{ search: 'language:es' }]), [es]); +}); + Deno.test('delete events', async () => { await using db = await createTestDB(); const { store } = db; diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 07be8067..b4dc0b9b 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -1,27 +1,17 @@ // deno-lint-ignore-file require-await -import { NDatabase, NPostgres } from '@nostrify/db'; -import { - NIP50, - NKinds, - NostrEvent, - NostrFilter, - NostrRelayCLOSED, - NostrRelayEOSE, - NostrRelayEVENT, - NSchema as n, - NStore, -} from '@nostrify/nostrify'; +import { NPostgres, NPostgresSchema } from '@nostrify/db'; +import { NIP50, NKinds, NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { Stickynotes } from '@soapbox/stickynotes'; +import { Kysely, SelectQueryBuilder } from 'kysely'; import { nip27 } from 'nostr-tools'; -import { Conf } from '@/config.ts'; -import { DittoDatabase } from '@/db/DittoDB.ts'; -import { dbEventCounter } from '@/metrics.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; +import { dbEventsCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; -import { purifyEvent } from '@/storages/hydrate.ts'; import { isNostrId, isURL } from '@/utils.ts'; import { abortError } from '@/utils/abort.ts'; +import { purifyEvent } from '@/utils/purify.ts'; /** Function to decide whether or not to index a tag. */ type TagCondition = ({ event, count, value }: { @@ -30,9 +20,18 @@ type TagCondition = ({ event, count, value }: { value: string; }) => boolean; -/** SQLite database storage adapter for Nostr events. */ -class EventsDB implements NStore { - private store: NDatabase | NPostgres; +/** Options for the EventsDB store. */ +interface EventsDBOpts { + /** Kysely instance to use. */ + kysely: Kysely; + /** Pubkey of the admin account. */ + pubkey: string; + /** Timeout in milliseconds for database queries. */ + timeout: number; +} + +/** SQL database storage adapter for Nostr events. */ +class EventsDB extends NPostgres { private console = new Stickynotes('ditto:db:events'); /** Conditions for when to index certain tags. */ @@ -52,28 +51,18 @@ class EventsDB implements NStore { 't': ({ event, count, value }) => (event.kind === 1985 ? count < 20 : count < 5) && value.length < 50, }; - constructor(private database: DittoDatabase) { - const { dialect, kysely } = database; - - if (dialect === 'postgres') { - this.store = new NPostgres(kysely, { - indexTags: EventsDB.indexTags, - indexSearch: EventsDB.searchText, - }); - } else { - this.store = new NDatabase(kysely, { - fts: 'sqlite', - indexTags: EventsDB.indexTags, - searchText: EventsDB.searchText, - }); - } + constructor(private opts: EventsDBOpts) { + super(opts.kysely, { + indexTags: EventsDB.indexTags, + indexSearch: EventsDB.searchText, + }); } /** Insert an event (and its tags) into the database. */ async event(event: NostrEvent, opts: { signal?: AbortSignal; timeout?: number } = {}): Promise { event = purifyEvent(event); this.console.debug('EVENT', JSON.stringify(event)); - dbEventCounter.inc({ kind: event.kind }); + dbEventsCounter.inc({ kind: event.kind }); if (await this.isDeletedAdmin(event)) { throw new RelayError('blocked', 'event deleted by admin'); @@ -82,7 +71,7 @@ class EventsDB implements NStore { await this.deleteEventsAdmin(event); try { - await this.store.event(event, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); + await super.event(event, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } catch (e) { if (e.message === 'Cannot add a deleted event') { throw new RelayError('blocked', 'event deleted by user'); @@ -97,7 +86,7 @@ class EventsDB implements NStore { /** Check if an event has been deleted by the admin. */ private async isDeletedAdmin(event: NostrEvent): Promise { const filters: NostrFilter[] = [ - { kinds: [5], authors: [Conf.pubkey], '#e': [event.id], limit: 1 }, + { kinds: [5], authors: [this.opts.pubkey], '#e': [event.id], limit: 1 }, ]; if (NKinds.replaceable(event.kind) || NKinds.parameterizedReplaceable(event.kind)) { @@ -105,7 +94,7 @@ class EventsDB implements NStore { filters.push({ kinds: [5], - authors: [Conf.pubkey], + authors: [this.opts.pubkey], '#a': [`${event.kind}:${event.pubkey}:${d}`], since: event.created_at, limit: 1, @@ -118,7 +107,7 @@ class EventsDB implements NStore { /** The DITTO_NSEC can delete any event from the database. NDatabase already handles user deletions. */ private async deleteEventsAdmin(event: NostrEvent): Promise { - if (event.kind === 5 && event.pubkey === Conf.pubkey) { + if (event.kind === 5 && event.pubkey === this.opts.pubkey) { const ids = new Set(event.tags.filter(([name]) => name === 'e').map(([_name, value]) => value)); const addrs = new Set(event.tags.filter(([name]) => name === 'a').map(([_name, value]) => value)); @@ -155,12 +144,37 @@ class EventsDB implements NStore { } } - /** Stream events from the database. */ - req( - filters: NostrFilter[], - opts: { signal?: AbortSignal } = {}, - ): AsyncIterable { - return this.store.req(filters, opts); + protected getFilterQuery(trx: Kysely, filter: NostrFilter) { + if (filter.search) { + const tokens = NIP50.parseInput(filter.search); + + let query = super.getFilterQuery(trx, { + ...filter, + search: tokens.filter((t) => typeof t === 'string').join(' '), + }) as SelectQueryBuilder>; + + const data = tokens.filter((t) => typeof t === 'object').reduce( + (acc, t) => acc.set(t.key, t.value), + new Map(), + ); + + const domain = data.get('domain'); + const language = data.get('language'); + + if (domain) { + query = query + .innerJoin('pubkey_domains', 'nostr_events.pubkey', 'pubkey_domains.pubkey') + .where('pubkey_domains.domain', '=', domain); + } + + if (language) { + query = query.where('language', '=', language); + } + + return query; + } + + return super.getFilterQuery(trx, filter); } /** Get events for filters from the database. */ @@ -185,32 +199,28 @@ class EventsDB implements NStore { } if (opts.signal?.aborted) return Promise.resolve([]); - if (!filters.length) return Promise.resolve([]); this.console.debug('REQ', JSON.stringify(filters)); - return this.store.query(filters, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); + return super.query(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } /** Delete events based on filters from the database. */ async remove(filters: NostrFilter[], opts: { signal?: AbortSignal; timeout?: number } = {}): Promise { - if (!filters.length) return Promise.resolve(); this.console.debug('DELETE', JSON.stringify(filters)); - - return this.store.remove(filters, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); + return super.remove(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } /** Get number of events that would be returned by filters. */ async count( filters: NostrFilter[], opts: { signal?: AbortSignal; timeout?: number } = {}, - ): Promise<{ count: number; approximate: boolean }> { + ): Promise<{ count: number; approximate: any }> { if (opts.signal?.aborted) return Promise.reject(abortError()); - if (!filters.length) return Promise.resolve({ count: 0, approximate: false }); this.console.debug('COUNT', JSON.stringify(filters)); - return this.store.count(filters, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); + return super.count(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } /** Return only the tags that should be indexed. */ @@ -273,40 +283,11 @@ class EventsDB implements NStore { return tags.map(([_tag, value]) => value).join('\n'); } - /** Converts filters to more performant, simpler filters that are better for SQLite. */ + /** Converts filters to more performant, simpler filters. */ async expandFilters(filters: NostrFilter[]): Promise { filters = structuredClone(filters); for (const filter of filters) { - if (filter.search) { - const tokens = NIP50.parseInput(filter.search); - - const domain = (tokens.find((t) => - typeof t === 'object' && t.key === 'domain' - ) as { key: 'domain'; value: string } | undefined)?.value; - - if (domain) { - const query = this.database.kysely - .selectFrom('pubkey_domains') - .select('pubkey') - .where('domain', '=', domain); - - if (filter.authors) { - query.where('pubkey', 'in', filter.authors); - } - - const pubkeys = await query - .execute() - .then((rows) => - rows.map((row) => row.pubkey) - ); - - filter.authors = pubkeys; - } - - filter.search = tokens.filter((t) => typeof t === 'string').join(' '); - } - if (filter.kinds) { // Ephemeral events are not stored, so don't bother querying for them. // If this results in an empty kinds array, NDatabase will remove the filter before querying and return no results. @@ -316,6 +297,10 @@ class EventsDB implements NStore { return filters; } + + async transaction(callback: (store: NPostgres, kysely: Kysely) => Promise): Promise { + return super.transaction((store, kysely) => callback(store, kysely as unknown as Kysely)); + } } export { EventsDB }; diff --git a/src/storages/InternalRelay.ts b/src/storages/InternalRelay.ts index 233a095c..93a480e1 100644 --- a/src/storages/InternalRelay.ts +++ b/src/storages/InternalRelay.ts @@ -12,7 +12,7 @@ import { Machina } from '@nostrify/nostrify/utils'; import { matchFilter } from 'nostr-tools'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { purifyEvent } from '@/storages/hydrate.ts'; +import { purifyEvent } from '@/utils/purify.ts'; /** * PubSub event store for streaming events within the application. diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 19ba0db4..7b11cfb8 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -1,13 +1,13 @@ -import { NostrEvent, NStore } from '@nostrify/nostrify'; +import { NStore } from '@nostrify/nostrify'; +import { Kysely } from 'kysely'; import { matchFilter } from 'nostr-tools'; -import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Conf } from '@/config.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { findQuoteTag } from '@/utils/tags.ts'; import { findQuoteInContent } from '@/utils/note.ts'; -import { Kysely } from 'kysely'; +import { Storages } from '@/storages.ts'; interface HydrateOpts { events: DittoEvent[]; @@ -18,7 +18,7 @@ interface HydrateOpts { /** Hydrate events using the provided storage. */ async function hydrateEvents(opts: HydrateOpts): Promise { - const { events, store, signal, kysely = (await DittoDB.getInstance()).kysely } = opts; + const { events, store, signal, kysely = await Storages.kysely() } = opts; if (!events.length) { return events; @@ -338,17 +338,4 @@ async function gatherEventStats( })); } -/** Return a normalized event without any non-standard keys. */ -function purifyEvent(event: NostrEvent): NostrEvent { - return { - id: event.id, - pubkey: event.pubkey, - kind: event.kind, - content: event.content, - tags: event.tags, - sig: event.sig, - created_at: event.created_at, - }; -} - -export { hydrateEvents, purifyEvent }; +export { hydrateEvents }; diff --git a/src/test.ts b/src/test.ts index df6c84f6..00bf4354 100644 --- a/src/test.ts +++ b/src/test.ts @@ -1,21 +1,10 @@ -import fs from 'node:fs/promises'; -import path from 'node:path'; - -import { Database as Sqlite } from '@db/sqlite'; -import { NDatabase, NDatabaseSchema, NPostgresSchema } from '@nostrify/db'; import { NostrEvent } from '@nostrify/nostrify'; -import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite'; import { finalizeEvent, generateSecretKey } from 'nostr-tools'; -import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; -import { PostgresJSDialect, PostgresJSDialectConfig } from 'kysely-postgres-js'; -import postgres from 'postgres'; -import { DittoDatabase, DittoDB } from '@/db/DittoDB.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; -import { purifyEvent } from '@/storages/hydrate.ts'; -import { KyselyLogger } from '@/db/KyselyLogger.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; import { Conf } from '@/config.ts'; +import { DittoDB } from '@/db/DittoDB.ts'; +import { EventsDB } from '@/storages/EventsDB.ts'; +import { purifyEvent } from '@/utils/purify.ts'; /** Import an event fixture by name in tests. */ export async function eventFixture(name: string): Promise { @@ -42,97 +31,27 @@ export function genEvent(t: Partial = {}, sk: Uint8Array = generateS return purifyEvent(event); } -/** Get an in-memory SQLite database to use for testing. It's automatically destroyed when it goes out of scope. */ -export async function getTestDB() { - const kysely = new Kysely({ - dialect: new DenoSqlite3Dialect({ - database: new Sqlite(':memory:'), - }), +/** Create a database for testing. It uses `TEST_DATABASE_URL`, or creates an in-memory database by default. */ +export async function createTestDB() { + const { testDatabaseUrl } = Conf; + const { protocol } = new URL(testDatabaseUrl); + const { kysely } = DittoDB.create(testDatabaseUrl, { poolSize: 1 }); + + await DittoDB.migrate(kysely); + + const store = new EventsDB({ + kysely, + timeout: Conf.db.timeouts.default, + pubkey: Conf.pubkey, }); - const migrator = new Migrator({ - db: kysely, - provider: new FileMigrationProvider({ - fs, - path, - migrationFolder: new URL(import.meta.resolve('./db/migrations')).pathname, - }), - }); - - await migrator.migrateToLatest(); - - const store = new NDatabase(kysely); - return { store, kysely, - [Symbol.asyncDispose]: () => kysely.destroy(), - }; -} - -/** Create an database for testing. */ -export const createTestDB = async (databaseUrl?: string) => { - databaseUrl ??= Deno.env.get('DATABASE_URL') ?? 'sqlite://:memory:'; - - let dialect: 'sqlite' | 'postgres' = (() => { - const protocol = databaseUrl.split(':')[0]; - switch (protocol) { - case 'sqlite': - return 'sqlite'; - case 'postgres': - return protocol; - case 'postgresql': - return 'postgres'; - default: - throw new Error(`Unsupported protocol: ${protocol}`); - } - })(); - - const allowToUseDATABASE_URL = Deno.env.get('ALLOW_TO_USE_DATABASE_URL')?.toLowerCase() ?? ''; - if (allowToUseDATABASE_URL !== 'true' && dialect === 'postgres') { - console.warn( - '%cRunning tests with sqlite, if you meant to use Postgres, run again with ALLOW_TO_USE_DATABASE_URL environment variable set to true', - 'color: yellow;', - ); - dialect = 'sqlite'; - } - - console.warn(`Using: ${dialect}`); - - const db: DittoDatabase = { dialect } as DittoDatabase; - - if (dialect === 'sqlite') { - // migration 021_pgfts_index.ts calls 'Conf.db.dialect', - // and this calls the DATABASE_URL environment variable. - // The following line ensures to NOT use the DATABASE_URL that may exist in an .env file. - Deno.env.set('DATABASE_URL', 'sqlite://:memory:'); - - db.kysely = new Kysely({ - dialect: new DenoSqlite3Dialect({ - database: new Sqlite(':memory:'), - }), - }) as Kysely & Kysely; - } else { - db.kysely = new Kysely({ - // @ts-ignore Kysely version mismatch. - dialect: new PostgresJSDialect({ - postgres: postgres(Conf.databaseUrl, { - max: Conf.pg.poolSize, - }) as unknown as PostgresJSDialectConfig['postgres'], - }), - log: KyselyLogger, - }) as Kysely & Kysely; - } - - await DittoDB.migrate(db.kysely); - const store = new EventsDB(db); - - return { - dialect, - store, - kysely: db.kysely, [Symbol.asyncDispose]: async () => { - if (dialect === 'postgres') { + // If we're testing against real Postgres, we will reuse the database + // between tests, so we should drop the tables to keep each test fresh. + if (['postgres:', 'postgresql:'].includes(protocol)) { for ( const table of [ 'author_stats', @@ -142,20 +61,18 @@ export const createTestDB = async (databaseUrl?: string) => { 'kysely_migration_lock', 'nip46_tokens', 'pubkey_domains', - 'unattached_media', 'nostr_events', - 'nostr_tags', - 'nostr_pgfts', 'event_zaps', + 'author_search', ] ) { - await db.kysely.schema.dropTable(table).ifExists().cascade().execute(); + await kysely.schema.dropTable(table).ifExists().cascade().execute(); } - await db.kysely.destroy(); + await kysely.destroy(); } }, }; -}; +} export function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); diff --git a/src/trends.ts b/src/trends.ts index 91164143..de91a33d 100644 --- a/src/trends.ts +++ b/src/trends.ts @@ -1,11 +1,12 @@ import { NostrFilter } from '@nostrify/nostrify'; import { Stickynotes } from '@soapbox/stickynotes'; -import { sql } from 'kysely'; +import { Kysely, sql } from 'kysely'; import { Conf } from '@/config.ts'; -import { DittoDatabase, DittoDB } from '@/db/DittoDB.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; import { handleEvent } from '@/pipeline.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; +import { Storages } from '@/storages.ts'; import { Time } from '@/utils/time.ts'; const console = new Stickynotes('ditto:trends'); @@ -13,88 +14,50 @@ const console = new Stickynotes('ditto:trends'); /** Get trending tag values for a given tag in the given time frame. */ export async function getTrendingTagValues( /** Kysely instance to execute queries on. */ - { dialect, kysely }: DittoDatabase, + kysely: Kysely, /** Tag name to filter by, eg `t` or `r`. */ tagNames: string[], /** Filter of eligible events. */ filter: NostrFilter, ): Promise<{ value: string; authors: number; uses: number }[]> { - if (dialect === 'postgres') { - let query = kysely - .selectFrom([ - 'nostr_events', - sql<{ key: string; value: string }>`jsonb_each_text(nostr_events.tags_index)`.as('kv'), - sql<{ key: string; value: string }>`jsonb_array_elements_text(kv.value::jsonb)`.as('element'), - ]) - .select(({ fn }) => [ - fn('lower', ['element.value']).as('value'), - fn.agg('count', ['nostr_events.pubkey']).distinct().as('authors'), - fn.countAll().as('uses'), - ]) - .where('kv.key', '=', (eb) => eb.fn.any(eb.val(tagNames))) - .groupBy((eb) => eb.fn('lower', ['element.value'])) - .orderBy((eb) => eb.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc'); + let query = kysely + .selectFrom([ + 'nostr_events', + sql<{ key: string; value: string }>`jsonb_each_text(nostr_events.tags_index)`.as('kv'), + sql<{ key: string; value: string }>`jsonb_array_elements_text(kv.value::jsonb)`.as('element'), + ]) + .select(({ fn }) => [ + fn('lower', ['element.value']).as('value'), + fn.agg('count', ['nostr_events.pubkey']).distinct().as('authors'), + fn.countAll().as('uses'), + ]) + .where('kv.key', '=', (eb) => eb.fn.any(eb.val(tagNames))) + .groupBy((eb) => eb.fn('lower', ['element.value'])) + .orderBy((eb) => eb.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc'); - if (filter.kinds) { - query = query.where('nostr_events.kind', '=', ({ fn, val }) => fn.any(val(filter.kinds))); - } - if (filter.authors) { - query = query.where('nostr_events.pubkey', '=', ({ fn, val }) => fn.any(val(filter.authors))); - } - if (typeof filter.since === 'number') { - query = query.where('nostr_events.created_at', '>=', filter.since); - } - if (typeof filter.until === 'number') { - query = query.where('nostr_events.created_at', '<=', filter.until); - } - if (typeof filter.limit === 'number') { - query = query.limit(filter.limit); - } - - const rows = await query.execute(); - - return rows.map((row) => ({ - value: row.value, - authors: Number(row.authors), - uses: Number(row.uses), - })); + if (filter.kinds) { + query = query.where('nostr_events.kind', '=', ({ fn, val }) => fn.any(val(filter.kinds))); + } + if (filter.authors) { + query = query.where('nostr_events.pubkey', '=', ({ fn, val }) => fn.any(val(filter.authors))); + } + if (typeof filter.since === 'number') { + query = query.where('nostr_events.created_at', '>=', filter.since); + } + if (typeof filter.until === 'number') { + query = query.where('nostr_events.created_at', '<=', filter.until); + } + if (typeof filter.limit === 'number') { + query = query.limit(filter.limit); } - if (dialect === 'sqlite') { - let query = kysely - .selectFrom('nostr_tags') - .select(({ fn }) => [ - 'nostr_tags.value', - fn.agg('count', ['nostr_tags.pubkey']).distinct().as('authors'), - fn.countAll().as('uses'), - ]) - .where('nostr_tags.name', 'in', tagNames) - .groupBy('nostr_tags.value') - .orderBy((c) => c.fn.agg('count', ['nostr_tags.pubkey']).distinct(), 'desc'); + const rows = await query.execute(); - if (filter.kinds) { - query = query.where('nostr_tags.kind', 'in', filter.kinds); - } - if (typeof filter.since === 'number') { - query = query.where('nostr_tags.created_at', '>=', filter.since); - } - if (typeof filter.until === 'number') { - query = query.where('nostr_tags.created_at', '<=', filter.until); - } - if (typeof filter.limit === 'number') { - query = query.limit(filter.limit); - } - - const rows = await query.execute(); - - return rows.map((row) => ({ - value: row.value, - authors: Number(row.authors), - uses: Number(row.uses), - })); - } - - return []; + return rows.map((row) => ({ + value: row.value, + authors: Number(row.authors), + uses: Number(row.uses), + })); } /** Get trending tags and publish an event with them. */ @@ -107,7 +70,7 @@ export async function updateTrendingTags( aliases?: string[], ) { console.info(`Updating trending ${l}...`); - const db = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); const signal = AbortSignal.timeout(1000); const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); @@ -116,7 +79,7 @@ export async function updateTrendingTags( const tagNames = aliases ? [tagName, ...aliases] : [tagName]; try { - const trends = await getTrendingTagValues(db, tagNames, { + const trends = await getTrendingTagValues(kysely, tagNames, { kinds, since: yesterday, until: now, diff --git a/src/utils/SimpleLRU.ts b/src/utils/SimpleLRU.ts index f1bf6512..0f2b5b37 100644 --- a/src/utils/SimpleLRU.ts +++ b/src/utils/SimpleLRU.ts @@ -17,7 +17,7 @@ export class SimpleLRU< constructor(fetchFn: FetchFn, opts: LRUCache.Options) { this.cache = new LRUCache({ - fetchMethod: (key, _staleValue, { signal }) => fetchFn(key, { signal: signal as AbortSignal }), + fetchMethod: (key, _staleValue, { signal }) => fetchFn(key, { signal: signal as unknown as AbortSignal }), ...opts, }); } diff --git a/src/utils/api.ts b/src/utils/api.ts index b3b5a8b1..c6d3c6b6 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -13,7 +13,7 @@ import { RelayError } from '@/RelayError.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; -import { purifyEvent } from '@/storages/hydrate.ts'; +import { purifyEvent } from '@/utils/purify.ts'; const debug = Debug('ditto:api'); diff --git a/src/utils/note.test.ts b/src/utils/note.test.ts index 39be190b..e8d17e89 100644 --- a/src/utils/note.test.ts +++ b/src/utils/note.test.ts @@ -49,6 +49,22 @@ Deno.test('parseNoteContent renders empty for non-profile nostr URIs', () => { assertEquals(html, ''); }); +Deno.test("parseNoteContent doesn't fuck up links to my own post", () => { + const { html } = parseNoteContent( + 'Check this post: https://gleasonator.dev/@alex@gleasonator.dev/posts/a8badb480d88f9e7b6a090342279ef47ed0e0a3989ed85f898dfedc6be94225f', + [{ + id: '0461fcbecc4c3374439932d6b8f11269ccdb7cc973ad7a50ae362db135a474dd', + username: 'alex', + acct: 'alex@gleasonator.dev', + url: 'https://gleasonator.dev/@alex', + }], + ); + assertEquals( + html, + 'Check this post: https://gleasonator.dev/@alex@gleasonator.dev/posts/a8badb480d88f9e7b6a090342279ef47ed0e0a3989ed85f898dfedc6be94225f', + ); +}); + Deno.test('getMediaLinks', () => { const links = [ { href: 'https://example.com/image.png' }, diff --git a/src/utils/purify.ts b/src/utils/purify.ts new file mode 100644 index 00000000..84c1e44b --- /dev/null +++ b/src/utils/purify.ts @@ -0,0 +1,14 @@ +import { NostrEvent } from '@nostrify/nostrify'; + +/** Return a normalized event without any non-standard keys. */ +export function purifyEvent(event: NostrEvent): NostrEvent { + return { + id: event.id, + pubkey: event.pubkey, + kind: event.kind, + content: event.content, + tags: event.tags, + sig: event.sig, + created_at: event.created_at, + }; +} diff --git a/src/utils/stats.ts b/src/utils/stats.ts index ccba0a5b..e4d4d3f2 100644 --- a/src/utils/stats.ts +++ b/src/utils/stats.ts @@ -5,7 +5,6 @@ import { z } from 'zod'; import { DittoTables } from '@/db/DittoTables.ts'; import { findQuoteTag, findReplyTag, getTagSet } from '@/utils/tags.ts'; -import { Conf } from '@/config.ts'; interface UpdateStatsOpts { kysely: Kysely; @@ -197,16 +196,13 @@ export async function updateAuthorStats( notes_count: 0, }; - let query = kysely + const prev = await kysely .selectFrom('author_stats') .selectAll() - .where('pubkey', '=', pubkey); + .forUpdate() + .where('pubkey', '=', pubkey) + .executeTakeFirst(); - if (Conf.db.dialect === 'postgres') { - query = query.forUpdate(); - } - - const prev = await query.executeTakeFirst(); const stats = fn(prev ?? empty); if (prev) { @@ -249,16 +245,13 @@ export async function updateEventStats( reactions: '{}', }; - let query = kysely + const prev = await kysely .selectFrom('event_stats') .selectAll() - .where('event_id', '=', eventId); + .forUpdate() + .where('event_id', '=', eventId) + .executeTakeFirst(); - if (Conf.db.dialect === 'postgres') { - query = query.forUpdate(); - } - - const prev = await query.executeTakeFirst(); const stats = fn(prev ?? empty); if (prev) { diff --git a/src/utils/unfurl.ts b/src/utils/unfurl.ts index a0ab1d7b..8123c423 100644 --- a/src/utils/unfurl.ts +++ b/src/utils/unfurl.ts @@ -3,6 +3,7 @@ import Debug from '@soapbox/stickynotes/debug'; import DOMPurify from 'isomorphic-dompurify'; import { unfurl } from 'unfurl.js'; +import { Conf } from '@/config.ts'; import { PreviewCard } from '@/entities/PreviewCard.ts'; import { Time } from '@/utils/time.ts'; import { fetchWorker } from '@/workers/fetch.ts'; @@ -15,7 +16,10 @@ async function unfurlCard(url: string, signal: AbortSignal): Promise fetchWorker(url, { - headers: { 'User-Agent': 'WhatsApp/2' }, + headers: { + 'Accept': 'text/html, application/xhtml+xml', + 'User-Agent': Conf.fetchUserAgent, + }, signal, }), }); diff --git a/src/utils/upload.ts b/src/utils/upload.ts index 90603c5d..81c88c86 100644 --- a/src/utils/upload.ts +++ b/src/utils/upload.ts @@ -2,7 +2,7 @@ import { HTTPException } from '@hono/hono/http-exception'; import { AppContext } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { insertUnattachedMedia, UnattachedMedia } from '@/db/unattached-media.ts'; +import { DittoUpload, dittoUploads } from '@/DittoUploads.ts'; interface FileMeta { pubkey: string; @@ -15,7 +15,7 @@ export async function uploadFile( file: File, meta: FileMeta, signal?: AbortSignal, -): Promise { +): Promise { const uploader = c.get('uploader'); if (!uploader) { throw new HTTPException(500, { @@ -36,11 +36,15 @@ export async function uploadFile( tags.push(['alt', description]); } - return insertUnattachedMedia({ + const upload = { id: crypto.randomUUID(), - pubkey, url, - data: tags, - uploaded_at: Date.now(), - }); + tags, + pubkey, + uploadedAt: new Date(), + }; + + dittoUploads.set(upload.id, upload); + + return upload; } diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 9320f604..4e9401fd 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -3,9 +3,9 @@ import { getUrlMediaType } from '@/utils/media.ts'; /** Render Mastodon media attachment. */ function renderAttachment( - media: { id?: string; data: string[][] }, + media: { id?: string; tags: string[][] }, ): (MastodonAttachment & { cid?: string }) | undefined { - const { id, data: tags } = media; + const { id, tags } = media; const url = tags.find(([name]) => name === 'url')?.[1]; diff --git a/src/views/mastodon/notifications.ts b/src/views/mastodon/notifications.ts index 5fc20a2f..f2438ad2 100644 --- a/src/views/mastodon/notifications.ts +++ b/src/views/mastodon/notifications.ts @@ -1,13 +1,19 @@ import { NostrEvent } from '@nostrify/nostrify'; +import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { Conf } from '@/config.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { nostrDate } from '@/utils.ts'; -import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; -interface RenderNotificationOpts { +export interface RenderNotificationOpts { viewerPubkey: string; + zap?: { + zapSender?: NostrEvent | NostrEvent['pubkey']; // kind 0 or pubkey + zappedPost?: NostrEvent; + amount?: number; + message?: string; + }; } function renderNotification(event: DittoEvent, opts: RenderNotificationOpts) { @@ -32,6 +38,10 @@ function renderNotification(event: DittoEvent, opts: RenderNotificationOpts) { if (event.kind === 30360 && event.pubkey === Conf.pubkey) { return renderNameGrant(event); } + + if (event.kind === 9735) { + return renderZap(event, opts); + } } async function renderMention(event: DittoEvent, opts: RenderNotificationOpts) { @@ -109,6 +119,27 @@ async function renderNameGrant(event: DittoEvent) { }; } +async function renderZap(event: DittoEvent, opts: RenderNotificationOpts) { + if (!opts.zap?.zapSender) return; + + const { amount = 0, message = '' } = opts.zap; + if (amount < 1) return; + + const account = typeof opts.zap.zapSender !== 'string' + ? await renderAccount(opts.zap.zapSender) + : await accountFromPubkey(opts.zap.zapSender); + + return { + id: notificationId(event), + type: 'ditto:zap', + amount, + message, + created_at: nostrDate(event.created_at).toISOString(), + account, + ...(opts.zap?.zappedPost ? { status: await renderStatus(opts.zap?.zappedPost, opts) } : {}), + }; +} + /** This helps notifications be sorted in the correct order. */ function notificationId({ id, created_at }: NostrEvent): string { return `${created_at}-${id}`; diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index d1c02f1e..e21c9e1c 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -84,7 +84,12 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< const imeta: string[][][] = event.tags .filter(([name]) => name === 'imeta') - .map(([_, ...entries]) => entries.map((entry) => entry.split(' '))); + .map(([_, ...entries]) => + entries.map((entry) => { + const split = entry.split(' '); + return [split[0], split.splice(1).join(' ')]; + }) + ); const media = imeta.length ? imeta : getMediaLinks(links); @@ -120,9 +125,9 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< pinned: Boolean(pinEvent), reblog: null, application: null, - media_attachments: media.map((m) => renderAttachment({ data: m })).filter((m): m is MastodonAttachment => - Boolean(m) - ), + media_attachments: media + .map((m) => renderAttachment({ tags: m })) + .filter((m): m is MastodonAttachment => Boolean(m)), mentions, tags: [], emojis: renderEmojis(event), diff --git a/src/workers/fetch.worker.ts b/src/workers/fetch.worker.ts index 0012088b..e6f98455 100644 --- a/src/workers/fetch.worker.ts +++ b/src/workers/fetch.worker.ts @@ -1,7 +1,9 @@ +/// + import Debug from '@soapbox/stickynotes/debug'; import * as Comlink from 'comlink'; -import './handlers/abortsignal.ts'; +import '@/workers/handlers/abortsignal.ts'; import '@/sentry.ts'; const debug = Debug('ditto:fetch.worker'); diff --git a/src/workers/policy.ts b/src/workers/policy.ts index ef9aa2cd..f86f9d9b 100644 --- a/src/workers/policy.ts +++ b/src/workers/policy.ts @@ -13,8 +13,8 @@ export const policyWorker = Comlink.wrap( type: 'module', deno: { permissions: { - read: [Conf.policy], - write: false, + read: [Conf.denoDir, Conf.policy, Conf.dataDir], + write: [Conf.dataDir], net: 'inherit', env: false, }, @@ -24,7 +24,12 @@ export const policyWorker = Comlink.wrap( ); try { - await policyWorker.import(Conf.policy); + await policyWorker.init({ + path: Conf.policy, + cwd: Deno.cwd(), + databaseUrl: Conf.databaseUrl, + adminPubkey: Conf.pubkey, + }); console.debug(`Using custom policy: ${Conf.policy}`); } catch (e) { if (e.message.includes('Module not found')) { diff --git a/src/workers/policy.worker.ts b/src/workers/policy.worker.ts index 9f94a008..1d65f405 100644 --- a/src/workers/policy.worker.ts +++ b/src/workers/policy.worker.ts @@ -3,6 +3,24 @@ import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify'; import { NoOpPolicy, ReadOnlyPolicy } from '@nostrify/nostrify/policies'; import * as Comlink from 'comlink'; +import { DittoDB } from '@/db/DittoDB.ts'; +import { EventsDB } from '@/storages/EventsDB.ts'; + +// @ts-ignore Don't try to access the env from this worker. +Deno.env = new Map(); + +/** Serializable object the worker can use to set up the state. */ +interface PolicyInit { + /** Path to the policy module (https, jsr, file, etc) */ + path: string; + /** Current working directory. */ + cwd: string; + /** Database URL to connect to. */ + databaseUrl: string; + /** Admin pubkey to use for EventsDB checks. */ + adminPubkey: string; +} + export class CustomPolicy implements NPolicy { private policy: NPolicy = new ReadOnlyPolicy(); @@ -11,10 +29,22 @@ export class CustomPolicy implements NPolicy { return this.policy.call(event); } - async import(path: string): Promise { + async init({ path, cwd, databaseUrl, adminPubkey }: PolicyInit): Promise { + // HACK: PGlite uses `path.resolve`, which requires read permission on Deno (which we don't want to give). + // We can work around this getting the cwd from the caller and overwriting `Deno.cwd`. + Deno.cwd = () => cwd; + + const { kysely } = DittoDB.create(databaseUrl, { poolSize: 1 }); + + const store = new EventsDB({ + kysely, + pubkey: adminPubkey, + timeout: 1_000, + }); + try { const Policy = (await import(path)).default; - this.policy = new Policy(); + this.policy = new Policy({ store }); } catch (e) { if (e.message.includes('Module not found')) { this.policy = new NoOpPolicy(); diff --git a/src/workers/sqlite.ts b/src/workers/sqlite.ts deleted file mode 100644 index 154ec556..00000000 --- a/src/workers/sqlite.ts +++ /dev/null @@ -1,52 +0,0 @@ -import * as Comlink from 'comlink'; -import { asyncGeneratorTransferHandler } from 'comlink-async-generator'; -import { CompiledQuery, QueryResult } from 'kysely'; - -import type { SqliteWorker as _SqliteWorker } from './sqlite.worker.ts'; - -Comlink.transferHandlers.set('asyncGenerator', asyncGeneratorTransferHandler); - -class SqliteWorker { - #worker: Worker; - #client: ReturnType>; - #ready: Promise; - - constructor() { - this.#worker = new Worker(new URL('./sqlite.worker.ts', import.meta.url).href, { type: 'module' }); - this.#client = Comlink.wrap(this.#worker); - - this.#ready = new Promise((resolve) => { - const handleEvent = (event: MessageEvent) => { - if (event.data[0] === 'ready') { - this.#worker.removeEventListener('message', handleEvent); - resolve(); - } - }; - this.#worker.addEventListener('message', handleEvent); - }); - } - - async open(path: string): Promise { - await this.#ready; - return this.#client.open(path); - } - - async executeQuery(query: CompiledQuery): Promise> { - await this.#ready; - return this.#client.executeQuery(query) as Promise>; - } - - async *streamQuery(query: CompiledQuery): AsyncIterableIterator> { - await this.#ready; - - for await (const result of await this.#client.streamQuery(query)) { - yield result as QueryResult; - } - } - - destroy(): Promise { - return this.#client.destroy(); - } -} - -export default SqliteWorker; diff --git a/src/workers/sqlite.worker.ts b/src/workers/sqlite.worker.ts deleted file mode 100644 index 23839dbd..00000000 --- a/src/workers/sqlite.worker.ts +++ /dev/null @@ -1,42 +0,0 @@ -/// -import { Database as SQLite } from '@db/sqlite'; -import * as Comlink from 'comlink'; -import { CompiledQuery, QueryResult } from 'kysely'; -import { asyncGeneratorTransferHandler } from 'comlink-async-generator'; - -import '@/sentry.ts'; - -let db: SQLite | undefined; - -export const SqliteWorker = { - open(path: string): void { - db = new SQLite(path); - }, - executeQuery({ sql, parameters }: CompiledQuery): QueryResult { - if (!db) throw new Error('Database not open'); - - return { - rows: db!.prepare(sql).all(...parameters as any[]) as R[], - numAffectedRows: BigInt(db!.changes), - insertId: BigInt(db!.lastInsertRowId), - }; - }, - async *streamQuery({ sql, parameters }: CompiledQuery): AsyncIterableIterator> { - if (!db) throw new Error('Database not open'); - - const stmt = db.prepare(sql).bind(...parameters as any[]); - for (const row of stmt) { - yield { - rows: [row], - }; - } - }, - destroy() { - db?.close(); - }, -}; - -Comlink.transferHandlers.set('asyncGenerator', asyncGeneratorTransferHandler); -Comlink.expose(SqliteWorker); - -self.postMessage(['ready']);