mirror of
https://gitlab.com/soapbox-pub/ditto.git
synced 2025-12-06 11:29:46 +00:00
Fix lockfile
# Conflicts: # deno.lock
This commit is contained in:
commit
e86e26eea9
74 changed files with 5750 additions and 1027 deletions
|
|
@ -1,4 +1,4 @@
|
|||
image: denoland/deno:1.45.5
|
||||
image: denoland/deno:1.46.3
|
||||
|
||||
default:
|
||||
interruptible: true
|
||||
|
|
@ -35,11 +35,10 @@ test:
|
|||
|
||||
postgres:
|
||||
stage: test
|
||||
script: deno task db:migrate && deno task test
|
||||
script: sleep 1 && deno task test
|
||||
services:
|
||||
- postgres:16
|
||||
variables:
|
||||
DITTO_NSEC: nsec1zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs4rm7hz
|
||||
DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres
|
||||
TEST_DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres
|
||||
POSTGRES_HOST_AUTH_METHOD: trust
|
||||
ALLOW_TO_USE_DATABASE_URL: true
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
deno 1.45.5
|
||||
deno 1.46.3
|
||||
23
ansible/playbook.yml
Normal file
23
ansible/playbook.yml
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
---
|
||||
- name: Update Ditto
|
||||
hosts: all
|
||||
become: true
|
||||
tasks:
|
||||
- name: Update Soapbox
|
||||
shell:
|
||||
cmd: deno task soapbox
|
||||
chdir: /opt/ditto
|
||||
become_user: ditto
|
||||
|
||||
- name: Update ditto from the main branch
|
||||
git:
|
||||
repo: 'https://gitlab.com/soapbox-pub/ditto.git'
|
||||
dest: '/opt/ditto'
|
||||
version: main
|
||||
become_user: ditto
|
||||
|
||||
- name: Restart ditto service
|
||||
systemd:
|
||||
name: ditto
|
||||
state: restarted
|
||||
become_user: root
|
||||
16
deno.json
16
deno.json
|
|
@ -1,5 +1,4 @@
|
|||
{
|
||||
"$schema": "https://deno.land/x/deno@v1.41.0/cli/schemas/config-file.v1.json",
|
||||
"version": "1.1.0",
|
||||
"tasks": {
|
||||
"start": "deno run -A src/server.ts",
|
||||
|
|
@ -19,8 +18,10 @@
|
|||
"admin:role": "deno run -A scripts/admin-role.ts",
|
||||
"setup": "deno run -A scripts/setup.ts",
|
||||
"stats:recompute": "deno run -A scripts/stats-recompute.ts",
|
||||
"soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip soapbox.zip && rm soapbox.zip",
|
||||
"trends": "deno run -A scripts/trends.ts"
|
||||
"soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip -o soapbox.zip && rm soapbox.zip",
|
||||
"trends": "deno run -A scripts/trends.ts",
|
||||
"clean:deps": "deno cache --reload src/app.ts",
|
||||
"db:populate-search": "deno run -A scripts/db-populate-search.ts"
|
||||
},
|
||||
"unstable": ["cron", "ffi", "kv", "worker-options"],
|
||||
"exclude": ["./public"],
|
||||
|
|
@ -28,16 +29,16 @@
|
|||
"@/": "./src/",
|
||||
"@b-fuze/deno-dom": "jsr:@b-fuze/deno-dom@^0.1.47",
|
||||
"@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/s3-lite-client@^0.7.4",
|
||||
"@db/sqlite": "jsr:@db/sqlite@^0.11.1",
|
||||
"@electric-sql/pglite": "npm:@soapbox.pub/pglite@^0.2.10",
|
||||
"@hono/hono": "jsr:@hono/hono@^4.4.6",
|
||||
"@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1",
|
||||
"@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1",
|
||||
"@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0",
|
||||
"@nostrify/db": "jsr:@nostrify/db@^0.31.2",
|
||||
"@nostrify/db": "jsr:@nostrify/db@^0.32.2",
|
||||
"@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.30.1",
|
||||
"@scure/base": "npm:@scure/base@^1.1.6",
|
||||
"@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs",
|
||||
"@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0",
|
||||
"@soapbox/kysely-pglite": "jsr:@soapbox/kysely-pglite@^0.0.1",
|
||||
"@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0",
|
||||
"@std/assert": "jsr:@std/assert@^0.225.1",
|
||||
"@std/cli": "jsr:@std/cli@^0.223.0",
|
||||
|
|
@ -62,6 +63,7 @@
|
|||
"isomorphic-dompurify": "npm:isomorphic-dompurify@^2.11.0",
|
||||
"kysely": "npm:kysely@^0.27.4",
|
||||
"kysely-postgres-js": "npm:kysely-postgres-js@2.0.0",
|
||||
"lande": "npm:lande@^1.0.10",
|
||||
"light-bolt11-decoder": "npm:light-bolt11-decoder",
|
||||
"linkify-plugin-hashtag": "npm:linkify-plugin-hashtag@^4.1.1",
|
||||
"linkify-string": "npm:linkify-string@^4.1.1",
|
||||
|
|
@ -71,7 +73,7 @@
|
|||
"nostr-tools": "npm:nostr-tools@2.5.1",
|
||||
"nostr-wasm": "npm:nostr-wasm@^0.1.0",
|
||||
"path-to-regexp": "npm:path-to-regexp@^7.1.0",
|
||||
"postgres": "https://raw.githubusercontent.com/xyzshantaram/postgres.js/8a9bbce88b3f6425ecaacd99a80372338b157a53/deno/mod.js",
|
||||
"postgres": "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/mod.js",
|
||||
"prom-client": "npm:prom-client@^15.1.2",
|
||||
"question-deno": "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/mod.ts",
|
||||
"tldts": "npm:tldts@^6.0.14",
|
||||
|
|
|
|||
242
deno.lock
generated
242
deno.lock
generated
|
|
@ -2,63 +2,68 @@
|
|||
"version": "3",
|
||||
"packages": {
|
||||
"specifiers": {
|
||||
"jsr:@b-fuze/deno-dom@^0.1.47": "jsr:@b-fuze/deno-dom@0.1.47",
|
||||
"jsr:@b-fuze/deno-dom@^0.1.47": "jsr:@b-fuze/deno-dom@0.1.48",
|
||||
"jsr:@bradenmacdonald/s3-lite-client@^0.7.4": "jsr:@bradenmacdonald/s3-lite-client@0.7.6",
|
||||
"jsr:@db/sqlite@^0.11.1": "jsr:@db/sqlite@0.11.1",
|
||||
"jsr:@denosaurs/plug@1": "jsr:@denosaurs/plug@1.0.6",
|
||||
"jsr:@denosaurs/plug@1.0.3": "jsr:@denosaurs/plug@1.0.3",
|
||||
"jsr:@gleasonator/policy": "jsr:@gleasonator/policy@0.2.0",
|
||||
"jsr:@gleasonator/policy@0.2.0": "jsr:@gleasonator/policy@0.2.0",
|
||||
"jsr:@gleasonator/policy@0.4.0": "jsr:@gleasonator/policy@0.4.0",
|
||||
"jsr:@gleasonator/policy@0.4.1": "jsr:@gleasonator/policy@0.4.1",
|
||||
"jsr:@gleasonator/policy@0.4.2": "jsr:@gleasonator/policy@0.4.2",
|
||||
"jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.9",
|
||||
"jsr:@gleasonator/policy@0.5.0": "jsr:@gleasonator/policy@0.5.0",
|
||||
"jsr:@gleasonator/policy@0.5.1": "jsr:@gleasonator/policy@0.5.1",
|
||||
"jsr:@gleasonator/policy@0.5.2": "jsr:@gleasonator/policy@0.5.2",
|
||||
"jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.11",
|
||||
"jsr:@lambdalisue/async@^2.1.1": "jsr:@lambdalisue/async@2.1.1",
|
||||
"jsr:@nostrify/db@^0.31.2": "jsr:@nostrify/db@0.31.2",
|
||||
"jsr:@nostrify/db@^0.32.2": "jsr:@nostrify/db@0.32.2",
|
||||
"jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5",
|
||||
"jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4",
|
||||
"jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5",
|
||||
"jsr:@nostrify/nostrify@^0.30.0": "jsr:@nostrify/nostrify@0.30.1",
|
||||
"jsr:@nostrify/nostrify@^0.30.1": "jsr:@nostrify/nostrify@0.30.1",
|
||||
"jsr:@nostrify/types@^0.30.0": "jsr:@nostrify/types@0.30.0",
|
||||
"jsr:@soapbox/kysely-deno-sqlite@^2.1.0": "jsr:@soapbox/kysely-deno-sqlite@2.2.0",
|
||||
"jsr:@nostrify/nostrify@^0.31.0": "jsr:@nostrify/nostrify@0.31.0",
|
||||
"jsr:@nostrify/policies@^0.33.0": "jsr:@nostrify/policies@0.33.0",
|
||||
"jsr:@nostrify/policies@^0.33.1": "jsr:@nostrify/policies@0.33.1",
|
||||
"jsr:@nostrify/types@^0.30.0": "jsr:@nostrify/types@0.30.1",
|
||||
"jsr:@nostrify/types@^0.30.1": "jsr:@nostrify/types@0.30.1",
|
||||
"jsr:@soapbox/kysely-pglite@^0.0.1": "jsr:@soapbox/kysely-pglite@0.0.1",
|
||||
"jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0",
|
||||
"jsr:@std/assert@^0.213.1": "jsr:@std/assert@0.213.1",
|
||||
"jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0",
|
||||
"jsr:@std/assert@^0.221.0": "jsr:@std/assert@0.221.0",
|
||||
"jsr:@std/assert@^0.223.0": "jsr:@std/assert@0.223.0",
|
||||
"jsr:@std/assert@^0.224.0": "jsr:@std/assert@0.224.0",
|
||||
"jsr:@std/assert@^0.225.1": "jsr:@std/assert@0.225.3",
|
||||
"jsr:@std/bytes@^0.223.0": "jsr:@std/bytes@0.223.0",
|
||||
"jsr:@std/bytes@^0.224.0": "jsr:@std/bytes@0.224.0",
|
||||
"jsr:@std/bytes@^1.0.0-rc.3": "jsr:@std/bytes@1.0.0",
|
||||
"jsr:@std/bytes@^1.0.1-rc.3": "jsr:@std/bytes@1.0.2",
|
||||
"jsr:@std/bytes@^1.0.2": "jsr:@std/bytes@1.0.2",
|
||||
"jsr:@std/bytes@^1.0.2-rc.3": "jsr:@std/bytes@1.0.2",
|
||||
"jsr:@std/cli@^0.223.0": "jsr:@std/cli@0.223.0",
|
||||
"jsr:@std/crypto@^0.224.0": "jsr:@std/crypto@0.224.0",
|
||||
"jsr:@std/dotenv@^0.224.0": "jsr:@std/dotenv@0.224.2",
|
||||
"jsr:@std/encoding@0.213.1": "jsr:@std/encoding@0.213.1",
|
||||
"jsr:@std/encoding@^0.221.0": "jsr:@std/encoding@0.221.0",
|
||||
"jsr:@std/encoding@^0.224.0": "jsr:@std/encoding@0.224.3",
|
||||
"jsr:@std/encoding@^0.224.1": "jsr:@std/encoding@0.224.3",
|
||||
"jsr:@std/fmt@0.213.1": "jsr:@std/fmt@0.213.1",
|
||||
"jsr:@std/fmt@^0.221.0": "jsr:@std/fmt@0.221.0",
|
||||
"jsr:@std/fs@0.213.1": "jsr:@std/fs@0.213.1",
|
||||
"jsr:@std/fs@^0.221.0": "jsr:@std/fs@0.221.0",
|
||||
"jsr:@std/fs@^0.229.3": "jsr:@std/fs@0.229.3",
|
||||
"jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.1",
|
||||
"jsr:@std/io@^0.224": "jsr:@std/io@0.224.6",
|
||||
"jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.3",
|
||||
"jsr:@std/io@^0.223.0": "jsr:@std/io@0.223.0",
|
||||
"jsr:@std/io@^0.224": "jsr:@std/io@0.224.7",
|
||||
"jsr:@std/json@^0.223.0": "jsr:@std/json@0.223.0",
|
||||
"jsr:@std/media-types@^0.224.1": "jsr:@std/media-types@0.224.1",
|
||||
"jsr:@std/path@0.213.1": "jsr:@std/path@0.213.1",
|
||||
"jsr:@std/path@0.217": "jsr:@std/path@0.217.0",
|
||||
"jsr:@std/path@0.217.0": "jsr:@std/path@0.217.0",
|
||||
"jsr:@std/path@1.0.0-rc.1": "jsr:@std/path@1.0.0-rc.1",
|
||||
"jsr:@std/path@^0.213.1": "jsr:@std/path@0.213.1",
|
||||
"jsr:@std/path@^0.221.0": "jsr:@std/path@0.221.0",
|
||||
"jsr:@std/streams@^0.223.0": "jsr:@std/streams@0.223.0",
|
||||
"npm:@isaacs/ttlcache@^1.4.1": "npm:@isaacs/ttlcache@1.4.1",
|
||||
"npm:@noble/hashes@^1.4.0": "npm:@noble/hashes@1.4.0",
|
||||
"npm:@noble/secp256k1@^2.0.0": "npm:@noble/secp256k1@2.1.0",
|
||||
"npm:@scure/base@^1.1.6": "npm:@scure/base@1.1.6",
|
||||
"npm:@scure/bip32@^1.4.0": "npm:@scure/bip32@1.4.0",
|
||||
"npm:@scure/bip39@^1.3.0": "npm:@scure/bip39@1.3.0",
|
||||
"npm:@soapbox.pub/pglite@^0.2.10": "npm:@soapbox.pub/pglite@0.2.10",
|
||||
"npm:@types/node": "npm:@types/node@18.16.19",
|
||||
"npm:comlink-async-generator": "npm:comlink-async-generator@0.0.1",
|
||||
"npm:comlink-async-generator@^0.0.1": "npm:comlink-async-generator@0.0.1",
|
||||
|
|
@ -74,6 +79,7 @@
|
|||
"npm:kysely@^0.27.2": "npm:kysely@0.27.4",
|
||||
"npm:kysely@^0.27.3": "npm:kysely@0.27.4",
|
||||
"npm:kysely@^0.27.4": "npm:kysely@0.27.4",
|
||||
"npm:lande@^1.0.10": "npm:lande@1.0.10",
|
||||
"npm:light-bolt11-decoder": "npm:light-bolt11-decoder@3.1.1",
|
||||
"npm:linkify-plugin-hashtag@^4.1.1": "npm:linkify-plugin-hashtag@4.1.3_linkifyjs@4.1.3",
|
||||
"npm:linkify-string@^4.1.1": "npm:linkify-string@4.1.3_linkifyjs@4.1.3",
|
||||
|
|
@ -90,6 +96,7 @@
|
|||
"npm:postgres@3.4.4": "npm:postgres@3.4.4",
|
||||
"npm:prom-client@^15.1.2": "npm:prom-client@15.1.2",
|
||||
"npm:tldts@^6.0.14": "npm:tldts@6.1.18",
|
||||
"npm:tseep@^1.2.1": "npm:tseep@1.2.1",
|
||||
"npm:type-fest@^4.3.0": "npm:type-fest@4.18.2",
|
||||
"npm:unfurl.js@^6.4.0": "npm:unfurl.js@6.4.0",
|
||||
"npm:websocket-ts@^2.1.5": "npm:websocket-ts@2.1.5",
|
||||
|
|
@ -102,19 +109,15 @@
|
|||
"jsr:@denosaurs/plug@1.0.3"
|
||||
]
|
||||
},
|
||||
"@b-fuze/deno-dom@0.1.48": {
|
||||
"integrity": "bf5b591aef2e9e9c59adfcbb93a9ecd45bab5b7c8263625beafa5c8f1662e7da"
|
||||
},
|
||||
"@bradenmacdonald/s3-lite-client@0.7.6": {
|
||||
"integrity": "2b5976dca95d207dc88e23f9807e3eecbc441b0cf547dcda5784afe6668404d1",
|
||||
"dependencies": [
|
||||
"jsr:@std/io@^0.224"
|
||||
]
|
||||
},
|
||||
"@db/sqlite@0.11.1": {
|
||||
"integrity": "546434e7ed762db07e6ade0f963540dd5e06723b802937bf260ff855b21ef9c5",
|
||||
"dependencies": [
|
||||
"jsr:@denosaurs/plug@1",
|
||||
"jsr:@std/path@0.217"
|
||||
]
|
||||
},
|
||||
"@denosaurs/plug@1.0.3": {
|
||||
"integrity": "b010544e386bea0ff3a1d05e0c88f704ea28cbd4d753439c2f1ee021a85d4640",
|
||||
"dependencies": [
|
||||
|
|
@ -124,15 +127,6 @@
|
|||
"jsr:@std/path@0.213.1"
|
||||
]
|
||||
},
|
||||
"@denosaurs/plug@1.0.6": {
|
||||
"integrity": "6cf5b9daba7799837b9ffbe89f3450510f588fafef8115ddab1ff0be9cb7c1a7",
|
||||
"dependencies": [
|
||||
"jsr:@std/encoding@^0.221.0",
|
||||
"jsr:@std/fmt@^0.221.0",
|
||||
"jsr:@std/fs@^0.221.0",
|
||||
"jsr:@std/path@^0.221.0"
|
||||
]
|
||||
},
|
||||
"@gleasonator/policy@0.2.0": {
|
||||
"integrity": "3fe58b853ab203b2b67e65b64391dbcf5c07bc1caaf46e97b2f8ed5b14f30fdf",
|
||||
"dependencies": [
|
||||
|
|
@ -157,6 +151,27 @@
|
|||
"jsr:@nostrify/nostrify@^0.22.1"
|
||||
]
|
||||
},
|
||||
"@gleasonator/policy@0.5.0": {
|
||||
"integrity": "c2882eb3b4147dfe96b6ec2870b012b5a614f686770d1d4b2f778fdc44e8b1f5",
|
||||
"dependencies": [
|
||||
"jsr:@nostrify/nostrify@^0.31.0",
|
||||
"jsr:@nostrify/policies@^0.33.0"
|
||||
]
|
||||
},
|
||||
"@gleasonator/policy@0.5.1": {
|
||||
"integrity": "2d687c5166556ce13ac05c4542f61ef8a47d8b96b57f6e43d52035805f895551",
|
||||
"dependencies": [
|
||||
"jsr:@nostrify/nostrify@^0.31.0",
|
||||
"jsr:@nostrify/policies@^0.33.0"
|
||||
]
|
||||
},
|
||||
"@gleasonator/policy@0.5.2": {
|
||||
"integrity": "cdd3add87be3132eb05736bca640dfb3bbb1aa79928a44d3563cde20bab7c0d3",
|
||||
"dependencies": [
|
||||
"jsr:@nostrify/nostrify@^0.31.0",
|
||||
"jsr:@nostrify/policies@^0.33.1"
|
||||
]
|
||||
},
|
||||
"@hono/hono@4.4.6": {
|
||||
"integrity": "aa557ca9930787ee86b9ca1730691f1ce1c379174c2cb244d5934db2b6314453"
|
||||
},
|
||||
|
|
@ -166,6 +181,9 @@
|
|||
"@hono/hono@4.5.1": {
|
||||
"integrity": "459748ed4d4146c6e4bdff0213ff1ac44749904066ae02e7550d6c7f28c9bc4c"
|
||||
},
|
||||
"@hono/hono@4.5.11": {
|
||||
"integrity": "5bd6b1a3a503efb746fdcf0aae3ac536dd09229d372988bde5db0798ef64ae4f"
|
||||
},
|
||||
"@hono/hono@4.5.3": {
|
||||
"integrity": "429923b2b3c6586a1450862328d61a1346fee5841e8ae86c494250475057213c"
|
||||
},
|
||||
|
|
@ -181,11 +199,11 @@
|
|||
"@lambdalisue/async@2.1.1": {
|
||||
"integrity": "1fc9bc6f4ed50215cd2f7217842b18cea80f81c25744f88f8c5eb4be5a1c9ab4"
|
||||
},
|
||||
"@nostrify/db@0.31.2": {
|
||||
"integrity": "a906b64edbf84a6b482cd7c9f5df2d2237c4ec42589116097d99ceb41347b1f5",
|
||||
"@nostrify/db@0.32.2": {
|
||||
"integrity": "265fb41e9d5810b99f1003ce56c89e4b468e6d0c04e7b9d9e3126c4efd49c1c2",
|
||||
"dependencies": [
|
||||
"jsr:@nostrify/nostrify@^0.30.0",
|
||||
"jsr:@nostrify/types@^0.30.0",
|
||||
"jsr:@nostrify/nostrify@^0.31.0",
|
||||
"jsr:@nostrify/types@^0.30.1",
|
||||
"npm:kysely@^0.27.3",
|
||||
"npm:nostr-tools@^2.7.0"
|
||||
]
|
||||
|
|
@ -249,13 +267,43 @@
|
|||
"npm:zod@^3.23.8"
|
||||
]
|
||||
},
|
||||
"@nostrify/nostrify@0.31.0": {
|
||||
"integrity": "1c1b686bb9ca3ad8d19807e3b96ef3793a65d70fd0f433fe6ef8b3fdb9f45557",
|
||||
"dependencies": [
|
||||
"jsr:@nostrify/types@^0.30.1",
|
||||
"jsr:@std/encoding@^0.224.1",
|
||||
"npm:@scure/bip32@^1.4.0",
|
||||
"npm:@scure/bip39@^1.3.0",
|
||||
"npm:lru-cache@^10.2.0",
|
||||
"npm:nostr-tools@^2.7.0",
|
||||
"npm:websocket-ts@^2.1.5",
|
||||
"npm:zod@^3.23.8"
|
||||
]
|
||||
},
|
||||
"@nostrify/policies@0.33.0": {
|
||||
"integrity": "c946b06d0527298b4d7c9819d142a10f522ba09eee76c37525aa4acfc5d87aee",
|
||||
"dependencies": [
|
||||
"jsr:@nostrify/types@^0.30.1",
|
||||
"npm:nostr-tools@^2.7.0"
|
||||
]
|
||||
},
|
||||
"@nostrify/policies@0.33.1": {
|
||||
"integrity": "381e1f9406a6da22da03a254e46b1aa07d5491b9761961cda3a4aeb5bf3f5286",
|
||||
"dependencies": [
|
||||
"jsr:@nostrify/types@^0.30.1",
|
||||
"npm:nostr-tools@^2.7.0"
|
||||
]
|
||||
},
|
||||
"@nostrify/types@0.30.0": {
|
||||
"integrity": "1f38fa849cff930bd709edbf94ef9ac02f46afb8b851f86c8736517b354616da"
|
||||
},
|
||||
"@soapbox/kysely-deno-sqlite@2.2.0": {
|
||||
"integrity": "668ec94600bc4b4d7bd618dd7ca65d4ef30ee61c46ffcb379b6f45203c08517a",
|
||||
"@nostrify/types@0.30.1": {
|
||||
"integrity": "245da176f6893a43250697db51ad32bfa29bf9b1cdc1ca218043d9abf6de5ae5"
|
||||
},
|
||||
"@soapbox/kysely-pglite@0.0.1": {
|
||||
"integrity": "7a4221aa780aad6fba9747c45c59dfb1c62017ba8cad9db5607f6e5822c058d5",
|
||||
"dependencies": [
|
||||
"npm:kysely@^0.27.2"
|
||||
"npm:kysely@^0.27.4"
|
||||
]
|
||||
},
|
||||
"@soapbox/stickynotes@0.4.0": {
|
||||
|
|
@ -264,11 +312,8 @@
|
|||
"@std/assert@0.213.1": {
|
||||
"integrity": "24c28178b30c8e0782c18e8e94ea72b16282207569cdd10ffb9d1d26f2edebfe"
|
||||
},
|
||||
"@std/assert@0.217.0": {
|
||||
"integrity": "c98e279362ca6982d5285c3b89517b757c1e3477ee9f14eb2fdf80a45aaa9642"
|
||||
},
|
||||
"@std/assert@0.221.0": {
|
||||
"integrity": "a5f1aa6e7909dbea271754fd4ab3f4e687aeff4873b4cef9a320af813adb489a"
|
||||
"@std/assert@0.223.0": {
|
||||
"integrity": "eb8d6d879d76e1cc431205bd346ed4d88dc051c6366365b1af47034b0670be24"
|
||||
},
|
||||
"@std/assert@0.224.0": {
|
||||
"integrity": "8643233ec7aec38a940a8264a6e3eed9bfa44e7a71cc6b3c8874213ff401967f"
|
||||
|
|
@ -279,6 +324,9 @@
|
|||
"jsr:@std/internal@^1.0.0"
|
||||
]
|
||||
},
|
||||
"@std/bytes@0.223.0": {
|
||||
"integrity": "84b75052cd8680942c397c2631318772b295019098f40aac5c36cead4cba51a8"
|
||||
},
|
||||
"@std/bytes@0.224.0": {
|
||||
"integrity": "a2250e1d0eb7d1c5a426f21267ab9bdeac2447fa87a3d0d1a467d3f7a6058e49"
|
||||
},
|
||||
|
|
@ -288,6 +336,12 @@
|
|||
"@std/bytes@1.0.2": {
|
||||
"integrity": "fbdee322bbd8c599a6af186a1603b3355e59a5fb1baa139f8f4c3c9a1b3e3d57"
|
||||
},
|
||||
"@std/cli@0.223.0": {
|
||||
"integrity": "2feb7970f2028904c3edc22ea916ce9538113dfc170844f3eae03578c333c356",
|
||||
"dependencies": [
|
||||
"jsr:@std/assert@^0.223.0"
|
||||
]
|
||||
},
|
||||
"@std/crypto@0.224.0": {
|
||||
"integrity": "154ef3ff08ef535562ef1a718718c5b2c5fc3808f0f9100daad69e829bfcdf2d",
|
||||
"dependencies": [
|
||||
|
|
@ -304,18 +358,12 @@
|
|||
"@std/encoding@0.213.1": {
|
||||
"integrity": "fcbb6928713dde941a18ca5db88ca1544d0755ec8fb20fe61e2dc8144b390c62"
|
||||
},
|
||||
"@std/encoding@0.221.0": {
|
||||
"integrity": "d1dd76ef0dc5d14088411e6dc1dede53bf8308c95d1537df1214c97137208e45"
|
||||
},
|
||||
"@std/encoding@0.224.3": {
|
||||
"integrity": "5e861b6d81be5359fad4155e591acf17c0207b595112d1840998bb9f476dbdaf"
|
||||
},
|
||||
"@std/fmt@0.213.1": {
|
||||
"integrity": "a06d31777566d874b9c856c10244ac3e6b660bdec4c82506cd46be052a1082c3"
|
||||
},
|
||||
"@std/fmt@0.221.0": {
|
||||
"integrity": "379fed69bdd9731110f26b9085aeb740606b20428ce6af31ef6bd45ef8efa62a"
|
||||
},
|
||||
"@std/fs@0.213.1": {
|
||||
"integrity": "fbcaf099f8a85c27ab0712b666262cda8fe6d02e9937bf9313ecaea39a22c501",
|
||||
"dependencies": [
|
||||
|
|
@ -323,15 +371,11 @@
|
|||
"jsr:@std/path@^0.213.1"
|
||||
]
|
||||
},
|
||||
"@std/fs@0.221.0": {
|
||||
"integrity": "028044450299de8ed5a716ade4e6d524399f035513b85913794f4e81f07da286",
|
||||
"dependencies": [
|
||||
"jsr:@std/assert@^0.221.0",
|
||||
"jsr:@std/path@^0.221.0"
|
||||
]
|
||||
},
|
||||
"@std/fs@0.229.3": {
|
||||
"integrity": "783bca21f24da92e04c3893c9e79653227ab016c48e96b3078377ebd5222e6eb"
|
||||
"integrity": "783bca21f24da92e04c3893c9e79653227ab016c48e96b3078377ebd5222e6eb",
|
||||
"dependencies": [
|
||||
"jsr:@std/path@1.0.0-rc.1"
|
||||
]
|
||||
},
|
||||
"@std/internal@1.0.0": {
|
||||
"integrity": "ac6a6dfebf838582c4b4f61a6907374e27e05bedb6ce276e0f1608fe84e7cd9a"
|
||||
|
|
@ -339,6 +383,16 @@
|
|||
"@std/internal@1.0.1": {
|
||||
"integrity": "6f8c7544d06a11dd256c8d6ba54b11ed870aac6c5aeafff499892662c57673e6"
|
||||
},
|
||||
"@std/internal@1.0.3": {
|
||||
"integrity": "208e9b94a3d5649bd880e9ca38b885ab7651ab5b5303a56ed25de4755fb7b11e"
|
||||
},
|
||||
"@std/io@0.223.0": {
|
||||
"integrity": "2d8c3c2ab3a515619b90da2c6ff5ea7b75a94383259ef4d02116b228393f84f1",
|
||||
"dependencies": [
|
||||
"jsr:@std/assert@^0.223.0",
|
||||
"jsr:@std/bytes@^0.223.0"
|
||||
]
|
||||
},
|
||||
"@std/io@0.224.0": {
|
||||
"integrity": "0aff885d21d829c050b8a08b1d71b54aed5841aecf227f8d77e99ec529a11e8e",
|
||||
"dependencies": [
|
||||
|
|
@ -369,8 +423,17 @@
|
|||
"jsr:@std/bytes@^1.0.2"
|
||||
]
|
||||
},
|
||||
"@std/io@0.224.7": {
|
||||
"integrity": "a70848793c44a7c100926571a8c9be68ba85487bfcd4d0540d86deabe1123dc9",
|
||||
"dependencies": [
|
||||
"jsr:@std/bytes@^1.0.2"
|
||||
]
|
||||
},
|
||||
"@std/json@0.223.0": {
|
||||
"integrity": "9a4a255931dd0397924c6b10bb6a72fe3e28ddd876b981ada2e3b8dd0764163f"
|
||||
"integrity": "9a4a255931dd0397924c6b10bb6a72fe3e28ddd876b981ada2e3b8dd0764163f",
|
||||
"dependencies": [
|
||||
"jsr:@std/streams@^0.223.0"
|
||||
]
|
||||
},
|
||||
"@std/media-types@0.224.1": {
|
||||
"integrity": "9e69a5daed37c5b5c6d3ce4731dc191f80e67f79bed392b0957d1d03b87f11e1"
|
||||
|
|
@ -381,20 +444,16 @@
|
|||
"jsr:@std/assert@^0.213.1"
|
||||
]
|
||||
},
|
||||
"@std/path@0.217.0": {
|
||||
"integrity": "1217cc25534bca9a2f672d7fe7c6f356e4027df400c0e85c0ef3e4343bc67d11",
|
||||
"dependencies": [
|
||||
"jsr:@std/assert@^0.217.0"
|
||||
]
|
||||
},
|
||||
"@std/path@0.221.0": {
|
||||
"integrity": "0a36f6b17314ef653a3a1649740cc8db51b25a133ecfe838f20b79a56ebe0095",
|
||||
"dependencies": [
|
||||
"jsr:@std/assert@^0.221.0"
|
||||
]
|
||||
"@std/path@1.0.0-rc.1": {
|
||||
"integrity": "b8c00ae2f19106a6bb7cbf1ab9be52aa70de1605daeb2dbdc4f87a7cbaf10ff6"
|
||||
},
|
||||
"@std/streams@0.223.0": {
|
||||
"integrity": "d6b28e498ced3960b04dc5d251f2dcfc1df244b5ec5a48dc23a8f9b490be3b99"
|
||||
"integrity": "d6b28e498ced3960b04dc5d251f2dcfc1df244b5ec5a48dc23a8f9b490be3b99",
|
||||
"dependencies": [
|
||||
"jsr:@std/assert@^0.223.0",
|
||||
"jsr:@std/bytes@^0.223.0",
|
||||
"jsr:@std/io@^0.223.0"
|
||||
]
|
||||
}
|
||||
},
|
||||
"npm": {
|
||||
|
|
@ -440,6 +499,10 @@
|
|||
"integrity": "sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg==",
|
||||
"dependencies": {}
|
||||
},
|
||||
"@noble/secp256k1@2.1.0": {
|
||||
"integrity": "sha512-XLEQQNdablO0XZOIniFQimiXsZDNwaYgL96dZwC54Q30imSbAOFf3NKtepc+cXyuZf5Q1HCgbqgZ2UFFuHVcEw==",
|
||||
"dependencies": {}
|
||||
},
|
||||
"@opentelemetry/api@1.9.0": {
|
||||
"integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==",
|
||||
"dependencies": {}
|
||||
|
|
@ -482,6 +545,10 @@
|
|||
"@scure/base": "@scure/base@1.1.6"
|
||||
}
|
||||
},
|
||||
"@soapbox.pub/pglite@0.2.10": {
|
||||
"integrity": "sha512-DEHejCr+R99RNdyOo34Nbl1FKLmpBCc0pMlPhH3yTyc/KH5HV7dPYbTGCgqRXPxODVkQhvaEuIF2266KsUlZcg==",
|
||||
"dependencies": {}
|
||||
},
|
||||
"@types/dompurify@3.0.5": {
|
||||
"integrity": "sha512-1Wg0g3BtQF7sSb27fJQAKck1HECM6zV1EB66j8JH9i3LCjYabJa0FSdiSgsD5K/RbrsR0SiraKacLB+T8ZVYAg==",
|
||||
"dependencies": {
|
||||
|
|
@ -846,6 +913,12 @@
|
|||
"integrity": "sha512-dyNKv2KRvYOQPLCAOCjjQuCk4YFd33BvGdf/o5bC7FiW+BB6snA81Zt+2wT9QDFzKqxKa5rrOmvlK/anehCcgA==",
|
||||
"dependencies": {}
|
||||
},
|
||||
"lande@1.0.10": {
|
||||
"integrity": "sha512-yT52DQh+UV2pEp08jOYrA4drDv0DbjpiRyZYgl25ak9G2cVR2AimzrqkYQWrD9a7Ud+qkAcaiDDoNH9DXfHPmw==",
|
||||
"dependencies": {
|
||||
"toygrad": "toygrad@2.6.0"
|
||||
}
|
||||
},
|
||||
"light-bolt11-decoder@3.1.1": {
|
||||
"integrity": "sha512-sLg/KCwYkgsHWkefWd6KqpCHrLFWWaXTOX3cf6yD2hAzL0SLpX+lFcaFK2spkjbgzG6hhijKfORDc9WoUHwX0A==",
|
||||
"dependencies": {
|
||||
|
|
@ -1195,6 +1268,10 @@
|
|||
"url-parse": "url-parse@1.5.10"
|
||||
}
|
||||
},
|
||||
"toygrad@2.6.0": {
|
||||
"integrity": "sha512-g4zBmlSbvzOE5FOILxYkAybTSxijKLkj1WoNqVGnbMcWDyj4wWQ+eYSr3ik7XOpIgMq/7eBcPRTJX3DM2E0YMg==",
|
||||
"dependencies": {}
|
||||
},
|
||||
"tr46@0.0.3": {
|
||||
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
|
||||
"dependencies": {}
|
||||
|
|
@ -1205,6 +1282,10 @@
|
|||
"punycode": "punycode@2.3.1"
|
||||
}
|
||||
},
|
||||
"tseep@1.2.1": {
|
||||
"integrity": "sha512-VFnsNcPGC4qFJ1nxbIPSjTmtRZOhlqLmtwRqtLVos8mbRHki8HO9cy9Z1e89EiWyxFmq6LBviI9TQjijxw/mEw==",
|
||||
"dependencies": {}
|
||||
},
|
||||
"type-fest@3.13.1": {
|
||||
"integrity": "sha512-tLq3bSNx+xSpwvAJnzrK0Ep5CLNWjvFTOp71URMaAEWBfRb9nnJiBoUe0tF8bI4ZFO3omgBR6NvnbzVUT3Ly4g==",
|
||||
"dependencies": {}
|
||||
|
|
@ -1863,6 +1944,18 @@
|
|||
"https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/main/mod.ts": "662438fd3909984bb8cbaf3fd44d2121e949d11301baf21d6c3f057ccf9887de",
|
||||
"https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/main/src/PostgreSQLDriver.ts": "590c2fa248cff38e6e0f623050983039b5fde61e9c7131593d2922fb1f0eb921",
|
||||
"https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/main/src/PostgreSQLDriverDatabaseConnection.ts": "2158de426860bfd4f8e73afff0289bd40a11e273c8d883d4fd6474db01a9c2a7",
|
||||
"https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/mod.js": "cb68f17d6d90df318934deccdb469d740be0888e7a597a9e7eea7100ce36a252",
|
||||
"https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/polyfills.js": "318eb01f2b4cc33a46c59f3ddc11f22a56d6b1db8b7719b2ad7decee63a5bd47",
|
||||
"https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/bytes.js": "f2de43bdc8fa5dc4b169f2c70d5d8b053a3dea8f85ef011d7b27dec69e14ebb7",
|
||||
"https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/connection.js": "63bb06ad07cf802d295b35788261c34e82a80cec30b0dffafe05ccd74af3716f",
|
||||
"https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/errors.js": "85cfbed9a5ab0db41ab8e97b806c881af29807dfe99bc656fdf1a18c1c13b6c6",
|
||||
"https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/index.js": "4e8b09c7d0ce6e9eea386f59337867266498d5bb60ccd567d0bea5da03f6094d",
|
||||
"https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/large.js": "f3e770cdb7cc695f7b50687b4c6c4b7252129515486ec8def98b7582ee7c54ef",
|
||||
"https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/query.js": "67c45a5151032aa46b587abc15381fe4efd97c696e5c1b53082b8161309c4ee2",
|
||||
"https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/queue.js": "709624843223ea842bf095f6934080f19f1a059a51cbbf82e9827f3bb1bf2ca7",
|
||||
"https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/result.js": "001ff5e0c8d634674f483d07fbcd620a797e3101f842d6c20ca3ace936260465",
|
||||
"https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/subscribe.js": "9e4d0c3e573a6048e77ee2f15abbd5bcd17da9ca85a78c914553472c6d6c169b",
|
||||
"https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/types.js": "471f4a6c35412aa202a7c177c0a7e5a7c3bd225f01bbde67c947894c1b8bf6ed",
|
||||
"https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/KeyCombo.ts": "a370b2dca76faa416d00e45479c8ce344971b5b86b44b4d0b213245c4bd2f8a3",
|
||||
"https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/checkbox.ts": "e337ee7396aaefe6cc8c6349a445542fe7f0760311773369c9012b3fa278d21e",
|
||||
"https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/config.ts": "a94a022c757f63ee7c410e29b97d3bfab1811889fb4483f56395cf376a911d1b",
|
||||
|
|
@ -1915,12 +2008,11 @@
|
|||
"dependencies": [
|
||||
"jsr:@b-fuze/deno-dom@^0.1.47",
|
||||
"jsr:@bradenmacdonald/s3-lite-client@^0.7.4",
|
||||
"jsr:@db/sqlite@^0.11.1",
|
||||
"jsr:@hono/hono@^4.4.6",
|
||||
"jsr:@lambdalisue/async@^2.1.1",
|
||||
"jsr:@nostrify/db@^0.31.2",
|
||||
"jsr:@nostrify/db@^0.32.2",
|
||||
"jsr:@nostrify/nostrify@^0.30.1",
|
||||
"jsr:@soapbox/kysely-deno-sqlite@^2.1.0",
|
||||
"jsr:@soapbox/kysely-pglite@^0.0.1",
|
||||
"jsr:@soapbox/stickynotes@^0.4.0",
|
||||
"jsr:@std/assert@^0.225.1",
|
||||
"jsr:@std/cli@^0.223.0",
|
||||
|
|
@ -1935,6 +2027,7 @@
|
|||
"npm:@isaacs/ttlcache@^1.4.1",
|
||||
"npm:@noble/secp256k1@^2.0.0",
|
||||
"npm:@scure/base@^1.1.6",
|
||||
"npm:@soapbox.pub/pglite@^0.2.10",
|
||||
"npm:comlink-async-generator@^0.0.1",
|
||||
"npm:comlink@^4.4.1",
|
||||
"npm:commander@12.1.0",
|
||||
|
|
@ -1946,6 +2039,7 @@
|
|||
"npm:isomorphic-dompurify@^2.11.0",
|
||||
"npm:kysely-postgres-js@2.0.0",
|
||||
"npm:kysely@^0.27.4",
|
||||
"npm:lande@^1.0.10",
|
||||
"npm:light-bolt11-decoder",
|
||||
"npm:linkify-plugin-hashtag@^4.1.1",
|
||||
"npm:linkify-string@^4.1.1",
|
||||
|
|
|
|||
|
|
@ -16,9 +16,9 @@ ssh -L 9229:localhost:9229 <user>@<host>
|
|||
|
||||
Then, in Chromium, go to `chrome://inspect` and the Ditto server should be available.
|
||||
|
||||
## SQLite performance
|
||||
## SQL performance
|
||||
|
||||
To track slow queries, first set `DEBUG=ditto:sqlite.worker` in the environment so only SQLite logs are shown.
|
||||
To track slow queries, first set `DEBUG=ditto:sql` in the environment so only SQL logs are shown.
|
||||
|
||||
Then, grep for any logs above 0.001s:
|
||||
|
||||
|
|
|
|||
4570
grafana/Ditto-Dashboard.json
Normal file
4570
grafana/Ditto-Dashboard.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -1,16 +1,13 @@
|
|||
import { JsonParseStream } from '@std/json/json-parse-stream';
|
||||
import { TextLineStream } from '@std/streams/text-line-stream';
|
||||
|
||||
import { DittoDB } from '@/db/DittoDB.ts';
|
||||
import { AdminSigner } from '@/signers/AdminSigner.ts';
|
||||
import { EventsDB } from '@/storages/EventsDB.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
import { type EventStub } from '@/utils/api.ts';
|
||||
import { nostrNow } from '@/utils.ts';
|
||||
|
||||
const signer = new AdminSigner();
|
||||
|
||||
const db = await DittoDB.getInstance();
|
||||
const eventsDB = new EventsDB(db);
|
||||
const store = await Storages.db();
|
||||
|
||||
const readable = Deno.stdin.readable
|
||||
.pipeThrough(new TextDecoderStream())
|
||||
|
|
@ -25,7 +22,7 @@ for await (const t of readable) {
|
|||
...t as EventStub,
|
||||
});
|
||||
|
||||
await eventsDB.event(event);
|
||||
await store.event(event);
|
||||
}
|
||||
|
||||
Deno.exit(0);
|
||||
|
|
|
|||
|
|
@ -1,13 +1,11 @@
|
|||
import { NSchema } from '@nostrify/nostrify';
|
||||
import { nip19 } from 'nostr-tools';
|
||||
|
||||
import { DittoDB } from '@/db/DittoDB.ts';
|
||||
import { AdminSigner } from '@/signers/AdminSigner.ts';
|
||||
import { EventsDB } from '@/storages/EventsDB.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
import { nostrNow } from '@/utils.ts';
|
||||
|
||||
const db = await DittoDB.getInstance();
|
||||
const eventsDB = new EventsDB(db);
|
||||
const store = await Storages.db();
|
||||
|
||||
const [pubkeyOrNpub, role] = Deno.args;
|
||||
const pubkey = pubkeyOrNpub.startsWith('npub1') ? nip19.decode(pubkeyOrNpub as `npub1${string}`).data : pubkeyOrNpub;
|
||||
|
|
@ -25,7 +23,7 @@ if (!['admin', 'user'].includes(role)) {
|
|||
const signer = new AdminSigner();
|
||||
const admin = await signer.getPublicKey();
|
||||
|
||||
const [existing] = await eventsDB.query([{
|
||||
const [existing] = await store.query([{
|
||||
kinds: [30382],
|
||||
authors: [admin],
|
||||
'#d': [pubkey],
|
||||
|
|
@ -59,6 +57,6 @@ const event = await signer.signEvent({
|
|||
created_at: nostrNow(),
|
||||
});
|
||||
|
||||
await eventsDB.event(event);
|
||||
await store.event(event);
|
||||
|
||||
Deno.exit(0);
|
||||
|
|
|
|||
|
|
@ -1,14 +1,7 @@
|
|||
import { Conf } from '@/config.ts';
|
||||
import { DittoDB } from '@/db/DittoDB.ts';
|
||||
import { sleep } from '@/test.ts';
|
||||
|
||||
if (Deno.env.get('CI') && Conf.db.dialect === 'postgres') {
|
||||
console.info('Waiting 1 second for postgres to start...');
|
||||
await sleep(1_000);
|
||||
}
|
||||
import { Storages } from '@/storages.ts';
|
||||
|
||||
// This migrates kysely internally.
|
||||
const { kysely } = await DittoDB.getInstance();
|
||||
const kysely = await Storages.kysely();
|
||||
|
||||
// Close the connection before exiting.
|
||||
await kysely.destroy();
|
||||
|
|
|
|||
32
scripts/db-populate-search.ts
Normal file
32
scripts/db-populate-search.ts
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
import { NSchema as n } from '@nostrify/nostrify';
|
||||
import { Storages } from '@/storages.ts';
|
||||
|
||||
const store = await Storages.db();
|
||||
const kysely = await Storages.kysely();
|
||||
|
||||
for await (const msg of store.req([{ kinds: [0] }])) {
|
||||
if (msg[0] === 'EVENT') {
|
||||
const { pubkey, content } = msg[2];
|
||||
|
||||
const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(content);
|
||||
const search = [name, nip05].filter(Boolean).join(' ').trim();
|
||||
|
||||
try {
|
||||
await kysely.insertInto('author_search').values({
|
||||
pubkey,
|
||||
search,
|
||||
}).onConflict(
|
||||
(oc) =>
|
||||
oc.column('pubkey')
|
||||
.doUpdateSet((eb) => ({ search: eb.ref('excluded.search') })),
|
||||
)
|
||||
.execute();
|
||||
} catch {
|
||||
// do nothing
|
||||
}
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Deno.exit();
|
||||
|
|
@ -6,11 +6,9 @@
|
|||
import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify';
|
||||
import { nip19 } from 'nostr-tools';
|
||||
|
||||
import { DittoDB } from '@/db/DittoDB.ts';
|
||||
import { EventsDB } from '@/storages/EventsDB.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
|
||||
const db = await DittoDB.getInstance();
|
||||
const eventsDB = new EventsDB(db);
|
||||
const store = await Storages.db();
|
||||
|
||||
interface ImportEventsOpts {
|
||||
profilesOnly: boolean;
|
||||
|
|
@ -21,7 +19,7 @@ const importUsers = async (
|
|||
authors: string[],
|
||||
relays: string[],
|
||||
opts?: Partial<ImportEventsOpts>,
|
||||
doEvent: DoEvent = async (event: NostrEvent) => await eventsDB.event(event),
|
||||
doEvent: DoEvent = async (event: NostrEvent) => await store.event(event),
|
||||
) => {
|
||||
// Kind 0s + follow lists.
|
||||
const profiles: Record<string, Record<number, NostrEvent>> = {};
|
||||
|
|
@ -29,6 +27,18 @@ const importUsers = async (
|
|||
const notes = new Set<string>();
|
||||
const { profilesOnly = false } = opts || {};
|
||||
|
||||
const put = async (event: NostrEvent) => {
|
||||
try {
|
||||
await doEvent(event);
|
||||
} catch (error) {
|
||||
if (error.message.includes('violates unique constraint')) {
|
||||
console.warn(`Skipping existing event ${event.id}...`);
|
||||
} else {
|
||||
console.error(error);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
await Promise.all(relays.map(async (relay) => {
|
||||
if (!relay.startsWith('wss://')) console.error(`Invalid relay url ${relay}`);
|
||||
const conn = new NRelay1(relay);
|
||||
|
|
@ -49,7 +59,7 @@ const importUsers = async (
|
|||
if (kind === 1 && !notes.has(event.id)) {
|
||||
// add the event to eventsDB only if it has not been found already.
|
||||
notes.add(event.id);
|
||||
await doEvent(event);
|
||||
await put(event);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -64,7 +74,7 @@ const importUsers = async (
|
|||
for (const user in profiles) {
|
||||
const profile = profiles[user];
|
||||
for (const kind in profile) {
|
||||
await doEvent(profile[kind]);
|
||||
await put(profile[kind]);
|
||||
}
|
||||
|
||||
let name = user;
|
||||
|
|
|
|||
|
|
@ -45,16 +45,16 @@ const DATABASE_URL = Deno.env.get('DATABASE_URL');
|
|||
if (DATABASE_URL) {
|
||||
vars.DATABASE_URL = await question('input', 'Database URL', DATABASE_URL);
|
||||
} else {
|
||||
const database = await question('list', 'Which database do you want to use?', ['postgres', 'sqlite']);
|
||||
if (database === 'sqlite') {
|
||||
const path = await question('input', 'Path to SQLite database', 'data/db.sqlite3');
|
||||
vars.DATABASE_URL = `sqlite://${path}`;
|
||||
const database = await question('list', 'Which database do you want to use?', ['postgres', 'pglite']);
|
||||
if (database === 'pglite') {
|
||||
const path = await question('input', 'Path to PGlite data directory', 'data/pgdata');
|
||||
vars.DATABASE_URL = `file://${path}`;
|
||||
}
|
||||
if (database === 'postgres') {
|
||||
const host = await question('input', 'Postgres host', 'localhost');
|
||||
const port = await question('input', 'Postgres port', '5432');
|
||||
const user = await question('input', 'Postgres user', 'ditto');
|
||||
const password = await question('input', 'Postgres password', 'ditto');
|
||||
const password = await question('password', 'Postgres password', true);
|
||||
const database = await question('input', 'Postgres database', 'ditto');
|
||||
vars.DATABASE_URL = `postgres://${user}:${password}@${host}:${port}/${database}`;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import { nip19 } from 'nostr-tools';
|
||||
|
||||
import { DittoDB } from '@/db/DittoDB.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
import { refreshAuthorStats } from '@/utils/stats.ts';
|
||||
|
||||
|
|
@ -18,6 +17,6 @@ try {
|
|||
}
|
||||
|
||||
const store = await Storages.db();
|
||||
const kysely = await DittoDB.getInstance();
|
||||
const kysely = await Storages.kysely();
|
||||
|
||||
await refreshAuthorStats({ pubkey, kysely, store });
|
||||
|
|
|
|||
16
src/DittoUploads.ts
Normal file
16
src/DittoUploads.ts
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
import { LRUCache } from 'lru-cache';
|
||||
|
||||
import { Time } from '@/utils/time.ts';
|
||||
|
||||
export interface DittoUpload {
|
||||
id: string;
|
||||
pubkey: string;
|
||||
url: string;
|
||||
tags: string[][];
|
||||
uploadedAt: Date;
|
||||
}
|
||||
|
||||
export const dittoUploads = new LRUCache<string, DittoUpload>({
|
||||
max: 1000,
|
||||
ttl: Time.minutes(15),
|
||||
});
|
||||
|
|
@ -53,7 +53,7 @@ import {
|
|||
instanceV2Controller,
|
||||
} from '@/controllers/api/instance.ts';
|
||||
import { markersController, updateMarkersController } from '@/controllers/api/markers.ts';
|
||||
import { mediaController } from '@/controllers/api/media.ts';
|
||||
import { mediaController, updateMediaController } from '@/controllers/api/media.ts';
|
||||
import { mutesController } from '@/controllers/api/mutes.ts';
|
||||
import { notificationsController } from '@/controllers/api/notifications.ts';
|
||||
import { createTokenController, oauthAuthorizeController, oauthController } from '@/controllers/api/oauth.ts';
|
||||
|
|
@ -226,6 +226,10 @@ app.delete('/api/v1/statuses/:id{[0-9a-f]{64}}', requireSigner, deleteStatusCont
|
|||
app.get('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/quotes', quotesController);
|
||||
|
||||
app.post('/api/v1/media', mediaController);
|
||||
app.put(
|
||||
'/api/v1/media/:id{[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}',
|
||||
updateMediaController,
|
||||
);
|
||||
app.post('/api/v2/media', mediaController);
|
||||
|
||||
app.get('/api/v1/timelines/home', requireSigner, homeTimelineController);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import url from 'node:url';
|
||||
|
||||
import os from 'node:os';
|
||||
import * as dotenv from '@std/dotenv';
|
||||
import { getPublicKey, nip19 } from 'nostr-tools';
|
||||
import { z } from 'zod';
|
||||
|
|
@ -36,21 +35,11 @@ class Conf {
|
|||
}
|
||||
return this._pubkey;
|
||||
}
|
||||
/** Ditto admin secret key as a Web Crypto key. */
|
||||
static get cryptoKey(): Promise<CryptoKey> {
|
||||
return crypto.subtle.importKey(
|
||||
'raw',
|
||||
Conf.seckey,
|
||||
{ name: 'HMAC', hash: 'SHA-256' },
|
||||
false,
|
||||
['sign', 'verify'],
|
||||
);
|
||||
}
|
||||
|
||||
/** Port to use when serving the HTTP server. */
|
||||
static get port(): number {
|
||||
return parseInt(Deno.env.get('PORT') || '4036');
|
||||
}
|
||||
|
||||
/** Relay URL to the Ditto server's relay. */
|
||||
static get relay(): `wss://${string}` | `ws://${string}` {
|
||||
const { protocol, host } = Conf.url;
|
||||
return `${protocol === 'https:' ? 'wss:' : 'ws:'}//${host}/relay`;
|
||||
|
|
@ -82,22 +71,17 @@ class Conf {
|
|||
* ```
|
||||
*/
|
||||
static get databaseUrl(): string {
|
||||
return Deno.env.get('DATABASE_URL') ?? 'sqlite://data/db.sqlite3';
|
||||
return Deno.env.get('DATABASE_URL') ?? 'file://data/pgdata';
|
||||
}
|
||||
/** Database to use in tests. */
|
||||
static get testDatabaseUrl(): string {
|
||||
return Deno.env.get('TEST_DATABASE_URL') ?? 'memory://';
|
||||
}
|
||||
/** PGlite debug level. 0 disables logging. */
|
||||
static get pgliteDebug(): 0 | 1 | 2 | 3 | 4 | 5 {
|
||||
return Number(Deno.env.get('PGLITE_DEBUG') || 0) as 0 | 1 | 2 | 3 | 4 | 5;
|
||||
}
|
||||
static db = {
|
||||
get url(): url.UrlWithStringQuery {
|
||||
return url.parse(Conf.databaseUrl);
|
||||
},
|
||||
get dialect(): 'sqlite' | 'postgres' | undefined {
|
||||
switch (Conf.db.url.protocol) {
|
||||
case 'sqlite:':
|
||||
return 'sqlite';
|
||||
case 'postgres:':
|
||||
case 'postgresql:':
|
||||
return 'postgres';
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
/** Database query timeout configurations. */
|
||||
timeouts: {
|
||||
/** Default query timeout when another setting isn't more specific. */
|
||||
|
|
@ -198,12 +182,6 @@ class Conf {
|
|||
'system',
|
||||
];
|
||||
}
|
||||
/** Proof-of-work configuration. */
|
||||
static pow = {
|
||||
get registrations(): number {
|
||||
return Number(Deno.env.get('DITTO_POW_REGISTRATIONS') ?? 20);
|
||||
},
|
||||
};
|
||||
/** Domain of the Ditto server as a `URL` object, for easily grabbing the `hostname`, etc. */
|
||||
static get url(): URL {
|
||||
return new URL(Conf.localDomain);
|
||||
|
|
@ -216,21 +194,6 @@ class Conf {
|
|||
static get sentryDsn(): string | undefined {
|
||||
return Deno.env.get('SENTRY_DSN');
|
||||
}
|
||||
/** SQLite settings. */
|
||||
static sqlite = {
|
||||
/**
|
||||
* Number of bytes to use for memory-mapped IO.
|
||||
* https://www.sqlite.org/pragma.html#pragma_mmap_size
|
||||
*/
|
||||
get mmapSize(): number {
|
||||
const value = Deno.env.get('SQLITE_MMAP_SIZE');
|
||||
if (value) {
|
||||
return Number(value);
|
||||
} else {
|
||||
return 1024 * 1024 * 1024;
|
||||
}
|
||||
},
|
||||
};
|
||||
/** Postgres settings. */
|
||||
static pg = {
|
||||
/** Number of connections to use in the pool. */
|
||||
|
|
@ -258,10 +221,22 @@ class Conf {
|
|||
'i',
|
||||
);
|
||||
}
|
||||
/** User-Agent to use when fetching link previews. Pretend to be Facebook by default. */
|
||||
static get fetchUserAgent(): string {
|
||||
return Deno.env.get('DITTO_FETCH_USER_AGENT') ?? 'facebookexternalhit';
|
||||
}
|
||||
/** Path to the custom policy module. Must be an absolute path, https:, npm:, or jsr: URI. */
|
||||
static get policy(): string {
|
||||
return Deno.env.get('DITTO_POLICY') || new URL('../data/policy.ts', import.meta.url).pathname;
|
||||
}
|
||||
/** Absolute path to the data directory used by Ditto. */
|
||||
static get dataDir(): string {
|
||||
return Deno.env.get('DITTO_DATA_DIR') || new URL('../data', import.meta.url).pathname;
|
||||
}
|
||||
/** Absolute path of the Deno directory. */
|
||||
static get denoDir(): string {
|
||||
return Deno.env.get('DENO_DIR') || `${os.userInfo().homedir}/.cache/deno`;
|
||||
}
|
||||
/** Whether zap splits should be enabled. */
|
||||
static get zapSplitsEnabled(): boolean {
|
||||
return optionalBooleanSchema.parse(Deno.env.get('ZAP_SPLITS_ENABLED')) ?? false;
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { type AppController } from '@/app.ts';
|
|||
import { Conf } from '@/config.ts';
|
||||
import { getAuthor, getFollowedPubkeys } from '@/queries.ts';
|
||||
import { booleanParamSchema, fileSchema } from '@/schema.ts';
|
||||
import { getPubkeysBySearch } from '@/controllers/api/search.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
import { uploadFile } from '@/utils/upload.ts';
|
||||
import { nostrNow } from '@/utils.ts';
|
||||
|
|
@ -115,6 +116,7 @@ const accountSearchQuerySchema = z.object({
|
|||
const accountSearchController: AppController = async (c) => {
|
||||
const { signal } = c.req.raw;
|
||||
const { limit } = c.get('pagination');
|
||||
const kysely = await Storages.kysely();
|
||||
|
||||
const result = accountSearchQuerySchema.safeParse(c.req.query());
|
||||
|
||||
|
|
@ -133,8 +135,17 @@ const accountSearchController: AppController = async (c) => {
|
|||
return c.json(pubkey ? [await accountFromPubkey(pubkey)] : []);
|
||||
}
|
||||
|
||||
const events = event ? [event] : await store.query([{ kinds: [0], search: query, limit }], { signal });
|
||||
const pubkeys = await getPubkeysBySearch(kysely, { q: query, limit });
|
||||
|
||||
let events = event ? [event] : await store.query([{ kinds: [0], authors: pubkeys, limit }], {
|
||||
signal,
|
||||
});
|
||||
|
||||
if (!event) {
|
||||
events = pubkeys
|
||||
.map((pubkey) => events.find((event) => event.pubkey === pubkey))
|
||||
.filter((event) => !!event);
|
||||
}
|
||||
const accounts = await hydrateEvents({ events, store, signal }).then(
|
||||
(events) =>
|
||||
Promise.all(
|
||||
|
|
|
|||
|
|
@ -63,9 +63,6 @@ const instanceV1Controller: AppController = async (c) => {
|
|||
nostr: {
|
||||
pubkey: Conf.pubkey,
|
||||
relay: `${wsProtocol}//${host}/relay`,
|
||||
pow: {
|
||||
registrations: Conf.pow.registrations,
|
||||
},
|
||||
},
|
||||
rules: [],
|
||||
});
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { fileSchema } from '@/schema.ts';
|
|||
import { parseBody } from '@/utils/api.ts';
|
||||
import { renderAttachment } from '@/views/mastodon/attachments.ts';
|
||||
import { uploadFile } from '@/utils/upload.ts';
|
||||
import { dittoUploads } from '@/DittoUploads.ts';
|
||||
|
||||
const mediaBodySchema = z.object({
|
||||
file: fileSchema,
|
||||
|
|
@ -13,6 +14,10 @@ const mediaBodySchema = z.object({
|
|||
focus: z.string().optional(),
|
||||
});
|
||||
|
||||
const mediaUpdateSchema = z.object({
|
||||
description: z.string(),
|
||||
});
|
||||
|
||||
const mediaController: AppController = async (c) => {
|
||||
const pubkey = await c.get('signer')?.getPublicKey()!;
|
||||
const result = mediaBodySchema.safeParse(await parseBody(c.req.raw));
|
||||
|
|
@ -32,4 +37,27 @@ const mediaController: AppController = async (c) => {
|
|||
}
|
||||
};
|
||||
|
||||
export { mediaController };
|
||||
const updateMediaController: AppController = async (c) => {
|
||||
const result = mediaUpdateSchema.safeParse(await parseBody(c.req.raw));
|
||||
|
||||
if (!result.success) {
|
||||
return c.json({ error: 'Bad request.', schema: result.error }, 422);
|
||||
}
|
||||
|
||||
const id = c.req.param('id');
|
||||
const { description } = result.data;
|
||||
const upload = dittoUploads.get(id);
|
||||
|
||||
if (!upload) {
|
||||
return c.json({ error: 'File with specified ID not found.' }, 404);
|
||||
}
|
||||
|
||||
dittoUploads.set(id, {
|
||||
...upload,
|
||||
tags: upload.tags.filter(([name]) => name !== 'alt').concat([['alt', description]]),
|
||||
});
|
||||
|
||||
return c.json({ message: 'ok' }, 200);
|
||||
};
|
||||
|
||||
export { mediaController, updateMediaController };
|
||||
|
|
|
|||
|
|
@ -4,9 +4,10 @@ import { z } from 'zod';
|
|||
import { AppContext, AppController } from '@/app.ts';
|
||||
import { Conf } from '@/config.ts';
|
||||
import { DittoPagination } from '@/interfaces/DittoPagination.ts';
|
||||
import { getAmount } from '@/utils/bolt11.ts';
|
||||
import { hydrateEvents } from '@/storages/hydrate.ts';
|
||||
import { paginated } from '@/utils/api.ts';
|
||||
import { renderNotification } from '@/views/mastodon/notifications.ts';
|
||||
import { renderNotification, RenderNotificationOpts } from '@/views/mastodon/notifications.ts';
|
||||
|
||||
/** Set of known notification types across backends. */
|
||||
const notificationTypes = new Set([
|
||||
|
|
@ -23,6 +24,7 @@ const notificationTypes = new Set([
|
|||
'severed_relationships',
|
||||
'pleroma:emoji_reaction',
|
||||
'ditto:name_grant',
|
||||
'ditto:zap',
|
||||
]);
|
||||
|
||||
const notificationsSchema = z.object({
|
||||
|
|
@ -50,6 +52,9 @@ const notificationsController: AppController = async (c) => {
|
|||
if (types.has('favourite') || types.has('pleroma:emoji_reaction')) {
|
||||
kinds.add(7);
|
||||
}
|
||||
if (types.has('ditto:zap')) {
|
||||
kinds.add(9735);
|
||||
}
|
||||
|
||||
const filter: NostrFilter = {
|
||||
kinds: [...kinds],
|
||||
|
|
@ -81,16 +86,55 @@ async function renderNotifications(
|
|||
const { signal } = c.req.raw;
|
||||
const opts = { signal, limit: params.limit, timeout: Conf.db.timeouts.timelines };
|
||||
|
||||
const zapsRelatedFilter: NostrFilter[] = [];
|
||||
|
||||
const events = await store
|
||||
.query(filters, opts)
|
||||
.then((events) => events.filter((event) => event.pubkey !== pubkey))
|
||||
.then((events) =>
|
||||
events.filter((event) => {
|
||||
if (event.kind === 9735) {
|
||||
const zappedEventId = event.tags.find(([name]) => name === 'e')?.[1];
|
||||
if (zappedEventId) zapsRelatedFilter.push({ kinds: [1], ids: [zappedEventId] });
|
||||
const zapSender = event.tags.find(([name]) => name === 'P')?.[1];
|
||||
if (zapSender) zapsRelatedFilter.push({ kinds: [0], authors: [zapSender] });
|
||||
}
|
||||
|
||||
return event.pubkey !== pubkey;
|
||||
})
|
||||
)
|
||||
.then((events) => hydrateEvents({ events, store, signal }));
|
||||
|
||||
if (!events.length) {
|
||||
return c.json([]);
|
||||
}
|
||||
|
||||
const notifications = (await Promise.all(events.map((event) => renderNotification(event, { viewerPubkey: pubkey }))))
|
||||
const zapSendersAndPosts = await store
|
||||
.query(zapsRelatedFilter, opts)
|
||||
.then((events) => hydrateEvents({ events, store, signal }));
|
||||
|
||||
const notifications = (await Promise.all(events.map((event) => {
|
||||
const opts: RenderNotificationOpts = { viewerPubkey: pubkey };
|
||||
if (event.kind === 9735) {
|
||||
const zapRequestString = event?.tags?.find(([name]) => name === 'description')?.[1];
|
||||
const zapRequest = n.json().pipe(n.event()).optional().catch(undefined).parse(zapRequestString);
|
||||
// By getting the pubkey from the zap request we guarantee who is the sender
|
||||
// some clients don't put the P tag in the zap receipt...
|
||||
const zapSender = zapRequest?.pubkey;
|
||||
const zappedPost = event.tags.find(([name]) => name === 'e')?.[1];
|
||||
|
||||
const amountSchema = z.coerce.number().int().nonnegative().catch(0);
|
||||
// amount in millisats
|
||||
const amount = amountSchema.parse(getAmount(event?.tags.find(([name]) => name === 'bolt11')?.[1]));
|
||||
|
||||
opts['zap'] = {
|
||||
zapSender: zapSendersAndPosts.find(({ pubkey, kind }) => kind === 0 && pubkey === zapSender) ?? zapSender,
|
||||
zappedPost: zapSendersAndPosts.find(({ id }) => id === zappedPost),
|
||||
amount,
|
||||
message: zapRequest?.content,
|
||||
};
|
||||
}
|
||||
return renderNotification(event, opts);
|
||||
})))
|
||||
.filter((notification) => notification && types.has(notification.type));
|
||||
|
||||
if (!notifications.length) {
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ import { z } from 'zod';
|
|||
|
||||
import { AppController } from '@/app.ts';
|
||||
import { Conf } from '@/config.ts';
|
||||
import { DittoDB } from '@/db/DittoDB.ts';
|
||||
import { nostrNow } from '@/utils.ts';
|
||||
import { parseBody } from '@/utils/api.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
|
|
@ -82,7 +81,7 @@ const createTokenController: AppController = async (c) => {
|
|||
async function getToken(
|
||||
{ pubkey, secret, relays = [] }: { pubkey: string; secret?: string; relays?: string[] },
|
||||
): Promise<`token1${string}`> {
|
||||
const { kysely } = await DittoDB.getInstance();
|
||||
const kysely = await Storages.kysely();
|
||||
const token = generateToken();
|
||||
|
||||
const serverSeckey = generateSecretKey();
|
||||
|
|
|
|||
21
src/controllers/api/search.test.ts
Normal file
21
src/controllers/api/search.test.ts
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
import { assertEquals } from '@std/assert';
|
||||
|
||||
import { createTestDB } from '@/test.ts';
|
||||
import { getPubkeysBySearch } from '@/controllers/api/search.ts';
|
||||
|
||||
Deno.test('fuzzy search works', async () => {
|
||||
await using db = await createTestDB();
|
||||
|
||||
await db.kysely.insertInto('author_search').values({
|
||||
pubkey: '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4',
|
||||
search: 'patrickReiis patrickdosreis.com',
|
||||
}).execute();
|
||||
|
||||
assertEquals(await getPubkeysBySearch(db.kysely, { q: 'pat rick', limit: 1 }), []);
|
||||
assertEquals(await getPubkeysBySearch(db.kysely, { q: 'patrick dos reis', limit: 1 }), [
|
||||
'47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4',
|
||||
]);
|
||||
assertEquals(await getPubkeysBySearch(db.kysely, { q: 'dosreis.com', limit: 1 }), [
|
||||
'47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4',
|
||||
]);
|
||||
});
|
||||
|
|
@ -1,8 +1,10 @@
|
|||
import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify';
|
||||
import { nip19 } from 'nostr-tools';
|
||||
import { Kysely, sql } from 'kysely';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { AppController } from '@/app.ts';
|
||||
import { DittoTables } from '@/db/DittoTables.ts';
|
||||
import { booleanParamSchema } from '@/schema.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
import { hydrateEvents } from '@/storages/hydrate.ts';
|
||||
|
|
@ -47,9 +49,8 @@ const searchController: AppController = async (c) => {
|
|||
|
||||
if (event) {
|
||||
events = [event];
|
||||
} else {
|
||||
events = await searchEvents(result.data, signal);
|
||||
}
|
||||
events.push(...(await searchEvents(result.data, signal)));
|
||||
|
||||
const viewerPubkey = await c.get('signer')?.getPublicKey();
|
||||
|
||||
|
|
@ -89,10 +90,33 @@ async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal:
|
|||
filter.authors = [account_id];
|
||||
}
|
||||
|
||||
const pubkeys: string[] = [];
|
||||
if (type === 'accounts') {
|
||||
const kysely = await Storages.kysely();
|
||||
|
||||
pubkeys.push(...(await getPubkeysBySearch(kysely, { q, limit })));
|
||||
|
||||
if (!filter?.authors) {
|
||||
filter.authors = pubkeys;
|
||||
} else {
|
||||
filter.authors.push(...pubkeys);
|
||||
}
|
||||
|
||||
filter.search = undefined;
|
||||
}
|
||||
|
||||
const store = await Storages.search();
|
||||
|
||||
return store.query([filter], { signal })
|
||||
let events = await store.query([filter], { signal })
|
||||
.then((events) => hydrateEvents({ events, store, signal }));
|
||||
|
||||
if (type !== 'accounts') return events;
|
||||
|
||||
events = pubkeys
|
||||
.map((pubkey) => events.find((event) => event.pubkey === pubkey))
|
||||
.filter((event) => !!event);
|
||||
|
||||
return events;
|
||||
}
|
||||
|
||||
/** Get event kinds to search from `type` query param. */
|
||||
|
|
@ -170,4 +194,16 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort
|
|||
return [];
|
||||
}
|
||||
|
||||
export { searchController };
|
||||
/** Get pubkeys whose name and NIP-05 is similar to 'q' */
|
||||
async function getPubkeysBySearch(kysely: Kysely<DittoTables>, { q, limit }: Pick<SearchQuery, 'q' | 'limit'>) {
|
||||
const pubkeys = (await sql<{ pubkey: string }>`
|
||||
SELECT *, word_similarity(${q}, search) AS sml
|
||||
FROM author_search
|
||||
WHERE ${q} % search
|
||||
ORDER BY sml DESC, search LIMIT ${limit}
|
||||
`.execute(kysely)).rows.map(({ pubkey }) => pubkey);
|
||||
|
||||
return pubkeys;
|
||||
}
|
||||
|
||||
export { getPubkeysBySearch, searchController };
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { HTTPException } from '@hono/hono/http-exception';
|
||||
import { NostrEvent, NSchema as n } from '@nostrify/nostrify';
|
||||
import ISO6391 from 'iso-639-1';
|
||||
import 'linkify-plugin-hashtag';
|
||||
|
|
@ -6,22 +7,22 @@ import { nip19 } from 'nostr-tools';
|
|||
import { z } from 'zod';
|
||||
|
||||
import { type AppController } from '@/app.ts';
|
||||
import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts';
|
||||
import { addTag, deleteTag } from '@/utils/tags.ts';
|
||||
import { asyncReplaceAll } from '@/utils/text.ts';
|
||||
import { Conf } from '@/config.ts';
|
||||
import { DittoDB } from '@/db/DittoDB.ts';
|
||||
import { DittoUpload, dittoUploads } from '@/DittoUploads.ts';
|
||||
import { DittoEvent } from '@/interfaces/DittoEvent.ts';
|
||||
import { getAncestors, getAuthor, getDescendants, getEvent } from '@/queries.ts';
|
||||
import { getUnattachedMediaByIds } from '@/db/unattached-media.ts';
|
||||
import { addTag, deleteTag } from '@/utils/tags.ts';
|
||||
import { asyncReplaceAll } from '@/utils/text.ts';
|
||||
import { lookupPubkey } from '@/utils/lookup.ts';
|
||||
import { renderEventAccounts } from '@/views.ts';
|
||||
import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
import { hydrateEvents, purifyEvent } from '@/storages/hydrate.ts';
|
||||
import { hydrateEvents } from '@/storages/hydrate.ts';
|
||||
import { createEvent, paginated, paginatedList, parseBody, updateListEvent } from '@/utils/api.ts';
|
||||
import { getInvoice, getLnurl } from '@/utils/lnurl.ts';
|
||||
import { purifyEvent } from '@/utils/purify.ts';
|
||||
import { getZapSplits } from '@/utils/zap-split.ts';
|
||||
import { renderEventAccounts } from '@/views.ts';
|
||||
import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts';
|
||||
import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts';
|
||||
|
||||
const createStatusSchema = z.object({
|
||||
in_reply_to_id: n.id().nullish(),
|
||||
|
|
@ -49,7 +50,6 @@ const statusController: AppController = async (c) => {
|
|||
const id = c.req.param('id');
|
||||
|
||||
const event = await getEvent(id, {
|
||||
kind: 1,
|
||||
signal: AbortSignal.timeout(1500),
|
||||
});
|
||||
|
||||
|
|
@ -63,7 +63,6 @@ const statusController: AppController = async (c) => {
|
|||
const createStatusController: AppController = async (c) => {
|
||||
const body = await parseBody(c.req.raw);
|
||||
const result = createStatusSchema.safeParse(body);
|
||||
const { kysely } = await DittoDB.getInstance();
|
||||
const store = c.get('store');
|
||||
|
||||
if (!result.success) {
|
||||
|
|
@ -112,10 +111,18 @@ const createStatusController: AppController = async (c) => {
|
|||
tags.push(['l', data.language, 'ISO-639-1']);
|
||||
}
|
||||
|
||||
const media = data.media_ids?.length ? await getUnattachedMediaByIds(kysely, data.media_ids) : [];
|
||||
const media: DittoUpload[] = (data.media_ids ?? []).map((id) => {
|
||||
const upload = dittoUploads.get(id);
|
||||
|
||||
const imeta: string[][] = media.map(({ data }) => {
|
||||
const values: string[] = data.map((tag) => tag.join(' '));
|
||||
if (!upload) {
|
||||
throw new HTTPException(422, { message: 'Uploaded attachment is no longer available.' });
|
||||
}
|
||||
|
||||
return upload;
|
||||
});
|
||||
|
||||
const imeta: string[][] = media.map(({ tags }) => {
|
||||
const values: string[] = tags.map((tag) => tag.join(' '));
|
||||
return ['imeta', ...values];
|
||||
});
|
||||
|
||||
|
|
@ -123,21 +130,25 @@ const createStatusController: AppController = async (c) => {
|
|||
|
||||
const pubkeys = new Set<string>();
|
||||
|
||||
const content = await asyncReplaceAll(data.status ?? '', /@([\w@+._]+)/g, async (match, username) => {
|
||||
const pubkey = await lookupPubkey(username);
|
||||
if (!pubkey) return match;
|
||||
const content = await asyncReplaceAll(
|
||||
data.status ?? '',
|
||||
/(?<![\w/])@([\w@+._]+)(?![\w/\.])/g,
|
||||
async (match, username) => {
|
||||
const pubkey = await lookupPubkey(username);
|
||||
if (!pubkey) return match;
|
||||
|
||||
// Content addressing (default)
|
||||
if (!data.to) {
|
||||
pubkeys.add(pubkey);
|
||||
}
|
||||
// Content addressing (default)
|
||||
if (!data.to) {
|
||||
pubkeys.add(pubkey);
|
||||
}
|
||||
|
||||
try {
|
||||
return `nostr:${nip19.npubEncode(pubkey)}`;
|
||||
} catch {
|
||||
return match;
|
||||
}
|
||||
});
|
||||
try {
|
||||
return `nostr:${nip19.npubEncode(pubkey)}`;
|
||||
} catch {
|
||||
return match;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// Explicit addressing
|
||||
for (const to of data.to ?? []) {
|
||||
|
|
@ -161,7 +172,7 @@ const createStatusController: AppController = async (c) => {
|
|||
}
|
||||
|
||||
const mediaUrls: string[] = media
|
||||
.map(({ data }) => data.find(([name]) => name === 'url')?.[1])
|
||||
.map(({ url }) => url)
|
||||
.filter((url): url is string => Boolean(url));
|
||||
|
||||
const quoteCompat = data.quote_id ? `\n\nnostr:${nip19.noteEncode(data.quote_id)}` : '';
|
||||
|
|
@ -567,7 +578,7 @@ const zappedByController: AppController = async (c) => {
|
|||
const id = c.req.param('id');
|
||||
const params = c.get('listPagination');
|
||||
const store = await Storages.db();
|
||||
const { kysely } = await DittoDB.getInstance();
|
||||
const kysely = await Storages.kysely();
|
||||
|
||||
const zaps = await kysely.selectFrom('event_zaps')
|
||||
.selectAll()
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ import { z } from 'zod';
|
|||
|
||||
import { type AppController } from '@/app.ts';
|
||||
import { Conf } from '@/config.ts';
|
||||
import { DittoDB } from '@/db/DittoDB.ts';
|
||||
import { streamingConnectionsGauge } from '@/metrics.ts';
|
||||
import { MuteListPolicy } from '@/policies/MuteListPolicy.ts';
|
||||
import { getFeedPubkeys } from '@/queries.ts';
|
||||
|
|
@ -38,6 +37,25 @@ const streamSchema = z.enum([
|
|||
|
||||
type Stream = z.infer<typeof streamSchema>;
|
||||
|
||||
/** https://docs.joinmastodon.org/methods/streaming/#events-11 */
|
||||
interface StreamingEvent {
|
||||
/** https://docs.joinmastodon.org/methods/streaming/#events */
|
||||
event:
|
||||
| 'update'
|
||||
| 'delete'
|
||||
| 'notification'
|
||||
| 'filters_changed'
|
||||
| 'conversation'
|
||||
| 'announcement'
|
||||
| 'announcement.reaction'
|
||||
| 'announcement.delete'
|
||||
| 'status.update'
|
||||
| 'encrypted_message'
|
||||
| 'notifications_merged';
|
||||
payload: string;
|
||||
stream: Stream[];
|
||||
}
|
||||
|
||||
const LIMITER_WINDOW = Time.minutes(5);
|
||||
const LIMITER_LIMIT = 100;
|
||||
|
||||
|
|
@ -73,18 +91,14 @@ const streamingController: AppController = async (c) => {
|
|||
|
||||
const policy = pubkey ? new MuteListPolicy(pubkey, await Storages.admin()) : undefined;
|
||||
|
||||
function send(name: string, payload: object) {
|
||||
function send(e: StreamingEvent) {
|
||||
if (socket.readyState === WebSocket.OPEN) {
|
||||
debug('send', name, JSON.stringify(payload));
|
||||
socket.send(JSON.stringify({
|
||||
event: name,
|
||||
payload: JSON.stringify(payload),
|
||||
stream: [stream],
|
||||
}));
|
||||
debug('send', e.event, e.payload);
|
||||
socket.send(JSON.stringify(e));
|
||||
}
|
||||
}
|
||||
|
||||
async function sub(type: string, filters: NostrFilter[], render: (event: NostrEvent) => Promise<unknown>) {
|
||||
async function sub(filters: NostrFilter[], render: (event: NostrEvent) => Promise<StreamingEvent | undefined>) {
|
||||
try {
|
||||
for await (const msg of pubsub.req(filters, { signal: controller.signal })) {
|
||||
if (msg[0] === 'EVENT') {
|
||||
|
|
@ -102,7 +116,7 @@ const streamingController: AppController = async (c) => {
|
|||
const result = await render(event);
|
||||
|
||||
if (result) {
|
||||
send(type, result);
|
||||
send(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -118,19 +132,37 @@ const streamingController: AppController = async (c) => {
|
|||
const topicFilter = await topicToFilter(stream, c.req.query(), pubkey);
|
||||
|
||||
if (topicFilter) {
|
||||
sub('update', [topicFilter], async (event) => {
|
||||
sub([topicFilter], async (event) => {
|
||||
let payload: object | undefined;
|
||||
|
||||
if (event.kind === 1) {
|
||||
return await renderStatus(event, { viewerPubkey: pubkey });
|
||||
payload = await renderStatus(event, { viewerPubkey: pubkey });
|
||||
}
|
||||
if (event.kind === 6) {
|
||||
return await renderReblog(event, { viewerPubkey: pubkey });
|
||||
payload = await renderReblog(event, { viewerPubkey: pubkey });
|
||||
}
|
||||
|
||||
if (payload) {
|
||||
return {
|
||||
event: 'update',
|
||||
payload: JSON.stringify(payload),
|
||||
stream: [stream],
|
||||
};
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (['user', 'user:notification'].includes(stream) && pubkey) {
|
||||
sub('notification', [{ '#p': [pubkey] }], async (event) => {
|
||||
return await renderNotification(event, { viewerPubkey: pubkey });
|
||||
sub([{ '#p': [pubkey] }], async (event) => {
|
||||
if (event.pubkey === pubkey) return; // skip own events
|
||||
const payload = await renderNotification(event, { viewerPubkey: pubkey });
|
||||
if (payload) {
|
||||
return {
|
||||
event: 'notification',
|
||||
payload: JSON.stringify(payload),
|
||||
stream: [stream],
|
||||
};
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
|
@ -189,7 +221,7 @@ async function topicToFilter(
|
|||
|
||||
async function getTokenPubkey(token: string): Promise<string | undefined> {
|
||||
if (token.startsWith('token1')) {
|
||||
const { kysely } = await DittoDB.getInstance();
|
||||
const kysely = await Storages.kysely();
|
||||
|
||||
const { user_pubkey } = await kysely
|
||||
.selectFrom('nip46_tokens')
|
||||
|
|
|
|||
|
|
@ -1,14 +1,16 @@
|
|||
import { register } from 'prom-client';
|
||||
|
||||
import { AppController } from '@/app.ts';
|
||||
import { DittoDB } from '@/db/DittoDB.ts';
|
||||
import { dbAvailableConnectionsGauge, dbPoolSizeGauge } from '@/metrics.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
|
||||
/** Prometheus/OpenMetrics controller. */
|
||||
export const metricsController: AppController = async (c) => {
|
||||
const db = await Storages.database();
|
||||
|
||||
// Update some metrics at request time.
|
||||
dbPoolSizeGauge.set(DittoDB.poolSize);
|
||||
dbAvailableConnectionsGauge.set(DittoDB.availableConnections);
|
||||
dbPoolSizeGauge.set(db.poolSize);
|
||||
dbAvailableConnectionsGauge.set(db.availableConnections);
|
||||
|
||||
const metrics = await register.metrics();
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import {
|
|||
import { AppController } from '@/app.ts';
|
||||
import { Conf } from '@/config.ts';
|
||||
import { relayInfoController } from '@/controllers/nostr/relay-info.ts';
|
||||
import { relayConnectionsGauge, relayEventCounter, relayMessageCounter } from '@/metrics.ts';
|
||||
import { relayConnectionsGauge, relayEventsCounter, relayMessagesCounter } from '@/metrics.ts';
|
||||
import * as pipeline from '@/pipeline.ts';
|
||||
import { RelayError } from '@/RelayError.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
|
|
@ -54,10 +54,10 @@ function connectStream(socket: WebSocket, ip: string | undefined) {
|
|||
|
||||
const result = n.json().pipe(n.clientMsg()).safeParse(e.data);
|
||||
if (result.success) {
|
||||
relayMessageCounter.inc({ verb: result.data[0] });
|
||||
relayMessagesCounter.inc({ verb: result.data[0] });
|
||||
handleMsg(result.data);
|
||||
} else {
|
||||
relayMessageCounter.inc();
|
||||
relayMessagesCounter.inc();
|
||||
send(['NOTICE', 'Invalid message.']);
|
||||
}
|
||||
};
|
||||
|
|
@ -130,7 +130,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) {
|
|||
|
||||
/** Handle EVENT. Store the event. */
|
||||
async function handleEvent([_, event]: NostrClientEVENT): Promise<void> {
|
||||
relayEventCounter.inc({ kind: event.kind.toString() });
|
||||
relayEventsCounter.inc({ kind: event.kind.toString() });
|
||||
try {
|
||||
// This will store it (if eligible) and run other side-effects.
|
||||
await pipeline.handleEvent(event, AbortSignal.timeout(1000));
|
||||
|
|
|
|||
|
|
@ -1,69 +1,32 @@
|
|||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
import { NDatabaseSchema, NPostgresSchema } from '@nostrify/db';
|
||||
import { FileMigrationProvider, Kysely, Migrator } from 'kysely';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
import { DittoPglite } from '@/db/adapters/DittoPglite.ts';
|
||||
import { DittoPostgres } from '@/db/adapters/DittoPostgres.ts';
|
||||
import { DittoSQLite } from '@/db/adapters/DittoSQLite.ts';
|
||||
import { DittoDatabase, DittoDatabaseOpts } from '@/db/DittoDatabase.ts';
|
||||
import { DittoTables } from '@/db/DittoTables.ts';
|
||||
|
||||
export type DittoDatabase = {
|
||||
dialect: 'sqlite';
|
||||
kysely: Kysely<DittoTables> & Kysely<NDatabaseSchema>;
|
||||
} | {
|
||||
dialect: 'postgres';
|
||||
kysely: Kysely<DittoTables> & Kysely<NPostgresSchema>;
|
||||
};
|
||||
|
||||
export class DittoDB {
|
||||
private static db: Promise<DittoDatabase> | undefined;
|
||||
/** Open a new database connection. */
|
||||
static create(databaseUrl: string, opts?: DittoDatabaseOpts): DittoDatabase {
|
||||
const { protocol } = new URL(databaseUrl);
|
||||
|
||||
static getInstance(): Promise<DittoDatabase> {
|
||||
if (!this.db) {
|
||||
this.db = this._getInstance();
|
||||
}
|
||||
return this.db;
|
||||
}
|
||||
|
||||
static async _getInstance(): Promise<DittoDatabase> {
|
||||
const result = {} as DittoDatabase;
|
||||
|
||||
switch (Conf.db.dialect) {
|
||||
case 'sqlite':
|
||||
result.dialect = 'sqlite';
|
||||
result.kysely = await DittoSQLite.getInstance();
|
||||
break;
|
||||
case 'postgres':
|
||||
result.dialect = 'postgres';
|
||||
result.kysely = await DittoPostgres.getInstance();
|
||||
break;
|
||||
switch (protocol) {
|
||||
case 'file:':
|
||||
case 'memory:':
|
||||
return DittoPglite.create(databaseUrl, opts);
|
||||
case 'postgres:':
|
||||
case 'postgresql:':
|
||||
return DittoPostgres.create(databaseUrl, opts);
|
||||
default:
|
||||
throw new Error('Unsupported database URL.');
|
||||
}
|
||||
|
||||
await this.migrate(result.kysely);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
static get poolSize(): number {
|
||||
if (Conf.db.dialect === 'postgres') {
|
||||
return DittoPostgres.poolSize;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static get availableConnections(): number {
|
||||
if (Conf.db.dialect === 'postgres') {
|
||||
return DittoPostgres.availableConnections;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
/** Migrate the database to the latest version. */
|
||||
static async migrate(kysely: DittoDatabase['kysely']) {
|
||||
static async migrate(kysely: Kysely<DittoTables>) {
|
||||
const migrator = new Migrator({
|
||||
db: kysely,
|
||||
provider: new FileMigrationProvider({
|
||||
|
|
|
|||
14
src/db/DittoDatabase.ts
Normal file
14
src/db/DittoDatabase.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
import { Kysely } from 'kysely';
|
||||
|
||||
import { DittoTables } from '@/db/DittoTables.ts';
|
||||
|
||||
export interface DittoDatabase {
|
||||
readonly kysely: Kysely<DittoTables>;
|
||||
readonly poolSize: number;
|
||||
readonly availableConnections: number;
|
||||
}
|
||||
|
||||
export interface DittoDatabaseOpts {
|
||||
poolSize?: number;
|
||||
debug?: 0 | 1 | 2 | 3 | 4 | 5;
|
||||
}
|
||||
|
|
@ -1,12 +1,21 @@
|
|||
export interface DittoTables {
|
||||
import { Nullable } from 'kysely';
|
||||
|
||||
import { NPostgresSchema } from '@nostrify/db';
|
||||
|
||||
export interface DittoTables extends NPostgresSchema {
|
||||
nostr_events: NostrEventsRow;
|
||||
nip46_tokens: NIP46TokenRow;
|
||||
unattached_media: UnattachedMediaRow;
|
||||
author_stats: AuthorStatsRow;
|
||||
event_stats: EventStatsRow;
|
||||
pubkey_domains: PubkeyDomainRow;
|
||||
event_zaps: EventZapRow;
|
||||
author_search: AuthorSearch;
|
||||
}
|
||||
|
||||
type NostrEventsRow = NPostgresSchema['nostr_events'] & {
|
||||
language: Nullable<string>;
|
||||
};
|
||||
|
||||
interface AuthorStatsRow {
|
||||
pubkey: string;
|
||||
followers_count: number;
|
||||
|
|
@ -33,14 +42,6 @@ interface NIP46TokenRow {
|
|||
connected_at: Date;
|
||||
}
|
||||
|
||||
interface UnattachedMediaRow {
|
||||
id: string;
|
||||
pubkey: string;
|
||||
url: string;
|
||||
data: string;
|
||||
uploaded_at: number;
|
||||
}
|
||||
|
||||
interface PubkeyDomainRow {
|
||||
pubkey: string;
|
||||
domain: string;
|
||||
|
|
@ -54,3 +55,8 @@ interface EventZapRow {
|
|||
amount_millisats: number;
|
||||
comment: string;
|
||||
}
|
||||
|
||||
interface AuthorSearch {
|
||||
pubkey: string;
|
||||
search: string;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { Stickynotes } from '@soapbox/stickynotes';
|
||||
import { Logger } from 'kysely';
|
||||
import { dbQueryCounter, dbQueryTimeHistogram } from '@/metrics.ts';
|
||||
import { dbQueriesCounter, dbQueryDurationHistogram } from '@/metrics.ts';
|
||||
|
||||
/** Log the SQL for queries. */
|
||||
export const KyselyLogger: Logger = (event) => {
|
||||
|
|
@ -9,8 +9,8 @@ export const KyselyLogger: Logger = (event) => {
|
|||
const { query, queryDurationMillis } = event;
|
||||
const { sql, parameters } = query;
|
||||
|
||||
dbQueryCounter.inc();
|
||||
dbQueryTimeHistogram.observe(queryDurationMillis);
|
||||
dbQueriesCounter.inc();
|
||||
dbQueryDurationHistogram.observe(queryDurationMillis);
|
||||
|
||||
console.debug(
|
||||
sql,
|
||||
|
|
|
|||
28
src/db/adapters/DittoPglite.ts
Normal file
28
src/db/adapters/DittoPglite.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
import { PGlite } from '@electric-sql/pglite';
|
||||
import { pg_trgm } from '@electric-sql/pglite/contrib/pg_trgm';
|
||||
import { PgliteDialect } from '@soapbox/kysely-pglite';
|
||||
import { Kysely } from 'kysely';
|
||||
|
||||
import { DittoDatabase, DittoDatabaseOpts } from '@/db/DittoDatabase.ts';
|
||||
import { DittoTables } from '@/db/DittoTables.ts';
|
||||
import { KyselyLogger } from '@/db/KyselyLogger.ts';
|
||||
|
||||
export class DittoPglite {
|
||||
static create(databaseUrl: string, opts?: DittoDatabaseOpts): DittoDatabase {
|
||||
const kysely = new Kysely<DittoTables>({
|
||||
dialect: new PgliteDialect({
|
||||
database: new PGlite(databaseUrl, {
|
||||
extensions: { pg_trgm },
|
||||
debug: opts?.debug,
|
||||
}),
|
||||
}),
|
||||
log: KyselyLogger,
|
||||
});
|
||||
|
||||
return {
|
||||
kysely,
|
||||
poolSize: 1,
|
||||
availableConnections: 1,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +1,3 @@
|
|||
import { NPostgresSchema } from '@nostrify/db';
|
||||
import {
|
||||
BinaryOperationNode,
|
||||
FunctionNode,
|
||||
|
|
@ -13,51 +12,43 @@ import {
|
|||
import { PostgresJSDialectConfig, PostgresJSDriver } from 'kysely-postgres-js';
|
||||
import postgres from 'postgres';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
import { DittoDatabase, DittoDatabaseOpts } from '@/db/DittoDatabase.ts';
|
||||
import { DittoTables } from '@/db/DittoTables.ts';
|
||||
import { KyselyLogger } from '@/db/KyselyLogger.ts';
|
||||
|
||||
export class DittoPostgres {
|
||||
static db: Kysely<DittoTables> & Kysely<NPostgresSchema> | undefined;
|
||||
static postgres?: postgres.Sql;
|
||||
static create(databaseUrl: string, opts?: DittoDatabaseOpts): DittoDatabase {
|
||||
const pg = postgres(databaseUrl, { max: opts?.poolSize });
|
||||
|
||||
// deno-lint-ignore require-await
|
||||
static async getInstance(): Promise<Kysely<DittoTables> & Kysely<NPostgresSchema>> {
|
||||
if (!this.postgres) {
|
||||
this.postgres = postgres(Conf.databaseUrl, { max: Conf.pg.poolSize });
|
||||
}
|
||||
|
||||
if (!this.db) {
|
||||
this.db = new Kysely({
|
||||
dialect: {
|
||||
createAdapter() {
|
||||
return new PostgresAdapter();
|
||||
},
|
||||
createDriver() {
|
||||
return new PostgresJSDriver({
|
||||
postgres: DittoPostgres.postgres as unknown as PostgresJSDialectConfig['postgres'],
|
||||
});
|
||||
},
|
||||
createIntrospector(db) {
|
||||
return new PostgresIntrospector(db);
|
||||
},
|
||||
createQueryCompiler() {
|
||||
return new DittoPostgresQueryCompiler();
|
||||
},
|
||||
const kysely = new Kysely<DittoTables>({
|
||||
dialect: {
|
||||
createAdapter() {
|
||||
return new PostgresAdapter();
|
||||
},
|
||||
log: KyselyLogger,
|
||||
}) as Kysely<DittoTables> & Kysely<NPostgresSchema>;
|
||||
}
|
||||
createDriver() {
|
||||
return new PostgresJSDriver({
|
||||
postgres: pg as unknown as PostgresJSDialectConfig['postgres'],
|
||||
});
|
||||
},
|
||||
createIntrospector(db) {
|
||||
return new PostgresIntrospector(db);
|
||||
},
|
||||
createQueryCompiler() {
|
||||
return new DittoPostgresQueryCompiler();
|
||||
},
|
||||
},
|
||||
log: KyselyLogger,
|
||||
});
|
||||
|
||||
return this.db;
|
||||
}
|
||||
|
||||
static get poolSize() {
|
||||
return this.postgres?.connections.open ?? 0;
|
||||
}
|
||||
|
||||
static get availableConnections(): number {
|
||||
return this.postgres?.connections.idle ?? 0;
|
||||
return {
|
||||
kysely,
|
||||
get poolSize() {
|
||||
return pg.connections.open;
|
||||
},
|
||||
get availableConnections() {
|
||||
return pg.connections.idle;
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,59 +0,0 @@
|
|||
import { NDatabaseSchema } from '@nostrify/db';
|
||||
import { PolySqliteDialect } from '@soapbox/kysely-deno-sqlite';
|
||||
import { Kysely, sql } from 'kysely';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
import { DittoTables } from '@/db/DittoTables.ts';
|
||||
import { KyselyLogger } from '@/db/KyselyLogger.ts';
|
||||
import SqliteWorker from '@/workers/sqlite.ts';
|
||||
|
||||
export class DittoSQLite {
|
||||
static db: Kysely<DittoTables> & Kysely<NDatabaseSchema> | undefined;
|
||||
|
||||
static async getInstance(): Promise<Kysely<DittoTables> & Kysely<NDatabaseSchema>> {
|
||||
if (!this.db) {
|
||||
const sqliteWorker = new SqliteWorker();
|
||||
await sqliteWorker.open(this.path);
|
||||
|
||||
this.db = new Kysely({
|
||||
dialect: new PolySqliteDialect({
|
||||
database: sqliteWorker,
|
||||
}),
|
||||
log: KyselyLogger,
|
||||
}) as Kysely<DittoTables> & Kysely<NDatabaseSchema>;
|
||||
|
||||
// Set PRAGMA values.
|
||||
await Promise.all([
|
||||
sql`PRAGMA synchronous = normal`.execute(this.db),
|
||||
sql`PRAGMA temp_store = memory`.execute(this.db),
|
||||
sql`PRAGMA foreign_keys = ON`.execute(this.db),
|
||||
sql`PRAGMA auto_vacuum = FULL`.execute(this.db),
|
||||
sql`PRAGMA journal_mode = WAL`.execute(this.db),
|
||||
sql.raw(`PRAGMA mmap_size = ${Conf.sqlite.mmapSize}`).execute(this.db),
|
||||
]);
|
||||
}
|
||||
return this.db;
|
||||
}
|
||||
|
||||
/** Get the relative or absolute path based on the `DATABASE_URL`. */
|
||||
static get path() {
|
||||
if (Conf.databaseUrl === 'sqlite://:memory:') {
|
||||
return ':memory:';
|
||||
}
|
||||
|
||||
const { host, pathname } = Conf.db.url;
|
||||
|
||||
if (!pathname) return '';
|
||||
|
||||
// Get relative path.
|
||||
if (host === '') {
|
||||
return pathname;
|
||||
} else if (host === '.') {
|
||||
return pathname;
|
||||
} else if (host) {
|
||||
return host + pathname;
|
||||
}
|
||||
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
|
@ -1,13 +1,8 @@
|
|||
import { Kysely, sql } from 'kysely';
|
||||
import { Kysely } from 'kysely';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
if (Conf.db.dialect === 'sqlite') {
|
||||
await sql`CREATE VIRTUAL TABLE events_fts USING fts5(id, content)`.execute(db);
|
||||
}
|
||||
export async function up(_db: Kysely<any>): Promise<void> {
|
||||
// This migration used to create an FTS table for SQLite, but SQLite support was removed.
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await db.schema.dropTable('events_fts').ifExists().execute();
|
||||
export async function down(_db: Kysely<any>): Promise<void> {
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,25 +1,13 @@
|
|||
import { Kysely, sql } from 'kysely';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
import { Kysely } from 'kysely';
|
||||
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await db.schema.alterTable('events').renameTo('nostr_events').execute();
|
||||
await db.schema.alterTable('tags').renameTo('nostr_tags').execute();
|
||||
await db.schema.alterTable('nostr_tags').renameColumn('tag', 'name').execute();
|
||||
|
||||
if (Conf.db.dialect === 'sqlite') {
|
||||
await db.schema.dropTable('events_fts').execute();
|
||||
await sql`CREATE VIRTUAL TABLE nostr_fts5 USING fts5(event_id, content)`.execute(db);
|
||||
}
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await db.schema.alterTable('nostr_events').renameTo('events').execute();
|
||||
await db.schema.alterTable('nostr_tags').renameTo('tags').execute();
|
||||
await db.schema.alterTable('tags').renameColumn('name', 'tag').execute();
|
||||
|
||||
if (Conf.db.dialect === 'sqlite') {
|
||||
await db.schema.dropTable('nostr_fts5').execute();
|
||||
await sql`CREATE VIRTUAL TABLE events_fts USING fts5(id, content)`.execute(db);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,19 +1,13 @@
|
|||
import { Kysely, sql } from 'kysely';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
if (Conf.db.dialect === 'postgres') {
|
||||
await db.schema.createTable('nostr_pgfts')
|
||||
.ifNotExists()
|
||||
.addColumn('event_id', 'text', (c) => c.primaryKey().references('nostr_events.id').onDelete('cascade'))
|
||||
.addColumn('search_vec', sql`tsvector`, (c) => c.notNull())
|
||||
.execute();
|
||||
}
|
||||
await db.schema.createTable('nostr_pgfts')
|
||||
.ifNotExists()
|
||||
.addColumn('event_id', 'text', (c) => c.primaryKey().references('nostr_events.id').onDelete('cascade'))
|
||||
.addColumn('search_vec', sql`tsvector`, (c) => c.notNull())
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
if (Conf.db.dialect === 'postgres') {
|
||||
await db.schema.dropTable('nostr_pgfts').ifExists().execute();
|
||||
}
|
||||
await db.schema.dropTable('nostr_pgfts').ifExists().execute();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,21 +1,15 @@
|
|||
import { Kysely } from 'kysely';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
if (Conf.db.dialect === 'postgres') {
|
||||
await db.schema
|
||||
.createIndex('nostr_pgfts_gin_search_vec')
|
||||
.ifNotExists()
|
||||
.on('nostr_pgfts')
|
||||
.using('gin')
|
||||
.column('search_vec')
|
||||
.execute();
|
||||
}
|
||||
await db.schema
|
||||
.createIndex('nostr_pgfts_gin_search_vec')
|
||||
.ifNotExists()
|
||||
.on('nostr_pgfts')
|
||||
.using('gin')
|
||||
.column('search_vec')
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
if (Conf.db.dialect === 'postgres') {
|
||||
await db.schema.dropIndex('nostr_pgfts_gin_search_vec').ifExists().execute();
|
||||
}
|
||||
await db.schema.dropIndex('nostr_pgfts_gin_search_vec').ifExists().execute();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,6 @@
|
|||
import { Kysely, sql } from 'kysely';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
if (Conf.db.dialect !== 'postgres') return;
|
||||
|
||||
// Create new table and indexes.
|
||||
await db.schema
|
||||
.createTable('nostr_events_new')
|
||||
|
|
|
|||
34
src/db/migrations/031_rm_unattached_media.ts
Normal file
34
src/db/migrations/031_rm_unattached_media.ts
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
import { Kysely } from 'kysely';
|
||||
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await db.schema.dropTable('unattached_media').execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await db.schema
|
||||
.createTable('unattached_media')
|
||||
.addColumn('id', 'text', (c) => c.primaryKey())
|
||||
.addColumn('pubkey', 'text', (c) => c.notNull())
|
||||
.addColumn('url', 'text', (c) => c.notNull())
|
||||
.addColumn('data', 'text', (c) => c.notNull())
|
||||
.addColumn('uploaded_at', 'bigint', (c) => c.notNull())
|
||||
.execute();
|
||||
|
||||
await db.schema
|
||||
.createIndex('unattached_media_id')
|
||||
.on('unattached_media')
|
||||
.column('id')
|
||||
.execute();
|
||||
|
||||
await db.schema
|
||||
.createIndex('unattached_media_pubkey')
|
||||
.on('unattached_media')
|
||||
.column('pubkey')
|
||||
.execute();
|
||||
|
||||
await db.schema
|
||||
.createIndex('unattached_media_url')
|
||||
.on('unattached_media')
|
||||
.column('url')
|
||||
.execute();
|
||||
}
|
||||
18
src/db/migrations/032_add_author_search.ts
Normal file
18
src/db/migrations/032_add_author_search.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
import { Kysely, sql } from 'kysely';
|
||||
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await db.schema
|
||||
.createTable('author_search')
|
||||
.addColumn('pubkey', 'char(64)', (col) => col.primaryKey())
|
||||
.addColumn('search', 'text', (col) => col.notNull())
|
||||
.ifNotExists()
|
||||
.execute();
|
||||
|
||||
await sql`CREATE EXTENSION IF NOT EXISTS pg_trgm`.execute(db);
|
||||
await sql`CREATE INDEX author_search_search_idx ON author_search USING GIN (search gin_trgm_ops)`.execute(db);
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await db.schema.dropIndex('author_search_search_idx').ifExists().execute();
|
||||
await db.schema.dropTable('author_search').execute();
|
||||
}
|
||||
15
src/db/migrations/033_add_language.ts
Normal file
15
src/db/migrations/033_add_language.ts
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
import { Kysely } from 'kysely';
|
||||
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await db.schema.alterTable('nostr_events').addColumn('language', 'char(2)').execute();
|
||||
|
||||
await db.schema.createIndex('nostr_events_language_created_idx')
|
||||
.on('nostr_events')
|
||||
.columns(['language', 'created_at desc', 'id asc', 'kind'])
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await db.schema.alterTable('nostr_events').dropColumn('language').execute();
|
||||
await db.schema.dropIndex('nostr_events_language_created_idx').execute();
|
||||
}
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
import { Kysely } from 'kysely';
|
||||
|
||||
import { DittoDB } from '@/db/DittoDB.ts';
|
||||
import { DittoTables } from '@/db/DittoTables.ts';
|
||||
|
||||
interface UnattachedMedia {
|
||||
id: string;
|
||||
pubkey: string;
|
||||
url: string;
|
||||
/** NIP-94 tags. */
|
||||
data: string[][];
|
||||
uploaded_at: number;
|
||||
}
|
||||
|
||||
/** Add unattached media into the database. */
|
||||
async function insertUnattachedMedia(media: UnattachedMedia) {
|
||||
const { kysely } = await DittoDB.getInstance();
|
||||
await kysely.insertInto('unattached_media')
|
||||
.values({ ...media, data: JSON.stringify(media.data) })
|
||||
.execute();
|
||||
|
||||
return media;
|
||||
}
|
||||
|
||||
/** Select query for unattached media. */
|
||||
function selectUnattachedMediaQuery(kysely: Kysely<DittoTables>) {
|
||||
return kysely.selectFrom('unattached_media')
|
||||
.select([
|
||||
'unattached_media.id',
|
||||
'unattached_media.pubkey',
|
||||
'unattached_media.url',
|
||||
'unattached_media.data',
|
||||
'unattached_media.uploaded_at',
|
||||
]);
|
||||
}
|
||||
|
||||
/** Delete unattached media by URL. */
|
||||
async function deleteUnattachedMediaByUrl(url: string) {
|
||||
const { kysely } = await DittoDB.getInstance();
|
||||
return kysely.deleteFrom('unattached_media')
|
||||
.where('url', '=', url)
|
||||
.execute();
|
||||
}
|
||||
|
||||
/** Get unattached media by IDs. */
|
||||
async function getUnattachedMediaByIds(kysely: Kysely<DittoTables>, ids: string[]): Promise<UnattachedMedia[]> {
|
||||
if (!ids.length) return [];
|
||||
|
||||
const results = await selectUnattachedMediaQuery(kysely)
|
||||
.where('id', 'in', ids)
|
||||
.execute();
|
||||
|
||||
return results.map((row) => ({
|
||||
...row,
|
||||
data: JSON.parse(row.data),
|
||||
}));
|
||||
}
|
||||
|
||||
/** Delete rows as an event with media is being created. */
|
||||
async function deleteAttachedMedia(pubkey: string, urls: string[]): Promise<void> {
|
||||
if (!urls.length) return;
|
||||
const { kysely } = await DittoDB.getInstance();
|
||||
await kysely.deleteFrom('unattached_media')
|
||||
.where('pubkey', '=', pubkey)
|
||||
.where('url', 'in', urls)
|
||||
.execute();
|
||||
}
|
||||
|
||||
export {
|
||||
deleteAttachedMedia,
|
||||
deleteUnattachedMediaByUrl,
|
||||
getUnattachedMediaByIds,
|
||||
insertUnattachedMedia,
|
||||
type UnattachedMedia,
|
||||
};
|
||||
|
|
@ -2,7 +2,7 @@ import { Semaphore } from '@lambdalisue/async';
|
|||
import { Stickynotes } from '@soapbox/stickynotes';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
import { firehoseEventCounter } from '@/metrics.ts';
|
||||
import { firehoseEventsCounter } from '@/metrics.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
import { nostrNow } from '@/utils.ts';
|
||||
|
||||
|
|
@ -23,7 +23,7 @@ export async function startFirehose(): Promise<void> {
|
|||
if (msg[0] === 'EVENT') {
|
||||
const event = msg[2];
|
||||
console.debug(`NostrEvent<${event.kind}> ${event.id}`);
|
||||
firehoseEventCounter.inc({ kind: event.kind });
|
||||
firehoseEventsCounter.inc({ kind: event.kind });
|
||||
|
||||
sem.lock(async () => {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -1,86 +1,86 @@
|
|||
import { Counter, Gauge, Histogram } from 'prom-client';
|
||||
|
||||
export const httpRequestCounter = new Counter({
|
||||
name: 'http_requests_total',
|
||||
export const httpRequestsCounter = new Counter({
|
||||
name: 'ditto_http_requests_total',
|
||||
help: 'Total number of HTTP requests',
|
||||
labelNames: ['method'],
|
||||
});
|
||||
|
||||
export const httpResponseCounter = new Counter({
|
||||
name: 'http_responses_total',
|
||||
export const httpResponsesCounter = new Counter({
|
||||
name: 'ditto_http_responses_total',
|
||||
help: 'Total number of HTTP responses',
|
||||
labelNames: ['method', 'path', 'status'],
|
||||
});
|
||||
|
||||
export const streamingConnectionsGauge = new Gauge({
|
||||
name: 'streaming_connections',
|
||||
name: 'ditto_streaming_connections',
|
||||
help: 'Number of active connections to the streaming API',
|
||||
});
|
||||
|
||||
export const fetchCounter = new Counter({
|
||||
name: 'fetch_total',
|
||||
name: 'ditto_fetch_total',
|
||||
help: 'Total number of fetch requests',
|
||||
labelNames: ['method'],
|
||||
});
|
||||
|
||||
export const firehoseEventCounter = new Counter({
|
||||
name: 'firehose_events_total',
|
||||
export const firehoseEventsCounter = new Counter({
|
||||
name: 'ditto_firehose_events_total',
|
||||
help: 'Total number of Nostr events processed by the firehose',
|
||||
labelNames: ['kind'],
|
||||
});
|
||||
|
||||
export const pipelineEventCounter = new Counter({
|
||||
name: 'pipeline_events_total',
|
||||
export const pipelineEventsCounter = new Counter({
|
||||
name: 'ditto_pipeline_events_total',
|
||||
help: 'Total number of Nostr events processed by the pipeline',
|
||||
labelNames: ['kind'],
|
||||
});
|
||||
|
||||
export const policyEventCounter = new Counter({
|
||||
name: 'policy_events_total',
|
||||
export const policyEventsCounter = new Counter({
|
||||
name: 'ditto_policy_events_total',
|
||||
help: 'Total number of policy OK responses',
|
||||
labelNames: ['ok'],
|
||||
});
|
||||
|
||||
export const relayEventCounter = new Counter({
|
||||
name: 'relay_events_total',
|
||||
export const relayEventsCounter = new Counter({
|
||||
name: 'ditto_relay_events_total',
|
||||
help: 'Total number of EVENT messages processed by the relay',
|
||||
labelNames: ['kind'],
|
||||
});
|
||||
|
||||
export const relayMessageCounter = new Counter({
|
||||
name: 'relay_messages_total',
|
||||
export const relayMessagesCounter = new Counter({
|
||||
name: 'ditto_relay_messages_total',
|
||||
help: 'Total number of Nostr messages processed by the relay',
|
||||
labelNames: ['verb'],
|
||||
});
|
||||
|
||||
export const relayConnectionsGauge = new Gauge({
|
||||
name: 'relay_connections',
|
||||
name: 'ditto_relay_connections',
|
||||
help: 'Number of active connections to the relay',
|
||||
});
|
||||
|
||||
export const dbQueryCounter = new Counter({
|
||||
name: 'db_query_total',
|
||||
export const dbQueriesCounter = new Counter({
|
||||
name: 'ditto_db_queries_total',
|
||||
help: 'Total number of database queries',
|
||||
labelNames: ['kind'],
|
||||
});
|
||||
|
||||
export const dbEventCounter = new Counter({
|
||||
name: 'db_events_total',
|
||||
export const dbEventsCounter = new Counter({
|
||||
name: 'ditto_db_events_total',
|
||||
help: 'Total number of database inserts',
|
||||
labelNames: ['kind'],
|
||||
});
|
||||
|
||||
export const dbPoolSizeGauge = new Gauge({
|
||||
name: 'db_pool_size',
|
||||
name: 'ditto_db_pool_size',
|
||||
help: 'Number of connections in the database pool',
|
||||
});
|
||||
|
||||
export const dbAvailableConnectionsGauge = new Gauge({
|
||||
name: 'db_available_connections',
|
||||
name: 'ditto_db_available_connections',
|
||||
help: 'Number of available connections in the database pool',
|
||||
});
|
||||
|
||||
export const dbQueryTimeHistogram = new Histogram({
|
||||
name: 'db_query_duration_ms',
|
||||
export const dbQueryDurationHistogram = new Histogram({
|
||||
name: 'ditto_db_query_duration_ms',
|
||||
help: 'Duration of database queries',
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
import { MiddlewareHandler } from '@hono/hono';
|
||||
|
||||
import { httpRequestCounter, httpResponseCounter } from '@/metrics.ts';
|
||||
import { httpRequestsCounter, httpResponsesCounter } from '@/metrics.ts';
|
||||
|
||||
/** Prometheus metrics middleware that tracks HTTP requests by methods and responses by status code. */
|
||||
export const metricsMiddleware: MiddlewareHandler = async (c, next) => {
|
||||
// HTTP Request.
|
||||
const { method } = c.req;
|
||||
httpRequestCounter.inc({ method });
|
||||
httpRequestsCounter.inc({ method });
|
||||
|
||||
// Wait for other handlers to run.
|
||||
await next();
|
||||
|
|
@ -16,5 +16,5 @@ export const metricsMiddleware: MiddlewareHandler = async (c, next) => {
|
|||
// Get a parameterized path name like `/posts/:id` instead of `/posts/1234`.
|
||||
// Tries to find actual route names first before falling back on potential middleware handlers like `app.use('*')`.
|
||||
const path = c.req.matchedRoutes.find((r) => r.method !== 'ALL')?.path ?? c.req.routePath;
|
||||
httpResponseCounter.inc({ method, status, path });
|
||||
httpResponsesCounter.inc({ method, status, path });
|
||||
};
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import { nip19 } from 'nostr-tools';
|
|||
import { AppMiddleware } from '@/app.ts';
|
||||
import { ConnectSigner } from '@/signers/ConnectSigner.ts';
|
||||
import { ReadOnlySigner } from '@/signers/ReadOnlySigner.ts';
|
||||
import { DittoDB } from '@/db/DittoDB.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
|
||||
/** We only accept "Bearer" type. */
|
||||
const BEARER_REGEX = new RegExp(`^Bearer (${nip19.BECH32_REGEX.source})$`);
|
||||
|
|
@ -20,7 +20,7 @@ export const signerMiddleware: AppMiddleware = async (c, next) => {
|
|||
|
||||
if (bech32.startsWith('token1')) {
|
||||
try {
|
||||
const { kysely } = await DittoDB.getInstance();
|
||||
const kysely = await Storages.kysely();
|
||||
|
||||
const { user_pubkey, server_seckey, relays } = await kysely
|
||||
.selectFrom('nip46_tokens')
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { assertEquals } from '@std/assert';
|
||||
import { generateSecretKey } from 'nostr-tools';
|
||||
|
||||
import { createTestDB, genEvent, getTestDB } from '@/test.ts';
|
||||
import { createTestDB, genEvent } from '@/test.ts';
|
||||
import { handleZaps } from '@/pipeline.ts';
|
||||
|
||||
Deno.test('store one zap receipt in nostr_events; convert it into event_zaps table format and store it', async () => {
|
||||
|
|
@ -58,7 +58,7 @@ Deno.test('store one zap receipt in nostr_events; convert it into event_zaps tab
|
|||
// If no error happens = ok
|
||||
|
||||
Deno.test('zap receipt does not have a "description" tag', async () => {
|
||||
await using db = await getTestDB();
|
||||
await using db = await createTestDB();
|
||||
const kysely = db.kysely;
|
||||
|
||||
const sk = generateSecretKey();
|
||||
|
|
@ -71,7 +71,7 @@ Deno.test('zap receipt does not have a "description" tag', async () => {
|
|||
});
|
||||
|
||||
Deno.test('zap receipt does not have a zap request stringified value in the "description" tag', async () => {
|
||||
await using db = await getTestDB();
|
||||
await using db = await createTestDB();
|
||||
const kysely = db.kysely;
|
||||
|
||||
const sk = generateSecretKey();
|
||||
|
|
@ -84,7 +84,7 @@ Deno.test('zap receipt does not have a zap request stringified value in the "des
|
|||
});
|
||||
|
||||
Deno.test('zap receipt does not have a "bolt11" tag', async () => {
|
||||
await using db = await getTestDB();
|
||||
await using db = await createTestDB();
|
||||
const kysely = db.kysely;
|
||||
|
||||
const sk = generateSecretKey();
|
||||
|
|
@ -103,7 +103,7 @@ Deno.test('zap receipt does not have a "bolt11" tag', async () => {
|
|||
});
|
||||
|
||||
Deno.test('zap request inside zap receipt does not have an "e" tag', async () => {
|
||||
await using db = await getTestDB();
|
||||
await using db = await createTestDB();
|
||||
const kysely = db.kysely;
|
||||
|
||||
const sk = generateSecretKey();
|
||||
|
|
|
|||
107
src/pipeline.ts
107
src/pipeline.ts
|
|
@ -1,14 +1,15 @@
|
|||
import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify';
|
||||
import Debug from '@soapbox/stickynotes/debug';
|
||||
import ISO6391 from 'iso-639-1';
|
||||
import { Kysely, sql } from 'kysely';
|
||||
import lande from 'lande';
|
||||
import { LRUCache } from 'lru-cache';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
import { DittoDB } from '@/db/DittoDB.ts';
|
||||
import { deleteAttachedMedia } from '@/db/unattached-media.ts';
|
||||
import { DittoTables } from '@/db/DittoTables.ts';
|
||||
import { DittoEvent } from '@/interfaces/DittoEvent.ts';
|
||||
import { pipelineEventCounter, policyEventCounter } from '@/metrics.ts';
|
||||
import { pipelineEventsCounter, policyEventsCounter } from '@/metrics.ts';
|
||||
import { RelayError } from '@/RelayError.ts';
|
||||
import { AdminSigner } from '@/signers/AdminSigner.ts';
|
||||
import { hydrateEvents } from '@/storages/hydrate.ts';
|
||||
|
|
@ -16,11 +17,11 @@ import { Storages } from '@/storages.ts';
|
|||
import { eventAge, parseNip05, Time } from '@/utils.ts';
|
||||
import { policyWorker } from '@/workers/policy.ts';
|
||||
import { verifyEventWorker } from '@/workers/verify.ts';
|
||||
import { getAmount } from '@/utils/bolt11.ts';
|
||||
import { nip05Cache } from '@/utils/nip05.ts';
|
||||
import { purifyEvent } from '@/utils/purify.ts';
|
||||
import { updateStats } from '@/utils/stats.ts';
|
||||
import { getTagSet } from '@/utils/tags.ts';
|
||||
import { getAmount } from '@/utils/bolt11.ts';
|
||||
import { DittoTables } from '@/db/DittoTables.ts';
|
||||
|
||||
const debug = Debug('ditto:pipeline');
|
||||
|
||||
|
|
@ -40,7 +41,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise<void
|
|||
if (encounterEvent(event)) return;
|
||||
if (await existsInDB(event)) return;
|
||||
debug(`NostrEvent<${event.kind}> ${event.id}`);
|
||||
pipelineEventCounter.inc({ kind: event.kind });
|
||||
pipelineEventsCounter.inc({ kind: event.kind });
|
||||
|
||||
if (event.kind !== 24133) {
|
||||
await policyFilter(event);
|
||||
|
|
@ -54,14 +55,14 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise<void
|
|||
throw new RelayError('blocked', 'user is disabled');
|
||||
}
|
||||
|
||||
const { kysely } = await DittoDB.getInstance();
|
||||
const kysely = await Storages.kysely();
|
||||
|
||||
await storeEvent(purifyEvent(event), signal);
|
||||
await Promise.all([
|
||||
storeEvent(event, signal),
|
||||
handleZaps(kysely, event),
|
||||
parseMetadata(event, signal),
|
||||
setLanguage(event),
|
||||
generateSetEvents(event),
|
||||
processMedia(event),
|
||||
streamOut(event),
|
||||
]);
|
||||
}
|
||||
|
|
@ -71,7 +72,7 @@ async function policyFilter(event: NostrEvent): Promise<void> {
|
|||
|
||||
try {
|
||||
const result = await policyWorker.call(event);
|
||||
policyEventCounter.inc({ ok: String(result[2]) });
|
||||
policyEventsCounter.inc({ ok: String(result[2]) });
|
||||
debug(JSON.stringify(result));
|
||||
RelayError.assert(result);
|
||||
} catch (e) {
|
||||
|
|
@ -106,7 +107,7 @@ async function existsInDB(event: DittoEvent): Promise<boolean> {
|
|||
async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise<void> {
|
||||
await hydrateEvents({ events: [event], store: await Storages.db(), signal });
|
||||
|
||||
const { kysely } = await DittoDB.getInstance();
|
||||
const kysely = await Storages.kysely();
|
||||
const domain = await kysely
|
||||
.selectFrom('pubkey_domains')
|
||||
.select('domain')
|
||||
|
|
@ -120,10 +121,11 @@ async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise<voi
|
|||
async function storeEvent(event: DittoEvent, signal?: AbortSignal): Promise<undefined> {
|
||||
if (NKinds.ephemeral(event.kind)) return;
|
||||
const store = await Storages.db();
|
||||
const { kysely } = await DittoDB.getInstance();
|
||||
|
||||
await updateStats({ event, store, kysely }).catch(debug);
|
||||
await store.event(event, { signal });
|
||||
await store.transaction(async (store, kysely) => {
|
||||
await updateStats({ event, store, kysely });
|
||||
await store.event(event, { signal });
|
||||
});
|
||||
}
|
||||
|
||||
/** Parse kind 0 metadata and track indexes in the database. */
|
||||
|
|
@ -134,41 +136,64 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise<vo
|
|||
const metadata = n.json().pipe(n.metadata()).catch({}).safeParse(event.content);
|
||||
if (!metadata.success) return;
|
||||
|
||||
const kysely = await Storages.kysely();
|
||||
|
||||
// Get nip05.
|
||||
const { nip05 } = metadata.data;
|
||||
if (!nip05) return;
|
||||
const { name, nip05 } = metadata.data;
|
||||
const result = nip05 ? await nip05Cache.fetch(nip05, { signal }).catch(() => undefined) : undefined;
|
||||
|
||||
// Fetch nip05.
|
||||
const result = await nip05Cache.fetch(nip05, { signal }).catch(() => undefined);
|
||||
if (!result) return;
|
||||
|
||||
// Ensure pubkey matches event.
|
||||
const { pubkey } = result;
|
||||
if (pubkey !== event.pubkey) return;
|
||||
|
||||
// Track pubkey domain.
|
||||
// Populate author_search.
|
||||
try {
|
||||
const { kysely } = await DittoDB.getInstance();
|
||||
const { domain } = parseNip05(nip05);
|
||||
const search = result?.pubkey === event.pubkey ? [name, nip05].filter(Boolean).join(' ').trim() : name ?? '';
|
||||
|
||||
await sql`
|
||||
INSERT INTO pubkey_domains (pubkey, domain, last_updated_at)
|
||||
VALUES (${pubkey}, ${domain}, ${event.created_at})
|
||||
ON CONFLICT(pubkey) DO UPDATE SET
|
||||
domain = excluded.domain,
|
||||
last_updated_at = excluded.last_updated_at
|
||||
WHERE excluded.last_updated_at > pubkey_domains.last_updated_at
|
||||
`.execute(kysely);
|
||||
} catch (_e) {
|
||||
if (search) {
|
||||
await kysely.insertInto('author_search')
|
||||
.values({ pubkey: event.pubkey, search })
|
||||
.onConflict((oc) => oc.column('pubkey').doUpdateSet({ search }))
|
||||
.execute();
|
||||
}
|
||||
} catch {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
if (nip05 && result && result.pubkey === event.pubkey) {
|
||||
// Track pubkey domain.
|
||||
try {
|
||||
const { domain } = parseNip05(nip05);
|
||||
|
||||
await sql`
|
||||
INSERT INTO pubkey_domains (pubkey, domain, last_updated_at)
|
||||
VALUES (${event.pubkey}, ${domain}, ${event.created_at})
|
||||
ON CONFLICT(pubkey) DO UPDATE SET
|
||||
domain = excluded.domain,
|
||||
last_updated_at = excluded.last_updated_at
|
||||
WHERE excluded.last_updated_at > pubkey_domains.last_updated_at
|
||||
`.execute(kysely);
|
||||
} catch (_e) {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Delete unattached media entries that are attached to the event. */
|
||||
function processMedia({ tags, pubkey, user }: DittoEvent) {
|
||||
if (user) {
|
||||
const urls = getTagSet(tags, 'media');
|
||||
return deleteAttachedMedia(pubkey, [...urls]);
|
||||
/** Update the event in the database and set its language. */
|
||||
async function setLanguage(event: NostrEvent): Promise<void> {
|
||||
const [topResult] = lande(event.content);
|
||||
|
||||
if (topResult) {
|
||||
const [iso6393, confidence] = topResult;
|
||||
const locale = new Intl.Locale(iso6393);
|
||||
|
||||
if (confidence >= 0.95 && ISO6391.validate(locale.language)) {
|
||||
const kysely = await Storages.kysely();
|
||||
try {
|
||||
await kysely.updateTable('nostr_events')
|
||||
.set('language', locale.language)
|
||||
.where('id', '=', event.id)
|
||||
.execute();
|
||||
} catch {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ async function getAncestors(store: NStore, event: NostrEvent, result: NostrEvent
|
|||
const inReplyTo = replyTag ? replyTag[1] : undefined;
|
||||
|
||||
if (inReplyTo) {
|
||||
const [parentEvent] = await store.query([{ kinds: [1], ids: [inReplyTo], until: event.created_at, limit: 1 }]);
|
||||
const [parentEvent] = await store.query([{ ids: [inReplyTo], until: event.created_at, limit: 1 }]);
|
||||
|
||||
if (parentEvent) {
|
||||
result.push(parentEvent);
|
||||
|
|
|
|||
|
|
@ -2,7 +2,10 @@ import { z } from 'zod';
|
|||
|
||||
/** Schema to parse pagination query params. */
|
||||
export const paginationSchema = z.object({
|
||||
max_id: z.string().optional().catch(undefined),
|
||||
max_id: z.string().transform((val) => {
|
||||
if (!val.includes('-')) return val;
|
||||
return val.split('-')[1];
|
||||
}).optional().catch(undefined),
|
||||
min_id: z.string().optional().catch(undefined),
|
||||
since: z.coerce.number().nonnegative().optional().catch(undefined),
|
||||
until: z.coerce.number().nonnegative().optional().catch(undefined),
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
// deno-lint-ignore-file require-await
|
||||
import { Conf } from '@/config.ts';
|
||||
import { DittoDatabase } from '@/db/DittoDatabase.ts';
|
||||
import { DittoDB } from '@/db/DittoDB.ts';
|
||||
import { AdminStore } from '@/storages/AdminStore.ts';
|
||||
import { EventsDB } from '@/storages/EventsDB.ts';
|
||||
|
|
@ -11,17 +12,37 @@ import { seedZapSplits } from '@/utils/zap-split.ts';
|
|||
|
||||
export class Storages {
|
||||
private static _db: Promise<EventsDB> | undefined;
|
||||
private static _database: Promise<DittoDatabase> | undefined;
|
||||
private static _admin: Promise<AdminStore> | undefined;
|
||||
private static _client: Promise<NPool> | undefined;
|
||||
private static _pubsub: Promise<InternalRelay> | undefined;
|
||||
private static _search: Promise<SearchStore> | undefined;
|
||||
|
||||
/** SQLite database to store events this Ditto server cares about. */
|
||||
public static async database(): Promise<DittoDatabase> {
|
||||
if (!this._database) {
|
||||
this._database = (async () => {
|
||||
const db = DittoDB.create(Conf.databaseUrl, {
|
||||
poolSize: Conf.pg.poolSize,
|
||||
debug: Conf.pgliteDebug,
|
||||
});
|
||||
await DittoDB.migrate(db.kysely);
|
||||
return db;
|
||||
})();
|
||||
}
|
||||
return this._database;
|
||||
}
|
||||
|
||||
public static async kysely(): Promise<DittoDatabase['kysely']> {
|
||||
const { kysely } = await this.database();
|
||||
return kysely;
|
||||
}
|
||||
|
||||
/** SQL database to store events this Ditto server cares about. */
|
||||
public static async db(): Promise<EventsDB> {
|
||||
if (!this._db) {
|
||||
this._db = (async () => {
|
||||
const db = await DittoDB.getInstance();
|
||||
const store = new EventsDB(db);
|
||||
const kysely = await this.kysely();
|
||||
const store = new EventsDB({ kysely, pubkey: Conf.pubkey, timeout: Conf.db.timeouts.default });
|
||||
await seedZapSplits(store);
|
||||
return store;
|
||||
})();
|
||||
|
|
|
|||
|
|
@ -54,6 +54,23 @@ Deno.test('query events with domain search filter', async () => {
|
|||
assertEquals(await store.query([{ kinds: [1], search: 'domain:example.com' }]), []);
|
||||
});
|
||||
|
||||
Deno.test('query events with language search filter', async () => {
|
||||
await using db = await createTestDB();
|
||||
const { store, kysely } = db;
|
||||
|
||||
const en = genEvent({ kind: 1, content: 'hello world!' });
|
||||
const es = genEvent({ kind: 1, content: 'hola mundo!' });
|
||||
|
||||
await store.event(en);
|
||||
await store.event(es);
|
||||
|
||||
await kysely.updateTable('nostr_events').set('language', 'en').where('id', '=', en.id).execute();
|
||||
await kysely.updateTable('nostr_events').set('language', 'es').where('id', '=', es.id).execute();
|
||||
|
||||
assertEquals(await store.query([{ search: 'language:en' }]), [en]);
|
||||
assertEquals(await store.query([{ search: 'language:es' }]), [es]);
|
||||
});
|
||||
|
||||
Deno.test('delete events', async () => {
|
||||
await using db = await createTestDB();
|
||||
const { store } = db;
|
||||
|
|
|
|||
|
|
@ -1,27 +1,17 @@
|
|||
// deno-lint-ignore-file require-await
|
||||
|
||||
import { NDatabase, NPostgres } from '@nostrify/db';
|
||||
import {
|
||||
NIP50,
|
||||
NKinds,
|
||||
NostrEvent,
|
||||
NostrFilter,
|
||||
NostrRelayCLOSED,
|
||||
NostrRelayEOSE,
|
||||
NostrRelayEVENT,
|
||||
NSchema as n,
|
||||
NStore,
|
||||
} from '@nostrify/nostrify';
|
||||
import { NPostgres, NPostgresSchema } from '@nostrify/db';
|
||||
import { NIP50, NKinds, NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify';
|
||||
import { Stickynotes } from '@soapbox/stickynotes';
|
||||
import { Kysely, SelectQueryBuilder } from 'kysely';
|
||||
import { nip27 } from 'nostr-tools';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
import { DittoDatabase } from '@/db/DittoDB.ts';
|
||||
import { dbEventCounter } from '@/metrics.ts';
|
||||
import { DittoTables } from '@/db/DittoTables.ts';
|
||||
import { dbEventsCounter } from '@/metrics.ts';
|
||||
import { RelayError } from '@/RelayError.ts';
|
||||
import { purifyEvent } from '@/storages/hydrate.ts';
|
||||
import { isNostrId, isURL } from '@/utils.ts';
|
||||
import { abortError } from '@/utils/abort.ts';
|
||||
import { purifyEvent } from '@/utils/purify.ts';
|
||||
|
||||
/** Function to decide whether or not to index a tag. */
|
||||
type TagCondition = ({ event, count, value }: {
|
||||
|
|
@ -30,9 +20,18 @@ type TagCondition = ({ event, count, value }: {
|
|||
value: string;
|
||||
}) => boolean;
|
||||
|
||||
/** SQLite database storage adapter for Nostr events. */
|
||||
class EventsDB implements NStore {
|
||||
private store: NDatabase | NPostgres;
|
||||
/** Options for the EventsDB store. */
|
||||
interface EventsDBOpts {
|
||||
/** Kysely instance to use. */
|
||||
kysely: Kysely<DittoTables>;
|
||||
/** Pubkey of the admin account. */
|
||||
pubkey: string;
|
||||
/** Timeout in milliseconds for database queries. */
|
||||
timeout: number;
|
||||
}
|
||||
|
||||
/** SQL database storage adapter for Nostr events. */
|
||||
class EventsDB extends NPostgres {
|
||||
private console = new Stickynotes('ditto:db:events');
|
||||
|
||||
/** Conditions for when to index certain tags. */
|
||||
|
|
@ -52,28 +51,18 @@ class EventsDB implements NStore {
|
|||
't': ({ event, count, value }) => (event.kind === 1985 ? count < 20 : count < 5) && value.length < 50,
|
||||
};
|
||||
|
||||
constructor(private database: DittoDatabase) {
|
||||
const { dialect, kysely } = database;
|
||||
|
||||
if (dialect === 'postgres') {
|
||||
this.store = new NPostgres(kysely, {
|
||||
indexTags: EventsDB.indexTags,
|
||||
indexSearch: EventsDB.searchText,
|
||||
});
|
||||
} else {
|
||||
this.store = new NDatabase(kysely, {
|
||||
fts: 'sqlite',
|
||||
indexTags: EventsDB.indexTags,
|
||||
searchText: EventsDB.searchText,
|
||||
});
|
||||
}
|
||||
constructor(private opts: EventsDBOpts) {
|
||||
super(opts.kysely, {
|
||||
indexTags: EventsDB.indexTags,
|
||||
indexSearch: EventsDB.searchText,
|
||||
});
|
||||
}
|
||||
|
||||
/** Insert an event (and its tags) into the database. */
|
||||
async event(event: NostrEvent, opts: { signal?: AbortSignal; timeout?: number } = {}): Promise<void> {
|
||||
event = purifyEvent(event);
|
||||
this.console.debug('EVENT', JSON.stringify(event));
|
||||
dbEventCounter.inc({ kind: event.kind });
|
||||
dbEventsCounter.inc({ kind: event.kind });
|
||||
|
||||
if (await this.isDeletedAdmin(event)) {
|
||||
throw new RelayError('blocked', 'event deleted by admin');
|
||||
|
|
@ -82,7 +71,7 @@ class EventsDB implements NStore {
|
|||
await this.deleteEventsAdmin(event);
|
||||
|
||||
try {
|
||||
await this.store.event(event, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default });
|
||||
await super.event(event, { ...opts, timeout: opts.timeout ?? this.opts.timeout });
|
||||
} catch (e) {
|
||||
if (e.message === 'Cannot add a deleted event') {
|
||||
throw new RelayError('blocked', 'event deleted by user');
|
||||
|
|
@ -97,7 +86,7 @@ class EventsDB implements NStore {
|
|||
/** Check if an event has been deleted by the admin. */
|
||||
private async isDeletedAdmin(event: NostrEvent): Promise<boolean> {
|
||||
const filters: NostrFilter[] = [
|
||||
{ kinds: [5], authors: [Conf.pubkey], '#e': [event.id], limit: 1 },
|
||||
{ kinds: [5], authors: [this.opts.pubkey], '#e': [event.id], limit: 1 },
|
||||
];
|
||||
|
||||
if (NKinds.replaceable(event.kind) || NKinds.parameterizedReplaceable(event.kind)) {
|
||||
|
|
@ -105,7 +94,7 @@ class EventsDB implements NStore {
|
|||
|
||||
filters.push({
|
||||
kinds: [5],
|
||||
authors: [Conf.pubkey],
|
||||
authors: [this.opts.pubkey],
|
||||
'#a': [`${event.kind}:${event.pubkey}:${d}`],
|
||||
since: event.created_at,
|
||||
limit: 1,
|
||||
|
|
@ -118,7 +107,7 @@ class EventsDB implements NStore {
|
|||
|
||||
/** The DITTO_NSEC can delete any event from the database. NDatabase already handles user deletions. */
|
||||
private async deleteEventsAdmin(event: NostrEvent): Promise<void> {
|
||||
if (event.kind === 5 && event.pubkey === Conf.pubkey) {
|
||||
if (event.kind === 5 && event.pubkey === this.opts.pubkey) {
|
||||
const ids = new Set(event.tags.filter(([name]) => name === 'e').map(([_name, value]) => value));
|
||||
const addrs = new Set(event.tags.filter(([name]) => name === 'a').map(([_name, value]) => value));
|
||||
|
||||
|
|
@ -155,12 +144,37 @@ class EventsDB implements NStore {
|
|||
}
|
||||
}
|
||||
|
||||
/** Stream events from the database. */
|
||||
req(
|
||||
filters: NostrFilter[],
|
||||
opts: { signal?: AbortSignal } = {},
|
||||
): AsyncIterable<NostrRelayEVENT | NostrRelayEOSE | NostrRelayCLOSED> {
|
||||
return this.store.req(filters, opts);
|
||||
protected getFilterQuery(trx: Kysely<NPostgresSchema>, filter: NostrFilter) {
|
||||
if (filter.search) {
|
||||
const tokens = NIP50.parseInput(filter.search);
|
||||
|
||||
let query = super.getFilterQuery(trx, {
|
||||
...filter,
|
||||
search: tokens.filter((t) => typeof t === 'string').join(' '),
|
||||
}) as SelectQueryBuilder<DittoTables, 'nostr_events', Pick<DittoTables['nostr_events'], keyof NostrEvent>>;
|
||||
|
||||
const data = tokens.filter((t) => typeof t === 'object').reduce(
|
||||
(acc, t) => acc.set(t.key, t.value),
|
||||
new Map<string, string>(),
|
||||
);
|
||||
|
||||
const domain = data.get('domain');
|
||||
const language = data.get('language');
|
||||
|
||||
if (domain) {
|
||||
query = query
|
||||
.innerJoin('pubkey_domains', 'nostr_events.pubkey', 'pubkey_domains.pubkey')
|
||||
.where('pubkey_domains.domain', '=', domain);
|
||||
}
|
||||
|
||||
if (language) {
|
||||
query = query.where('language', '=', language);
|
||||
}
|
||||
|
||||
return query;
|
||||
}
|
||||
|
||||
return super.getFilterQuery(trx, filter);
|
||||
}
|
||||
|
||||
/** Get events for filters from the database. */
|
||||
|
|
@ -185,32 +199,28 @@ class EventsDB implements NStore {
|
|||
}
|
||||
|
||||
if (opts.signal?.aborted) return Promise.resolve([]);
|
||||
if (!filters.length) return Promise.resolve([]);
|
||||
|
||||
this.console.debug('REQ', JSON.stringify(filters));
|
||||
|
||||
return this.store.query(filters, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default });
|
||||
return super.query(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout });
|
||||
}
|
||||
|
||||
/** Delete events based on filters from the database. */
|
||||
async remove(filters: NostrFilter[], opts: { signal?: AbortSignal; timeout?: number } = {}): Promise<void> {
|
||||
if (!filters.length) return Promise.resolve();
|
||||
this.console.debug('DELETE', JSON.stringify(filters));
|
||||
|
||||
return this.store.remove(filters, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default });
|
||||
return super.remove(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout });
|
||||
}
|
||||
|
||||
/** Get number of events that would be returned by filters. */
|
||||
async count(
|
||||
filters: NostrFilter[],
|
||||
opts: { signal?: AbortSignal; timeout?: number } = {},
|
||||
): Promise<{ count: number; approximate: boolean }> {
|
||||
): Promise<{ count: number; approximate: any }> {
|
||||
if (opts.signal?.aborted) return Promise.reject(abortError());
|
||||
if (!filters.length) return Promise.resolve({ count: 0, approximate: false });
|
||||
|
||||
this.console.debug('COUNT', JSON.stringify(filters));
|
||||
|
||||
return this.store.count(filters, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default });
|
||||
return super.count(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout });
|
||||
}
|
||||
|
||||
/** Return only the tags that should be indexed. */
|
||||
|
|
@ -273,40 +283,11 @@ class EventsDB implements NStore {
|
|||
return tags.map(([_tag, value]) => value).join('\n');
|
||||
}
|
||||
|
||||
/** Converts filters to more performant, simpler filters that are better for SQLite. */
|
||||
/** Converts filters to more performant, simpler filters. */
|
||||
async expandFilters(filters: NostrFilter[]): Promise<NostrFilter[]> {
|
||||
filters = structuredClone(filters);
|
||||
|
||||
for (const filter of filters) {
|
||||
if (filter.search) {
|
||||
const tokens = NIP50.parseInput(filter.search);
|
||||
|
||||
const domain = (tokens.find((t) =>
|
||||
typeof t === 'object' && t.key === 'domain'
|
||||
) as { key: 'domain'; value: string } | undefined)?.value;
|
||||
|
||||
if (domain) {
|
||||
const query = this.database.kysely
|
||||
.selectFrom('pubkey_domains')
|
||||
.select('pubkey')
|
||||
.where('domain', '=', domain);
|
||||
|
||||
if (filter.authors) {
|
||||
query.where('pubkey', 'in', filter.authors);
|
||||
}
|
||||
|
||||
const pubkeys = await query
|
||||
.execute()
|
||||
.then((rows) =>
|
||||
rows.map((row) => row.pubkey)
|
||||
);
|
||||
|
||||
filter.authors = pubkeys;
|
||||
}
|
||||
|
||||
filter.search = tokens.filter((t) => typeof t === 'string').join(' ');
|
||||
}
|
||||
|
||||
if (filter.kinds) {
|
||||
// Ephemeral events are not stored, so don't bother querying for them.
|
||||
// If this results in an empty kinds array, NDatabase will remove the filter before querying and return no results.
|
||||
|
|
@ -316,6 +297,10 @@ class EventsDB implements NStore {
|
|||
|
||||
return filters;
|
||||
}
|
||||
|
||||
async transaction(callback: (store: NPostgres, kysely: Kysely<any>) => Promise<void>): Promise<void> {
|
||||
return super.transaction((store, kysely) => callback(store, kysely as unknown as Kysely<DittoTables>));
|
||||
}
|
||||
}
|
||||
|
||||
export { EventsDB };
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import { Machina } from '@nostrify/nostrify/utils';
|
|||
import { matchFilter } from 'nostr-tools';
|
||||
|
||||
import { DittoEvent } from '@/interfaces/DittoEvent.ts';
|
||||
import { purifyEvent } from '@/storages/hydrate.ts';
|
||||
import { purifyEvent } from '@/utils/purify.ts';
|
||||
|
||||
/**
|
||||
* PubSub event store for streaming events within the application.
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
import { NostrEvent, NStore } from '@nostrify/nostrify';
|
||||
import { NStore } from '@nostrify/nostrify';
|
||||
import { Kysely } from 'kysely';
|
||||
import { matchFilter } from 'nostr-tools';
|
||||
|
||||
import { DittoDB } from '@/db/DittoDB.ts';
|
||||
import { DittoTables } from '@/db/DittoTables.ts';
|
||||
import { Conf } from '@/config.ts';
|
||||
import { type DittoEvent } from '@/interfaces/DittoEvent.ts';
|
||||
import { findQuoteTag } from '@/utils/tags.ts';
|
||||
import { findQuoteInContent } from '@/utils/note.ts';
|
||||
import { Kysely } from 'kysely';
|
||||
import { Storages } from '@/storages.ts';
|
||||
|
||||
interface HydrateOpts {
|
||||
events: DittoEvent[];
|
||||
|
|
@ -18,7 +18,7 @@ interface HydrateOpts {
|
|||
|
||||
/** Hydrate events using the provided storage. */
|
||||
async function hydrateEvents(opts: HydrateOpts): Promise<DittoEvent[]> {
|
||||
const { events, store, signal, kysely = (await DittoDB.getInstance()).kysely } = opts;
|
||||
const { events, store, signal, kysely = await Storages.kysely() } = opts;
|
||||
|
||||
if (!events.length) {
|
||||
return events;
|
||||
|
|
@ -338,17 +338,4 @@ async function gatherEventStats(
|
|||
}));
|
||||
}
|
||||
|
||||
/** Return a normalized event without any non-standard keys. */
|
||||
function purifyEvent(event: NostrEvent): NostrEvent {
|
||||
return {
|
||||
id: event.id,
|
||||
pubkey: event.pubkey,
|
||||
kind: event.kind,
|
||||
content: event.content,
|
||||
tags: event.tags,
|
||||
sig: event.sig,
|
||||
created_at: event.created_at,
|
||||
};
|
||||
}
|
||||
|
||||
export { hydrateEvents, purifyEvent };
|
||||
export { hydrateEvents };
|
||||
|
|
|
|||
127
src/test.ts
127
src/test.ts
|
|
@ -1,21 +1,10 @@
|
|||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
import { Database as Sqlite } from '@db/sqlite';
|
||||
import { NDatabase, NDatabaseSchema, NPostgresSchema } from '@nostrify/db';
|
||||
import { NostrEvent } from '@nostrify/nostrify';
|
||||
import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite';
|
||||
import { finalizeEvent, generateSecretKey } from 'nostr-tools';
|
||||
import { FileMigrationProvider, Kysely, Migrator } from 'kysely';
|
||||
import { PostgresJSDialect, PostgresJSDialectConfig } from 'kysely-postgres-js';
|
||||
import postgres from 'postgres';
|
||||
|
||||
import { DittoDatabase, DittoDB } from '@/db/DittoDB.ts';
|
||||
import { DittoTables } from '@/db/DittoTables.ts';
|
||||
import { purifyEvent } from '@/storages/hydrate.ts';
|
||||
import { KyselyLogger } from '@/db/KyselyLogger.ts';
|
||||
import { EventsDB } from '@/storages/EventsDB.ts';
|
||||
import { Conf } from '@/config.ts';
|
||||
import { DittoDB } from '@/db/DittoDB.ts';
|
||||
import { EventsDB } from '@/storages/EventsDB.ts';
|
||||
import { purifyEvent } from '@/utils/purify.ts';
|
||||
|
||||
/** Import an event fixture by name in tests. */
|
||||
export async function eventFixture(name: string): Promise<NostrEvent> {
|
||||
|
|
@ -42,97 +31,27 @@ export function genEvent(t: Partial<NostrEvent> = {}, sk: Uint8Array = generateS
|
|||
return purifyEvent(event);
|
||||
}
|
||||
|
||||
/** Get an in-memory SQLite database to use for testing. It's automatically destroyed when it goes out of scope. */
|
||||
export async function getTestDB() {
|
||||
const kysely = new Kysely<DittoTables>({
|
||||
dialect: new DenoSqlite3Dialect({
|
||||
database: new Sqlite(':memory:'),
|
||||
}),
|
||||
/** Create a database for testing. It uses `TEST_DATABASE_URL`, or creates an in-memory database by default. */
|
||||
export async function createTestDB() {
|
||||
const { testDatabaseUrl } = Conf;
|
||||
const { protocol } = new URL(testDatabaseUrl);
|
||||
const { kysely } = DittoDB.create(testDatabaseUrl, { poolSize: 1 });
|
||||
|
||||
await DittoDB.migrate(kysely);
|
||||
|
||||
const store = new EventsDB({
|
||||
kysely,
|
||||
timeout: Conf.db.timeouts.default,
|
||||
pubkey: Conf.pubkey,
|
||||
});
|
||||
|
||||
const migrator = new Migrator({
|
||||
db: kysely,
|
||||
provider: new FileMigrationProvider({
|
||||
fs,
|
||||
path,
|
||||
migrationFolder: new URL(import.meta.resolve('./db/migrations')).pathname,
|
||||
}),
|
||||
});
|
||||
|
||||
await migrator.migrateToLatest();
|
||||
|
||||
const store = new NDatabase(kysely);
|
||||
|
||||
return {
|
||||
store,
|
||||
kysely,
|
||||
[Symbol.asyncDispose]: () => kysely.destroy(),
|
||||
};
|
||||
}
|
||||
|
||||
/** Create an database for testing. */
|
||||
export const createTestDB = async (databaseUrl?: string) => {
|
||||
databaseUrl ??= Deno.env.get('DATABASE_URL') ?? 'sqlite://:memory:';
|
||||
|
||||
let dialect: 'sqlite' | 'postgres' = (() => {
|
||||
const protocol = databaseUrl.split(':')[0];
|
||||
switch (protocol) {
|
||||
case 'sqlite':
|
||||
return 'sqlite';
|
||||
case 'postgres':
|
||||
return protocol;
|
||||
case 'postgresql':
|
||||
return 'postgres';
|
||||
default:
|
||||
throw new Error(`Unsupported protocol: ${protocol}`);
|
||||
}
|
||||
})();
|
||||
|
||||
const allowToUseDATABASE_URL = Deno.env.get('ALLOW_TO_USE_DATABASE_URL')?.toLowerCase() ?? '';
|
||||
if (allowToUseDATABASE_URL !== 'true' && dialect === 'postgres') {
|
||||
console.warn(
|
||||
'%cRunning tests with sqlite, if you meant to use Postgres, run again with ALLOW_TO_USE_DATABASE_URL environment variable set to true',
|
||||
'color: yellow;',
|
||||
);
|
||||
dialect = 'sqlite';
|
||||
}
|
||||
|
||||
console.warn(`Using: ${dialect}`);
|
||||
|
||||
const db: DittoDatabase = { dialect } as DittoDatabase;
|
||||
|
||||
if (dialect === 'sqlite') {
|
||||
// migration 021_pgfts_index.ts calls 'Conf.db.dialect',
|
||||
// and this calls the DATABASE_URL environment variable.
|
||||
// The following line ensures to NOT use the DATABASE_URL that may exist in an .env file.
|
||||
Deno.env.set('DATABASE_URL', 'sqlite://:memory:');
|
||||
|
||||
db.kysely = new Kysely({
|
||||
dialect: new DenoSqlite3Dialect({
|
||||
database: new Sqlite(':memory:'),
|
||||
}),
|
||||
}) as Kysely<DittoTables> & Kysely<NDatabaseSchema>;
|
||||
} else {
|
||||
db.kysely = new Kysely({
|
||||
// @ts-ignore Kysely version mismatch.
|
||||
dialect: new PostgresJSDialect({
|
||||
postgres: postgres(Conf.databaseUrl, {
|
||||
max: Conf.pg.poolSize,
|
||||
}) as unknown as PostgresJSDialectConfig['postgres'],
|
||||
}),
|
||||
log: KyselyLogger,
|
||||
}) as Kysely<DittoTables> & Kysely<NPostgresSchema>;
|
||||
}
|
||||
|
||||
await DittoDB.migrate(db.kysely);
|
||||
const store = new EventsDB(db);
|
||||
|
||||
return {
|
||||
dialect,
|
||||
store,
|
||||
kysely: db.kysely,
|
||||
[Symbol.asyncDispose]: async () => {
|
||||
if (dialect === 'postgres') {
|
||||
// If we're testing against real Postgres, we will reuse the database
|
||||
// between tests, so we should drop the tables to keep each test fresh.
|
||||
if (['postgres:', 'postgresql:'].includes(protocol)) {
|
||||
for (
|
||||
const table of [
|
||||
'author_stats',
|
||||
|
|
@ -142,20 +61,18 @@ export const createTestDB = async (databaseUrl?: string) => {
|
|||
'kysely_migration_lock',
|
||||
'nip46_tokens',
|
||||
'pubkey_domains',
|
||||
'unattached_media',
|
||||
'nostr_events',
|
||||
'nostr_tags',
|
||||
'nostr_pgfts',
|
||||
'event_zaps',
|
||||
'author_search',
|
||||
]
|
||||
) {
|
||||
await db.kysely.schema.dropTable(table).ifExists().cascade().execute();
|
||||
await kysely.schema.dropTable(table).ifExists().cascade().execute();
|
||||
}
|
||||
await db.kysely.destroy();
|
||||
await kysely.destroy();
|
||||
}
|
||||
},
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
export function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
|
|
|||
117
src/trends.ts
117
src/trends.ts
|
|
@ -1,11 +1,12 @@
|
|||
import { NostrFilter } from '@nostrify/nostrify';
|
||||
import { Stickynotes } from '@soapbox/stickynotes';
|
||||
import { sql } from 'kysely';
|
||||
import { Kysely, sql } from 'kysely';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
import { DittoDatabase, DittoDB } from '@/db/DittoDB.ts';
|
||||
import { DittoTables } from '@/db/DittoTables.ts';
|
||||
import { handleEvent } from '@/pipeline.ts';
|
||||
import { AdminSigner } from '@/signers/AdminSigner.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
import { Time } from '@/utils/time.ts';
|
||||
|
||||
const console = new Stickynotes('ditto:trends');
|
||||
|
|
@ -13,88 +14,50 @@ const console = new Stickynotes('ditto:trends');
|
|||
/** Get trending tag values for a given tag in the given time frame. */
|
||||
export async function getTrendingTagValues(
|
||||
/** Kysely instance to execute queries on. */
|
||||
{ dialect, kysely }: DittoDatabase,
|
||||
kysely: Kysely<DittoTables>,
|
||||
/** Tag name to filter by, eg `t` or `r`. */
|
||||
tagNames: string[],
|
||||
/** Filter of eligible events. */
|
||||
filter: NostrFilter,
|
||||
): Promise<{ value: string; authors: number; uses: number }[]> {
|
||||
if (dialect === 'postgres') {
|
||||
let query = kysely
|
||||
.selectFrom([
|
||||
'nostr_events',
|
||||
sql<{ key: string; value: string }>`jsonb_each_text(nostr_events.tags_index)`.as('kv'),
|
||||
sql<{ key: string; value: string }>`jsonb_array_elements_text(kv.value::jsonb)`.as('element'),
|
||||
])
|
||||
.select(({ fn }) => [
|
||||
fn<string>('lower', ['element.value']).as('value'),
|
||||
fn.agg<number>('count', ['nostr_events.pubkey']).distinct().as('authors'),
|
||||
fn.countAll<number>().as('uses'),
|
||||
])
|
||||
.where('kv.key', '=', (eb) => eb.fn.any(eb.val(tagNames)))
|
||||
.groupBy((eb) => eb.fn<string>('lower', ['element.value']))
|
||||
.orderBy((eb) => eb.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc');
|
||||
let query = kysely
|
||||
.selectFrom([
|
||||
'nostr_events',
|
||||
sql<{ key: string; value: string }>`jsonb_each_text(nostr_events.tags_index)`.as('kv'),
|
||||
sql<{ key: string; value: string }>`jsonb_array_elements_text(kv.value::jsonb)`.as('element'),
|
||||
])
|
||||
.select(({ fn }) => [
|
||||
fn<string>('lower', ['element.value']).as('value'),
|
||||
fn.agg<number>('count', ['nostr_events.pubkey']).distinct().as('authors'),
|
||||
fn.countAll<number>().as('uses'),
|
||||
])
|
||||
.where('kv.key', '=', (eb) => eb.fn.any(eb.val(tagNames)))
|
||||
.groupBy((eb) => eb.fn<string>('lower', ['element.value']))
|
||||
.orderBy((eb) => eb.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc');
|
||||
|
||||
if (filter.kinds) {
|
||||
query = query.where('nostr_events.kind', '=', ({ fn, val }) => fn.any(val(filter.kinds)));
|
||||
}
|
||||
if (filter.authors) {
|
||||
query = query.where('nostr_events.pubkey', '=', ({ fn, val }) => fn.any(val(filter.authors)));
|
||||
}
|
||||
if (typeof filter.since === 'number') {
|
||||
query = query.where('nostr_events.created_at', '>=', filter.since);
|
||||
}
|
||||
if (typeof filter.until === 'number') {
|
||||
query = query.where('nostr_events.created_at', '<=', filter.until);
|
||||
}
|
||||
if (typeof filter.limit === 'number') {
|
||||
query = query.limit(filter.limit);
|
||||
}
|
||||
|
||||
const rows = await query.execute();
|
||||
|
||||
return rows.map((row) => ({
|
||||
value: row.value,
|
||||
authors: Number(row.authors),
|
||||
uses: Number(row.uses),
|
||||
}));
|
||||
if (filter.kinds) {
|
||||
query = query.where('nostr_events.kind', '=', ({ fn, val }) => fn.any(val(filter.kinds)));
|
||||
}
|
||||
if (filter.authors) {
|
||||
query = query.where('nostr_events.pubkey', '=', ({ fn, val }) => fn.any(val(filter.authors)));
|
||||
}
|
||||
if (typeof filter.since === 'number') {
|
||||
query = query.where('nostr_events.created_at', '>=', filter.since);
|
||||
}
|
||||
if (typeof filter.until === 'number') {
|
||||
query = query.where('nostr_events.created_at', '<=', filter.until);
|
||||
}
|
||||
if (typeof filter.limit === 'number') {
|
||||
query = query.limit(filter.limit);
|
||||
}
|
||||
|
||||
if (dialect === 'sqlite') {
|
||||
let query = kysely
|
||||
.selectFrom('nostr_tags')
|
||||
.select(({ fn }) => [
|
||||
'nostr_tags.value',
|
||||
fn.agg<number>('count', ['nostr_tags.pubkey']).distinct().as('authors'),
|
||||
fn.countAll<number>().as('uses'),
|
||||
])
|
||||
.where('nostr_tags.name', 'in', tagNames)
|
||||
.groupBy('nostr_tags.value')
|
||||
.orderBy((c) => c.fn.agg('count', ['nostr_tags.pubkey']).distinct(), 'desc');
|
||||
const rows = await query.execute();
|
||||
|
||||
if (filter.kinds) {
|
||||
query = query.where('nostr_tags.kind', 'in', filter.kinds);
|
||||
}
|
||||
if (typeof filter.since === 'number') {
|
||||
query = query.where('nostr_tags.created_at', '>=', filter.since);
|
||||
}
|
||||
if (typeof filter.until === 'number') {
|
||||
query = query.where('nostr_tags.created_at', '<=', filter.until);
|
||||
}
|
||||
if (typeof filter.limit === 'number') {
|
||||
query = query.limit(filter.limit);
|
||||
}
|
||||
|
||||
const rows = await query.execute();
|
||||
|
||||
return rows.map((row) => ({
|
||||
value: row.value,
|
||||
authors: Number(row.authors),
|
||||
uses: Number(row.uses),
|
||||
}));
|
||||
}
|
||||
|
||||
return [];
|
||||
return rows.map((row) => ({
|
||||
value: row.value,
|
||||
authors: Number(row.authors),
|
||||
uses: Number(row.uses),
|
||||
}));
|
||||
}
|
||||
|
||||
/** Get trending tags and publish an event with them. */
|
||||
|
|
@ -107,7 +70,7 @@ export async function updateTrendingTags(
|
|||
aliases?: string[],
|
||||
) {
|
||||
console.info(`Updating trending ${l}...`);
|
||||
const db = await DittoDB.getInstance();
|
||||
const kysely = await Storages.kysely();
|
||||
const signal = AbortSignal.timeout(1000);
|
||||
|
||||
const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000);
|
||||
|
|
@ -116,7 +79,7 @@ export async function updateTrendingTags(
|
|||
const tagNames = aliases ? [tagName, ...aliases] : [tagName];
|
||||
|
||||
try {
|
||||
const trends = await getTrendingTagValues(db, tagNames, {
|
||||
const trends = await getTrendingTagValues(kysely, tagNames, {
|
||||
kinds,
|
||||
since: yesterday,
|
||||
until: now,
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ export class SimpleLRU<
|
|||
|
||||
constructor(fetchFn: FetchFn<K, V, { signal: AbortSignal }>, opts: LRUCache.Options<K, V, void>) {
|
||||
this.cache = new LRUCache({
|
||||
fetchMethod: (key, _staleValue, { signal }) => fetchFn(key, { signal: signal as AbortSignal }),
|
||||
fetchMethod: (key, _staleValue, { signal }) => fetchFn(key, { signal: signal as unknown as AbortSignal }),
|
||||
...opts,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import { RelayError } from '@/RelayError.ts';
|
|||
import { AdminSigner } from '@/signers/AdminSigner.ts';
|
||||
import { Storages } from '@/storages.ts';
|
||||
import { nostrNow } from '@/utils.ts';
|
||||
import { purifyEvent } from '@/storages/hydrate.ts';
|
||||
import { purifyEvent } from '@/utils/purify.ts';
|
||||
|
||||
const debug = Debug('ditto:api');
|
||||
|
||||
|
|
|
|||
|
|
@ -49,6 +49,22 @@ Deno.test('parseNoteContent renders empty for non-profile nostr URIs', () => {
|
|||
assertEquals(html, '');
|
||||
});
|
||||
|
||||
Deno.test("parseNoteContent doesn't fuck up links to my own post", () => {
|
||||
const { html } = parseNoteContent(
|
||||
'Check this post: https://gleasonator.dev/@alex@gleasonator.dev/posts/a8badb480d88f9e7b6a090342279ef47ed0e0a3989ed85f898dfedc6be94225f',
|
||||
[{
|
||||
id: '0461fcbecc4c3374439932d6b8f11269ccdb7cc973ad7a50ae362db135a474dd',
|
||||
username: 'alex',
|
||||
acct: 'alex@gleasonator.dev',
|
||||
url: 'https://gleasonator.dev/@alex',
|
||||
}],
|
||||
);
|
||||
assertEquals(
|
||||
html,
|
||||
'Check this post: <a href="https://gleasonator.dev/@alex@gleasonator.dev/posts/a8badb480d88f9e7b6a090342279ef47ed0e0a3989ed85f898dfedc6be94225f">https://gleasonator.dev/@alex@gleasonator.dev/posts/a8badb480d88f9e7b6a090342279ef47ed0e0a3989ed85f898dfedc6be94225f</a>',
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test('getMediaLinks', () => {
|
||||
const links = [
|
||||
{ href: 'https://example.com/image.png' },
|
||||
|
|
|
|||
14
src/utils/purify.ts
Normal file
14
src/utils/purify.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
import { NostrEvent } from '@nostrify/nostrify';
|
||||
|
||||
/** Return a normalized event without any non-standard keys. */
|
||||
export function purifyEvent(event: NostrEvent): NostrEvent {
|
||||
return {
|
||||
id: event.id,
|
||||
pubkey: event.pubkey,
|
||||
kind: event.kind,
|
||||
content: event.content,
|
||||
tags: event.tags,
|
||||
sig: event.sig,
|
||||
created_at: event.created_at,
|
||||
};
|
||||
}
|
||||
|
|
@ -5,7 +5,6 @@ import { z } from 'zod';
|
|||
|
||||
import { DittoTables } from '@/db/DittoTables.ts';
|
||||
import { findQuoteTag, findReplyTag, getTagSet } from '@/utils/tags.ts';
|
||||
import { Conf } from '@/config.ts';
|
||||
|
||||
interface UpdateStatsOpts {
|
||||
kysely: Kysely<DittoTables>;
|
||||
|
|
@ -197,16 +196,13 @@ export async function updateAuthorStats(
|
|||
notes_count: 0,
|
||||
};
|
||||
|
||||
let query = kysely
|
||||
const prev = await kysely
|
||||
.selectFrom('author_stats')
|
||||
.selectAll()
|
||||
.where('pubkey', '=', pubkey);
|
||||
.forUpdate()
|
||||
.where('pubkey', '=', pubkey)
|
||||
.executeTakeFirst();
|
||||
|
||||
if (Conf.db.dialect === 'postgres') {
|
||||
query = query.forUpdate();
|
||||
}
|
||||
|
||||
const prev = await query.executeTakeFirst();
|
||||
const stats = fn(prev ?? empty);
|
||||
|
||||
if (prev) {
|
||||
|
|
@ -249,16 +245,13 @@ export async function updateEventStats(
|
|||
reactions: '{}',
|
||||
};
|
||||
|
||||
let query = kysely
|
||||
const prev = await kysely
|
||||
.selectFrom('event_stats')
|
||||
.selectAll()
|
||||
.where('event_id', '=', eventId);
|
||||
.forUpdate()
|
||||
.where('event_id', '=', eventId)
|
||||
.executeTakeFirst();
|
||||
|
||||
if (Conf.db.dialect === 'postgres') {
|
||||
query = query.forUpdate();
|
||||
}
|
||||
|
||||
const prev = await query.executeTakeFirst();
|
||||
const stats = fn(prev ?? empty);
|
||||
|
||||
if (prev) {
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import Debug from '@soapbox/stickynotes/debug';
|
|||
import DOMPurify from 'isomorphic-dompurify';
|
||||
import { unfurl } from 'unfurl.js';
|
||||
|
||||
import { Conf } from '@/config.ts';
|
||||
import { PreviewCard } from '@/entities/PreviewCard.ts';
|
||||
import { Time } from '@/utils/time.ts';
|
||||
import { fetchWorker } from '@/workers/fetch.ts';
|
||||
|
|
@ -15,7 +16,10 @@ async function unfurlCard(url: string, signal: AbortSignal): Promise<PreviewCard
|
|||
const result = await unfurl(url, {
|
||||
fetch: (url) =>
|
||||
fetchWorker(url, {
|
||||
headers: { 'User-Agent': 'WhatsApp/2' },
|
||||
headers: {
|
||||
'Accept': 'text/html, application/xhtml+xml',
|
||||
'User-Agent': Conf.fetchUserAgent,
|
||||
},
|
||||
signal,
|
||||
}),
|
||||
});
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { HTTPException } from '@hono/hono/http-exception';
|
|||
|
||||
import { AppContext } from '@/app.ts';
|
||||
import { Conf } from '@/config.ts';
|
||||
import { insertUnattachedMedia, UnattachedMedia } from '@/db/unattached-media.ts';
|
||||
import { DittoUpload, dittoUploads } from '@/DittoUploads.ts';
|
||||
|
||||
interface FileMeta {
|
||||
pubkey: string;
|
||||
|
|
@ -15,7 +15,7 @@ export async function uploadFile(
|
|||
file: File,
|
||||
meta: FileMeta,
|
||||
signal?: AbortSignal,
|
||||
): Promise<UnattachedMedia> {
|
||||
): Promise<DittoUpload> {
|
||||
const uploader = c.get('uploader');
|
||||
if (!uploader) {
|
||||
throw new HTTPException(500, {
|
||||
|
|
@ -36,11 +36,15 @@ export async function uploadFile(
|
|||
tags.push(['alt', description]);
|
||||
}
|
||||
|
||||
return insertUnattachedMedia({
|
||||
const upload = {
|
||||
id: crypto.randomUUID(),
|
||||
pubkey,
|
||||
url,
|
||||
data: tags,
|
||||
uploaded_at: Date.now(),
|
||||
});
|
||||
tags,
|
||||
pubkey,
|
||||
uploadedAt: new Date(),
|
||||
};
|
||||
|
||||
dittoUploads.set(upload.id, upload);
|
||||
|
||||
return upload;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@ import { getUrlMediaType } from '@/utils/media.ts';
|
|||
|
||||
/** Render Mastodon media attachment. */
|
||||
function renderAttachment(
|
||||
media: { id?: string; data: string[][] },
|
||||
media: { id?: string; tags: string[][] },
|
||||
): (MastodonAttachment & { cid?: string }) | undefined {
|
||||
const { id, data: tags } = media;
|
||||
const { id, tags } = media;
|
||||
|
||||
const url = tags.find(([name]) => name === 'url')?.[1];
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,19 @@
|
|||
import { NostrEvent } from '@nostrify/nostrify';
|
||||
|
||||
import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts';
|
||||
import { Conf } from '@/config.ts';
|
||||
import { DittoEvent } from '@/interfaces/DittoEvent.ts';
|
||||
import { nostrDate } from '@/utils.ts';
|
||||
import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts';
|
||||
import { renderStatus } from '@/views/mastodon/statuses.ts';
|
||||
|
||||
interface RenderNotificationOpts {
|
||||
export interface RenderNotificationOpts {
|
||||
viewerPubkey: string;
|
||||
zap?: {
|
||||
zapSender?: NostrEvent | NostrEvent['pubkey']; // kind 0 or pubkey
|
||||
zappedPost?: NostrEvent;
|
||||
amount?: number;
|
||||
message?: string;
|
||||
};
|
||||
}
|
||||
|
||||
function renderNotification(event: DittoEvent, opts: RenderNotificationOpts) {
|
||||
|
|
@ -32,6 +38,10 @@ function renderNotification(event: DittoEvent, opts: RenderNotificationOpts) {
|
|||
if (event.kind === 30360 && event.pubkey === Conf.pubkey) {
|
||||
return renderNameGrant(event);
|
||||
}
|
||||
|
||||
if (event.kind === 9735) {
|
||||
return renderZap(event, opts);
|
||||
}
|
||||
}
|
||||
|
||||
async function renderMention(event: DittoEvent, opts: RenderNotificationOpts) {
|
||||
|
|
@ -109,6 +119,27 @@ async function renderNameGrant(event: DittoEvent) {
|
|||
};
|
||||
}
|
||||
|
||||
async function renderZap(event: DittoEvent, opts: RenderNotificationOpts) {
|
||||
if (!opts.zap?.zapSender) return;
|
||||
|
||||
const { amount = 0, message = '' } = opts.zap;
|
||||
if (amount < 1) return;
|
||||
|
||||
const account = typeof opts.zap.zapSender !== 'string'
|
||||
? await renderAccount(opts.zap.zapSender)
|
||||
: await accountFromPubkey(opts.zap.zapSender);
|
||||
|
||||
return {
|
||||
id: notificationId(event),
|
||||
type: 'ditto:zap',
|
||||
amount,
|
||||
message,
|
||||
created_at: nostrDate(event.created_at).toISOString(),
|
||||
account,
|
||||
...(opts.zap?.zappedPost ? { status: await renderStatus(opts.zap?.zappedPost, opts) } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** This helps notifications be sorted in the correct order. */
|
||||
function notificationId({ id, created_at }: NostrEvent): string {
|
||||
return `${created_at}-${id}`;
|
||||
|
|
|
|||
|
|
@ -84,7 +84,12 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
|
|||
|
||||
const imeta: string[][][] = event.tags
|
||||
.filter(([name]) => name === 'imeta')
|
||||
.map(([_, ...entries]) => entries.map((entry) => entry.split(' ')));
|
||||
.map(([_, ...entries]) =>
|
||||
entries.map((entry) => {
|
||||
const split = entry.split(' ');
|
||||
return [split[0], split.splice(1).join(' ')];
|
||||
})
|
||||
);
|
||||
|
||||
const media = imeta.length ? imeta : getMediaLinks(links);
|
||||
|
||||
|
|
@ -120,9 +125,9 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
|
|||
pinned: Boolean(pinEvent),
|
||||
reblog: null,
|
||||
application: null,
|
||||
media_attachments: media.map((m) => renderAttachment({ data: m })).filter((m): m is MastodonAttachment =>
|
||||
Boolean(m)
|
||||
),
|
||||
media_attachments: media
|
||||
.map((m) => renderAttachment({ tags: m }))
|
||||
.filter((m): m is MastodonAttachment => Boolean(m)),
|
||||
mentions,
|
||||
tags: [],
|
||||
emojis: renderEmojis(event),
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
/// <reference lib="webworker" />
|
||||
|
||||
import Debug from '@soapbox/stickynotes/debug';
|
||||
import * as Comlink from 'comlink';
|
||||
|
||||
import './handlers/abortsignal.ts';
|
||||
import '@/workers/handlers/abortsignal.ts';
|
||||
import '@/sentry.ts';
|
||||
|
||||
const debug = Debug('ditto:fetch.worker');
|
||||
|
|
|
|||
|
|
@ -13,8 +13,8 @@ export const policyWorker = Comlink.wrap<CustomPolicy>(
|
|||
type: 'module',
|
||||
deno: {
|
||||
permissions: {
|
||||
read: [Conf.policy],
|
||||
write: false,
|
||||
read: [Conf.denoDir, Conf.policy, Conf.dataDir],
|
||||
write: [Conf.dataDir],
|
||||
net: 'inherit',
|
||||
env: false,
|
||||
},
|
||||
|
|
@ -24,7 +24,12 @@ export const policyWorker = Comlink.wrap<CustomPolicy>(
|
|||
);
|
||||
|
||||
try {
|
||||
await policyWorker.import(Conf.policy);
|
||||
await policyWorker.init({
|
||||
path: Conf.policy,
|
||||
cwd: Deno.cwd(),
|
||||
databaseUrl: Conf.databaseUrl,
|
||||
adminPubkey: Conf.pubkey,
|
||||
});
|
||||
console.debug(`Using custom policy: ${Conf.policy}`);
|
||||
} catch (e) {
|
||||
if (e.message.includes('Module not found')) {
|
||||
|
|
|
|||
|
|
@ -3,6 +3,24 @@ import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify';
|
|||
import { NoOpPolicy, ReadOnlyPolicy } from '@nostrify/nostrify/policies';
|
||||
import * as Comlink from 'comlink';
|
||||
|
||||
import { DittoDB } from '@/db/DittoDB.ts';
|
||||
import { EventsDB } from '@/storages/EventsDB.ts';
|
||||
|
||||
// @ts-ignore Don't try to access the env from this worker.
|
||||
Deno.env = new Map<string, string>();
|
||||
|
||||
/** Serializable object the worker can use to set up the state. */
|
||||
interface PolicyInit {
|
||||
/** Path to the policy module (https, jsr, file, etc) */
|
||||
path: string;
|
||||
/** Current working directory. */
|
||||
cwd: string;
|
||||
/** Database URL to connect to. */
|
||||
databaseUrl: string;
|
||||
/** Admin pubkey to use for EventsDB checks. */
|
||||
adminPubkey: string;
|
||||
}
|
||||
|
||||
export class CustomPolicy implements NPolicy {
|
||||
private policy: NPolicy = new ReadOnlyPolicy();
|
||||
|
||||
|
|
@ -11,10 +29,22 @@ export class CustomPolicy implements NPolicy {
|
|||
return this.policy.call(event);
|
||||
}
|
||||
|
||||
async import(path: string): Promise<void> {
|
||||
async init({ path, cwd, databaseUrl, adminPubkey }: PolicyInit): Promise<void> {
|
||||
// HACK: PGlite uses `path.resolve`, which requires read permission on Deno (which we don't want to give).
|
||||
// We can work around this getting the cwd from the caller and overwriting `Deno.cwd`.
|
||||
Deno.cwd = () => cwd;
|
||||
|
||||
const { kysely } = DittoDB.create(databaseUrl, { poolSize: 1 });
|
||||
|
||||
const store = new EventsDB({
|
||||
kysely,
|
||||
pubkey: adminPubkey,
|
||||
timeout: 1_000,
|
||||
});
|
||||
|
||||
try {
|
||||
const Policy = (await import(path)).default;
|
||||
this.policy = new Policy();
|
||||
this.policy = new Policy({ store });
|
||||
} catch (e) {
|
||||
if (e.message.includes('Module not found')) {
|
||||
this.policy = new NoOpPolicy();
|
||||
|
|
|
|||
|
|
@ -1,52 +0,0 @@
|
|||
import * as Comlink from 'comlink';
|
||||
import { asyncGeneratorTransferHandler } from 'comlink-async-generator';
|
||||
import { CompiledQuery, QueryResult } from 'kysely';
|
||||
|
||||
import type { SqliteWorker as _SqliteWorker } from './sqlite.worker.ts';
|
||||
|
||||
Comlink.transferHandlers.set('asyncGenerator', asyncGeneratorTransferHandler);
|
||||
|
||||
class SqliteWorker {
|
||||
#worker: Worker;
|
||||
#client: ReturnType<typeof Comlink.wrap<typeof _SqliteWorker>>;
|
||||
#ready: Promise<void>;
|
||||
|
||||
constructor() {
|
||||
this.#worker = new Worker(new URL('./sqlite.worker.ts', import.meta.url).href, { type: 'module' });
|
||||
this.#client = Comlink.wrap<typeof _SqliteWorker>(this.#worker);
|
||||
|
||||
this.#ready = new Promise<void>((resolve) => {
|
||||
const handleEvent = (event: MessageEvent) => {
|
||||
if (event.data[0] === 'ready') {
|
||||
this.#worker.removeEventListener('message', handleEvent);
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
this.#worker.addEventListener('message', handleEvent);
|
||||
});
|
||||
}
|
||||
|
||||
async open(path: string): Promise<void> {
|
||||
await this.#ready;
|
||||
return this.#client.open(path);
|
||||
}
|
||||
|
||||
async executeQuery<R>(query: CompiledQuery): Promise<QueryResult<R>> {
|
||||
await this.#ready;
|
||||
return this.#client.executeQuery(query) as Promise<QueryResult<R>>;
|
||||
}
|
||||
|
||||
async *streamQuery<R>(query: CompiledQuery): AsyncIterableIterator<QueryResult<R>> {
|
||||
await this.#ready;
|
||||
|
||||
for await (const result of await this.#client.streamQuery(query)) {
|
||||
yield result as QueryResult<R>;
|
||||
}
|
||||
}
|
||||
|
||||
destroy(): Promise<void> {
|
||||
return this.#client.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
export default SqliteWorker;
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
/// <reference lib="webworker" />
|
||||
import { Database as SQLite } from '@db/sqlite';
|
||||
import * as Comlink from 'comlink';
|
||||
import { CompiledQuery, QueryResult } from 'kysely';
|
||||
import { asyncGeneratorTransferHandler } from 'comlink-async-generator';
|
||||
|
||||
import '@/sentry.ts';
|
||||
|
||||
let db: SQLite | undefined;
|
||||
|
||||
export const SqliteWorker = {
|
||||
open(path: string): void {
|
||||
db = new SQLite(path);
|
||||
},
|
||||
executeQuery<R>({ sql, parameters }: CompiledQuery): QueryResult<R> {
|
||||
if (!db) throw new Error('Database not open');
|
||||
|
||||
return {
|
||||
rows: db!.prepare(sql).all(...parameters as any[]) as R[],
|
||||
numAffectedRows: BigInt(db!.changes),
|
||||
insertId: BigInt(db!.lastInsertRowId),
|
||||
};
|
||||
},
|
||||
async *streamQuery<R>({ sql, parameters }: CompiledQuery): AsyncIterableIterator<QueryResult<R>> {
|
||||
if (!db) throw new Error('Database not open');
|
||||
|
||||
const stmt = db.prepare(sql).bind(...parameters as any[]);
|
||||
for (const row of stmt) {
|
||||
yield {
|
||||
rows: [row],
|
||||
};
|
||||
}
|
||||
},
|
||||
destroy() {
|
||||
db?.close();
|
||||
},
|
||||
};
|
||||
|
||||
Comlink.transferHandlers.set('asyncGenerator', asyncGeneratorTransferHandler);
|
||||
Comlink.expose(SqliteWorker);
|
||||
|
||||
self.postMessage(['ready']);
|
||||
Loading…
Add table
Reference in a new issue