diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b2140dbf..58a34532 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,4 +28,14 @@ test: paths: - deno-test.xml reports: - junit: deno-test.xml \ No newline at end of file + junit: deno-test.xml + +postgres: + stage: test + script: deno task db:migrate + services: + - postgres:16 + variables: + DITTO_NSEC: nsec1zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs4rm7hz + DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres + POSTGRES_HOST_AUTH_METHOD: trust \ No newline at end of file diff --git a/deno.json b/deno.json index d5f6119c..77acaf66 100644 --- a/deno.json +++ b/deno.json @@ -5,6 +5,7 @@ "start": "deno run -A src/server.ts", "dev": "deno run -A --watch src/server.ts", "hook": "deno run --allow-read --allow-run --allow-write https://deno.land/x/deno_hooks@0.1.1/mod.ts", + "db:migrate": "deno run -A scripts/db-migrate.ts", "debug": "deno run -A --inspect src/server.ts", "test": "DATABASE_URL=\"sqlite://:memory:\" deno test -A --junit-path=./deno-test.xml", "check": "deno check src/server.ts", @@ -21,7 +22,6 @@ "@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "@db/sqlite": "jsr:@db/sqlite@^0.11.1", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", - "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.23.0", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", @@ -57,7 +57,7 @@ "tseep": "npm:tseep@^1.2.1", "type-fest": "npm:type-fest@^4.3.0", "unfurl.js": "npm:unfurl.js@^6.4.0", - "zod": "npm:zod@^3.23.5", + "zod": "npm:zod@^3.23.8", "~/fixtures/": "./fixtures/" }, "lint": { diff --git a/fixtures/hydrated.jsonl b/fixtures/hydrated.jsonl new file mode 100644 index 00000000..27c35986 --- /dev/null +++ b/fixtures/hydrated.jsonl @@ -0,0 +1,32 @@ +{"id":"63d54e69da65af273683e62a8afe35bb125901d6f9c38817f2db38850dcad38f","kind":1,"pubkey":"17538dc2a62769d09443f18c37cbe358fab5bbf981173542aa7c5ff171ed77c4","content":"Fran had beef empanadas for breakfast and came after mandatory algos","created_at":1716651366,"tags":[["e","d14564e5f13e7ea2d090fcb301cfc71e214b2e9348f98dbda18f930e1ee91453","","root"],["p","726a1e261cc6474674e8285e3951b3bb139be9a773d1acf49dc868db861a1c11"]],"sig":"4f592599e9adcf4b4160626ebe06f061ade4309ac683ef8f0fa04bf0514ea3f90171993dd8c755ca419f487ded496cce9d40a7b340cb1ed6ca213da2980f3051"} +{"id":"d72f4995ebadb841e75afa08d28626c7fb275515e5e8cca29842c34018476522","kind":1,"pubkey":"3f770d65d3a764a9c5cb503ae123e62ec7598ad035d836e2a810f3877a745b24","content":"๐Ÿ˜‚","created_at":1716651307,"tags":[["e","b1ed8ea342d0023a8bedcb79de77633f8f550a21e363e707dd260e411977cff4","","root"],["e","518977b03781fbcf4383cc7fa76f8d3a5326ba687d37d0567db13e7be88e12cb","","reply"],["p","cfe3b4316d905335b6ce056ba0ec230b587a334381e82bf9a02a184f2d068f8d"],["p","3f770d65d3a764a9c5cb503ae123e62ec7598ad035d836e2a810f3877a745b24"],["p","4b74667f89358cd582ad82b16a2d24d5bfcb89ac4b1347ee80e5674a13ba78b2"]],"sig":"7c535641d7b405c74857dd6beddb639554e8a8e5cb0b61b1e172b3865a69a7208d38e22558693d8630516e54f23171df00055fc8923ab90a5ad22f2f4453a62a"} +{"id":"879784117ef204bacc0fdc7fa306b81e9f76f6793f743639550239074ec28373","kind":1,"pubkey":"3f770d65d3a764a9c5cb503ae123e62ec7598ad035d836e2a810f3877a745b24","content":"Loves me some fried chicken.","created_at":1716651290,"tags":[["e","b1ed8ea342d0023a8bedcb79de77633f8f550a21e363e707dd260e411977cff4","","root"],["e","b1ed8ea342d0023a8bedcb79de77633f8f550a21e363e707dd260e411977cff4","","root"],["e","262e209395ea847fa4ade5e4370394276ae95cf908fff8d29fb86f8a360c00d2","","reply"],["p","3f770d65d3a764a9c5cb503ae123e62ec7598ad035d836e2a810f3877a745b24"],["p","af387a6c488c5484088ba715dbb42b55ce72b475e1e2b86be791b24b8d51e215"]],"sig":"dca7b3443d6db6d21eac2a3570f68b8854dc8edd7eb948c4d903569304d9e4865388cd77362ba508dff974d9e684308a454f0f766cdb63ba120e033a3024a9ef"} +{"id":"38c755514d9d7e1d8c8fbfd8ff6d475c97d1fba504e2c90431cd3e3e84c71056","kind":1,"pubkey":"17538dc2a62769d09443f18c37cbe358fab5bbf981173542aa7c5ff171ed77c4","content":"๐Ÿ˜‚๐Ÿ‘","created_at":1716651146,"tags":[["e","bd2afa1348221ee72205e77368ad0341f2cedb13150928bf9dadbd7a58d68c5e","","root"],["p","13a9e5d2a683cb4690ffb83f12848adc9c3423e2fcd786e86d35ee25faacbfbd"]],"sig":"0cbb28192f853f879d51f470f7c8a338ae1f759829fd1a88aa1e4b4f166b71b5bacf98f654cf3f9a8e40282b20a21e939b38aa9e445b1c2af535e36658e2149f"} +{"id":"e088a883c95ca15c2ac245512fd9b1307136101ecb98eeea9621cd22d9cce854","kind":1,"pubkey":"3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d","content":"Use a combined key with MuSig to publish.\n\nOr just create a new key that all collaborators know, publish from it, then add it to your list of good wiki people and/or defer from your articles to their articles.","created_at":1716650955,"tags":[["p","fa984bd7dbb282f07e16e7ae87b26a2a7b9b90b7246a44771f0cf5ae58018f52","wss://relay.damus.io/","PABLOF7z"],["e","cc127dc2528ad97eaa88ff37d6c5d6bbe94b163ca873701db64f9e1bcfaa40cb","wss://relay.damus.io/","root","fa984bd7dbb282f07e16e7ae87b26a2a7b9b90b7246a44771f0cf5ae58018f52"]],"sig":"b316e9fdba6dbbf547ffea4449bf4c2c06ba880da1b05aa8947633748d1b5ddf06e085d9ca728ebff296c123082ad36178728a3ff075c52bead12cba71fbd93c"} +{"id":"c77d684d95f0babf382ccc6d7cf2c01111ad28a6f3892594da1c565d8f42be00","kind":1,"pubkey":"3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d","content":"Maybe you should just defer at the same time you're pushing your edit? If you trust the person enough to give them \"write access\" to your wiki you probably already trust their article enough and/or trust that they will merge your edit.","created_at":1716650124,"tags":[["p","d91191e30e00444b942c0e82cad470b32af171764c2275bee0bd99377efd4075","wss://relay.damus.io/","gsovereignty"],["p","266815e0c9210dfa324c6cba3573b14bee49da4209a9456f9484e5106cd408a5","wss://nostr.wine/","hzrd149"],["p","dd664d5e4016433a8cd69f005ae1480804351789b59de5af06276de65633d319","wss://theforest.nostr1.com/","Laeserin"],["p","1bc70a0148b3f316da33fe3c89f23e3e71ac4ff998027ec712b905cd24f6a411","wss://eden.nostr.land/","Karnage"],["p","fa984bd7dbb282f07e16e7ae87b26a2a7b9b90b7246a44771f0cf5ae58018f52","wss://relay.damus.io/","PABLOF7z"],["e","cc127dc2528ad97eaa88ff37d6c5d6bbe94b163ca873701db64f9e1bcfaa40cb","wss://relay.damus.io/","root"],["e","1c9ca83dfdc96bd795e0420904bdfeb81d9434aa69c88ce773f0e6e849b8e6cd","","mention"],["e","ad680fadfc4c0a5e5ab62475c5bdb6a55af05fcd2b2ec7b5e340cf03fdc36a1d","wss://relay.damus.io/","reply","fa984bd7dbb282f07e16e7ae87b26a2a7b9b90b7246a44771f0cf5ae58018f52"]],"sig":"7e9f9ac0e763ea5a3a45f1cbbead272396f508f9fa9abd2677123e98ab49f4d4bac9ee28e8f2d812b75ef2eae2277edca2c3e11eb406ec54e6918075bd3f9558"} +{"id":"319b1708200eeea2887981ff4d32a8e8ac0b35ce3b3029936a13a8bd6626b4df","kind":1,"pubkey":"3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d","content":"For example: how would I list all your articles now? I can't, because some of them are now hosted in a different pubkey, so for every article of yours I have to make a new request.","created_at":1716649794,"tags":[["p","fa984bd7dbb282f07e16e7ae87b26a2a7b9b90b7246a44771f0cf5ae58018f52","wss://relay.damus.io/","PABLOF7z"],["p","1b5c7f2f8a8f7e7ea7775879fd02d098db6ec588114521f1a1edfccd635c1fe9","wss://nostr.wine/","Zach"],["e","cc127dc2528ad97eaa88ff37d6c5d6bbe94b163ca873701db64f9e1bcfaa40cb","wss://140.f7z.io/","root"],["e","742ca34a0b51fa3ccc10ef89ae190b7c25c4ece0f0776554c0ca985705dbdd27","wss://relay.nostr.band/","mention"],["e","aba8036a7de241777a7a5a2c7b015450da795ba0275ed589edf98abda683dca2","wss://relay.damus.io/","mention","fa984bd7dbb282f07e16e7ae87b26a2a7b9b90b7246a44771f0cf5ae58018f52"],["e","3b0b5ba7c23c1af922c89151b947858bf441163857d2c2bcc113570e73ff9303","wss://pyramid.fiatjaf.com/","reply","3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d"]],"sig":"449d9a4239dbde9978eb5517143693edcbaa7e437a73283c92b5291d474f55f3311fe6b9dd9438059b50926eec5c289f150bc17e5d102585810bd75e06a0eaa3"} +{"id":"3b0b5ba7c23c1af922c89151b947858bf441163857d2c2bcc113570e73ff9303","kind":1,"pubkey":"3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d","content":"More overhead for people writing the articles, much less overhead for people reading and much less complexity in-protocol.","created_at":1716649678,"tags":[["p","fa984bd7dbb282f07e16e7ae87b26a2a7b9b90b7246a44771f0cf5ae58018f52","wss://relay.damus.io/","PABLOF7z"],["p","1b5c7f2f8a8f7e7ea7775879fd02d098db6ec588114521f1a1edfccd635c1fe9","wss://nostr.wine/","Zach"],["e","cc127dc2528ad97eaa88ff37d6c5d6bbe94b163ca873701db64f9e1bcfaa40cb","wss://relay.damus.io/","root"],["e","742ca34a0b51fa3ccc10ef89ae190b7c25c4ece0f0776554c0ca985705dbdd27","wss://relay.nostr.band/","mention"],["e","aba8036a7de241777a7a5a2c7b015450da795ba0275ed589edf98abda683dca2","wss://relay.damus.io/","reply","fa984bd7dbb282f07e16e7ae87b26a2a7b9b90b7246a44771f0cf5ae58018f52"]],"sig":"2609ced7c4dc101d00ebfd6dc6c4255f828af1f0d9ffc876543c815c8d24ce5131f61d8bde0fa184aa06ea1c549b3bd3cfc0d579bcbf6caee86794484d903b64"} +{"id":"512992206a039a3079f5e892eb14368e3d29c4609722911e77919c93b7a68b15","kind":6,"pubkey":"c37b6a82a98de368c104bbc6da365571ec5a263b07057d0a3977b4c05afa7e63","content":"{\"pubkey\":\"9be0be0e64d38a29a9cec9a5c8ef5d873c2bfa5362a4b558da5ff69bc3cbb81e\",\"content\":\"nostr protocol allows anyone to change their name and profile picture at any time.\\n\\nNostur will show previous name and profile pic if available\\nhttps://media.utxo.nl/wp-content/uploads/nostr/7/f/7fcfe9ea3eb7dc850828cc6906dd91ca839de2184982954b4479952d85a1d447.webp\",\"id\":\"b8d7eff16cd1ead7c28032f9e36fd4ef2e29682f84e2a89f2fca8c2bec13385d\",\"created_at\":1716645465,\"sig\":\"2bc2a9eeffa50c5b48c388bdbf2c9622aa2cfdf7484625ef859a027f788983bbde481be7b647121a784ebd0417f109da84a2e43e124b028028c8e792787f7588\",\"kind\":1,\"tags\":[[\"imeta\",\"url https://media.utxo.nl/wp-content/uploads/nostr/7/f/7fcfe9ea3eb7dc850828cc6906dd91ca839de2184982954b4479952d85a1d447.webp\",\"dim 314x260\",\"sha256 7fcfe9ea3eb7dc850828cc6906dd91ca839de2184982954b4479952d85a1d447\"],[\"client\",\"Nostur\",\"31990:9be0be0fc079548233231614e4e1efc9f28b0db398011efeecf05fe570e5dd33:1685868693432\"]]}","created_at":1716649603,"tags":[["e","b8d7eff16cd1ead7c28032f9e36fd4ef2e29682f84e2a89f2fca8c2bec13385d","wss://relay.damus.io","mention"],["p","9be0be0e64d38a29a9cec9a5c8ef5d873c2bfa5362a4b558da5ff69bc3cbb81e"]],"sig":"7af3d238027e58504fe515b8cfb087751485235525a8ce46aa6a43728b97f7681d7b7180cc76cfa5f00a57aafba9ed7b20b301bce59943c3d2e6a9bb7f203e91"} +{"id":"ba9b2c033eba0b14108473773d4529c7f2ad5e8f3d3267a90cd161f94b2b948f","kind":1,"pubkey":"c37b6a82a98de368c104bbc6da365571ec5a263b07057d0a3977b4c05afa7e63","content":"this doesnt look good for manchester city, might not be their day","created_at":1716649355,"tags":[["client","Nostur","31990:9be0be0fc079548233231614e4e1efc9f28b0db398011efeecf05fe570e5dd33:1685868693432"]],"sig":"e748efa0c267aee2c58a58f2031ef14abf74eaf104e110902363f26e04aba56e37813ac8d1ab1d02592ee0546a5c5644092f3e74a2e30c236c4de56449731b56"} +{"id":"f331dc1c3985cf76b997dc9fadcb46241ba0ccb9d20159b0a3ca6f77f4316f58","kind":1,"pubkey":"c37b6a82a98de368c104bbc6da365571ec5a263b07057d0a3977b4c05afa7e63","content":"woman on top","created_at":1716649278,"tags":[["e","88fddb8d21d8f1198ce2e0e1b24c6897a06458576ac57f93b901c01eda862b21","","root"],["e","e43414005ac14ee53e83ed06ec6292dca5beef24077bb4b33546e348f1fb84b0","","reply"],["p","45f195cffcb8c9724efc248f0507a2fb65b579dfabe7cd35398598163cab7627"],["client","Nostur","31990:9be0be0fc079548233231614e4e1efc9f28b0db398011efeecf05fe570e5dd33:1685868693432"]],"sig":"fb492636ab58e604b66a53635f56f9b62a9162fae700a5296313f30ddfe3a3eac8c2019b5b09cf58db5b961a36da7cfcfc08a79f98af5b16d592fe5f76af8e58"} +{"id":"21a8f27145674b0664a4b7ccd0d0ff8c36e4ceb7e35db44df1c3df6f0d593aea","kind":1,"pubkey":"3f770d65d3a764a9c5cb503ae123e62ec7598ad035d836e2a810f3877a745b24","content":"Found some on the shore. https://i.nostr.build/AaRYE.jpg","created_at":1716649252,"tags":[["e","b1ed8ea342d0023a8bedcb79de77633f8f550a21e363e707dd260e411977cff4","","root"],["e","3232c85dcc455e4c323ffc3742a0eb8b4bbe38418b7c430203e694b4416fce50"],["e","6f3f1713f6c82561cd757ba3b2a2233d63f49478f30b90f2eddc71a1a25d49e9","","reply"],["p","cfe3b4316d905335b6ce056ba0ec230b587a334381e82bf9a02a184f2d068f8d"],["p","3f770d65d3a764a9c5cb503ae123e62ec7598ad035d836e2a810f3877a745b24"],["p","20d29810d6a5f92b045ade02ebbadc9036d741cc686b00415c42b4236fe4ad2f"],["p","7b394902eeadb8370931f1903d00569545e84113fb6a09634664763be232009c"],["r","https://i.nostr.build/AaRYE.jpg"],["imeta","url https://i.nostr.build/AaRYE.jpg","m image/jpeg","alt Verifiable file url","x c86385263058d896c409ec5644a4a993a52d7decac3b7e052901bd42c2f2cb89","size 603150","dim 2040x1536","blurhash #9Gu5ZJ50f=x,@57I=~BV@0LD*$+I:Io=|M|9axa02RkrrbvNwn$oJozWVR*W-WUofWDazxGxaR+,?kDNxw{niSgf+WAWUJBWARPX9%Mw]t6W?spemWWxvRP$$kXNHRPo2","ox 1951abbc1ee6bf76771f70f1fd305775cec89d30cd5a3947d31656d852efa9bd"]],"sig":"206c40361e578f5cec2c7726bfb99812d706df48057e67eb42f76b7bed8c8269a489be324df9b1f925c241757128394fb60eb494733d3c96cf0ac23569f3680c"} +{"id":"b1ed8ea342d0023a8bedcb79de77633f8f550a21e363e707dd260e411977cff4","kind":1,"pubkey":"3f770d65d3a764a9c5cb503ae123e62ec7598ad035d836e2a810f3877a745b24","content":"nostr:npub1el3mgvtdjpfntdkwq446pmprpdv85v6rs85zh7dq9gvy7tgx37xs2kl27r it's that time of year again. #thighstr https://i.nostr.build/BRGYD.jpg","created_at":1716648863,"tags":[["p","cfe3b4316d905335b6ce056ba0ec230b587a334381e82bf9a02a184f2d068f8d","","mention"],["t","thighstr"],["r","https://i.nostr.build/BRGYD.jpg"],["imeta","url https://i.nostr.build/BRGYD.jpg","m image/jpeg","alt Verifiable file url","x 376255322f1798e621985bbf028c70c346287404535edbab0aee830094b46673","size 82827","dim 768x1020","blurhash _AEfTfJI0:x[Q,${fN00q?=Y9bx^ohRkK-9~-C~AS0I;t7#=_3OF9_WUi^%1=DVXI.Io%3t7NGO@NH$%?Gi_ogNHJBo#%MNHS2aJsl-7$gRPWnI:ozs.adobR%ja%2M|Si","ox bf1c31404466531f3d58331b404f1116bc9e7a12d6cb41012275f3d36cb53f98"]],"sig":"cd1e87d85f0fd257eac1195ee5c565023d81aa4d17f2d6c0ac3538179db9112b215b951555dcc7bec3e317ad93b08a9f7635c32c9f101986231e898f64075108"} +{"id":"c8d97acccc86babc0884216a3962f4c1beddc9ee16d5dac3b72a2acabad0fdfa","kind":6,"pubkey":"7579076d9aff0a4cfdefa7e2045f2486c7e5d8bc63bfc6b45397233e1bbfcb19","content":"{\"id\":\"cc127dc2528ad97eaa88ff37d6c5d6bbe94b163ca873701db64f9e1bcfaa40cb\",\"pubkey\":\"fa984bd7dbb282f07e16e7ae87b26a2a7b9b90b7246a44771f0cf5ae58018f52\",\"created_at\":1716648312,\"kind\":1,\"tags\":[[\"client\",\"highlighter\",\"31990:73c6bb92440a9344279f7a36aa3de1710c9198b1e9e8a394cd13e0dd5c994c63:1704502265408\"]],\"content\":\"people interested in nostr wiki:\\n\\nthoughts on allowing collaboration on an entry?\\n\",\"sig\":\"eeac1fa0347e3e240d564e04504d2b01900fd90e7eeab94cfd5034777230e3b593569e771ae0ac9077ae705225faa6cd001a8a43a540b69b69240605484e4ae3\"}","created_at":1716648793,"tags":[["e","cc127dc2528ad97eaa88ff37d6c5d6bbe94b163ca873701db64f9e1bcfaa40cb"],["p","fa984bd7dbb282f07e16e7ae87b26a2a7b9b90b7246a44771f0cf5ae58018f52"],["alt","Repost event"]],"sig":"05d131b8bae0756cf54bb961a27a1dc23109c329d2a84e6d826c11ff5f363536e123d7770f26ea88c4f3580b6c1df6e0cd0dea3470b821fd4e23b52da04086ca"} +{"id":"e8fae24ac25cd3d2d5f5a8efc08e5fe9f0f45ddd975e960805d020bb5f2eb119","kind":1,"pubkey":"17538dc2a62769d09443f18c37cbe358fab5bbf981173542aa7c5ff171ed77c4","content":"Voltage is focused on b2b - I am not sure folks use them for a personal node. \n\nThat said managing your own node is a pain in the ๐ŸŒ.","created_at":1716648296,"tags":[["e","2253e1fed036e0a86eac892477552b8908684fae69696d89275119bb4e5c42c7","","root"],["e","b136b92f53f4b5ac6988755ba8256d227c3b7a7f687c87cfb256932df359cc60","","reply"],["p","49f586188679af2b31e8d62ff153baf767fd0c586939f4142ef65606d236ff75"]],"sig":"f9659da7b43eda0992418d20a219735162bfa81524b4dba4bce42b31aba7ba41d5be84376a98c37d0bca0321f090326896d44b1fbfb964fb3aa5cff667db8d37"} +{"id":"52cfedd86e7693e3533900f6d6d444ee5b64e5d571679f1a1c60ae25c4d1fbf8","kind":1,"pubkey":"9ca0bd7450742d6a20319c0e3d4c679c9e046a9dc70e8ef55c2905e24052340b","content":"He's here in substack and YouTube RSS form lol","created_at":1716648028,"tags":[["e","514341e9cfd55b9ce955897a0f5dd1bc1b165ea45868cc31824dc945fdaa7841","","root"],["e","37183ef6e232453c05b7b9ffe831a76704f07b4ac123484094707d184dadf569","","reply"],["p","9ca0bd7450742d6a20319c0e3d4c679c9e046a9dc70e8ef55c2905e24052340b"],["p","95c7f867802a9c7cfb1d2b243be152234864e2cf9f60f467ae32bf1cf05f89fb"]],"sig":"97758c7b494fff1fbc3a2e88618eb8d6510dc93a7d3aef2bb5bdda977c34ef34a16ddeb59aefc95c7f6a94f49c0c41a2edede768409a361db5561f76f55aa9d2"} +{"id":"2a6860b01ac1cb31c081a6cf93d2d83f7bb4a54d669414db61ead5602688a03e","kind":1,"pubkey":"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322","content":"PSA: DVM feeds on Coracle were broken this week. They may still be broken, but in a different way. I'll continue to refine the latest release this coming week (and make a tutorial for nostr:nprofile1qy2hwumn8ghj7un9d3shjtnyv9kh2uewd9hj7qgwwaehxw309ahx7uewd3hkctcpzemhxue69uhhyetvv9ujumt0wd68ytnsw43z7qgnwaehxw309aex2mrp0yhxvdm69e5k7tcpz9mhxue69uhkummnw3ezuamfdejj7qgkwaehxw309ajkgetw9ehx7um5wghxcctwvshszxthwden5te0wfjkccte9ekk7mt0wd68ytnsd9hxktcprpmhxue69uhkummnv3exjan99eshqup0wfjkccteqyw8wumn8ghj7un9d3shjtngd9nksmrfva58getj9e3k7mf0qqsx8zd7vjg70d5na8ek3m8g3lx3ghc8cp5d9sdm4epy0wd4aape6vsavyafj )","created_at":1716647834,"tags":[["p","6389be6491e7b693e9f368ece88fcd145f07c068d2c1bbae4247b9b5ef439d32","wss://relay.damus.io/","NunyaBidness"],["client","Coracle","31990:97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322:1685968093690"]],"sig":"2b7a5c612f74532e73b967f5d3b4cc9950fb5055873dd1893aeadf58b379cd5e743a45ec321cd36cfd1b62f32bd27ebcea93c640963fbab37ede063d3f5b5c43"} +{"id":"b0ca7c5df23236a14fd8949d0b032252fa0090904d36885a314ff16cade03591","kind":1,"pubkey":"9ca0bd7450742d6a20319c0e3d4c679c9e046a9dc70e8ef55c2905e24052340b","content":"I'll probably just ignore it as best I can lol. Look at a nice tree and a pleasant stream:) \nMaybe take up beekeeping. That sort of thing.","created_at":1716647725,"tags":[["e","386666836de6df61f2104bc7b0552ac8e2c2e841a99d3a70885452df7f0865b0","","root"],["e","49857012475717e98c7713609dc7e4d95b2339d16eba644f95152ad2feb22e3a","","reply"],["p","9ca0bd7450742d6a20319c0e3d4c679c9e046a9dc70e8ef55c2905e24052340b"],["p","95c7f867802a9c7cfb1d2b243be152234864e2cf9f60f467ae32bf1cf05f89fb"]],"sig":"998c657f78dc112d2b826fed284f78e8059dd65c0e12d55606d3438acd2f2e685f90fca965f0a30018679c82835246a15af0a8faeee262400d9e79bde156771d"} +{"id":"ccb4828a0955d366f3479a7e9374416f089b2692c80e7bb2a52da27834dbaed9","kind":1,"pubkey":"9ca0bd7450742d6a20319c0e3d4c679c9e046a9dc70e8ef55c2905e24052340b","content":"Think so yeah, there was a few hood ones. But yeah sounds like it","created_at":1716647663,"tags":[["e","5013462f3f82a32c0ed1f749c4e90a9073a263ecf505fe373d0549f9575d0115","","root"],["e","c7df680aa4e977ff5130e5f4f6765d95bd92dbcd0c13cbc00b16d7681bd8de70"],["e","563671b66257ccfab60abbc9ef3be63a92765efdc7efa3032de68fd8daa68eb2","","reply"],["p","9ca0bd7450742d6a20319c0e3d4c679c9e046a9dc70e8ef55c2905e24052340b"],["p","dd664d5e4016433a8cd69f005ae1480804351789b59de5af06276de65633d319"]],"sig":"379bc8f177b927b11f4d6527967d3d703a90f830f6264fb3c131dbc2f07aa269fb6629263f3b19207dc1952e1e9a064841092d0846ed69ca486af67743e88130"} +{"id":"88fd33948257b3edbe5fe5e597848252968726d998b81126faff0fbd2eed3a07","kind":1,"pubkey":"17538dc2a62769d09443f18c37cbe358fab5bbf981173542aa7c5ff171ed77c4","content":"For whom?","created_at":1716647197,"tags":[["e","2253e1fed036e0a86eac892477552b8908684fae69696d89275119bb4e5c42c7","","root"],["p","49f586188679af2b31e8d62ff153baf767fd0c586939f4142ef65606d236ff75"]],"sig":"685ba60871b169f7b732b3ff9c8f0bcdc7c5294ec1b479729d5bc409b3d51029465f7b330bd8f4ecb7625b4e8fb6fbb8cf472f78a38069899bbd2b590d68fda7"} +{"id":"cc127dc2528ad97eaa88ff37d6c5d6bbe94b163ca873701db64f9e1bcfaa40cb","kind":1,"pubkey":"fa984bd7dbb282f07e16e7ae87b26a2a7b9b90b7246a44771f0cf5ae58018f52","content":"people interested in nostr wiki:\n\nthoughts on allowing collaboration on an entry?\n","created_at":1716648312,"tags":[["client","highlighter","31990:73c6bb92440a9344279f7a36aa3de1710c9198b1e9e8a394cd13e0dd5c994c63:1704502265408"]],"sig":"eeac1fa0347e3e240d564e04504d2b01900fd90e7eeab94cfd5034777230e3b593569e771ae0ac9077ae705225faa6cd001a8a43a540b69b69240605484e4ae3"} +{"id":"b8d7eff16cd1ead7c28032f9e36fd4ef2e29682f84e2a89f2fca8c2bec13385d","kind":1,"pubkey":"9be0be0e64d38a29a9cec9a5c8ef5d873c2bfa5362a4b558da5ff69bc3cbb81e","content":"nostr protocol allows anyone to change their name and profile picture at any time.\n\nNostur will show previous name and profile pic if available\nhttps://media.utxo.nl/wp-content/uploads/nostr/7/f/7fcfe9ea3eb7dc850828cc6906dd91ca839de2184982954b4479952d85a1d447.webp","created_at":1716645465,"tags":[["imeta","url https://media.utxo.nl/wp-content/uploads/nostr/7/f/7fcfe9ea3eb7dc850828cc6906dd91ca839de2184982954b4479952d85a1d447.webp","dim 314x260","sha256 7fcfe9ea3eb7dc850828cc6906dd91ca839de2184982954b4479952d85a1d447"],["client","Nostur","31990:9be0be0fc079548233231614e4e1efc9f28b0db398011efeecf05fe570e5dd33:1685868693432"]],"sig":"2bc2a9eeffa50c5b48c388bdbf2c9622aa2cfdf7484625ef859a027f788983bbde481be7b647121a784ebd0417f109da84a2e43e124b028028c8e792787f7588"} +{"id":"1c9ca83dfdc96bd795e0420904bdfeb81d9434aa69c88ce773f0e6e849b8e6cd","kind":1,"pubkey":"fa984bd7dbb282f07e16e7ae87b26a2a7b9b90b7246a44771f0cf5ae58018f52","content":"say I create a document about \"Stalin\" and I add [\"p\", \"\", \"editor\" ]\n\nthis would mean that the most recent version of my version of Stalin is whatever comes back from the REQ { \"#d\": [\"stalin\"], authors: [, ] }\n","created_at":1716648317,"tags":[["e","cc127dc2528ad97eaa88ff37d6c5d6bbe94b163ca873701db64f9e1bcfaa40cb","","root"],["p","d91191e30e00444b942c0e82cad470b32af171764c2275bee0bd99377efd4075"],["client","highlighter","31990:73c6bb92440a9344279f7a36aa3de1710c9198b1e9e8a394cd13e0dd5c994c63:1704502265408"]],"sig":"0756b98ad88cc041c9299922a5da43748c15266640924b7f760897e0386cb0fbdde6338e561ad2af2fee6c72051ee6e6254c6abad9aac6f0d2018d9b07f09cda"} +{"id":"9161f8f465fc4b0462995149ceb23d024c885caf66bc9a421f0c0f842325b021","kind":0,"pubkey":"c37b6a82a98de368c104bbc6da365571ec5a263b07057d0a3977b4c05afa7e63","content":"{\"about\":\"#nobridge #noDM #noSTR #noRMIE\\nโœ‰๏ธ no.str@aol.com \\n๐Ÿ” https://s.id/no-str\\n๐Ÿ”— https://gleasonator.dev/@Bac0t.my.id\",\"banner\":\"https://pbs.twimg.com/profile_banners/573176539/1697452071/1500x500\",\"lud16\":\"feelingisrael15@walletofsatoshi.com\",\"name\":\"๐Ÿฆ–\",\"nip05\":\"_@Bac0t.my.id\",\"picture\":\"https://pbs.twimg.com/profile_images/1788948029283950592/m9PMKCZO_400x400.png\"}","created_at":1716603430,"tags":[],"sig":"8e955263a526522bbd3a60db8522a0d859fd5bb59a0e0bb03a41cdf2f0e5fb4a43048052871a7338a7f198806107bf005e19767a5719e8aa806ba9cd49183ce8"} +{"id":"e50020597a6aebf6b704686206a682121a9612b6294242baa895437752d62010","kind":0,"pubkey":"9ca0bd7450742d6a20319c0e3d4c679c9e046a9dc70e8ef55c2905e24052340b","content":"{\"name\":\"7fqx\",\"about\":\"๐“Š๐Ÿ€\",\"picture\":\"https://image.nostr.build/f89f134a56d932f91c5b79175c22bc749bace2cf94f7334f8427d24b210cb876.jpg\",\"banner\":\"https://image.nostr.build/c21af2ac126f85f8b1eacdf04ae47df8fcda4f5916e7d27c7f2f0a5f35df3fcc.jpg\",\"lud16\":\"glhf@getalby.com\",\"image\":\"https://image.nostr.build/86b5aee8f3c7bb5293819e7dd9049fcdc2f912d54150b9b24e041eb5b66aef19.jpg\",\"displayName\":\"7fqx\",\"display_name\":\"7fqx\",\"pubkey\":\"9ca0bd7450742d6a20319c0e3d4c679c9e046a9dc70e8ef55c2905e24052340b\",\"npub\":\"npub1njst6azswskk5gp3ns8r6nr8nj0qg65acu8gaa2u9yz7yszjxs9s6k7fqx\",\"created_at\":1714857730}","created_at":1716498460,"tags":[],"sig":"3920d47a7c6314bafaa24ba876a300c9cf3875c3ac775bc5a4ead1e0643daa7a5f10c49e563d32846e313a82d809a05e7297367d873e746908bf9476df1125c3"} +{"id":"ec6b70dcb1714d8a887e468305af0ef2d002466385f2e875cdd306f750521b0f","kind":0,"pubkey":"17538dc2a62769d09443f18c37cbe358fab5bbf981173542aa7c5ff171ed77c4","content":"{\"website\":\"https://github.com/alltheseas\",\"lud16\":\"elsat@mutiny.plus\",\"display_name\":\"\",\"about\":\"Damus and freedom tech Product Janitor ๐Ÿงน\",\"picture\":\"https://nostr.build/i/p/nostr.build_7b9579da60a52c32a61cfe48e7b55b9fbd58d389ca29128d6c6851b00bb23d0a.jpg\",\"name\":\"elsat\",\"banner\":\"https://cdn.nostr.build/i/9b853a8461d114ec2c353b7caaab598286406d1ab4e47f9ebffda4db757bdaa5.jpg\",\"reactions\":false}","created_at":1716219324,"tags":[],"sig":"ea0f6c0563fa2bf549f141e6d769d432dae4767d739968814b24f841b4a68cbaf36c848d383453fa2a46d530a08d8c70eb6c9865d941c3ca1ef78057a740f40a"} +{"id":"93e845c76ee32784733bc1dbba5e45270fde733c4207eb5832e5ba30a98f20ca","kind":0,"pubkey":"3f770d65d3a764a9c5cb503ae123e62ec7598ad035d836e2a810f3877a745b24","content":"{\"created_at\":1707238393,\"picture\":\"https://i.nostr.build/Z7j8.jpg\",\"name\":\"Derek Ross\",\"about\":\"Building NostrPlebs.com and NostrNests.com. The purple pill helps the orange pill go down. Nostr is the social glue that binds all of your apps together.\",\"lud16\":\"pay@derekross.me\",\"display_name\":\"Derek Ross\",\"banner\":\"https://i.nostr.build/O2JE.jpg\",\"website\":\"https://nostrplebs.com\",\"nip05\":\"derekross@nostrplebs.com\"}","created_at":1715808097,"tags":[["alt","User profile for Derek Ross"],["i","twitter:derekmross","1634343988407726081"],["i","github:derekross","3edaf845975fa4500496a15039323fa3"]],"sig":"2862018e2ca23d9a376691a40c306494786148560d28f9799e4137dfa6a2f1bead2d35b21b68fe09837c4e6cf858f5642f38a5d97234e37b1a5e002ca1d37a8e"} +{"id":"0810aed567c2cc7a0caccab17ea51fe30ea71862a71c13516777f9842b1f7532","kind":0,"pubkey":"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322","content":"{\"picture\":\"https://i.nostr.build/AZ0L.jpg\",\"about\":\"Christian Bitcoiner and developer of the coracle.social nostr client.\\nLearn more at https://coracle.tools\",\"name\":\"hodlbod\",\"nip05\":\"hodlbod@coracle.social\",\"nip05_updated_at\":1676671261,\"banner\":\"https://i.nostr.build/axYJ.jpg\",\"lud16\":\"hodlbod@getalby.com\",\"website\":\"coracle.social\"}","created_at":1714776395,"tags":[["client","Coracle","31990:97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322:1685968093690"]],"sig":"04f515ac035b94b940a6c91f876de1532708d1e4118491fd16519d9fbed096d327aa644bd14008629f7572c405e27abab0865805255a0ea526259c802ad31dcb"} +{"id":"5967d8b638bf20b305eafcbaf4aa2cf317d38696e9e86853cf4e079c1f463f24","kind":0,"pubkey":"3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d","content":"{\"name\":\"fiatjaf\",\"about\":\"~\",\"picture\":\"https://fiatjaf.com/static/favicon.jpg\",\"nip05\":\"_@fiatjaf.com\",\"lud06\":\"\",\"lud16\":\"fiatjaf@zbd.gg\"}","created_at":1714591065,"tags":[],"sig":"066666c5f1f4127816e7634165799cb7780634c96539bb75c56183f59d7a3cd54245df4fce696da320256f5c19746389d75206ff127f6f08a0927c444cf8d38f"} +{"id":"22191979d7f21129c97a6740909e53f3df554f39667f7853d087623e5cf1a11d","kind":0,"pubkey":"7579076d9aff0a4cfdefa7e2045f2486c7e5d8bc63bfc6b45397233e1bbfcb19","content":"{\"name\":\"greenart7c3\",\"nip05\":\"greenart7c3@greenart7c3.com\",\"about\":\"PGP\\n\\n44F0AAEB77F373747E3D5444885822EED3A26A6D\\n\\nDeveloping Amber\\n\\nhttps://github.com/greenart7c3/Amber\",\"lud16\":\"greenart7c3@greenart7c3.com\",\"display_name\":\"greenart7c3\",\"picture\":\"https://pfp.nostr.build/a40c078816657986911bd2ec73cf9db6bd68af60bea6eaddbf14bbce7424feb8.png\",\"website\":\"https://paynym.is/+florallake7D2\"}","created_at":1714136365,"tags":[["alt","User profile for greenart7c3"]],"sig":"ea12a8605e84040a021a6929f5d898aa4537080f10a4e281a5d37bd04b923a02273da1dac1abb14ef39f5588246fab10e631fa9860b49cf459255e00f9324625"} +{"id":"5db62e87cbb8dfdea54e61713a1ea6647bf164aea5a58afd620b220d22e20b22","kind":0,"pubkey":"fa984bd7dbb282f07e16e7ae87b26a2a7b9b90b7246a44771f0cf5ae58018f52","content":"{\"banner\":\"https://pablof7z.com/images/banner.jpg\",\"about\":\"Magical Other Stuff Maximalist\",\"name\":\"PABLOF7z\",\"website\":\"https://pablof7z.com\",\"display_name\":\"PABLOF7z\",\"lud16\":\"pablof7z@primal.net\",\"picture\":\"https://pablof7z.com/images/me.jpg\",\"nip05\":\"_@f7z.io\",\"created_at\":1712782129,\"categories\":[]}","created_at":1712947216,"tags":[["c","Business & Entrepreneurship"],["c","Development & Engineering"],["client","highlighter","31990:73c6bb92440a9344279f7a36aa3de1710c9198b1e9e8a394cd13e0dd5c994c63:1704502265408"]],"sig":"99b38ad955ecb1b05fb6488d9890851f2f86603558d6631e0cf32cb0bfaba1bdcc1c33a2f97c5cd46c811d08a04e97b273bd5677eb89f93f22332ad58d01ac34"} +{"id":"f89a514d9d547528e229def6ff869bd9f50963d79a728409c2d43f0207e7ca94","kind":0,"pubkey":"9be0be0e64d38a29a9cec9a5c8ef5d873c2bfa5362a4b558da5ff69bc3cbb81e","content":"{\"name\":\"Fabian\",\"nip05\":\"fabian@nostur.com\",\"picture\":\"https:\\/\\/profilepics.nostur.com\\/profilepic_v1\\/9512c8b31c97ec0ebc8e66b44de8bafef498d335fd542a0aa1400bbc19fec9d5\\/profilepic.jpg?1710414069\",\"lud16\":\"weathereddarkness25@getalby.com\",\"about\":\"https:\\/\\/nostur.com\",\"banner\":\"https:\\/\\/profilepics.nostur.com\\/banner_v1\\/e358d89477e2303af113a2c0023f6e77bd5b73d502cf1dbdb432ec59a25bfc0f\\/banner.jpg?1682440972\"}","created_at":1712841341,"tags":[],"sig":"39a9b80361bcf8c6a13504a694037016ac5570b1f49e339521bd7068c46edf64352179ba797190f1ceeb94c0b6809977b65e2cd7e5bfe1367aa967eb614e93ef"} \ No newline at end of file diff --git a/fixtures/stats.json b/fixtures/stats.json new file mode 100644 index 00000000..065ce76e --- /dev/null +++ b/fixtures/stats.json @@ -0,0 +1 @@ +{"authors":[{"pubkey":"17538dc2a62769d09443f18c37cbe358fab5bbf981173542aa7c5ff171ed77c4","followers_count":1386,"following_count":2108,"notes_count":805},{"pubkey":"3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d","followers_count":7420,"following_count":478,"notes_count":446},{"pubkey":"3f770d65d3a764a9c5cb503ae123e62ec7598ad035d836e2a810f3877a745b24","followers_count":6999,"following_count":1428,"notes_count":801},{"pubkey":"7579076d9aff0a4cfdefa7e2045f2486c7e5d8bc63bfc6b45397233e1bbfcb19","followers_count":535,"following_count":962,"notes_count":59},{"pubkey":"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322","followers_count":4199,"following_count":398,"notes_count":176},{"pubkey":"9be0be0e64d38a29a9cec9a5c8ef5d873c2bfa5362a4b558da5ff69bc3cbb81e","followers_count":695,"following_count":242,"notes_count":49},{"pubkey":"9ca0bd7450742d6a20319c0e3d4c679c9e046a9dc70e8ef55c2905e24052340b","followers_count":614,"following_count":301,"notes_count":566},{"pubkey":"c37b6a82a98de368c104bbc6da365571ec5a263b07057d0a3977b4c05afa7e63","followers_count":270,"following_count":361,"notes_count":589},{"pubkey":"fa984bd7dbb282f07e16e7ae87b26a2a7b9b90b7246a44771f0cf5ae58018f52","followers_count":6902,"following_count":1,"notes_count":536}],"events":[{"event_id":"1c9ca83dfdc96bd795e0420904bdfeb81d9434aa69c88ce773f0e6e849b8e6cd","reposts_count":0,"replies_count":0,"reactions_count":3,"reactions":"{\"๐Ÿ”ฅ\":2,\"๐Ÿค™\":1}"},{"event_id":"2a6860b01ac1cb31c081a6cf93d2d83f7bb4a54d669414db61ead5602688a03e","reposts_count":0,"replies_count":0,"reactions_count":2,"reactions":"{\"๐Ÿงก\":1,\"+\":1}"},{"event_id":"b1ed8ea342d0023a8bedcb79de77633f8f550a21e363e707dd260e411977cff4","reposts_count":0,"replies_count":0,"reactions_count":4,"reactions":"{\"๐Ÿ”ฅ\":2,\"+\":2}"},{"event_id":"b8d7eff16cd1ead7c28032f9e36fd4ef2e29682f84e2a89f2fca8c2bec13385d","reposts_count":1,"replies_count":0,"reactions_count":4,"reactions":"{\"๐Ÿค™\":1,\"+\":2,\"๐Ÿ‘Œ\":1}"},{"event_id":"cc127dc2528ad97eaa88ff37d6c5d6bbe94b163ca873701db64f9e1bcfaa40cb","reposts_count":2,"replies_count":0,"reactions_count":5,"reactions":"{\"๐Ÿ’œ\":1,\"๐Ÿค™\":3,\"+\":1}"},{"event_id":"f331dc1c3985cf76b997dc9fadcb46241ba0ccb9d20159b0a3ca6f77f4316f58","reposts_count":0,"replies_count":0,"reactions_count":1,"reactions":"{\"+\":1}"}]} \ No newline at end of file diff --git a/scripts/db-migrate.ts b/scripts/db-migrate.ts new file mode 100644 index 00000000..b9f63bfe --- /dev/null +++ b/scripts/db-migrate.ts @@ -0,0 +1,6 @@ +import { DittoDB } from '@/db/DittoDB.ts'; + +const kysely = await DittoDB.getInstance(); +await kysely.destroy(); + +Deno.exit(); diff --git a/scripts/stats-recompute.ts b/scripts/stats-recompute.ts index 4037a85b..107a3167 100644 --- a/scripts/stats-recompute.ts +++ b/scripts/stats-recompute.ts @@ -1,6 +1,8 @@ import { nip19 } from 'nostr-tools'; -import { refreshAuthorStats } from '@/stats.ts'; +import { DittoDB } from '@/db/DittoDB.ts'; +import { Storages } from '@/storages.ts'; +import { refreshAuthorStats } from '@/utils/stats.ts'; let pubkey: string; try { @@ -15,4 +17,7 @@ try { Deno.exit(1); } -await refreshAuthorStats(pubkey); +const store = await Storages.db(); +const kysely = await DittoDB.getInstance(); + +await refreshAuthorStats({ pubkey, kysely, store }); diff --git a/src/app.ts b/src/app.ts index fbc1ef2e..82e438d5 100644 --- a/src/app.ts +++ b/src/app.ts @@ -13,6 +13,7 @@ import { accountSearchController, accountStatusesController, createAccountController, + familiarFollowersController, favouritesController, followController, followersController, @@ -156,6 +157,7 @@ app.patch('/api/v1/accounts/update_credentials', requireSigner, updateCredential app.get('/api/v1/accounts/search', accountSearchController); app.get('/api/v1/accounts/lookup', accountLookupController); app.get('/api/v1/accounts/relationships', requireSigner, relationshipsController); +app.get('/api/v1/accounts/familiar_followers', requireSigner, familiarFollowersController); app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/block', requireSigner, blockController); app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/unblock', requireSigner, unblockController); app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/mute', requireSigner, muteController); @@ -245,11 +247,29 @@ app.get('/api/v1/lists', emptyArrayController); app.use('/api/*', notImplementedController); -app.get('*', serveStatic({ root: './public/' })); -app.get('*', serveStatic({ root: './static/' })); -app.get('*', serveStatic({ path: './public/index.html' })); +const publicFiles = serveStatic({ root: './public/' }); +const staticFiles = serveStatic({ root: './static/' }); +const frontendController = serveStatic({ path: './public/index.html' }); -app.get('/', indexController); +// Known frontend routes +app.get('/@:acct', frontendController); +app.get('/@:acct/*', frontendController); +app.get('/users/*', frontendController); +app.get('/statuses/*', frontendController); +app.get('/notice/*', frontendController); + +// Known static file routes +app.get('/favicon.ico', publicFiles, staticFiles); +app.get('/images/*', publicFiles, staticFiles); +app.get('/instance/*', publicFiles); +app.get('/packs/*', publicFiles); +app.get('/sw.js', publicFiles); + +// Site index +app.get('/', frontendController, indexController); + +// Fallback +app.get('*', publicFiles, staticFiles, frontendController); export default app; diff --git a/src/config.ts b/src/config.ts index cc149983..d8e322df 100644 --- a/src/config.ts +++ b/src/config.ts @@ -61,27 +61,6 @@ class Conf { static get externalDomain() { return Deno.env.get('NOSTR_EXTERNAL') || Conf.localDomain; } - /** Path to the main SQLite database which stores users, events, and more. */ - static get dbPath() { - if (Deno.env.get('DATABASE_URL') === 'sqlite://:memory:') { - return ':memory:'; - } - - const { host, pathname } = Conf.databaseUrl; - - if (!pathname) return ''; - - // Get relative path. - if (host === '') { - return pathname; - } else if (host === '.') { - return pathname; - } else if (host) { - return host + pathname; - } - - return ''; - } /** * Heroku-style database URL. This is used in production to connect to the * database. @@ -92,9 +71,24 @@ class Conf { * protocol://username:password@host:port/database_name * ``` */ - static get databaseUrl(): url.UrlWithStringQuery { - return url.parse(Deno.env.get('DATABASE_URL') ?? 'sqlite://data/db.sqlite3'); + static get databaseUrl(): string { + return Deno.env.get('DATABASE_URL') ?? 'sqlite://data/db.sqlite3'; } + static db = { + get url(): url.UrlWithStringQuery { + return url.parse(Deno.env.get('DATABASE_URL') ?? 'sqlite://data/db.sqlite3'); + }, + get dialect(): 'sqlite' | 'postgres' | undefined { + switch (Conf.db.url.protocol) { + case 'sqlite:': + return 'sqlite'; + case 'postgres:': + case 'postgresql:': + return 'postgres'; + } + return undefined; + }, + }; /** Character limit to enforce for posts made through Mastodon API. */ static get postCharLimit() { return Number(Deno.env.get('POST_CHAR_LIMIT') || 5000); diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 71665751..4b8a5697 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -66,8 +66,11 @@ const verifyCredentialsController: AppController = async (c) => { : await accountFromPubkey(pubkey, { withSource: true }); if (settingsStore) { - const data = await signer.nip44!.decrypt(pubkey, settingsStore.content); - account.pleroma.settings_store = JSON.parse(data); + try { + account.pleroma.settings_store = JSON.parse(settingsStore.content); + } catch { + // Ignore + } } return c.json(account); @@ -220,15 +223,15 @@ const accountStatusesController: AppController = async (c) => { const updateCredentialsSchema = z.object({ display_name: z.string().optional(), note: z.string().optional(), - avatar: fileSchema.optional(), - header: fileSchema.optional(), + avatar: fileSchema.or(z.literal('')).optional(), + header: fileSchema.or(z.literal('')).optional(), locked: z.boolean().optional(), bot: z.boolean().optional(), discoverable: z.boolean().optional(), - nip05: z.string().optional(), + nip05: z.string().email().or(z.literal('')).optional(), pleroma_settings_store: z.unknown().optional(), - lud16: z.string().email().optional().catch(''), - website: z.string().url().optional().catch(''), + lud16: z.string().email().or(z.literal('')).optional(), + website: z.string().url().or(z.literal('')).optional(), }); const updateCredentialsController: AppController = async (c) => { @@ -252,6 +255,7 @@ const updateCredentialsController: AppController = async (c) => { nip05, lud16, website, + bot, } = result.data; const [avatar, header] = await Promise.all([ @@ -266,6 +270,13 @@ const updateCredentialsController: AppController = async (c) => { meta.nip05 = nip05 ?? meta.nip05; meta.lud16 = lud16 ?? meta.lud16; meta.website = website ?? meta.website; + meta.bot = bot ?? meta.bot; + + if (avatarFile === '') delete meta.picture; + if (headerFile === '') delete meta.banner; + if (nip05 === '') delete meta.nip05; + if (lud16 === '') delete meta.lud16; + if (website === '') delete meta.website; const event = await createEvent({ kind: 0, @@ -280,7 +291,7 @@ const updateCredentialsController: AppController = async (c) => { await createEvent({ kind: 30078, tags: [['d', 'pub.ditto.pleroma_settings_store']], - content: await signer.nip44!.encrypt(pubkey, JSON.stringify(settingsStore)), + content: JSON.stringify(settingsStore), }, c); } @@ -400,6 +411,28 @@ const favouritesController: AppController = async (c) => { return paginated(c, events1, statuses); }; +const familiarFollowersController: AppController = async (c) => { + const store = await Storages.db(); + const signer = c.get('signer')!; + const pubkey = await signer.getPublicKey(); + + const ids = z.array(z.string()).parse(c.req.queries('id[]')); + const follows = await getFollowedPubkeys(pubkey); + + const results = await Promise.all(ids.map(async (id) => { + const followLists = await store.query([{ kinds: [3], authors: follows, '#p': [id] }]) + .then((events) => hydrateEvents({ events, store })); + + const accounts = await Promise.all( + followLists.map((event) => event.author ? renderAccount(event.author) : accountFromPubkey(event.pubkey)), + ); + + return { id, accounts }; + })); + + return c.json(results); +}; + export { accountController, accountLookupController, @@ -407,6 +440,7 @@ export { accountStatusesController, blockController, createAccountController, + familiarFollowersController, favouritesController, followController, followersController, diff --git a/src/controllers/api/pleroma.ts b/src/controllers/api/pleroma.ts index 3bbdd70e..31b4fc58 100644 --- a/src/controllers/api/pleroma.ts +++ b/src/controllers/api/pleroma.ts @@ -51,7 +51,10 @@ const updateConfigController: AppController = async (c) => { await createAdminEvent({ kind: 30078, content: await new AdminSigner().nip44.encrypt(pubkey, JSON.stringify(configs)), - tags: [['d', 'pub.ditto.pleroma.config']], + tags: [ + ['d', 'pub.ditto.pleroma.config'], + ['encrypted', 'nip44'], + ], }, c); return c.json({ configs: newConfigs, need_reboot: false }); diff --git a/src/controllers/api/reactions.ts b/src/controllers/api/reactions.ts index f88d36e5..7c65672a 100644 --- a/src/controllers/api/reactions.ts +++ b/src/controllers/api/reactions.ts @@ -33,7 +33,9 @@ const reactionController: AppController = async (c) => { tags: [['e', id]], }, c); - const status = renderStatus(event, { viewerPubkey: await signer.getPublicKey() }); + await hydrateEvents({ events: [event], store }); + + const status = await renderStatus(event, { viewerPubkey: await signer.getPublicKey() }); return c.json(status); }; diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index e3852d97..a69fbb04 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -49,7 +49,7 @@ const streamingController: AppController = (c) => { return c.json({ error: 'Invalid access token' }, 401); } - const { socket, response } = Deno.upgradeWebSocket(c.req.raw, { protocol: token }); + const { socket, response } = Deno.upgradeWebSocket(c.req.raw, { protocol: token, idleTimeout: 30 }); function send(name: string, payload: object) { if (socket.readyState === WebSocket.OPEN) { @@ -69,8 +69,8 @@ const streamingController: AppController = (c) => { if (!filter) return; try { + const db = await Storages.db(); const pubsub = await Storages.pubsub(); - const optimizer = await Storages.optimizer(); for await (const msg of pubsub.req([filter], { signal: controller.signal })) { if (msg[0] === 'EVENT') { @@ -86,7 +86,7 @@ const streamingController: AppController = (c) => { await hydrateEvents({ events: [event], - store: optimizer, + store: db, signal: AbortSignal.timeout(1000), }); diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 259f5e94..4d239990 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -154,7 +154,7 @@ const relayController: AppController = (c, next) => { return c.text('Please use a Nostr client to connect.', 400); } - const { socket, response } = Deno.upgradeWebSocket(c.req.raw); + const { socket, response } = Deno.upgradeWebSocket(c.req.raw, { idleTimeout: 30 }); connectStream(socket); return response; diff --git a/src/db/DittoDB.ts b/src/db/DittoDB.ts index 68fdc627..d06b3318 100644 --- a/src/db/DittoDB.ts +++ b/src/db/DittoDB.ts @@ -19,16 +19,13 @@ export class DittoDB { } static async _getInstance(): Promise> { - const { databaseUrl } = Conf; - let kysely: Kysely; - switch (databaseUrl.protocol) { - case 'sqlite:': + switch (Conf.db.dialect) { + case 'sqlite': kysely = await DittoSQLite.getInstance(); break; - case 'postgres:': - case 'postgresql:': + case 'postgres': kysely = await DittoPostgres.getInstance(); break; default: @@ -47,7 +44,7 @@ export class DittoDB { provider: new FileMigrationProvider({ fs, path, - migrationFolder: new URL(import.meta.resolve('../db/migrations')).pathname, + migrationFolder: new URL(import.meta.resolve('./migrations')).pathname, }), }); diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index 42d39ea9..c2d1f861 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -20,6 +20,7 @@ interface EventStatsRow { replies_count: number; reposts_count: number; reactions_count: number; + reactions: string; } interface EventRow { diff --git a/src/db/adapters/DittoPostgres.ts b/src/db/adapters/DittoPostgres.ts index d0abbf99..bfecd92d 100644 --- a/src/db/adapters/DittoPostgres.ts +++ b/src/db/adapters/DittoPostgres.ts @@ -19,7 +19,7 @@ export class DittoPostgres { // @ts-ignore mismatched kysely versions probably createDriver() { return new PostgreSQLDriver( - { connectionString: Deno.env.get('DATABASE_URL') }, + { connectionString: Conf.databaseUrl }, Conf.pg.poolSize, ); }, diff --git a/src/db/adapters/DittoSQLite.ts b/src/db/adapters/DittoSQLite.ts index fe225a20..d412ca31 100644 --- a/src/db/adapters/DittoSQLite.ts +++ b/src/db/adapters/DittoSQLite.ts @@ -36,11 +36,11 @@ export class DittoSQLite { /** Get the relative or absolute path based on the `DATABASE_URL`. */ static get path() { - if (Deno.env.get('DATABASE_URL') === 'sqlite://:memory:') { + if (Conf.databaseUrl === 'sqlite://:memory:') { return ':memory:'; } - const { host, pathname } = Conf.databaseUrl; + const { host, pathname } = Conf.db.url; if (!pathname) return ''; diff --git a/src/db/migrations/002_events_fts.ts b/src/db/migrations/002_events_fts.ts index ffaf5fbf..56abab5f 100644 --- a/src/db/migrations/002_events_fts.ts +++ b/src/db/migrations/002_events_fts.ts @@ -3,7 +3,7 @@ import { Kysely, sql } from 'kysely'; import { Conf } from '@/config.ts'; export async function up(db: Kysely): Promise { - if (Conf.databaseUrl.protocol === 'sqlite:') { + if (Conf.db.dialect === 'sqlite') { await sql`CREATE VIRTUAL TABLE events_fts USING fts5(id, content)`.execute(db); } } diff --git a/src/db/migrations/019_ndatabase_schema.ts b/src/db/migrations/019_ndatabase_schema.ts index 94378f00..31b86cd3 100644 --- a/src/db/migrations/019_ndatabase_schema.ts +++ b/src/db/migrations/019_ndatabase_schema.ts @@ -7,7 +7,7 @@ export async function up(db: Kysely): Promise { await db.schema.alterTable('tags').renameTo('nostr_tags').execute(); await db.schema.alterTable('nostr_tags').renameColumn('tag', 'name').execute(); - if (Conf.databaseUrl.protocol === 'sqlite:') { + if (Conf.db.dialect === 'sqlite') { await db.schema.dropTable('events_fts').execute(); await sql`CREATE VIRTUAL TABLE nostr_fts5 USING fts5(event_id, content)`.execute(db); } @@ -18,7 +18,7 @@ export async function down(db: Kysely): Promise { await db.schema.alterTable('nostr_tags').renameTo('tags').execute(); await db.schema.alterTable('tags').renameColumn('name', 'tag').execute(); - if (Conf.databaseUrl.protocol === 'sqlite:') { + if (Conf.db.dialect === 'sqlite') { await db.schema.dropTable('nostr_fts5').execute(); await sql`CREATE VIRTUAL TABLE events_fts USING fts5(id, content)`.execute(db); } diff --git a/src/db/migrations/020_pgfts.ts b/src/db/migrations/020_pgfts.ts index 8b3cfa0c..835de117 100644 --- a/src/db/migrations/020_pgfts.ts +++ b/src/db/migrations/020_pgfts.ts @@ -3,7 +3,7 @@ import { Kysely, sql } from 'kysely'; import { Conf } from '@/config.ts'; export async function up(db: Kysely): Promise { - if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + if (Conf.db.dialect === 'postgres') { await db.schema.createTable('nostr_pgfts') .ifNotExists() .addColumn('event_id', 'text', (c) => c.primaryKey().references('nostr_events.id').onDelete('cascade')) @@ -13,7 +13,7 @@ export async function up(db: Kysely): Promise { } export async function down(db: Kysely): Promise { - if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + if (Conf.db.dialect === 'postgres') { await db.schema.dropTable('nostr_pgfts').ifExists().execute(); } } diff --git a/src/db/migrations/021_pgfts_index.ts b/src/db/migrations/021_pgfts_index.ts index d18d110b..4b834995 100644 --- a/src/db/migrations/021_pgfts_index.ts +++ b/src/db/migrations/021_pgfts_index.ts @@ -3,7 +3,7 @@ import { Kysely } from 'kysely'; import { Conf } from '@/config.ts'; export async function up(db: Kysely): Promise { - if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + if (Conf.db.dialect === 'postgres') { await db.schema .createIndex('nostr_pgfts_gin_search_vec') .ifNotExists() @@ -15,7 +15,7 @@ export async function up(db: Kysely): Promise { } export async function down(db: Kysely): Promise { - if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + if (Conf.db.dialect === 'postgres') { await db.schema.dropIndex('nostr_pgfts_gin_search_vec').ifExists().execute(); } } diff --git a/src/db/migrations/022_event_stats_reactions.ts b/src/db/migrations/022_event_stats_reactions.ts new file mode 100644 index 00000000..0bc69147 --- /dev/null +++ b/src/db/migrations/022_event_stats_reactions.ts @@ -0,0 +1,12 @@ +import { Kysely } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .alterTable('event_stats') + .addColumn('reactions', 'text', (col) => col.defaultTo('{}')) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.alterTable('event_stats').dropColumn('reactions').execute(); +} diff --git a/src/interfaces/DittoEvent.ts b/src/interfaces/DittoEvent.ts index 41847fb1..b9f95e43 100644 --- a/src/interfaces/DittoEvent.ts +++ b/src/interfaces/DittoEvent.ts @@ -11,7 +11,7 @@ export interface AuthorStats { export interface EventStats { replies_count: number; reposts_count: number; - reactions_count: number; + reactions: Record; } /** Internal Event representation used by Ditto, including extra keys. */ diff --git a/src/pipeline.ts b/src/pipeline.ts index bfb0577e..fd5fc990 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -3,14 +3,15 @@ import { LNURL } from '@nostrify/nostrify/ln'; import { PipePolicy } from '@nostrify/nostrify/policies'; import Debug from '@soapbox/stickynotes/debug'; import { sql } from 'kysely'; +import { LRUCache } from 'lru-cache'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; import { deleteAttachedMedia } from '@/db/unattached-media.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { DVM } from '@/pipeline/DVM.ts'; +import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; import { RelayError } from '@/RelayError.ts'; -import { updateStats } from '@/stats.ts'; import { hydrateEvents, purifyEvent } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; import { eventAge, nostrDate, nostrNow, parseNip05, Time } from '@/utils.ts'; @@ -21,10 +22,9 @@ import { verifyEventWorker } from '@/workers/verify.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { lnurlCache } from '@/utils/lnurl.ts'; import { nip05Cache } from '@/utils/nip05.ts'; +import { updateStats } from '@/utils/stats.ts'; import { getTagSet } from '@/utils/tags.ts'; -import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; - const debug = Debug('ditto:pipeline'); /** @@ -33,7 +33,7 @@ const debug = Debug('ditto:pipeline'); */ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { if (!(await verifyEventWorker(event))) return; - if (await encounterEvent(event, signal)) return; + if (encounterEvent(event)) return; debug(`NostrEvent<${event.kind}> ${event.id}`); if (event.kind !== 24133) { @@ -47,7 +47,6 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { } } +const encounters = new LRUCache({ max: 1000 }); + /** Encounter the event, and return whether it has already been encountered. */ -async function encounterEvent(event: NostrEvent, signal: AbortSignal): Promise { - const cache = await Storages.cache(); - const reqmeister = await Storages.reqmeister(); - - const [existing] = await cache.query([{ ids: [event.id], limit: 1 }]); - - cache.event(event); - reqmeister.event(event, { signal }); - - return !!existing; +function encounterEvent(event: NostrEvent): boolean { + const encountered = !!encounters.get(event.id); + if (!encountered) { + encounters.set(event.id, true); + } + return encountered; } /** Hydrate the event with the user, if applicable. */ @@ -121,8 +118,9 @@ async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { if (NKinds.ephemeral(event.kind)) return; const store = await Storages.db(); + const kysely = await DittoDB.getInstance(); - await updateStats(event).catch(debug); + await updateStats({ event, store, kysely }).catch(debug); await store.event(event, { signal }); } @@ -183,31 +181,6 @@ async function trackHashtags(event: NostrEvent): Promise { } } -/** Queue related events to fetch. */ -async function fetchRelatedEvents(event: DittoEvent) { - const cache = await Storages.cache(); - const reqmeister = await Storages.reqmeister(); - - if (!event.author) { - const signal = AbortSignal.timeout(3000); - reqmeister.query([{ kinds: [0], authors: [event.pubkey] }], { signal }) - .then((events) => Promise.allSettled(events.map((event) => handleEvent(event, signal)))) - .catch(() => {}); - } - - for (const [name, id] of event.tags) { - if (name === 'e') { - const { count } = await cache.count([{ ids: [id] }]); - if (!count) { - const signal = AbortSignal.timeout(3000); - reqmeister.query([{ ids: [id] }], { signal }) - .then((events) => Promise.allSettled(events.map((event) => handleEvent(event, signal)))) - .catch(() => {}); - } - } - } -} - /** Delete unattached media entries that are attached to the event. */ function processMedia({ tags, pubkey, user }: DittoEvent) { if (user) { diff --git a/src/queries.ts b/src/queries.ts index 7407077e..1fccb68d 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -25,7 +25,7 @@ const getEvent = async ( opts: GetEventOpts = {}, ): Promise => { debug(`getEvent: ${id}`); - const store = await Storages.optimizer(); + const store = await Storages.db(); const { kind, signal = AbortSignal.timeout(1000) } = opts; const filter: NostrFilter = { ids: [id], limit: 1 }; @@ -40,7 +40,7 @@ const getEvent = async ( /** Get a Nostr `set_medatadata` event for a user's pubkey. */ const getAuthor = async (pubkey: string, opts: GetEventOpts = {}): Promise => { - const store = await Storages.optimizer(); + const store = await Storages.db(); const { signal = AbortSignal.timeout(1000) } = opts; return await store.query([{ authors: [pubkey], kinds: [0], limit: 1 }], { limit: 1, signal }) diff --git a/src/stats.ts b/src/stats.ts deleted file mode 100644 index 6ffe5f7e..00000000 --- a/src/stats.ts +++ /dev/null @@ -1,273 +0,0 @@ -import { Semaphore } from '@lambdalisue/async'; -import { NKinds, NostrEvent, NStore } from '@nostrify/nostrify'; -import Debug from '@soapbox/stickynotes/debug'; -import { InsertQueryBuilder, Kysely } from 'kysely'; -import { LRUCache } from 'lru-cache'; -import { SetRequired } from 'type-fest'; - -import { DittoDB } from '@/db/DittoDB.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; -import { Storages } from '@/storages.ts'; -import { findReplyTag, getTagSet } from '@/utils/tags.ts'; - -type AuthorStat = keyof Omit; -type EventStat = keyof Omit; - -type AuthorStatDiff = ['author_stats', pubkey: string, stat: AuthorStat, diff: number]; -type EventStatDiff = ['event_stats', eventId: string, stat: EventStat, diff: number]; -type StatDiff = AuthorStatDiff | EventStatDiff; - -const debug = Debug('ditto:stats'); - -/** Store stats for the event. */ -async function updateStats(event: NostrEvent) { - let prev: NostrEvent | undefined; - const queries: InsertQueryBuilder[] = []; - - // Kind 3 is a special case - replace the count with the new list. - if (event.kind === 3) { - prev = await getPrevEvent(event); - if (!prev || event.created_at >= prev.created_at) { - queries.push(await updateFollowingCountQuery(event)); - } - } - - const statDiffs = await getStatsDiff(event, prev); - const pubkeyDiffs = statDiffs.filter(([table]) => table === 'author_stats') as AuthorStatDiff[]; - const eventDiffs = statDiffs.filter(([table]) => table === 'event_stats') as EventStatDiff[]; - - if (statDiffs.length) { - debug(JSON.stringify({ id: event.id, pubkey: event.pubkey, kind: event.kind, tags: event.tags, statDiffs })); - } - - pubkeyDiffs.forEach(([_, pubkey]) => refreshAuthorStatsDebounced(pubkey)); - - const kysely = await DittoDB.getInstance(); - - if (pubkeyDiffs.length) queries.push(authorStatsQuery(kysely, pubkeyDiffs)); - if (eventDiffs.length) queries.push(eventStatsQuery(kysely, eventDiffs)); - - if (queries.length) { - await Promise.all(queries.map((query) => query.execute())); - } -} - -/** Calculate stats changes ahead of time so we can build an efficient query. */ -async function getStatsDiff(event: NostrEvent, prev: NostrEvent | undefined): Promise { - const store = await Storages.db(); - const statDiffs: StatDiff[] = []; - - const firstTaggedId = event.tags.find(([name]) => name === 'e')?.[1]; - const inReplyToId = findReplyTag(event.tags)?.[1]; - - switch (event.kind) { - case 1: - statDiffs.push(['author_stats', event.pubkey, 'notes_count', 1]); - if (inReplyToId) { - statDiffs.push(['event_stats', inReplyToId, 'replies_count', 1]); - } - break; - case 3: - statDiffs.push(...getFollowDiff(event, prev)); - break; - case 5: { - if (!firstTaggedId) break; - - const [repostedEvent] = await store.query( - [{ kinds: [6], ids: [firstTaggedId], authors: [event.pubkey] }], - { limit: 1 }, - ); - // Check if the event being deleted is of kind 6, - // if it is then proceed, else just break - if (!repostedEvent) break; - - const eventBeingRepostedId = repostedEvent.tags.find(([name]) => name === 'e')?.[1]; - const eventBeingRepostedPubkey = repostedEvent.tags.find(([name]) => name === 'p')?.[1]; - if (!eventBeingRepostedId || !eventBeingRepostedPubkey) break; - - const [eventBeingReposted] = await store.query( - [{ kinds: [1], ids: [eventBeingRepostedId], authors: [eventBeingRepostedPubkey] }], - { limit: 1 }, - ); - if (!eventBeingReposted) break; - - statDiffs.push(['event_stats', eventBeingRepostedId, 'reposts_count', -1]); - break; - } - case 6: - if (firstTaggedId) { - statDiffs.push(['event_stats', firstTaggedId, 'reposts_count', 1]); - } - break; - case 7: - if (firstTaggedId) { - statDiffs.push(['event_stats', firstTaggedId, 'reactions_count', 1]); - } - } - - return statDiffs; -} - -/** Create an author stats query from the list of diffs. */ -function authorStatsQuery(kysely: Kysely, diffs: AuthorStatDiff[]) { - const values: DittoTables['author_stats'][] = diffs.map(([_, pubkey, stat, diff]) => { - const row: DittoTables['author_stats'] = { - pubkey, - followers_count: 0, - following_count: 0, - notes_count: 0, - }; - row[stat] = diff; - return row; - }); - - return kysely.insertInto('author_stats') - .values(values) - .onConflict((oc) => - oc - .column('pubkey') - .doUpdateSet((eb) => ({ - followers_count: eb('author_stats.followers_count', '+', eb.ref('excluded.followers_count')), - following_count: eb('author_stats.following_count', '+', eb.ref('excluded.following_count')), - notes_count: eb('author_stats.notes_count', '+', eb.ref('excluded.notes_count')), - })) - ); -} - -/** Create an event stats query from the list of diffs. */ -function eventStatsQuery(kysely: Kysely, diffs: EventStatDiff[]) { - const values: DittoTables['event_stats'][] = diffs.map(([_, event_id, stat, diff]) => { - const row: DittoTables['event_stats'] = { - event_id, - replies_count: 0, - reposts_count: 0, - reactions_count: 0, - }; - row[stat] = diff; - return row; - }); - - return kysely.insertInto('event_stats') - .values(values) - .onConflict((oc) => - oc - .column('event_id') - .doUpdateSet((eb) => ({ - replies_count: eb('event_stats.replies_count', '+', eb.ref('excluded.replies_count')), - reposts_count: eb('event_stats.reposts_count', '+', eb.ref('excluded.reposts_count')), - reactions_count: eb('event_stats.reactions_count', '+', eb.ref('excluded.reactions_count')), - })) - ); -} - -/** Get the last version of the event, if any. */ -async function getPrevEvent(event: NostrEvent): Promise { - if (NKinds.replaceable(event.kind) || NKinds.parameterizedReplaceable(event.kind)) { - const store = await Storages.db(); - - const [prev] = await store.query([ - { kinds: [event.kind], authors: [event.pubkey], limit: 1 }, - ]); - - return prev; - } -} - -/** Set the following count to the total number of unique "p" tags in the follow list. */ -async function updateFollowingCountQuery({ pubkey, tags }: NostrEvent) { - const following_count = new Set( - tags - .filter(([name]) => name === 'p') - .map(([_, value]) => value), - ).size; - - const kysely = await DittoDB.getInstance(); - return kysely.insertInto('author_stats') - .values({ - pubkey, - following_count, - followers_count: 0, - notes_count: 0, - }) - .onConflict((oc) => - oc - .column('pubkey') - .doUpdateSet({ following_count }) - ); -} - -/** Compare the old and new follow events (if any), and return a diff array. */ -function getFollowDiff(event: NostrEvent, prev?: NostrEvent): AuthorStatDiff[] { - const prevTags = prev?.tags ?? []; - - const prevPubkeys = new Set( - prevTags - .filter(([name]) => name === 'p') - .map(([_, value]) => value), - ); - - const pubkeys = new Set( - event.tags - .filter(([name]) => name === 'p') - .map(([_, value]) => value), - ); - - const added = [...pubkeys].filter((pubkey) => !prevPubkeys.has(pubkey)); - const removed = [...prevPubkeys].filter((pubkey) => !pubkeys.has(pubkey)); - - return [ - ...added.map((pubkey): AuthorStatDiff => ['author_stats', pubkey, 'followers_count', 1]), - ...removed.map((pubkey): AuthorStatDiff => ['author_stats', pubkey, 'followers_count', -1]), - ]; -} - -/** Refresh the author's stats in the database. */ -async function refreshAuthorStats(pubkey: string): Promise { - const store = await Storages.db(); - const stats = await countAuthorStats(store, pubkey); - - const kysely = await DittoDB.getInstance(); - await kysely.insertInto('author_stats') - .values(stats) - .onConflict((oc) => oc.column('pubkey').doUpdateSet(stats)) - .execute(); - - return stats; -} - -/** Calculate author stats from the database. */ -async function countAuthorStats( - store: SetRequired, - pubkey: string, -): Promise { - const [{ count: followers_count }, { count: notes_count }, [followList]] = await Promise.all([ - store.count([{ kinds: [3], '#p': [pubkey] }]), - store.count([{ kinds: [1], authors: [pubkey] }]), - store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]), - ]); - - return { - pubkey, - followers_count, - following_count: getTagSet(followList?.tags ?? [], 'p').size, - notes_count, - }; -} - -const authorStatsSemaphore = new Semaphore(10); -const refreshedAuthors = new LRUCache({ max: 1000 }); - -/** Calls `refreshAuthorStats` only once per author. */ -function refreshAuthorStatsDebounced(pubkey: string): void { - if (refreshedAuthors.get(pubkey)) { - return; - } - - refreshedAuthors.set(pubkey, true); - debug('refreshing author stats:', pubkey); - - authorStatsSemaphore - .lock(() => refreshAuthorStats(pubkey).catch(() => {})); -} - -export { refreshAuthorStats, refreshAuthorStatsDebounced, updateStats }; diff --git a/src/storages.ts b/src/storages.ts index 10d5b05a..f8f206d1 100644 --- a/src/storages.ts +++ b/src/storages.ts @@ -1,25 +1,18 @@ // deno-lint-ignore-file require-await -import { NCache } from '@nostrify/nostrify'; import { RelayPoolWorker } from 'nostr-relaypool'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; -import { Optimizer } from '@/storages/optimizer.ts'; import { PoolStore } from '@/storages/pool-store.ts'; -import { Reqmeister } from '@/storages/reqmeister.ts'; import { SearchStore } from '@/storages/search-store.ts'; import { InternalRelay } from '@/storages/InternalRelay.ts'; import { UserStore } from '@/storages/UserStore.ts'; -import { Time } from '@/utils/time.ts'; export class Storages { private static _db: Promise | undefined; private static _admin: Promise | undefined; - private static _cache: Promise | undefined; private static _client: Promise | undefined; - private static _optimizer: Promise | undefined; - private static _reqmeister: Promise | undefined; private static _pubsub: Promise | undefined; private static _search: Promise | undefined; @@ -93,49 +86,13 @@ export class Storages { return this._client; } - /** In-memory data store for cached events. */ - public static async cache(): Promise { - if (!this._cache) { - this._cache = Promise.resolve(new NCache({ max: 3000 })); - } - return this._cache; - } - - /** Batches requests for single events. */ - public static async reqmeister(): Promise { - if (!this._reqmeister) { - this._reqmeister = Promise.resolve( - new Reqmeister({ - client: await this.client(), - delay: Time.seconds(1), - timeout: Time.seconds(1), - }), - ); - } - return this._reqmeister; - } - - /** Main Ditto storage adapter */ - public static async optimizer(): Promise { - if (!this._optimizer) { - this._optimizer = Promise.resolve( - new Optimizer({ - db: await this.db(), - cache: await this.cache(), - client: await this.reqmeister(), - }), - ); - } - return this._optimizer; - } - /** Storage to use for remote search. */ public static async search(): Promise { if (!this._search) { this._search = Promise.resolve( new SearchStore({ relay: Conf.searchRelay, - fallback: await this.optimizer(), + fallback: await this.db(), }), ); } diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 63d831fe..778a9e68 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -45,17 +45,8 @@ class EventsDB implements NStore { }; constructor(private kysely: Kysely) { - let fts: 'sqlite' | 'postgres' | undefined; - - if (Conf.databaseUrl.protocol === 'sqlite:') { - fts = 'sqlite'; - } - if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { - fts = 'postgres'; - } - this.store = new NDatabase(kysely, { - fts, + fts: Conf.db.dialect, indexTags: EventsDB.indexTags, searchText: EventsDB.searchText, }); diff --git a/src/storages/hydrate.bench.ts b/src/storages/hydrate.bench.ts new file mode 100644 index 00000000..eeacec50 --- /dev/null +++ b/src/storages/hydrate.bench.ts @@ -0,0 +1,13 @@ +import { assembleEvents } from '@/storages/hydrate.ts'; +import { jsonlEvents } from '@/test.ts'; + +const testEvents = await jsonlEvents('fixtures/hydrated.jsonl'); +const testStats = JSON.parse(await Deno.readTextFile('fixtures/stats.json')); + +// The first 20 events in this file are my home feed. +// The rest are events that would be hydrated by the store. +const events = testEvents.slice(0, 20); + +Deno.bench('assembleEvents with home feed', () => { + assembleEvents(events, testEvents, testStats); +}); diff --git a/src/storages/hydrate.test.ts b/src/storages/hydrate.test.ts index 1edafd7e..e3b1cf2e 100644 --- a/src/storages/hydrate.test.ts +++ b/src/storages/hydrate.test.ts @@ -1,8 +1,8 @@ -import { assertEquals } from '@std/assert'; -import { hydrateEvents } from '@/storages/hydrate.ts'; import { MockRelay } from '@nostrify/nostrify/test'; +import { assertEquals } from '@std/assert'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; +import { hydrateEvents } from '@/storages/hydrate.ts'; import { eventFixture } from '@/test.ts'; Deno.test('hydrateEvents(): author --- WITHOUT stats', async () => { diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 68dc0bdb..1f56590e 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -2,10 +2,9 @@ import { NostrEvent, NStore } from '@nostrify/nostrify'; import { matchFilter } from 'nostr-tools'; import { DittoDB } from '@/db/DittoDB.ts'; -import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Conf } from '@/config.ts'; -import { refreshAuthorStatsDebounced } from '@/stats.ts'; +import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { findQuoteTag } from '@/utils/tags.ts'; interface HydrateOpts { @@ -57,8 +56,6 @@ async function hydrateEvents(opts: HydrateOpts): Promise { events: await gatherEventStats(cache), }; - refreshMissingAuthorStats(events, stats.authors); - // Dedupe events. const results = [...new Map(cache.map((event) => [event.id, event])).values()]; @@ -70,13 +67,18 @@ async function hydrateEvents(opts: HydrateOpts): Promise { } /** Connect the events in list `b` to the DittoEvent fields in list `a`. */ -function assembleEvents( +export function assembleEvents( a: DittoEvent[], b: DittoEvent[], stats: { authors: DittoTables['author_stats'][]; events: DittoTables['event_stats'][] }, ): DittoEvent[] { const admin = Conf.pubkey; + const eventStats = stats.events.map((stat) => ({ + ...stat, + reactions: JSON.parse(stat.reactions), + })); + for (const event of a) { event.author = b.find((e) => matchFilter({ kinds: [0], authors: [event.pubkey] }, e)); event.user = b.find((e) => matchFilter({ kinds: [30361], authors: [admin], '#d': [event.pubkey] }, e)); @@ -120,7 +122,7 @@ function assembleEvents( } event.author_stats = stats.authors.find((stats) => stats.pubkey === event.pubkey); - event.event_stats = stats.events.find((stats) => stats.event_id === event.id); + event.event_stats = eventStats.find((stats) => stats.event_id === event.id); } return a; @@ -270,22 +272,6 @@ async function gatherAuthorStats(events: DittoEvent[]): Promise( - events - .filter((event) => event.kind === 0) - .map((event) => event.pubkey), - ); - - const missing = pubkeys.difference( - new Set(stats.map((stat) => stat.pubkey)), - ); - - for (const pubkey of missing) { - refreshAuthorStatsDebounced(pubkey); - } -} - /** Collect event stats from the events. */ async function gatherEventStats(events: DittoEvent[]): Promise { const ids = new Set( @@ -309,8 +295,9 @@ async function gatherEventStats(events: DittoEvent[]): Promise ({ event_id: row.event_id, reposts_count: Math.max(0, row.reposts_count), - reactions_count: Math.max(0, row.reactions_count), replies_count: Math.max(0, row.replies_count), + reactions_count: Math.max(0, row.reactions_count), + reactions: row.reactions, })); } diff --git a/src/storages/optimizer.ts b/src/storages/optimizer.ts deleted file mode 100644 index 7b4153e9..00000000 --- a/src/storages/optimizer.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { NostrFilter, NSet, NStore } from '@nostrify/nostrify'; -import Debug from '@soapbox/stickynotes/debug'; - -import { normalizeFilters } from '@/filter.ts'; -import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { abortError } from '@/utils/abort.ts'; - -interface OptimizerOpts { - db: NStore; - cache: NStore; - client: NStore; -} - -class Optimizer implements NStore { - #debug = Debug('ditto:optimizer'); - - #db: NStore; - #cache: NStore; - #client: NStore; - - constructor(opts: OptimizerOpts) { - this.#db = opts.db; - this.#cache = opts.cache; - this.#client = opts.client; - } - - async event(event: DittoEvent, opts?: { signal?: AbortSignal }): Promise { - if (opts?.signal?.aborted) return Promise.reject(abortError()); - - await Promise.all([ - this.#db.event(event, opts), - this.#cache.event(event, opts), - ]); - } - - async query(filters: NostrFilter[], opts: { signal?: AbortSignal; limit?: number } = {}): Promise { - if (opts?.signal?.aborted) return Promise.reject(abortError()); - - filters = normalizeFilters(filters); - this.#debug('REQ', JSON.stringify(filters)); - if (!filters.length) return Promise.resolve([]); - - const { limit = Infinity } = opts; - const results = new NSet(); - - // Filters with IDs are immutable, so we can take them straight from the cache if we have them. - for (let i = 0; i < filters.length; i++) { - const filter = filters[i]; - if (filter.ids) { - this.#debug(`Filter[${i}] is an IDs filter; querying cache...`); - const ids = new Set(filter.ids); - for (const event of await this.#cache.query([filter], opts)) { - ids.delete(event.id); - results.add(event); - if (results.size >= limit) return getResults(); - } - filters[i] = { ...filter, ids: [...ids] }; - } - } - - filters = normalizeFilters(filters); - if (!filters.length) return getResults(); - - // Query the database for events. - this.#debug('Querying database...'); - for (const dbEvent of await this.#db.query(filters, opts)) { - results.add(dbEvent); - if (results.size >= limit) return getResults(); - } - - // We already searched the DB, so stop if this is a search filter. - if (filters.some((filter) => typeof filter.search === 'string')) { - this.#debug(`Bailing early for search filter: "${filters[0]?.search}"`); - return getResults(); - } - - // Query the cache again. - this.#debug('Querying cache...'); - for (const cacheEvent of await this.#cache.query(filters, opts)) { - results.add(cacheEvent); - if (results.size >= limit) return getResults(); - } - - // Finally, query the client. - this.#debug('Querying client...'); - try { - for (const clientEvent of await this.#client.query(filters, opts)) { - results.add(clientEvent); - if (results.size >= limit) return getResults(); - } - } catch (_e) { - // do nothing - } - - /** Get return type from map. */ - function getResults() { - return [...results.values()]; - } - - return getResults(); - } -} - -export { Optimizer }; diff --git a/src/storages/reqmeister.ts b/src/storages/reqmeister.ts deleted file mode 100644 index e3833d37..00000000 --- a/src/storages/reqmeister.ts +++ /dev/null @@ -1,144 +0,0 @@ -import { NostrEvent, NostrFilter, NStore } from '@nostrify/nostrify'; -import Debug from '@soapbox/stickynotes/debug'; -import { EventEmitter } from 'tseep'; - -import { eventToMicroFilter, getFilterId, isMicrofilter, type MicroFilter } from '@/filter.ts'; -import { Time } from '@/utils/time.ts'; -import { abortError } from '@/utils/abort.ts'; - -interface ReqmeisterOpts { - client: NStore; - delay?: number; - timeout?: number; -} - -interface ReqmeisterReqOpts { - relays?: WebSocket['url'][]; - signal?: AbortSignal; -} - -type ReqmeisterQueueItem = [string, MicroFilter, WebSocket['url'][]]; - -/** Batches requests to Nostr relays using microfilters. */ -class Reqmeister extends EventEmitter<{ [filterId: string]: (event: NostrEvent) => any }> implements NStore { - #debug = Debug('ditto:reqmeister'); - - #opts: ReqmeisterOpts; - #queue: ReqmeisterQueueItem[] = []; - #promise!: Promise; - #resolve!: () => void; - - constructor(opts: ReqmeisterOpts) { - super(); - this.#opts = opts; - this.#tick(); - this.#perform(); - } - - #tick() { - this.#resolve?.(); - this.#promise = new Promise((resolve) => { - this.#resolve = resolve; - }); - } - - async #perform() { - const { client, delay, timeout = Time.seconds(1) } = this.#opts; - await new Promise((resolve) => setTimeout(resolve, delay)); - - const queue = this.#queue; - this.#queue = []; - - const wantedEvents = new Set(); - const wantedAuthors = new Set(); - - // TODO: batch by relays. - for (const [_filterId, filter, _relays] of queue) { - if ('ids' in filter) { - filter.ids.forEach((id) => wantedEvents.add(id)); - } else { - wantedAuthors.add(filter.authors[0]); - } - } - - const filters: NostrFilter[] = []; - - if (wantedEvents.size) filters.push({ ids: [...wantedEvents] }); - if (wantedAuthors.size) filters.push({ kinds: [0], authors: [...wantedAuthors] }); - - if (filters.length) { - try { - const events = await client.query(filters, { signal: AbortSignal.timeout(timeout) }); - - for (const event of events) { - this.event(event); - } - } catch (_e) { - // do nothing - } - } - - this.#tick(); - this.#perform(); - } - - private fetch(filter: MicroFilter, opts: ReqmeisterReqOpts = {}): Promise { - const { - relays = [], - signal = AbortSignal.timeout(this.#opts.timeout ?? 1000), - } = opts; - - if (signal.aborted) { - return Promise.reject(abortError()); - } - - const filterId = getFilterId(filter); - - this.#queue.push([filterId, filter, relays]); - - return new Promise((resolve, reject) => { - const handleEvent = (event: NostrEvent) => { - resolve(event); - this.removeListener(filterId, handleEvent); - }; - - const handleAbort = () => { - reject(new DOMException('Aborted', 'AbortError')); - this.removeListener(filterId, resolve); - signal.removeEventListener('abort', handleAbort); - }; - - this.once(filterId, handleEvent); - signal.addEventListener('abort', handleAbort, { once: true }); - }); - } - - event(event: NostrEvent, _opts?: { signal?: AbortSignal }): Promise { - const filterId = getFilterId(eventToMicroFilter(event)); - this.#queue = this.#queue.filter(([id]) => id !== filterId); - this.emit(filterId, event); - return Promise.resolve(); - } - - async query(filters: NostrFilter[], opts?: { signal?: AbortSignal }): Promise { - if (opts?.signal?.aborted) return Promise.reject(abortError()); - - this.#debug('REQ', JSON.stringify(filters)); - if (!filters.length) return Promise.resolve([]); - - const promises = filters.reduce[]>((result, filter) => { - if (isMicrofilter(filter)) { - result.push(this.fetch(filter, opts)); - } - return result; - }, []); - - const results = await Promise.allSettled(promises); - - return results - .filter((result): result is PromiseFulfilledResult => result.status === 'fulfilled') - .map((result) => result.value); - } -} - -export { Reqmeister }; diff --git a/src/test.ts b/src/test.ts index ea9c8fa4..b0172063 100644 --- a/src/test.ts +++ b/src/test.ts @@ -1,6 +1,13 @@ -import { NostrEvent } from '@nostrify/nostrify'; +import fs from 'node:fs/promises'; +import path from 'node:path'; + +import { Database as Sqlite } from '@db/sqlite'; +import { NDatabase, NostrEvent } from '@nostrify/nostrify'; +import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite'; +import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; import { finalizeEvent, generateSecretKey } from 'nostr-tools'; +import { DittoTables } from '@/db/DittoTables.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; /** Import an event fixture by name in tests. */ @@ -9,6 +16,12 @@ export async function eventFixture(name: string): Promise { return structuredClone(result.default); } +/** Import a JSONL fixture by name in tests. */ +export async function jsonlEvents(path: string): Promise { + const data = await Deno.readTextFile(path); + return data.split('\n').map((line) => JSON.parse(line)); +} + /** Generate an event for use in tests. */ export function genEvent(t: Partial = {}, sk: Uint8Array = generateSecretKey()): NostrEvent { const event = finalizeEvent({ @@ -21,3 +34,31 @@ export function genEvent(t: Partial = {}, sk: Uint8Array = generateS return purifyEvent(event); } + +/** Get an in-memory SQLite database to use for testing. It's automatically destroyed when it goes out of scope. */ +export async function getTestDB() { + const kysely = new Kysely({ + dialect: new DenoSqlite3Dialect({ + database: new Sqlite(':memory:'), + }), + }); + + const migrator = new Migrator({ + db: kysely, + provider: new FileMigrationProvider({ + fs, + path, + migrationFolder: new URL(import.meta.resolve('./db/migrations')).pathname, + }), + }); + + await migrator.migrateToLatest(); + + const store = new NDatabase(kysely); + + return { + store, + kysely, + [Symbol.asyncDispose]: () => kysely.destroy(), + }; +} diff --git a/src/utils/stats.test.ts b/src/utils/stats.test.ts new file mode 100644 index 00000000..5f57dc4d --- /dev/null +++ b/src/utils/stats.test.ts @@ -0,0 +1,160 @@ +import { assertEquals } from '@std/assert'; +import { generateSecretKey, getPublicKey } from 'nostr-tools'; + +import { genEvent, getTestDB } from '@/test.ts'; +import { countAuthorStats, getAuthorStats, getEventStats, getFollowDiff, updateStats } from '@/utils/stats.ts'; + +Deno.test('updateStats with kind 1 increments notes count', async () => { + await using db = await getTestDB(); + + const sk = generateSecretKey(); + const pubkey = getPublicKey(sk); + + await updateStats({ ...db, event: genEvent({ kind: 1 }, sk) }); + + const stats = await getAuthorStats(db.kysely, pubkey); + + assertEquals(stats!.notes_count, 1); +}); + +Deno.test('updateStats with kind 5 decrements notes count', async () => { + await using db = await getTestDB(); + + const sk = generateSecretKey(); + const pubkey = getPublicKey(sk); + + const create = genEvent({ kind: 1 }, sk); + const remove = genEvent({ kind: 5, tags: [['e', create.id]] }, sk); + + await updateStats({ ...db, event: create }); + assertEquals((await getAuthorStats(db.kysely, pubkey))!.notes_count, 1); + await db.store.event(create); + + await updateStats({ ...db, event: remove }); + assertEquals((await getAuthorStats(db.kysely, pubkey))!.notes_count, 0); + await db.store.event(remove); +}); + +Deno.test('updateStats with kind 3 increments followers count', async () => { + await using db = await getTestDB(); + + await updateStats({ ...db, event: genEvent({ kind: 3, tags: [['p', 'alex']] }) }); + await updateStats({ ...db, event: genEvent({ kind: 3, tags: [['p', 'alex']] }) }); + await updateStats({ ...db, event: genEvent({ kind: 3, tags: [['p', 'alex']] }) }); + + const stats = await getAuthorStats(db.kysely, 'alex'); + + assertEquals(stats!.followers_count, 3); +}); + +Deno.test('updateStats with kind 3 decrements followers count', async () => { + await using db = await getTestDB(); + + const sk = generateSecretKey(); + const follow = genEvent({ kind: 3, tags: [['p', 'alex']], created_at: 0 }, sk); + const remove = genEvent({ kind: 3, tags: [], created_at: 1 }, sk); + + await updateStats({ ...db, event: follow }); + assertEquals((await getAuthorStats(db.kysely, 'alex'))!.followers_count, 1); + await db.store.event(follow); + + await updateStats({ ...db, event: remove }); + assertEquals((await getAuthorStats(db.kysely, 'alex'))!.followers_count, 0); + await db.store.event(remove); +}); + +Deno.test('getFollowDiff returns added and removed followers', () => { + const prev = genEvent({ tags: [['p', 'alex'], ['p', 'bob']] }); + const next = genEvent({ tags: [['p', 'alex'], ['p', 'carol']] }); + + const { added, removed } = getFollowDiff(next.tags, prev.tags); + + assertEquals(added, new Set(['carol'])); + assertEquals(removed, new Set(['bob'])); +}); + +Deno.test('updateStats with kind 6 increments reposts count', async () => { + await using db = await getTestDB(); + + const note = genEvent({ kind: 1 }); + await updateStats({ ...db, event: note }); + await db.store.event(note); + + const repost = genEvent({ kind: 6, tags: [['e', note.id]] }); + await updateStats({ ...db, event: repost }); + await db.store.event(repost); + + const stats = await getEventStats(db.kysely, note.id); + + assertEquals(stats!.reposts_count, 1); +}); + +Deno.test('updateStats with kind 5 decrements reposts count', async () => { + await using db = await getTestDB(); + + const note = genEvent({ kind: 1 }); + await updateStats({ ...db, event: note }); + await db.store.event(note); + + const sk = generateSecretKey(); + const repost = genEvent({ kind: 6, tags: [['e', note.id]] }, sk); + await updateStats({ ...db, event: repost }); + await db.store.event(repost); + + await updateStats({ ...db, event: genEvent({ kind: 5, tags: [['e', repost.id]] }, sk) }); + + const stats = await getEventStats(db.kysely, note.id); + + assertEquals(stats!.reposts_count, 0); +}); + +Deno.test('updateStats with kind 7 increments reactions count', async () => { + await using db = await getTestDB(); + + const note = genEvent({ kind: 1 }); + await updateStats({ ...db, event: note }); + await db.store.event(note); + + await updateStats({ ...db, event: genEvent({ kind: 7, content: '+', tags: [['e', note.id]] }) }); + await updateStats({ ...db, event: genEvent({ kind: 7, content: '๐Ÿ˜‚', tags: [['e', note.id]] }) }); + + const stats = await getEventStats(db.kysely, note.id); + + assertEquals(stats!.reactions, JSON.stringify({ '+': 1, '๐Ÿ˜‚': 1 })); + assertEquals(stats!.reactions_count, 2); +}); + +Deno.test('updateStats with kind 5 decrements reactions count', async () => { + await using db = await getTestDB(); + + const note = genEvent({ kind: 1 }); + await updateStats({ ...db, event: note }); + await db.store.event(note); + + const sk = generateSecretKey(); + const reaction = genEvent({ kind: 7, content: '+', tags: [['e', note.id]] }, sk); + await updateStats({ ...db, event: reaction }); + await db.store.event(reaction); + + await updateStats({ ...db, event: genEvent({ kind: 5, tags: [['e', reaction.id]] }, sk) }); + + const stats = await getEventStats(db.kysely, note.id); + + assertEquals(stats!.reactions, JSON.stringify({})); +}); + +Deno.test('countAuthorStats counts author stats from the database', async () => { + await using db = await getTestDB(); + + const sk = generateSecretKey(); + const pubkey = getPublicKey(sk); + + await db.store.event(genEvent({ kind: 1, content: 'hello' }, sk)); + await db.store.event(genEvent({ kind: 1, content: 'yolo' }, sk)); + await db.store.event(genEvent({ kind: 3, tags: [['p', pubkey]] })); + + const stats = await countAuthorStats(db.store, pubkey); + + assertEquals(stats!.notes_count, 2); + assertEquals(stats!.followers_count, 1); +}); diff --git a/src/utils/stats.ts b/src/utils/stats.ts new file mode 100644 index 00000000..62135c0f --- /dev/null +++ b/src/utils/stats.ts @@ -0,0 +1,266 @@ +import { NostrEvent, NStore } from '@nostrify/nostrify'; +import { Kysely, UpdateObject } from 'kysely'; +import { SetRequired } from 'type-fest'; + +import { DittoTables } from '@/db/DittoTables.ts'; +import { getTagSet } from '@/utils/tags.ts'; +import { Conf } from '@/config.ts'; + +interface UpdateStatsOpts { + kysely: Kysely; + store: NStore; + event: NostrEvent; + x?: 1 | -1; +} + +/** Handle one event at a time and update relevant stats for it. */ +// deno-lint-ignore require-await +export async function updateStats({ event, kysely, store, x = 1 }: UpdateStatsOpts): Promise { + switch (event.kind) { + case 1: + return handleEvent1(kysely, event, x); + case 3: + return handleEvent3(kysely, event, x, store); + case 5: + return handleEvent5(kysely, event, -1, store); + case 6: + return handleEvent6(kysely, event, x); + case 7: + return handleEvent7(kysely, event, x); + } +} + +/** Update stats for kind 1 event. */ +async function handleEvent1(kysely: Kysely, event: NostrEvent, x: number): Promise { + await updateAuthorStats(kysely, event.pubkey, ({ notes_count }) => ({ notes_count: Math.max(0, notes_count + x) })); +} + +/** Update stats for kind 3 event. */ +async function handleEvent3(kysely: Kysely, event: NostrEvent, x: number, store: NStore): Promise { + const following = getTagSet(event.tags, 'p'); + + await updateAuthorStats(kysely, event.pubkey, () => ({ following_count: following.size })); + + const [prev] = await store.query([ + { kinds: [3], authors: [event.pubkey], limit: 1 }, + ]); + + const { added, removed } = getFollowDiff(event.tags, prev?.tags); + + for (const pubkey of added) { + await updateAuthorStats( + kysely, + pubkey, + ({ followers_count }) => ({ followers_count: Math.max(0, followers_count + x) }), + ); + } + + for (const pubkey of removed) { + await updateAuthorStats( + kysely, + pubkey, + ({ followers_count }) => ({ followers_count: Math.max(0, followers_count - x) }), + ); + } +} + +/** Update stats for kind 5 event. */ +async function handleEvent5(kysely: Kysely, event: NostrEvent, x: -1, store: NStore): Promise { + const id = event.tags.find(([name]) => name === 'e')?.[1]; + if (id) { + const [target] = await store.query([{ ids: [id], authors: [event.pubkey], limit: 1 }]); + if (target) { + await updateStats({ event: target, kysely, store, x }); + } + } +} + +/** Update stats for kind 6 event. */ +async function handleEvent6(kysely: Kysely, event: NostrEvent, x: number): Promise { + const id = event.tags.find(([name]) => name === 'e')?.[1]; + if (id) { + await updateEventStats(kysely, id, ({ reposts_count }) => ({ reposts_count: Math.max(0, reposts_count + x) })); + } +} + +/** Update stats for kind 7 event. */ +async function handleEvent7(kysely: Kysely, event: NostrEvent, x: number): Promise { + const id = event.tags.find(([name]) => name === 'e')?.[1]; + const emoji = event.content; + + if (id && emoji && (['+', '-'].includes(emoji) || /^\p{RGI_Emoji}$/v.test(emoji))) { + await updateEventStats(kysely, id, ({ reactions }) => { + const data: Record = JSON.parse(reactions); + + // Increment or decrement the emoji count. + data[emoji] = (data[emoji] ?? 0) + x; + + // Remove reactions with a count of 0 or less. + for (const key of Object.keys(data)) { + if (data[key] < 1) { + delete data[key]; + } + } + + // Total reactions count. + const count = Object.values(data).reduce((result, value) => result + value, 0); + + return { + reactions: JSON.stringify(data), + reactions_count: count, + }; + }); + } +} + +/** Get the pubkeys that were added and removed from a follow event. */ +export function getFollowDiff( + tags: string[][], + prevTags: string[][] = [], +): { added: Set; removed: Set } { + const pubkeys = getTagSet(tags, 'p'); + const prevPubkeys = getTagSet(prevTags, 'p'); + + return { + added: pubkeys.difference(prevPubkeys), + removed: prevPubkeys.difference(pubkeys), + }; +} + +/** Retrieve the author stats by the pubkey. */ +export function getAuthorStats( + kysely: Kysely, + pubkey: string, +): Promise { + return kysely + .selectFrom('author_stats') + .selectAll() + .where('pubkey', '=', pubkey) + .executeTakeFirst(); +} + +/** Retrieve the author stats by the pubkey, then call the callback to update it. */ +export async function updateAuthorStats( + kysely: Kysely, + pubkey: string, + fn: (prev: DittoTables['author_stats']) => UpdateObject, +): Promise { + const empty: DittoTables['author_stats'] = { + pubkey, + followers_count: 0, + following_count: 0, + notes_count: 0, + }; + + let query = kysely + .selectFrom('author_stats') + .selectAll() + .where('pubkey', '=', pubkey); + + if (Conf.db.dialect === 'postgres') { + query = query.forUpdate(); + } + + const prev = await query.executeTakeFirst(); + const stats = fn(prev ?? empty); + + if (prev) { + await kysely.updateTable('author_stats') + .set(stats) + .where('pubkey', '=', pubkey) + .execute(); + } else { + await kysely.insertInto('author_stats') + .values({ ...empty, ...stats }) + .execute(); + } +} + +/** Retrieve the event stats by the event ID. */ +export function getEventStats( + kysely: Kysely, + eventId: string, +): Promise { + return kysely + .selectFrom('event_stats') + .selectAll() + .where('event_id', '=', eventId) + .executeTakeFirst(); +} + +/** Retrieve the event stats by the event ID, then call the callback to update it. */ +export async function updateEventStats( + kysely: Kysely, + eventId: string, + fn: (prev: DittoTables['event_stats']) => UpdateObject, +): Promise { + const empty: DittoTables['event_stats'] = { + event_id: eventId, + replies_count: 0, + reposts_count: 0, + reactions_count: 0, + reactions: '{}', + }; + + let query = kysely + .selectFrom('event_stats') + .selectAll() + .where('event_id', '=', eventId); + + if (Conf.db.dialect === 'postgres') { + query = query.forUpdate(); + } + + const prev = await query.executeTakeFirst(); + const stats = fn(prev ?? empty); + + if (prev) { + await kysely.updateTable('event_stats') + .set(stats) + .where('event_id', '=', eventId) + .execute(); + } else { + await kysely.insertInto('event_stats') + .values({ ...empty, ...stats }) + .execute(); + } +} + +/** Calculate author stats from the database. */ +export async function countAuthorStats( + store: SetRequired, + pubkey: string, +): Promise { + const [{ count: followers_count }, { count: notes_count }, [followList]] = await Promise.all([ + store.count([{ kinds: [3], '#p': [pubkey] }]), + store.count([{ kinds: [1], authors: [pubkey] }]), + store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]), + ]); + + return { + pubkey, + followers_count, + following_count: getTagSet(followList?.tags ?? [], 'p').size, + notes_count, + }; +} + +export interface RefreshAuthorStatsOpts { + pubkey: string; + kysely: Kysely; + store: SetRequired; +} + +/** Refresh the author's stats in the database. */ +export async function refreshAuthorStats( + { pubkey, kysely, store }: RefreshAuthorStatsOpts, +): Promise { + const stats = await countAuthorStats(store, pubkey); + + await kysely.insertInto('author_stats') + .values(stats) + .onConflict((oc) => oc.column('pubkey').doUpdateSet(stats)) + .execute(); + + return stats; +} diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index f8294d85..918d03b9 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -43,7 +43,7 @@ async function renderAccount( bot: false, created_at: nostrDate(event.user?.created_at ?? event.created_at).toISOString(), discoverable: true, - display_name: name, + display_name: name ?? '', emojis: renderEmojis(event), fields: [], follow_requests_count: 0, @@ -86,7 +86,7 @@ async function renderAccount( pubkey, lud16, }, - website, + website: website && /^https?:\/\//.test(website) ? website : undefined, }; } diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index cc7cc36b..fe03857f 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -39,10 +39,9 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< ), ]; - const db = await Storages.db(); - const optimizer = await Storages.optimizer(); + const store = await Storages.db(); - const mentionedProfiles = await optimizer.query( + const mentionedProfiles = await store.query( [{ kinds: [0], authors: mentionedPubkeys, limit: mentionedPubkeys.length }], ); @@ -55,7 +54,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< ), firstUrl ? unfurlCardCached(firstUrl) : null, viewerPubkey - ? await db.query([ + ? await store.query([ { kinds: [6], '#e': [event.id], authors: [viewerPubkey], limit: 1 }, { kinds: [7], '#e': [event.id], authors: [viewerPubkey], limit: 1 }, { kinds: [9734], '#e': [event.id], authors: [viewerPubkey], limit: 1 }, @@ -82,6 +81,15 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< const media = imeta.length ? imeta : getMediaLinks(links); + /** Pleroma emoji reactions object. */ + const reactions = Object.entries(event.event_stats?.reactions ?? {}).reduce((acc, [emoji, count]) => { + if (['+', '-'].includes(emoji)) return acc; + acc.push({ name: emoji, count, me: reactionEvent?.content === emoji }); + return acc; + }, [] as { name: string; count: number; me: boolean }[]); + + const expiresAt = new Date(Number(event.tags.find(([name]) => name === 'expiration')?.[1]) * 1000); + return { id: event.id, account, @@ -96,7 +104,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< language: event.tags.find((tag) => tag[0] === 'lang')?.[1] || null, replies_count: event.event_stats?.replies_count ?? 0, reblogs_count: event.event_stats?.reposts_count ?? 0, - favourites_count: event.event_stats?.reactions_count ?? 0, + favourites_count: event.event_stats?.reactions['+'] ?? 0, favourited: reactionEvent?.content === '+', reblogged: Boolean(repostEvent), muted: false, @@ -114,23 +122,22 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< uri: Conf.external(note), url: Conf.external(note), zapped: Boolean(zapEvent), + pleroma: { + emoji_reactions: reactions, + expires_at: !isNaN(expiresAt.getTime()) ? expiresAt.toISOString() : undefined, + }, }; } async function renderReblog(event: DittoEvent, opts: RenderStatusOpts) { const { viewerPubkey } = opts; - - const repostId = event.tags.find(([name]) => name === 'e')?.[1]; - if (!repostId) return; - if (!event.repost) return; + const status = await renderStatus(event, {}); // omit viewerPubkey intentionally const reblog = await renderStatus(event.repost, { viewerPubkey }); return { - id: event.id, - account: event.author ? await renderAccount(event.author) : await accountFromPubkey(event.pubkey), - reblogged: true, + ...status, reblog, }; }