From 6f85c968be5b72aae941b1c2073c588992e8af34 Mon Sep 17 00:00:00 2001 From: Andrew McKenzie Date: Mon, 22 Jan 2024 17:15:05 +0000 Subject: [PATCH 1/2] hex boost support --- Cargo.lock | 641 +++++++--- Cargo.toml | 5 +- boost_manager/Cargo.toml | 52 + boost_manager/README.md | 10 + boost_manager/migrations/1_setup.sql | 27 + boost_manager/migrations/2_meta.sql | 4 + .../migrations/3_activated_hexes.sql | 20 + .../migrations/4_files_processed.sql | 7 + boost_manager/pkg/settings-template.toml | 44 + boost_manager/src/activator.rs | 153 +++ boost_manager/src/db.rs | 222 ++++ boost_manager/src/lib.rs | 20 + boost_manager/src/main.rs | 147 +++ boost_manager/src/settings.rs | 96 ++ boost_manager/src/telemetry.rs | 29 + boost_manager/src/updater.rs | 189 +++ boost_manager/src/watcher.rs | 114 ++ boost_manager/tests/activator_tests.rs | 182 +++ boost_manager/tests/common/mod.rs | 79 ++ boost_manager/tests/updater_tests.rs | 185 +++ boost_manager/tests/watcher_tests.rs | 114 ++ file_store/src/cli/dump.rs | 19 +- file_store/src/file_info.rs | 6 + file_store/src/hex_boost.rs | 9 + file_store/src/lib.rs | 1 + file_store/src/traits/msg_verify.rs | 3 + iot_packet_verifier/src/balances.rs | 2 +- iot_packet_verifier/src/burner.rs | 2 +- iot_packet_verifier/src/daemon.rs | 2 +- iot_packet_verifier/src/pending.rs | 2 +- iot_packet_verifier/src/settings.rs | 2 +- iot_packet_verifier/src/verifier.rs | 2 +- .../tests/integration_tests.rs | 5 +- iot_verifier/src/meta.rs | 6 +- mobile_config/src/boosted_hex_info.rs | 244 ++++ .../src/client/hex_boosting_client.rs | 105 ++ mobile_config/src/client/mod.rs | 1 + mobile_config/src/client/settings.rs | 12 + mobile_config/src/hex_boosting_service.rs | 162 +++ mobile_config/src/lib.rs | 3 + mobile_config/src/main.rs | 13 +- mobile_packet_verifier/src/burner.rs | 2 +- mobile_packet_verifier/src/daemon.rs | 2 +- mobile_packet_verifier/src/settings.rs | 2 +- mobile_verifier/src/boosted_hexes.rs | 41 + mobile_verifier/src/cli/reward_from_db.rs | 12 +- mobile_verifier/src/cli/server.rs | 5 +- mobile_verifier/src/coverage.rs | 167 ++- mobile_verifier/src/reward_shares.rs | 83 +- mobile_verifier/src/rewarder.rs | 37 +- mobile_verifier/tests/common/mod.rs | 6 + mobile_verifier/tests/hex_boosting.rs | 1086 +++++++++++++++++ mobile_verifier/tests/modeled_coverage.rs | 110 +- mobile_verifier/tests/rewarder_poc_dc.rs | 61 +- price/src/cli/check.rs | 3 +- price/src/price_generator.rs | 6 +- solana/Cargo.toml | 2 + solana/src/burn.rs | 414 +++++++ solana/src/lib.rs | 442 +------ solana/src/start_boost.rs | 198 +++ 60 files changed, 4944 insertions(+), 676 deletions(-) create mode 100644 boost_manager/Cargo.toml create mode 100644 boost_manager/README.md create mode 100644 boost_manager/migrations/1_setup.sql create mode 100644 boost_manager/migrations/2_meta.sql create mode 100644 boost_manager/migrations/3_activated_hexes.sql create mode 100644 boost_manager/migrations/4_files_processed.sql create mode 100644 boost_manager/pkg/settings-template.toml create mode 100644 boost_manager/src/activator.rs create mode 100644 boost_manager/src/db.rs create mode 100644 boost_manager/src/lib.rs create mode 100644 boost_manager/src/main.rs create mode 100644 boost_manager/src/settings.rs create mode 100644 boost_manager/src/telemetry.rs create mode 100644 boost_manager/src/updater.rs create mode 100644 boost_manager/src/watcher.rs create mode 100644 boost_manager/tests/activator_tests.rs create mode 100644 boost_manager/tests/common/mod.rs create mode 100644 boost_manager/tests/updater_tests.rs create mode 100644 boost_manager/tests/watcher_tests.rs create mode 100644 file_store/src/hex_boost.rs create mode 100644 mobile_config/src/boosted_hex_info.rs create mode 100644 mobile_config/src/client/hex_boosting_client.rs create mode 100644 mobile_config/src/hex_boosting_service.rs create mode 100644 mobile_verifier/src/boosted_hexes.rs create mode 100644 mobile_verifier/tests/hex_boosting.rs create mode 100644 solana/src/burn.rs create mode 100644 solana/src/start_boost.rs diff --git a/Cargo.lock b/Cargo.lock index fc112cc42..2cc7c7a13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,6 +130,20 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "anchor-attribute-access-control" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faa5be5b72abea167f87c868379ba3c2be356bfca9e6f474fd055fa0f7eeb4f2" +dependencies = [ + "anchor-syn 0.28.0", + "anyhow", + "proc-macro2 1.0.69", + "quote 1.0.33", + "regex", + "syn 1.0.109", +] + [[package]] name = "anchor-attribute-account" version = "0.26.0" @@ -145,6 +159,21 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "anchor-attribute-account" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f468970344c7c9f9d03b4da854fd7c54f21305059f53789d0045c1dd803f0018" +dependencies = [ + "anchor-syn 0.28.0", + "anyhow", + "bs58 0.5.0", + "proc-macro2 1.0.69", + "quote 1.0.33", + "rustversion", + "syn 1.0.109", +] + [[package]] name = "anchor-attribute-constant" version = "0.26.0" @@ -156,6 +185,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "anchor-attribute-constant" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59948e7f9ef8144c2aefb3f32a40c5fce2798baeec765ba038389e82301017ef" +dependencies = [ + "anchor-syn 0.28.0", + "proc-macro2 1.0.69", + "syn 1.0.109", +] + [[package]] name = "anchor-attribute-error" version = "0.26.0" @@ -168,6 +208,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "anchor-attribute-error" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc753c9d1c7981cb8948cf7e162fb0f64558999c0413058e2d43df1df5448086" +dependencies = [ + "anchor-syn 0.28.0", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 1.0.109", +] + [[package]] name = "anchor-attribute-event" version = "0.26.0" @@ -181,6 +233,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "anchor-attribute-event" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38b4e172ba1b52078f53fdc9f11e3dc0668ad27997838a0aad2d148afac8c97" +dependencies = [ + "anchor-syn 0.28.0", + "anyhow", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 1.0.109", +] + [[package]] name = "anchor-attribute-interface" version = "0.26.0" @@ -208,6 +273,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "anchor-attribute-program" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eebd21543606ab61e2d83d9da37d24d3886a49f390f9c43a1964735e8c0f0d5" +dependencies = [ + "anchor-syn 0.28.0", + "anyhow", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 1.0.109", +] + [[package]] name = "anchor-attribute-state" version = "0.26.0" @@ -227,7 +305,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "582dd4960f08a340a91ebe3cac5431338cfd2d2ccfa6520dcc8f2036a86f5125" dependencies = [ - "anchor-lang", + "anchor-lang 0.26.0", "anyhow", "regex", "serde", @@ -251,6 +329,30 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "anchor-derive-accounts" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4720d899b3686396cced9508f23dab420f1308344456ec78ef76f98fda42af" +dependencies = [ + "anchor-syn 0.28.0", + "anyhow", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 1.0.109", +] + +[[package]] +name = "anchor-derive-space" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f495e85480bd96ddeb77b71d499247c7d4e8b501e75ecb234e9ef7ae7bd6552a" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 1.0.109", +] + [[package]] name = "anchor-gen" version = "0.3.1" @@ -303,15 +405,15 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "662ceafe667448ee4199a4be2ee83b6bb76da28566eee5cea05f96ab38255af8" dependencies = [ - "anchor-attribute-access-control", - "anchor-attribute-account", - "anchor-attribute-constant", - "anchor-attribute-error", - "anchor-attribute-event", + "anchor-attribute-access-control 0.26.0", + "anchor-attribute-account 0.26.0", + "anchor-attribute-constant 0.26.0", + "anchor-attribute-error 0.26.0", + "anchor-attribute-event 0.26.0", "anchor-attribute-interface", - "anchor-attribute-program", + "anchor-attribute-program 0.26.0", "anchor-attribute-state", - "anchor-derive-accounts", + "anchor-derive-accounts 0.26.0", "arrayref", "base64 0.13.1", "bincode", @@ -321,6 +423,43 @@ dependencies = [ "thiserror", ] +[[package]] +name = "anchor-lang" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d2d4b20100f1310a774aba3471ef268e5c4ba4d5c28c0bbe663c2658acbc414" +dependencies = [ + "anchor-attribute-access-control 0.28.0", + "anchor-attribute-account 0.28.0", + "anchor-attribute-constant 0.28.0", + "anchor-attribute-error 0.28.0", + "anchor-attribute-event 0.28.0", + "anchor-attribute-program 0.28.0", + "anchor-derive-accounts 0.28.0", + "anchor-derive-space", + "arrayref", + "base64 0.13.1", + "bincode", + "borsh", + "bytemuck", + "getrandom 0.2.10", + "solana-program", + "thiserror", +] + +[[package]] +name = "anchor-spl" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78f860599da1c2354e7234c768783049eb42e2f54509ecfc942d2e0076a2da7b" +dependencies = [ + "anchor-lang 0.28.0", + "solana-program", + "spl-associated-token-account", + "spl-token", + "spl-token-2022 0.6.1", +] + [[package]] name = "anchor-syn" version = "0.24.2" @@ -359,6 +498,24 @@ dependencies = [ "thiserror", ] +[[package]] +name = "anchor-syn" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a125e4b0cc046cfec58f5aa25038e34cf440151d58f0db3afc55308251fe936d" +dependencies = [ + "anyhow", + "bs58 0.5.0", + "heck 0.3.3", + "proc-macro2 1.0.69", + "quote 1.0.33", + "serde", + "serde_json", + "sha2 0.10.6", + "syn 1.0.109", + "thiserror", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -518,19 +675,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" -[[package]] -name = "async-compression" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a" -dependencies = [ - "flate2", - "futures-core", - "memchr", - "pin-project-lite", - "tokio", -] - [[package]] name = "async-compression" version = "0.4.5" @@ -666,8 +810,8 @@ dependencies = [ "aws-types 0.51.0", "bytes", "hex", - "http", - "hyper", + "http 0.2.11", + "hyper 0.14.23", "ring 0.16.20", "time", "tokio", @@ -698,7 +842,7 @@ dependencies = [ "aws-smithy-http 0.51.0", "aws-smithy-types 0.51.0", "aws-types 0.51.0", - "http", + "http 0.2.11", "regex", "tracing", ] @@ -713,8 +857,8 @@ dependencies = [ "aws-smithy-types 0.51.0", "aws-types 0.51.0", "bytes", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.5", "lazy_static", "percent-encoding", "pin-project-lite", @@ -742,8 +886,8 @@ dependencies = [ "aws-types 0.51.0", "bytes", "bytes-utils", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.5", "tokio-stream", "tower", "tracing", @@ -766,7 +910,7 @@ dependencies = [ "aws-smithy-types 0.51.0", "aws-types 0.51.0", "bytes", - "http", + "http 0.2.11", "tokio-stream", "tower", ] @@ -789,7 +933,7 @@ dependencies = [ "aws-smithy-xml", "aws-types 0.51.0", "bytes", - "http", + "http 0.2.11", "tower", ] @@ -803,7 +947,7 @@ dependencies = [ "aws-smithy-eventstream", "aws-smithy-http 0.51.0", "aws-types 0.51.0", - "http", + "http 0.2.11", "tracing", ] @@ -817,7 +961,7 @@ dependencies = [ "aws-sigv4 0.54.1", "aws-smithy-http 0.54.4", "aws-types 0.54.1", - "http", + "http 0.2.11", "tracing", ] @@ -832,7 +976,7 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "http", + "http 0.2.11", "once_cell", "percent-encoding", "regex", @@ -851,7 +995,7 @@ dependencies = [ "form_urlencoded", "hex", "hmac 0.12.1", - "http", + "http 0.2.11", "once_cell", "percent-encoding", "regex", @@ -896,8 +1040,8 @@ dependencies = [ "crc32c", "crc32fast", "hex", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.5", "md-5", "pin-project-lite", "sha1", @@ -917,9 +1061,9 @@ dependencies = [ "aws-smithy-types 0.51.0", "bytes", "fastrand", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.5", + "hyper 0.14.23", "hyper-rustls 0.23.1", "lazy_static", "pin-project-lite", @@ -940,8 +1084,8 @@ dependencies = [ "aws-smithy-types 0.54.4", "bytes", "fastrand", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.5", "pin-project-lite", "tokio", "tower", @@ -970,9 +1114,9 @@ dependencies = [ "bytes", "bytes-utils", "futures-core", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.5", + "hyper 0.14.23", "once_cell", "percent-encoding", "pin-project-lite", @@ -992,9 +1136,9 @@ dependencies = [ "bytes", "bytes-utils", "futures-core", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.5", + "hyper 0.14.23", "once_cell", "percent-encoding", "pin-project-lite", @@ -1010,8 +1154,8 @@ checksum = "20c96d7bd35e7cf96aca1134b2f81b1b59ffe493f7c6539c051791cbbf7a42d3" dependencies = [ "aws-smithy-http 0.51.0", "bytes", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.5", "pin-project-lite", "tower", "tracing", @@ -1026,8 +1170,8 @@ dependencies = [ "aws-smithy-http 0.54.4", "aws-smithy-types 0.54.4", "bytes", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.5", "pin-project-lite", "tower", "tracing", @@ -1096,7 +1240,7 @@ dependencies = [ "aws-smithy-client 0.51.0", "aws-smithy-http 0.51.0", "aws-smithy-types 0.51.0", - "http", + "http 0.2.11", "rustc_version 0.4.0", "tracing", "zeroize", @@ -1113,7 +1257,7 @@ dependencies = [ "aws-smithy-client 0.54.4", "aws-smithy-http 0.54.4", "aws-smithy-types 0.54.4", - "http", + "http 0.2.11", "rustc_version 0.4.0", "tracing", ] @@ -1125,13 +1269,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "678c5130a507ae3a7c797f9a17393c14849300b8440eac47cdb90a5bdcb3a543" dependencies = [ "async-trait", - "axum-core", + "axum-core 0.3.2", "bitflags", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.5", + "hyper 0.14.23", "itoa 1.0.9", "matchit", "memchr", @@ -1147,6 +1291,40 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1236b4b292f6c4d6dc34604bb5120d85c3fe1d1aa596bd5cc52ca054d13e7b9e" +dependencies = [ + "async-trait", + "axum-core 0.4.3", + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.1.0", + "hyper-util", + "itoa 1.0.9", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "axum-core" version = "0.3.2" @@ -1156,12 +1334,33 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.5", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "http-body-util", "mime", + "pin-project-lite", "rustversion", + "sync_wrapper", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -1249,7 +1448,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=master#1cda75ccc8d8cf8a069ca95cbba6d43593e39869" +source = "git+https://github.com/helium/proto?branch=andymck/hex-boosting-support#fb07b06d55aa9bcba6facf0307a218ab63b5d6cd" dependencies = [ "base64 0.21.0", "byteorder", @@ -1259,7 +1458,7 @@ dependencies = [ "rand_chacha 0.3.0", "rust_decimal", "serde", - "sha2 0.10.6", + "sha2 0.9.9", "thiserror", ] @@ -1369,6 +1568,54 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" +[[package]] +name = "boost-manager" +version = "0.1.0" +dependencies = [ + "anchor-lang 0.28.0", + "anchor-spl", + "anyhow", + "async-trait", + "axum 0.7.4", + "base64 0.21.0", + "bs58 0.4.0", + "chrono", + "clap 4.4.8", + "config", + "db-store", + "file-store", + "futures", + "futures-util", + "helium-crypto", + "helium-proto", + "http 0.2.11", + "http-serde", + "lazy_static", + "metrics", + "metrics-exporter-prometheus", + "mobile-config", + "once_cell", + "poc-metrics", + "prost", + "rand 0.8.5", + "rust_decimal", + "rust_decimal_macros", + "serde", + "serde_json", + "sha2 0.10.6", + "solana", + "solana-sdk", + "sqlx", + "task-manager", + "thiserror", + "tokio", + "tokio-util", + "tonic", + "tracing", + "tracing-subscriber", + "triggered", +] + [[package]] name = "borsh" version = "0.9.3" @@ -1592,10 +1839,10 @@ dependencies = [ [[package]] name = "circuit-breaker" version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#24a2bd3f812edf5852517f7a3ab0859f993357b5" +source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang", + "anchor-lang 0.26.0", ] [[package]] @@ -2127,10 +2374,10 @@ dependencies = [ [[package]] name = "data-credits" version = "0.2.1" -source = "git+https://github.com/helium/helium-anchor-gen.git#24a2bd3f812edf5852517f7a3ab0859f993357b5" +source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang", + "anchor-lang 0.26.0", ] [[package]] @@ -2149,7 +2396,7 @@ dependencies = [ "aws-sig-auth 0.54.1", "aws-smithy-http 0.54.4", "aws-types 0.54.1", - "http", + "http 0.2.11", "metrics", "poc-metrics", "serde", @@ -2526,10 +2773,10 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "fanout" version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#24a2bd3f812edf5852517f7a3ab0859f993357b5" +source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang", + "anchor-lang 0.26.0", ] [[package]] @@ -2562,7 +2809,7 @@ name = "file-store" version = "0.1.0" dependencies = [ "anyhow", - "async-compression 0.3.15", + "async-compression", "async-trait", "aws-config", "aws-sdk-s3", @@ -2582,7 +2829,7 @@ dependencies = [ "helium-crypto", "helium-proto", "hex-literal", - "http", + "http 0.2.11", "lazy_static", "metrics", "poc-metrics", @@ -2905,7 +3152,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.11", "indexmap 1.9.3", "slab", "tokio", @@ -2913,6 +3160,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "h2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31d030e59af851932b72ceebadf4a2b5986dba4c3b99dd2493f8273a0f151943" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 1.0.0", + "indexmap 2.0.2", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h3o" version = "0.4.0" @@ -3015,10 +3281,10 @@ dependencies = [ [[package]] name = "helium-anchor-gen" version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#24a2bd3f812edf5852517f7a3ab0859f993357b5" +source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang", + "anchor-lang 0.26.0", "circuit-breaker", "data-credits", "fanout", @@ -3044,7 +3310,7 @@ dependencies = [ "bs58 0.5.0", "byteorder", "ed25519-compact", - "getrandom 0.2.10", + "getrandom 0.1.16", "k256", "lazy_static", "multihash", @@ -3062,16 +3328,16 @@ dependencies = [ [[package]] name = "helium-entity-manager" version = "0.2.4" -source = "git+https://github.com/helium/helium-anchor-gen.git#24a2bd3f812edf5852517f7a3ab0859f993357b5" +source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang", + "anchor-lang 0.26.0", ] [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=master#1cda75ccc8d8cf8a069ca95cbba6d43593e39869" +source = "git+https://github.com/helium/proto?branch=andymck/hex-boosting-support#fb07b06d55aa9bcba6facf0307a218ab63b5d6cd" dependencies = [ "bytes", "prost", @@ -3085,10 +3351,10 @@ dependencies = [ [[package]] name = "helium-sub-daos" version = "0.1.4" -source = "git+https://github.com/helium/helium-anchor-gen.git#24a2bd3f812edf5852517f7a3ab0859f993357b5" +source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang", + "anchor-lang 0.26.0", ] [[package]] @@ -3120,11 +3386,11 @@ checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" [[package]] name = "hexboosting" -version = "0.0.1" -source = "git+https://github.com/helium/helium-anchor-gen.git#24a2bd3f812edf5852517f7a3ab0859f993357b5" +version = "0.0.2" +source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang", + "anchor-lang 0.26.0", ] [[package]] @@ -3199,6 +3465,17 @@ dependencies = [ "itoa 1.0.9", ] +[[package]] +name = "http" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +dependencies = [ + "bytes", + "fnv", + "itoa 1.0.9", +] + [[package]] name = "http-body" version = "0.4.5" @@ -3206,7 +3483,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", - "http", + "http 0.2.11", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.0.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +dependencies = [ + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -3222,7 +3522,7 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e272971f774ba29341db2f686255ff8a979365a26fb9e4277f6b6d9ec0cdd5e" dependencies = [ - "http", + "http 0.2.11", "serde", ] @@ -3254,9 +3554,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.21", + "http 0.2.11", + "http-body 0.4.5", "httparse", "httpdate", "itoa 1.0.9", @@ -3268,14 +3568,33 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5aa53871fc917b1a9ed87b683a5d86db645e23acb32c2e0785a353e522fb75" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.2", + "http 1.0.0", + "http-body 1.0.0", + "httparse", + "httpdate", + "itoa 1.0.9", + "pin-project-lite", + "tokio", +] + [[package]] name = "hyper-rustls" version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59df7c4e19c950e6e0e868dcc0a300b09a9b88e9ec55bd879ca819087a77355d" dependencies = [ - "http", - "hyper", + "http 0.2.11", + "hyper 0.14.23", "log", "rustls 0.20.7", "rustls-native-certs", @@ -3291,8 +3610,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", - "hyper", + "http 0.2.11", + "hyper 0.14.23", "rustls 0.21.9", "tokio", "tokio-rustls 0.24.1", @@ -3304,12 +3623,30 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.23", "pin-project-lite", "tokio", "tokio-io-timeout", ] +[[package]] +name = "hyper-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdea9aac0dbe5a9240d68cfd9501e2db94222c6dc06843e06640b9e07f0fdc67" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "hyper 1.1.0", + "pin-project-lite", + "socket2 0.5.5", + "tokio", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.53" @@ -3415,7 +3752,7 @@ dependencies = [ "futures-util", "helium-crypto", "helium-proto", - "http", + "http 0.2.11", "metrics", "metrics-exporter-prometheus", "poc-metrics", @@ -3472,7 +3809,7 @@ dependencies = [ "helium-crypto", "helium-proto", "hextree", - "http", + "http 0.2.11", "http-serde", "libflate", "metrics", @@ -3511,7 +3848,7 @@ dependencies = [ "futures-util", "helium-crypto", "helium-proto", - "http", + "http 0.2.11", "http-serde", "iot-config", "metrics", @@ -3675,7 +4012,7 @@ dependencies = [ "futures-channel", "futures-timer", "futures-util", - "hyper", + "hyper 0.14.23", "jsonrpsee-types", "rustc-hash", "serde", @@ -3692,7 +4029,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42007820863ab29f3adeacf43886ef54abaedb35bc33dada25771db4e1f94de4" dependencies = [ "async-trait", - "hyper", + "hyper 0.14.23", "hyper-rustls 0.23.1", "jsonrpsee-core", "jsonrpsee-types", @@ -3763,19 +4100,19 @@ dependencies = [ [[package]] name = "lazy-distributor" version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#24a2bd3f812edf5852517f7a3ab0859f993357b5" +source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang", + "anchor-lang 0.26.0", ] [[package]] name = "lazy-transactions" version = "0.2.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#24a2bd3f812edf5852517f7a3ab0859f993357b5" +source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang", + "anchor-lang 0.26.0", ] [[package]] @@ -4010,7 +4347,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a4964177ddfdab1e3a2b37aec7cf320e14169abb0ed73999f558136409178d5" dependencies = [ "base64 0.21.0", - "hyper", + "hyper 0.14.23", "indexmap 1.9.3", "ipnet", "metrics", @@ -4106,7 +4443,7 @@ dependencies = [ "helium-crypto", "helium-proto", "hextree", - "http", + "http 0.2.11", "http-serde", "lazy_static", "metrics", @@ -4156,10 +4493,10 @@ dependencies = [ [[package]] name = "mobile-entity-manager" version = "0.1.1" -source = "git+https://github.com/helium/helium-anchor-gen.git#24a2bd3f812edf5852517f7a3ab0859f993357b5" +source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang", + "anchor-lang 0.26.0", ] [[package]] @@ -4177,7 +4514,7 @@ dependencies = [ "futures-util", "helium-crypto", "helium-proto", - "http", + "http 0.2.11", "http-serde", "metrics", "mobile-config", @@ -4728,8 +5065,8 @@ dependencies = [ "futures-util", "helium-crypto", "helium-proto", - "http", - "hyper", + "http 0.2.11", + "hyper 0.14.23", "jsonrpsee", "metrics", "metrics-exporter-prometheus", @@ -4796,7 +5133,7 @@ dependencies = [ name = "price" version = "0.1.0" dependencies = [ - "anchor-lang", + "anchor-lang 0.26.0", "anyhow", "chrono", "clap 4.4.8", @@ -4826,10 +5163,10 @@ dependencies = [ [[package]] name = "price-oracle" version = "0.2.1" -source = "git+https://github.com/helium/helium-anchor-gen.git#24a2bd3f812edf5852517f7a3ab0859f993357b5" +source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang", + "anchor-lang 0.26.0", ] [[package]] @@ -5256,16 +5593,16 @@ version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ - "async-compression 0.4.5", + "async-compression", "base64 0.21.0", "bytes", "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.21", + "http 0.2.11", + "http-body 0.4.5", + "hyper 0.14.23", "hyper-rustls 0.24.2", "ipnet", "js-sys", @@ -5353,10 +5690,10 @@ dependencies = [ [[package]] name = "rewards-oracle" version = "0.2.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#24a2bd3f812edf5852517f7a3ab0859f993357b5" +source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang", + "anchor-lang 0.26.0", ] [[package]] @@ -5739,6 +6076,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" +dependencies = [ + "itoa 1.0.9", + "serde", +] + [[package]] name = "serde_repr" version = "0.1.11" @@ -5944,10 +6291,12 @@ name = "solana" version = "0.1.0" dependencies = [ "anchor-client", - "anchor-lang", + "anchor-lang 0.26.0", "anyhow", "async-trait", + "chrono", "clap 4.4.8", + "file-store", "futures", "helium-anchor-gen", "helium-crypto", @@ -5984,7 +6333,7 @@ dependencies = [ "solana-sdk", "solana-vote-program", "spl-token", - "spl-token-2022", + "spl-token-2022 0.5.0", "thiserror", "zstd", ] @@ -6089,7 +6438,7 @@ dependencies = [ "solana-transaction-status", "solana-version", "solana-vote-program", - "spl-token-2022", + "spl-token-2022 0.5.0", "thiserror", "tokio", "tokio-stream", @@ -6489,7 +6838,7 @@ dependencies = [ "spl-associated-token-account", "spl-memo", "spl-token", - "spl-token-2022", + "spl-token-2022 0.5.0", "thiserror", ] @@ -6598,7 +6947,7 @@ dependencies = [ "num-traits", "solana-program", "spl-token", - "spl-token-2022", + "spl-token-2022 0.5.0", "thiserror", ] @@ -6644,6 +6993,24 @@ dependencies = [ "thiserror", ] +[[package]] +name = "spl-token-2022" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0043b590232c400bad5ee9eb983ced003d15163c4c5d56b090ac6d9a57457b47" +dependencies = [ + "arrayref", + "bytemuck", + "num-derive", + "num-traits", + "num_enum", + "solana-program", + "solana-zk-token-sdk", + "spl-memo", + "spl-token", + "thiserror", +] + [[package]] name = "sqlformat" version = "0.2.0" @@ -7149,15 +7516,15 @@ checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" dependencies = [ "async-stream", "async-trait", - "axum", + "axum 0.6.3", "base64 0.13.1", "bytes", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.21", + "http 0.2.11", + "http-body 0.4.5", + "hyper 0.14.23", "hyper-timeout", "percent-encoding", "pin-project", @@ -7219,8 +7586,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.5", "http-range-header", "pin-project-lite", "tower", @@ -7302,10 +7669,10 @@ dependencies = [ [[package]] name = "treasury-management" version = "0.2.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#24a2bd3f812edf5852517f7a3ab0859f993357b5" +source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang", + "anchor-lang 0.26.0", ] [[package]] @@ -7329,7 +7696,7 @@ dependencies = [ "base64 0.13.1", "byteorder", "bytes", - "http", + "http 0.2.11", "httparse", "log", "rand 0.8.5", @@ -7349,7 +7716,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", - "rand 0.8.5", + "rand 0.7.3", "static_assertions", ] @@ -7529,10 +7896,10 @@ dependencies = [ [[package]] name = "voter-stake-registry" version = "0.3.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#24a2bd3f812edf5852517f7a3ab0859f993357b5" +source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang", + "anchor-lang 0.26.0", ] [[package]] @@ -7953,7 +8320,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "sha2 0.10.6", + "sha2 0.9.9", "thiserror", "twox-hash", "xorf", diff --git a/Cargo.toml b/Cargo.toml index 2930f0467..f8ecfb336 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ debug = true [workspace] members = [ + "boost_manager", "db_store", "denylist", "file_store", @@ -61,14 +62,14 @@ sqlx = {version = "0", features = [ ]} helium-anchor-gen = {git = "https://github.com/helium/helium-anchor-gen.git"} helium-crypto = {version = "0.8.1", features=["sqlx-postgres", "multisig"]} -helium-proto = {git = "https://github.com/helium/proto", branch = "master", features = ["services"]} +helium-proto = {git = "https://github.com/helium/proto", branch = "andymck/hex-boosting-support", features = ["services"]} hextree = "*" solana-client = "1.14" solana-sdk = "1.14" solana-program = "1.11" spl-token = "3.5.0" reqwest = {version = "0", default-features=false, features = ["gzip", "json", "rustls-tls"]} -beacon = { git = "https://github.com/helium/proto", branch = "master" } +beacon = { git = "https://github.com/helium/proto", branch = "andymck/hex-boosting-support" } humantime = "2" metrics = "0" metrics-exporter-prometheus = "0" diff --git a/boost_manager/Cargo.toml b/boost_manager/Cargo.toml new file mode 100644 index 000000000..9f1c298d7 --- /dev/null +++ b/boost_manager/Cargo.toml @@ -0,0 +1,52 @@ +[package] +name = "boost-manager" +version = "0.1.0" +description = "Hex boosting manager" +edition.workspace = true +authors.workspace = true +license.workspace = true + + +[dependencies] +anyhow = {workspace = true} +anchor-lang = "0.28" +anchor-spl = "0.28" +axum = {version = "0", features = ["tracing"]} +bs58 = {workspace = true} +config = {workspace = true} +clap = {workspace = true} +thiserror = {workspace = true} +serde = {workspace = true} +serde_json = {workspace = true} +sqlx = {workspace = true} +base64 = {workspace = true} +sha2 = {workspace = true} +lazy_static = {workspace = true} +triggered = {workspace = true} +futures = {workspace = true} +futures-util = {workspace = true} +prost = {workspace = true} +once_cell = {workspace = true} +mobile-config = {path = "../mobile_config"} +file-store = {path = "../file_store"} +db-store = { path = "../db_store" } +poc-metrics = {path = "../metrics"} +tokio = { workspace = true } +tokio-util = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +chrono = { workspace = true, features = ["serde"] } +metrics = {workspace = true } +metrics-exporter-prometheus = { workspace = true } +helium-proto = { workspace = true } +helium-crypto = {workspace = true, features = ["sqlx-postgres", "multisig", "solana"]} +rust_decimal = {workspace = true} +rust_decimal_macros = {workspace = true} +tonic = {workspace = true} +rand = {workspace = true} +async-trait = {workspace = true} +task-manager = { path = "../task_manager" } +http = {workspace = true} +http-serde = {workspace = true} +solana = {path = "../solana"} +solana-sdk = {workspace = true} diff --git a/boost_manager/README.md b/boost_manager/README.md new file mode 100644 index 000000000..b7860d0e9 --- /dev/null +++ b/boost_manager/README.md @@ -0,0 +1,10 @@ +# Boost Manager + +### S3 Inputs + +| File Type | Pattern | | +| :--- |:-----------------------| :-- | +| RewardManifest | reward_manifest.\* | [Proto](https://github.com/helium/proto/blob/149997d2a74e08679e56c2c892d7e46f2d0d1c46/src/reward_manifest.proto#L5) | +| MobileRewardShare | mobile_reward_share.\* | [Proto](https://github.com/helium/proto/blob/149997d2a74e08679e56c2c892d7e46f2d0d1c46/src/service/poc_lora.proto#L171) | + + diff --git a/boost_manager/migrations/1_setup.sql b/boost_manager/migrations/1_setup.sql new file mode 100644 index 000000000..00bc52613 --- /dev/null +++ b/boost_manager/migrations/1_setup.sql @@ -0,0 +1,27 @@ +-- This extension gives us `uuid_generate_v1mc()` which generates UUIDs that cluster better than `gen_random_uuid()` +-- while still being difficult to predict and enumerate. +-- Also, while unlikely, `gen_random_uuid()` can in theory produce collisions which can trigger spurious errors on +-- insertion, whereas it's much less likely with `uuid_generate_v1mc()`. +create extension if not exists "uuid-ossp"; + +create or replace function set_updated_at() + returns trigger as +$$ +begin + NEW.updated_at = now(); + return NEW; +end; +$$ language plpgsql; + +create or replace function trigger_updated_at(tablename regclass) + returns void as +$$ +begin + execute format('CREATE TRIGGER set_updated_at + BEFORE UPDATE + ON %s + FOR EACH ROW + WHEN (OLD is distinct from NEW) + EXECUTE FUNCTION set_updated_at();', tablename); +end; +$$ language plpgsql; diff --git a/boost_manager/migrations/2_meta.sql b/boost_manager/migrations/2_meta.sql new file mode 100644 index 000000000..1b21089dc --- /dev/null +++ b/boost_manager/migrations/2_meta.sql @@ -0,0 +1,4 @@ +create table meta ( + key text primary key not null, + value text +); diff --git a/boost_manager/migrations/3_activated_hexes.sql b/boost_manager/migrations/3_activated_hexes.sql new file mode 100644 index 000000000..4575cf5a2 --- /dev/null +++ b/boost_manager/migrations/3_activated_hexes.sql @@ -0,0 +1,20 @@ +CREATE TYPE onchain_status AS ENUM ( + 'queued', + 'pending', + 'success', + 'failed', + 'cancelled' +); + +create table activated_hexes ( + location bigint primary key not null, + activation_ts timestamptz not null, + boosted_hex_pubkey text not null, + boost_config_pubkey text not null, + status onchain_status not null, + txn_id text, + retries integer not null default 0, + inserted_at timestamptz default now(), + updated_at timestamptz default now() +); + diff --git a/boost_manager/migrations/4_files_processed.sql b/boost_manager/migrations/4_files_processed.sql new file mode 100644 index 000000000..adefd6a80 --- /dev/null +++ b/boost_manager/migrations/4_files_processed.sql @@ -0,0 +1,7 @@ +CREATE TABLE files_processed ( + file_name VARCHAR PRIMARY KEY, + file_type VARCHAR NOT NULL, + file_timestamp TIMESTAMPTZ NOT NULL, + processed_at TIMESTAMPTZ NOT NULL, + process_name text not null default 'default' +); diff --git a/boost_manager/pkg/settings-template.toml b/boost_manager/pkg/settings-template.toml new file mode 100644 index 000000000..46cb7f764 --- /dev/null +++ b/boost_manager/pkg/settings-template.toml @@ -0,0 +1,44 @@ +log = "boost_manager=info,solana=debug" + +# Cache location for generated boost manager outputs; Required +cache = "/tmp/oracles/boost-manager" + +start_after = 1702602001 + +enable_solana_integration = true + +activation_check_interval = 30 + +[solana] +# Solana RPC. This may contain a secret +rpc_url = "https://api.devnet.solana.com" +# Path to the keypair used to sign data credit burn solana transactions +start_authority_keypair = "" +# Public key of the hex boost authority +hexboost_authority_pubkey = "" +# Solana cluster to use. "devnet" or "mainnet" +cluster = "devnet" + +# +[database] +url = "postgresql://postgres:postgres@localhost:5432/hexboosting" +# Max connections to the database. +max_connections = 10 + +[verifier] +bucket = "mobile-verified" + +[output] +bucket = "mobile-verified" + +[mobile_config_client] +url = "http://localhost:6090" +config_pubkey = "" +signing_keypair = "" + + +[metrics] + +# Endpoint for metrics. Default below +# +endpoint = "127.0.0.1:19001" diff --git a/boost_manager/src/activator.rs b/boost_manager/src/activator.rs new file mode 100644 index 000000000..b5f504276 --- /dev/null +++ b/boost_manager/src/activator.rs @@ -0,0 +1,153 @@ +use crate::{db, telemetry}; +use anyhow::Result; +use chrono::{DateTime, Utc}; +use file_store::{ + file_info_poller::FileInfoStream, reward_manifest::RewardManifest, FileInfo, FileStore, +}; +use futures::{future::LocalBoxFuture, stream, StreamExt, TryFutureExt, TryStreamExt}; +use helium_proto::{ + services::poc_mobile::{ + mobile_reward_share::Reward as MobileReward, BoostedHex as BoostedHexProto, + MobileRewardShare, + }, + Message, +}; +use mobile_config::{ + boosted_hex_info::BoostedHexes, + client::{hex_boosting_client::HexBoostingInfoResolver, ClientError}, +}; +use poc_metrics::record_duration; +use sqlx::{Pool, Postgres, Transaction}; +use std::str::FromStr; +use task_manager::ManagedTask; +use tokio::sync::mpsc::Receiver; + +pub struct Activator { + pool: Pool, + verifier_store: FileStore, + receiver: Receiver>, + hex_boosting_client: A, +} + +impl ManagedTask for Activator +where + A: HexBoostingInfoResolver, +{ + fn start_task( + self: Box, + shutdown: triggered::Listener, + ) -> LocalBoxFuture<'static, anyhow::Result<()>> { + let handle = tokio::spawn(self.run(shutdown)); + Box::pin( + handle + .map_err(anyhow::Error::from) + .and_then(|result| async move { result.map_err(anyhow::Error::from) }), + ) + } +} + +impl Activator +where + A: HexBoostingInfoResolver, +{ + pub async fn new( + pool: Pool, + receiver: Receiver>, + hex_boosting_client: A, + verifier_store: FileStore, + ) -> Result { + Ok(Self { + pool, + receiver, + hex_boosting_client, + verifier_store, + }) + } + + pub async fn run(mut self, shutdown: triggered::Listener) -> anyhow::Result<()> { + tracing::info!("starting Activator"); + loop { + tokio::select! { + biased; + _ = shutdown.clone() => break, + msg = self.receiver.recv() => if let Some(file_info_stream) = msg { + let key = &file_info_stream.file_info.key.clone(); + tracing::info!(file = %key, "Received reward manifest file"); + + let mut txn = self.pool.begin().await?; + let mut stream = file_info_stream.into_stream(&mut txn).await?; + + while let Some(reward_manifest) = stream.next().await { + record_duration!( + "reward_index_duration", + self.handle_rewards(&mut txn, reward_manifest).await? + ) + } + txn.commit().await?; + tracing::info!(file = %key, "Completed processing reward file"); + telemetry::last_reward_processed_time(&self.pool, Utc::now()).await?; + } + } + } + tracing::info!("stopping Activator"); + Ok(()) + } + + async fn handle_rewards( + &mut self, + txn: &mut Transaction<'_, Postgres>, + manifest: RewardManifest, + ) -> Result<()> { + // get latest boosted hexes info from mobile config + let boosted_hexes = BoostedHexes::get_all(&self.hex_boosting_client).await?; + + // get the rewards file from the manifest + let manifest_time = manifest.end_timestamp; + let reward_files = stream::iter( + manifest + .written_files + .into_iter() + .map(|file_name| FileInfo::from_str(&file_name)), + ) + .boxed(); + + // read in the rewards file + let mut reward_shares = self.verifier_store.source_unordered(5, reward_files); + + while let Some(msg) = reward_shares.try_next().await? { + let share = MobileRewardShare::decode(msg)?; + if let Some(MobileReward::RadioReward(r)) = share.reward { + for hex in r.boosted_hexes.into_iter() { + process_boosted_hex(txn, manifest_time, &boosted_hexes, &hex).await? + } + } + } + Ok(()) + } +} + +pub async fn process_boosted_hex( + txn: &mut Transaction<'_, Postgres>, + manifest_time: DateTime, + boosted_hexes: &BoostedHexes, + hex: &BoostedHexProto, +) -> Result<()> { + match boosted_hexes.hexes.get(&hex.location) { + Some(info) => { + if info.start_ts.is_none() { + db::insert_activated_hex( + txn, + hex.location, + &info.boosted_hex_pubkey, + &info.boost_config_pubkey, + manifest_time, + ) + .await?; + } + } + None => { + tracing::warn!(hex = %hex.location, "got an invalid boosted hex"); + } + } + Ok(()) +} diff --git a/boost_manager/src/db.rs b/boost_manager/src/db.rs new file mode 100644 index 000000000..07fb5bff2 --- /dev/null +++ b/boost_manager/src/db.rs @@ -0,0 +1,222 @@ +use crate::OnChainStatus; +use chrono::{DateTime, Utc}; +use file_store::hex_boost::BoostedHexActivation; +use sqlx::{postgres::PgRow, FromRow, Pool, Postgres, Row, Transaction}; + +const MAX_RETRIES: i32 = 10; +const MAX_BATCH_COUNT: i32 = 200; + +struct Boost(BoostedHexActivation); + +impl FromRow<'_, PgRow> for Boost { + fn from_row(row: &PgRow) -> sqlx::Result { + let boost = BoostedHexActivation { + location: row.get::("location") as u64, + activation_ts: row.get::, &str>("activation_ts"), + boosted_hex_pubkey: row.get::("boosted_hex_pubkey"), + boost_config_pubkey: row.get::("boost_config_pubkey"), + }; + Ok(Boost(boost)) + } +} + +#[derive(sqlx::FromRow, Debug, Clone)] +pub struct TxnRow { + pub txn_id: String, +} + +#[derive(sqlx::FromRow, Debug, Clone)] +pub struct StatusRow { + #[sqlx(try_from = "i64")] + pub location: u64, + pub status: OnChainStatus, +} + +pub async fn insert_activated_hex( + txn: &mut Transaction<'_, Postgres>, + location: u64, + boosted_hex_pubkey: &String, + boost_config_pubkey: &String, + activation_ts: DateTime, +) -> Result<(), sqlx::Error> { + sqlx::query( + r#" + insert into activated_hexes ( + location, + activation_ts, + boosted_hex_pubkey, + boost_config_pubkey, + status + ) values ($1, $2, $3, $4, $5) + on conflict do nothing + "#, + ) + .bind(location as i64) + .bind(activation_ts) + .bind(boosted_hex_pubkey) + .bind(boost_config_pubkey) + .bind(OnChainStatus::Queued) + .execute(txn) + .await?; + + Ok(()) +} + +pub async fn get_queued_batch( + db: &Pool, +) -> Result, sqlx::Error> { + Ok(sqlx::query_as::<_, Boost>( + r#" + SELECT location, activation_ts, boosted_hex_pubkey, boost_config_pubkey + FROM activated_hexes + WHERE status = $1 AND retries < $2 AND txn_id IS NULL + ORDER BY activation_ts ASC + LIMIT $3; + "#, + ) + .bind(OnChainStatus::Queued) + .bind(MAX_RETRIES) + .bind(MAX_BATCH_COUNT) + .fetch_all(db) + .await? + .into_iter() + .map(|boost| boost.0) + .collect::>()) +} + +pub async fn query_activation_statuses(db: &Pool) -> anyhow::Result> { + Ok(sqlx::query_as::<_, StatusRow>( + r#" + SELECT location, status + FROM activated_hexes + "#, + ) + .fetch_all(db) + .await?) +} + +pub async fn save_batch_txn_id( + db: &Pool, + txn_id: &str, + hexes: &[u64], +) -> anyhow::Result<()> { + let hexes = hexes.iter().map(|x| *x as i64).collect::>(); + Ok(sqlx::query( + r#" + UPDATE activated_hexes + SET txn_id = $1, updated_at = $2 + WHERE location IN (SELECT * FROM UNNEST($3)) + "#, + ) + .bind(txn_id) + .bind(Utc::now()) + .bind(hexes) + .execute(db) + .await + .map(|_| ())?) +} + +pub async fn update_success_batch(db: &Pool, hexes: &[u64]) -> anyhow::Result<()> { + let hexes = hexes.iter().map(|x| *x as i64).collect::>(); + Ok(sqlx::query( + r#" + UPDATE activated_hexes + SET status = $1, updated_at = $2 + WHERE location IN (SELECT * FROM UNNEST($3)) + "#, + ) + .bind(OnChainStatus::Success) + .bind(Utc::now()) + .bind(hexes) + .execute(db) + .await + .map(|_| ())?) +} + +pub async fn update_failed_batch(db: &Pool, hexes: &[u64]) -> anyhow::Result<()> { + let hexes = hexes.iter().map(|x| *x as i64).collect::>(); + Ok(sqlx::query( + r#" + UPDATE activated_hexes + SET updated_at = $1, retries = retries + 1, txn_id = NULL + WHERE location IN (SELECT * FROM UNNEST($2)) + "#, + ) + .bind(Utc::now()) + .bind(hexes) + .execute(db) + .await + .map(|_| ())?) +} + +pub async fn update_failed_activations(db: &Pool) -> anyhow::Result { + Ok(sqlx::query( + r#" + UPDATE activated_hexes + SET status = $1, updated_at = $2 + WHERE status = $3 AND retries >= $4 + "#, + ) + .bind(OnChainStatus::Failed) + .bind(Utc::now()) + .bind(OnChainStatus::Queued) + .bind(MAX_RETRIES) + .execute(db) + .await + .map(|result| result.rows_affected())?) +} + +pub async fn get_failed_activations_count(db: &Pool) -> Result { + sqlx::query_scalar::<_, i64>( + " select count(location) from activated_hexes where retries >= 10 and status != 'success' ", + ) + .bind(MAX_RETRIES) + .fetch_one(db) + .await + .map(|count| count as u64) +} + +pub async fn get_txns_ids_to_verify(db: &Pool) -> Result, sqlx::Error> { + sqlx::query_as::<_, TxnRow>( + r#" + select distinct(txn_id) from activated_hexes where status = $1 and txn_id is not null + "#, + ) + .bind(OnChainStatus::Queued) + .fetch_all(db) + .await +} + +pub async fn update_verified_txns_onchain(db: &Pool, txn_id: &str) -> anyhow::Result<()> { + Ok(sqlx::query( + r#" + UPDATE activated_hexes + SET status = $1, updated_at = $2 + WHERE txn_id = $3 + "#, + ) + .bind(OnChainStatus::Success) + .bind(Utc::now()) + .bind(txn_id) + .execute(db) + .await + .map(|_| ())?) +} + +pub async fn update_verified_txns_not_onchain( + db: &Pool, + txn_id: &str, +) -> anyhow::Result<()> { + Ok(sqlx::query( + r#" + UPDATE activated_hexes + SET txn_id = null, updated_at = $1 + WHERE txn_id = $2 + "#, + ) + .bind(Utc::now()) + .bind(txn_id) + .execute(db) + .await + .map(|_| ())?) +} diff --git a/boost_manager/src/lib.rs b/boost_manager/src/lib.rs new file mode 100644 index 000000000..ff4a37726 --- /dev/null +++ b/boost_manager/src/lib.rs @@ -0,0 +1,20 @@ +use serde::{Deserialize, Serialize}; + +pub mod activator; +pub mod db; +pub mod settings; +pub mod telemetry; +pub use settings::Settings; +pub mod updater; +pub mod watcher; + +#[derive(Debug, Eq, Hash, PartialEq, Copy, Clone, Deserialize, Serialize, sqlx::Type)] +#[sqlx(type_name = "onchain_status")] +#[sqlx(rename_all = "lowercase")] +pub enum OnChainStatus { + Queued = 0, + Pending = 1, + Success = 2, + Failed = 3, + Cancelled = 4, +} diff --git a/boost_manager/src/main.rs b/boost_manager/src/main.rs new file mode 100644 index 000000000..ac0256cd4 --- /dev/null +++ b/boost_manager/src/main.rs @@ -0,0 +1,147 @@ +use anyhow::{bail, Result}; +use boost_manager::{ + activator::Activator, settings::Settings, telemetry, updater::Updater, watcher::Watcher, +}; +use chrono::Duration; +use clap::Parser; +use file_store::{ + file_info_poller::LookbackBehavior, file_sink, file_source, file_upload, + reward_manifest::RewardManifest, FileStore, FileType, +}; +use mobile_config::client::hex_boosting_client::HexBoostingClient; +use solana::start_boost::SolanaRpc; +use std::path::{self, PathBuf}; +use task_manager::TaskManager; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; + +#[derive(Debug, clap::Parser)] +#[clap(version = env!("CARGO_PKG_VERSION"))] +#[clap(about = "Helium Boost Manager")] +pub struct Cli { + /// Optional configuration file to use. If present the toml file at the + /// given path will be loaded. Environemnt variables can override the + /// settins in the given file. + #[clap(short = 'c')] + config: Option, + + #[clap(subcommand)] + cmd: Cmd, +} + +impl Cli { + pub async fn run(self) -> Result<()> { + let settings = Settings::new(self.config)?; + self.cmd.run(settings).await + } +} + +#[derive(Debug, clap::Subcommand)] +pub enum Cmd { + Server(Server), +} + +impl Cmd { + pub async fn run(&self, settings: Settings) -> Result<()> { + match self { + Self::Server(cmd) => cmd.run(&settings).await, + } + } +} + +#[derive(Debug, clap::Args)] +pub struct Server {} + +impl Server { + pub async fn run(&self, settings: &Settings) -> Result<()> { + tracing_subscriber::registry() + .with(tracing_subscriber::EnvFilter::new(&settings.log)) + .with(tracing_subscriber::fmt::layer()) + .init(); + + // Install the prometheus metrics exporter + poc_metrics::start_metrics(&settings.metrics)?; + + // Set up the solana network: + let solana = if settings.enable_solana_integration { + let Some(ref solana_settings) = settings.solana else { + bail!("Missing solana section in settings"); + }; + // Set up the solana RpcClient: + Some(SolanaRpc::new(solana_settings).await?) + } else { + None + }; + + // Create database pool and run migrations + let pool = settings.database.connect(env!("CARGO_PKG_NAME")).await?; + sqlx::migrate!().run(&pool).await?; + + telemetry::initialize(&pool).await?; + + let hex_boosting_client = HexBoostingClient::from_settings(&settings.mobile_config_client)?; + + let (file_upload, file_upload_server) = + file_upload::FileUpload::from_settings_tm(&settings.output).await?; + let store_base_path = path::Path::new(&settings.cache); + + // setup the received for the rewards manifest files + let file_store = FileStore::from_settings(&settings.verifier).await?; + let (manifest_receiver, manifest_server) = + file_source::continuous_source::() + .state(pool.clone()) + .store(file_store) + .prefix(FileType::RewardManifest.to_string()) + .lookback(LookbackBehavior::StartAfter(settings.start_after())) + .poll_duration(settings.reward_check_interval()) + .offset(settings.reward_check_interval() * 2) + .create() + .await?; + + // setup the writer for our updated hexes + let (updated_hexes_sink, updated_hexes_sink_server) = file_sink::FileSinkBuilder::new( + FileType::BoostedHexUpdate, + store_base_path, + concat!(env!("CARGO_PKG_NAME"), "_boosted_hex_update"), + ) + .file_upload(Some(file_upload.clone())) + .roll_time(Duration::minutes(5)) + .create() + .await?; + + // The server to monitor rewards and activate any newly seen boosted hexes + let verifier_store = FileStore::from_settings(&settings.verifier).await?; + let activator = Activator::new( + pool.clone(), + manifest_receiver, + hex_boosting_client.clone(), + verifier_store, + ) + .await?; + + let watcher = Watcher::new(pool.clone(), updated_hexes_sink, hex_boosting_client).await?; + + let updater = Updater::new( + pool.clone(), + settings.enable_solana_integration, + settings.activation_check_interval(), + settings.txn_batch_size(), + solana, + )?; + + TaskManager::builder() + .add_task(file_upload_server) + .add_task(manifest_server) + .add_task(updated_hexes_sink_server) + .add_task(activator) + .add_task(watcher) + .add_task(updater) + .start() + .await + } +} + +#[tokio::main] +async fn main() -> Result<()> { + let cli = Cli::parse(); + cli.run().await +} diff --git a/boost_manager/src/settings.rs b/boost_manager/src/settings.rs new file mode 100644 index 000000000..04ee7a938 --- /dev/null +++ b/boost_manager/src/settings.rs @@ -0,0 +1,96 @@ +use chrono::{DateTime, Duration as ChronoDuration, TimeZone, Utc}; +use config::{Config, Environment, File}; +use serde::Deserialize; +use std::{path::Path, time::Duration}; + +#[derive(Debug, Deserialize)] +pub struct Settings { + /// RUST_LOG compatible settings string. Default to + /// "poc_entropy=debug,poc_store=info" + #[serde(default = "default_log")] + pub log: String, + /// Cache location for generated verified reports + pub cache: String, + /// Reward files check interval in seconds. (Default is 900; 15 minutes) + #[serde(default = "default_reward_check_interval")] + pub reward_check_interval: i64, + /// Hex Activation check interval in seconds. (Default is 900; 15 minutes) + /// determines how often we will check the DB for queued txns to solana + #[serde(default = "default_activation_check_interval")] + pub activation_check_interval: i64, + pub database: db_store::Settings, + pub verifier: file_store::Settings, + pub mobile_config_client: mobile_config::ClientSettings, + pub metrics: poc_metrics::Settings, + pub output: file_store::Settings, + #[serde(default)] + pub enable_solana_integration: bool, + pub solana: Option, + #[serde(default = "default_start_after")] + pub start_after: u64, + // the number of records to fit per solana txn + #[serde(default = "default_txn_batch_size")] + pub txn_batch_size: u32, +} + +fn default_txn_batch_size() -> u32 { + 18 +} + +fn default_reward_check_interval() -> i64 { + 900 +} + +fn default_activation_check_interval() -> i64 { + 900 +} + +pub fn default_start_after() -> u64 { + 0 +} + +pub fn default_log() -> String { + "boost_manager=info".to_string() +} + +impl Settings { + /// Load Settings from a given path. Settings are loaded from a given + /// optional path and can be overriden with environment variables. + /// + /// Environemnt overrides have the same name as the entries in the settings + /// file in uppercase and prefixed with "MI_". For example "MI_DATABASE_URL" + /// will override the data base url. + pub fn new>(path: Option

) -> Result { + let mut builder = Config::builder(); + + if let Some(file) = path { + // Add optional settings file + builder = builder + .add_source(File::with_name(&file.as_ref().to_string_lossy()).required(false)); + } + // Add in settings from the environment (with a prefix of APP) + // Eg.. `MI_DEBUG=1 ./target/app` would set the `debug` key + builder + .add_source(Environment::with_prefix("MI").separator("_")) + .build() + .and_then(|config| config.try_deserialize()) + } + + pub fn reward_check_interval(&self) -> ChronoDuration { + ChronoDuration::seconds(self.reward_check_interval) + } + + pub fn activation_check_interval(&self) -> Duration { + Duration::from_secs(self.activation_check_interval as u64) + } + + pub fn txn_batch_size(&self) -> usize { + self.txn_batch_size as usize + } + + pub fn start_after(&self) -> DateTime { + Utc.timestamp_opt(self.start_after as i64, 0) + .single() + .unwrap() + } +} diff --git a/boost_manager/src/telemetry.rs b/boost_manager/src/telemetry.rs new file mode 100644 index 000000000..7589b68df --- /dev/null +++ b/boost_manager/src/telemetry.rs @@ -0,0 +1,29 @@ +use chrono::{DateTime, TimeZone, Utc}; +use db_store::meta; +use sqlx::{Pool, Postgres}; + +const LAST_REWARD_PROCESSED_TIME: &str = "last_reward_processed_time"; + +pub async fn initialize(db: &Pool) -> anyhow::Result<()> { + match meta::fetch(db, LAST_REWARD_PROCESSED_TIME).await { + Ok(timestamp) => last_reward_processed_time(db, to_datetime(timestamp)?).await, + Err(db_store::Error::NotFound(_)) => Ok(()), + Err(err) => Err(err.into()), + } +} + +pub async fn last_reward_processed_time( + db: &Pool, + datetime: DateTime, +) -> anyhow::Result<()> { + metrics::gauge!(LAST_REWARD_PROCESSED_TIME, datetime.timestamp() as f64); + meta::store(db, LAST_REWARD_PROCESSED_TIME, datetime.timestamp()).await?; + + Ok(()) +} + +fn to_datetime(timestamp: i64) -> anyhow::Result> { + Utc.timestamp_opt(timestamp, 0) + .single() + .ok_or_else(|| anyhow::anyhow!("Unable to decode timestamp")) +} diff --git a/boost_manager/src/updater.rs b/boost_manager/src/updater.rs new file mode 100644 index 000000000..02b0350a1 --- /dev/null +++ b/boost_manager/src/updater.rs @@ -0,0 +1,189 @@ +use crate::db::{self, TxnRow}; +use anyhow::Result; +use futures::{future::LocalBoxFuture, TryFutureExt}; +use solana::{start_boost::SolanaNetwork, GetSignature}; +use sqlx::{Pool, Postgres}; +use std::time::Duration; +use task_manager::ManagedTask; +use tokio::time::{self, MissedTickBehavior}; + +pub struct Updater { + pool: Pool, + chain_enabled: bool, + interval: Duration, + batch_size: usize, + pub solana: S, +} + +impl ManagedTask for Updater +where + S: SolanaNetwork, +{ + fn start_task( + self: Box, + shutdown: triggered::Listener, + ) -> LocalBoxFuture<'static, Result<()>> { + let handle = tokio::spawn(self.run(shutdown)); + Box::pin( + handle + .map_err(anyhow::Error::from) + .and_then(|result| async move { result.map_err(anyhow::Error::from) }), + ) + } +} + +impl Updater +where + S: SolanaNetwork, +{ + pub fn new( + pool: Pool, + chain_enabled: bool, + interval: Duration, + batch_size: usize, + solana: S, + ) -> Result { + Ok(Self { + pool, + chain_enabled, + interval, + batch_size, + solana, + }) + } + + pub async fn run(self, mut shutdown: triggered::Listener) -> Result<()> { + tracing::info!("starting Updater"); + // on startup if there are activations in the DB with 'queued' status and WITH a txn id then + // it suggests we crashed out early of the updater tick + // after we started the solana activation flow + // we need to check if these txns are onchain and if not then null the txn id + // if they are on chain then update their status to success + let txns_ids_to_verify = db::get_txns_ids_to_verify(&self.pool).await?; + tracing::warn!( + "checking {} txn_ids on chain status", + txns_ids_to_verify.len() + ); + + if !txns_ids_to_verify.is_empty() { + // If we have activations that we need to verify, wait one minute to ensure + // every transaction has been confirmed on chain + tracing::info!("We have pending txn_id's to verify, sleeping for one minute to given them time to appear on-chain"); + tokio::time::sleep(Duration::from_secs(60)).await; + for p in txns_ids_to_verify { + self.confirm_txn(&p).await? + } + } + + let mut timer = time::interval(self.interval); + timer.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + tokio::select! { + biased; + _ = &mut shutdown => break, + _ = timer.tick() => { + if self.chain_enabled { + self.process_activations().await? + } + else { + tracing::info!("processing of activations is disabled, skipping...")} + } + } + } + tracing::info!("stopping Updater"); + Ok(()) + } + + pub async fn process_activations(&self) -> Result<()> { + self.check_failed_activations().await?; + // get the batch of queued activations to update on chain + let activations = db::get_queued_batch(&self.pool).await?; + + if activations.is_empty() { + tracing::info!("no activations in queue"); + return Ok(()); + } + + let activations_count = activations.len() as u64; + tracing::info!( + num_of_activations = activations_count, + "processing activations," + ); + + // slice the activations up into batches of N activations and submit to solana + for batch in activations.chunks(self.batch_size) { + let batch_size = batch.len(); + + // get a list of all the activations DB ids which form part of the batch + let ids: Vec = batch.iter().map(|sp| sp.location).collect(); + + let solana_txn = self.solana.make_start_boost_transaction(batch).await?; + let transaction_id = solana_txn.get_signature().to_string(); + // update the batch in the db with the txn id + db::save_batch_txn_id(&self.pool, &transaction_id, &ids).await?; + + // if activations were processed successfully then + // update their status in the DB to success + // if not processed successfully then bump their retry count + // if retry count is below max retries then the activations + // will be retried next tick + match self.solana.submit_transaction(&solana_txn).await { + Ok(()) => { + self.handle_submit_txn_success(&ids, batch_size, activations_count) + .await?; + } + Err(e) => { + tracing::warn!("submit txn failed, error: {}", e); + self.handle_submit_txn_failure(&ids, batch_size).await?; + } + }; + } + Ok(()) + } + + async fn check_failed_activations(&self) -> Result<()> { + let num_marked_failed = db::update_failed_activations(&self.pool).await?; + metrics::counter!("failed_activations", num_marked_failed); + let total_failed_count = db::get_failed_activations_count(&self.pool).await?; + metrics::gauge!("db_failed_row_count", total_failed_count as f64); + if total_failed_count > 0 { + tracing::warn!("{} failed status activations ", total_failed_count); + }; + Ok(()) + } + + async fn handle_submit_txn_success( + &self, + ids: &[u64], + batch_size: usize, + summed_activations_count: u64, + ) -> Result<()> { + tracing::info!("processed batch of {} activations successfully", batch_size); + metrics::counter!("success_activations", summed_activations_count); + db::update_success_batch(&self.pool, ids).await?; + Ok(()) + } + + async fn handle_submit_txn_failure(&self, ids: &[u64], batch_size: usize) -> Result<()> { + tracing::info!( + "failed to process batch of {} activations, retrying next tick", + batch_size + ); + db::update_failed_batch(&self.pool, ids).await?; + Ok(()) + } + + async fn confirm_txn<'a>(&self, txn_row: &TxnRow) -> Result<()> { + if self.solana.confirm_transaction(&txn_row.txn_id).await? { + tracing::info!("txn_id {} confirmed on chain, updated db", txn_row.txn_id); + db::update_verified_txns_onchain(&self.pool, &txn_row.txn_id).await? + } else { + tracing::info!( + "txn_id {} confirmed NOT on chain, updated db and requeued activations", + txn_row.txn_id + ); + db::update_verified_txns_not_onchain(&self.pool, &txn_row.txn_id).await? + } + Ok(()) + } +} diff --git a/boost_manager/src/watcher.rs b/boost_manager/src/watcher.rs new file mode 100644 index 000000000..82a35a078 --- /dev/null +++ b/boost_manager/src/watcher.rs @@ -0,0 +1,114 @@ +use anyhow::Result; +use chrono::{DateTime, TimeZone, Utc}; +use db_store::meta; +use file_store::file_sink::FileSinkClient; +use file_store::traits::TimestampEncode; +use futures::{future::LocalBoxFuture, TryFutureExt}; +use helium_proto::BoostedHexUpdateV1 as BoostedHexUpdateProto; +use mobile_config::{ + boosted_hex_info::BoostedHexes, + client::{hex_boosting_client::HexBoostingInfoResolver, ClientError}, +}; +use sqlx::{PgExecutor, Pool, Postgres}; +use task_manager::ManagedTask; +use tokio::time; + +const POLL_TIME: time::Duration = time::Duration::from_secs(60 * 30); +const LAST_PROCESSED_TIMESTAMP_KEY: &str = "last_processed_hex_boosting_info"; + +pub struct Watcher { + pub pool: Pool, + pub hex_boosting_client: A, + pub file_sink: FileSinkClient, +} + +impl ManagedTask for Watcher +where + A: HexBoostingInfoResolver, +{ + fn start_task( + self: Box, + shutdown: triggered::Listener, + ) -> LocalBoxFuture<'static, anyhow::Result<()>> { + let handle = tokio::spawn(self.run(shutdown)); + Box::pin( + handle + .map_err(anyhow::Error::from) + .and_then(|result| async move { result.map_err(anyhow::Error::from) }), + ) + } +} + +impl Watcher +where + A: HexBoostingInfoResolver, +{ + pub async fn new( + pool: Pool, + file_sink: FileSinkClient, + hex_boosting_client: A, + ) -> Result { + Ok(Self { + pool, + file_sink, + hex_boosting_client, + }) + } + + pub async fn run(mut self, shutdown: triggered::Listener) -> anyhow::Result<()> { + tracing::info!("starting Watcher"); + let mut timer = time::interval(POLL_TIME); + loop { + tokio::select! { + biased; + _ = shutdown.clone() => break, + _ = timer.tick() => { + match self.handle_tick().await { + Ok(()) => (), + Err(err) => { + tracing::error!("fatal Watcher error: {err:?}"); + } + } + } + } + } + tracing::info!("stopping Watcher"); + Ok(()) + } + + pub async fn handle_tick(&mut self) -> Result<()> { + let now = Utc::now(); + // get the last time we processed hex boosting info + let last_processed_ts = fetch_last_processed_timestamp(&self.pool).await?; + + // get modified hex info from mobile config + let boosted_hexes = + BoostedHexes::get_modified(&self.hex_boosting_client, last_processed_ts).await?; + tracing::info!("modified hexes count: {}", boosted_hexes.hexes.len()); + for info in boosted_hexes.hexes.values() { + let proto: BoostedHexUpdateProto = BoostedHexUpdateProto { + timestamp: now.encode_timestamp(), + update: Some(info.clone().try_into()?), + }; + self.file_sink.write(proto, []).await?.await??; + } + self.file_sink.commit().await?; + save_last_processed_timestamp(&self.pool, &now).await?; + Ok(()) + } +} + +pub async fn fetch_last_processed_timestamp( + db: impl PgExecutor<'_>, +) -> db_store::Result> { + Utc.timestamp_opt(meta::fetch(db, LAST_PROCESSED_TIMESTAMP_KEY).await?, 0) + .single() + .ok_or(db_store::Error::DecodeError) +} + +pub async fn save_last_processed_timestamp( + db: impl PgExecutor<'_>, + value: &DateTime, +) -> db_store::Result<()> { + meta::store(db, LAST_PROCESSED_TIMESTAMP_KEY, value.timestamp()).await +} diff --git a/boost_manager/tests/activator_tests.rs b/boost_manager/tests/activator_tests.rs new file mode 100644 index 000000000..b6cb0119d --- /dev/null +++ b/boost_manager/tests/activator_tests.rs @@ -0,0 +1,182 @@ +mod common; +use boost_manager::{activator, db, OnChainStatus}; +use chrono::{DateTime, Duration as ChronoDuration, Duration, Timelike, Utc}; +use helium_proto::services::poc_mobile::BoostedHex as BoostedHexProto; +use mobile_config::boosted_hex_info::{BoostedHexInfo, BoostedHexes}; +use sqlx::PgPool; +use std::collections::HashMap; + +const BOOST_CONFIG_PUBKEY: &str = "11hd7HoicRgBPjBGcqcT2Y9hRQovdZeff5eKFMbCSuDYQmuCiF1"; + +struct TestContext { + boosted_hexes: Vec, +} + +impl TestContext { + fn setup(now: DateTime) -> anyhow::Result { + let epoch = (now - ChronoDuration::hours(24))..now; + let boost_period_length = Duration::days(30); + + // setup boosted hex data to stream as updates + let multipliers1 = vec![2, 10, 15, 35]; + let start_ts_1 = epoch.start - boost_period_length; + let end_ts_1 = start_ts_1 + (boost_period_length * multipliers1.len() as i32); + let multipliers2 = vec![3, 10, 20]; + let start_ts_2 = epoch.start - (boost_period_length * 2); + let end_ts_2 = start_ts_2 + (boost_period_length * multipliers2.len() as i32); + let multipliers3 = vec![1, 10, 20]; + + let boosts = vec![ + BoostedHexInfo { + location: 0x8a1fb466d2dffff_u64, + start_ts: Some(start_ts_1), + end_ts: Some(end_ts_1), + period_length: boost_period_length, + multipliers: multipliers1, + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + BoostedHexInfo { + location: 0x8a1fb49642dffff_u64, + start_ts: Some(start_ts_2), + end_ts: Some(end_ts_2), + period_length: boost_period_length, + multipliers: multipliers2, + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + BoostedHexInfo { + // hotspot 3's location + location: 0x8c2681a306607ff_u64, + start_ts: None, + end_ts: None, + period_length: boost_period_length, + multipliers: multipliers3, + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + ]; + Ok(Self { + boosted_hexes: boosts, + }) + } +} +#[sqlx::test] +async fn test_activated_hex_insert(pool: PgPool) -> anyhow::Result<()> { + let now = Utc::now(); + let ctx = TestContext::setup(now)?; + let boosted_hexes_map = ctx + .boosted_hexes + .iter() + .map(|info| (info.location, info.clone())) + .collect::>(); + let boosted_hexes = BoostedHexes { + hexes: boosted_hexes_map, + }; + + // test a boosted hex derived from radio rewards + // with a non set start date, will result in a row being + // inserted to the activation table + let mut txn = pool.clone().begin().await?; + activator::process_boosted_hex( + &mut txn, + now, + &boosted_hexes, + &BoostedHexProto { + location: 0x8c2681a306607ff_u64, + multiplier: 10, + }, + ) + .await?; + txn.commit().await?; + let rows = db::get_queued_batch(&pool).await?; + assert_eq!(rows.len(), 1); + let status = db::query_activation_statuses(&pool).await?; + assert_eq!(status[0].status, OnChainStatus::Queued); + assert_eq!(status[0].location, 0x8c2681a306607ff_u64); + + Ok(()) +} + +#[sqlx::test] +async fn test_activated_hex_no_insert(pool: PgPool) -> anyhow::Result<()> { + let now = Utc::now(); + let ctx = TestContext::setup(now)?; + let boosted_hexes_map = ctx + .boosted_hexes + .iter() + .map(|info| (info.location, info.clone())) + .collect::>(); + let boosted_hexes = BoostedHexes { + hexes: boosted_hexes_map, + }; + + // test a boosted hex derived from radio rewards + // with an active start date, will result in no row being + // inserted to the activation table + let mut txn = pool.clone().begin().await?; + activator::process_boosted_hex( + &mut txn, + now, + &boosted_hexes, + &BoostedHexProto { + location: 0x8a1fb49642dffff_u64, + multiplier: 10, + }, + ) + .await?; + txn.commit().await?; + let rows = db::get_queued_batch(&pool).await?; + assert_eq!(rows.len(), 0); + Ok(()) +} + +#[sqlx::test] +async fn test_activated_dup_hex_insert(pool: PgPool) -> anyhow::Result<()> { + let now = Utc::now().with_second(0).unwrap(); + let ctx = TestContext::setup(now)?; + let boosted_hexes_map = ctx + .boosted_hexes + .iter() + .map(|info| (info.location, info.clone())) + .collect::>(); + let boosted_hexes = BoostedHexes { + hexes: boosted_hexes_map, + }; + + // test with DUPLICATE boosted hexes derived from radio rewards + // with a non set start date, will result in a single row being + // inserted to the activation table with an activation ts + // equal to the first hex processed + let mut txn = pool.clone().begin().await?; + activator::process_boosted_hex( + &mut txn, + now, + &boosted_hexes, + &BoostedHexProto { + location: 0x8c2681a306607ff_u64, + multiplier: 10, + }, + ) + .await?; + + activator::process_boosted_hex( + &mut txn, + now - ChronoDuration::days(1), + &boosted_hexes, + &BoostedHexProto { + location: 0x8c2681a306607ff_u64, + multiplier: 5, + }, + ) + .await?; + + txn.commit().await?; + let rows1 = db::get_queued_batch(&pool).await?; + assert_eq!(rows1.len(), 1); + // assert_eq!(rows1[0].activation_ts, now); + Ok(()) +} diff --git a/boost_manager/tests/common/mod.rs b/boost_manager/tests/common/mod.rs new file mode 100644 index 000000000..1c5d267b1 --- /dev/null +++ b/boost_manager/tests/common/mod.rs @@ -0,0 +1,79 @@ +use file_store::file_sink::{FileSinkClient, Message as SinkMessage}; +use helium_proto::BoostedHexInfoV1 as BoostedHexInfoProto; +use helium_proto::BoostedHexUpdateV1 as BoostedHexUpdateProto; +use helium_proto::Message; +use mobile_config::boosted_hex_info::BoostedHexInfo; +use tokio::{sync::mpsc::error::TryRecvError, time::timeout}; + +#[derive(Debug, Clone)] +pub struct MockHexBoostingClient { + pub boosted_hexes: Vec, +} + +pub struct MockFileSinkReceiver { + pub receiver: tokio::sync::mpsc::Receiver, +} + +#[allow(dead_code)] +impl MockFileSinkReceiver { + pub async fn receive(&mut self) -> Option> { + match timeout(seconds(2), self.receiver.recv()).await { + Ok(Some(SinkMessage::Data(on_write_tx, msg))) => { + let _ = on_write_tx.send(Ok(())); + Some(msg) + } + Ok(None) => None, + Err(e) => panic!("timeout while waiting for message1 {:?}", e), + Ok(Some(unexpected_msg)) => { + println!("ignoring unexpected msg {:?}", unexpected_msg); + None + } + } + } + + pub async fn get_all(&mut self) -> Vec> { + let mut buf = Vec::new(); + while let Ok(SinkMessage::Data(on_write_tx, msg)) = self.receiver.try_recv() { + let _ = on_write_tx.send(Ok(())); + buf.push(msg); + } + buf + } + + pub fn assert_no_messages(&mut self) { + let Err(TryRecvError::Empty) = self.receiver.try_recv() else { + panic!("receiver should have been empty") + }; + } + + pub async fn receive_updated_hex(&mut self) -> BoostedHexInfoProto { + match self.receive().await { + Some(bytes) => { + let boosted_hex_update = BoostedHexUpdateProto::decode(bytes.as_slice()) + .expect("failed to decode boosted hex update"); + println!("boosted hex update: {:?}", boosted_hex_update); + match boosted_hex_update.update { + Some(r) => r, + _ => panic!("failed to get boosted hex update"), + } + } + None => panic!("failed to receive boosted hex update"), + } + } +} + +#[allow(dead_code)] +pub fn create_file_sink() -> (FileSinkClient, MockFileSinkReceiver) { + let (tx, rx) = tokio::sync::mpsc::channel(20); + ( + FileSinkClient { + sender: tx, + metric: "metric", + }, + MockFileSinkReceiver { receiver: rx }, + ) +} + +pub fn seconds(s: u64) -> std::time::Duration { + std::time::Duration::from_secs(s) +} diff --git a/boost_manager/tests/updater_tests.rs b/boost_manager/tests/updater_tests.rs new file mode 100644 index 000000000..0ba20c73c --- /dev/null +++ b/boost_manager/tests/updater_tests.rs @@ -0,0 +1,185 @@ +use async_trait::async_trait; +use boost_manager::{db, updater::Updater, OnChainStatus}; +use chrono::{DateTime, Utc}; +use file_store::hex_boost::BoostedHexActivation; +use solana::{start_boost::SolanaNetwork, GetSignature}; +use solana_sdk::signature::Signature; +use sqlx::{PgPool, Postgres, Transaction}; +use std::{string::ToString, sync::Mutex, time::Duration}; + +const BOOSTED_HEX1_PUBKEY: &str = "112NqN2WWMwtK29PMzRby62fDydBJfsCLkCAf392stdok48ovNT6"; +const BOOSTED_HEX2_PUBKEY: &str = "11uJHS2YaEWJqgqC7yza9uvSmpv5FWoMQXiP8WbxBGgNUmifUJf"; +const BOOSTED_HEX3_PUBKEY: &str = "11hd7HoicRgBPjBGcqcT2Y9hRQovdZeff5eKFMbCSuDYQmuCiF1"; +const BOOSTED_HEX_CONFIG_PUBKEY: &str = "112QhnxqU8QZ3jUXpoRk51quuQVft9Pf5P5zzDDvLxj7Q9QqbMh7"; + +#[derive(Clone, Debug)] +#[allow(dead_code)] +pub struct MockTransaction { + signature: Signature, + activations: Vec, +} + +pub struct MockSolanaConnection { + submitted: Mutex>, + error: Option, +} + +#[derive(Clone, Debug)] +pub struct MockSignature { + pub signature: String, +} + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("not found")] + SubmitError(String), +} + +impl MockSolanaConnection { + fn ok() -> Self { + Self { + submitted: Mutex::new(vec![]), + error: None, + } + } + + fn with_error(error: String) -> Self { + Self { + submitted: Mutex::new(vec![]), + error: Some(error), + } + } +} + +#[async_trait] +impl SolanaNetwork for MockSolanaConnection { + type Error = Error; + type Transaction = MockTransaction; + + async fn make_start_boost_transaction( + &self, + batch: &[BoostedHexActivation], + ) -> Result { + Ok(MockTransaction { + signature: Signature::new_unique(), + activations: batch.to_owned(), + }) + } + + async fn submit_transaction(&self, txn: &Self::Transaction) -> Result<(), Self::Error> { + self.submitted.lock().unwrap().push(txn.clone()); + self.error + .as_ref() + .map(|str| Err(Error::SubmitError(str.clone()))) + .unwrap_or(Ok(())) + } + + async fn confirm_transaction(&self, _id: &str) -> Result { + Ok(true) + } +} + +impl GetSignature for MockTransaction { + fn get_signature(&self) -> &Signature { + &self.signature + } +} + +#[sqlx::test] +async fn test_process_activations_success(pool: PgPool) -> anyhow::Result<()> { + let now = Utc::now(); + let solana_connection = MockSolanaConnection::ok(); + let updater = Updater::new( + pool.clone(), + true, + Duration::from_secs(10), + 10, + solana_connection, + )?; + + let mut txn = pool.begin().await?; + seed_activations(&mut txn, now).await?; + txn.commit().await?; + + updater.process_activations().await?; + + let res = db::query_activation_statuses(&pool).await?; + println!("res: {:?}", res); + assert_eq!(res[0].status, OnChainStatus::Success); + assert_eq!(res[0].location, 0x8a1fb466d2dffff_u64); + assert_eq!(res[1].status, OnChainStatus::Success); + assert_eq!(res[1].location, 0x8a1fb49642dffff_u64); + assert_eq!(res[2].status, OnChainStatus::Success); + assert_eq!(res[2].location, 0x8c2681a306607ff_u64); + Ok(()) +} + +#[sqlx::test] +async fn test_process_activations_failure(pool: PgPool) -> anyhow::Result<()> { + let now = Utc::now(); + let solana_connection = MockSolanaConnection::with_error("txn failed".to_string()); + let updater = Updater::new( + pool.clone(), + true, + Duration::from_secs(10), + 10, + solana_connection, + )?; + + let mut txn = pool.begin().await?; + seed_activations(&mut txn, now).await?; + txn.commit().await?; + + // ensure the activations are processed at least 10 times + // submit_txn will bork each time + // pushing the retries value to exceed max and + // thus forcing it to FAILED status + for _ in 1..=11 { + updater.process_activations().await?; + } + let mut res = db::query_activation_statuses(&pool).await?; + res.sort_by(|a, b| b.location.cmp(&a.location)); + assert_eq!(res[0].status, OnChainStatus::Failed); + assert_eq!(res[0].location, 0x8c2681a306607ff_u64); + assert_eq!(res[1].status, OnChainStatus::Failed); + assert_eq!(res[1].location, 0x8a1fb49642dffff_u64); + assert_eq!(res[2].status, OnChainStatus::Failed); + assert_eq!(res[2].location, 0x8a1fb466d2dffff_u64); + + // should return zero queued activations + let rows = db::get_queued_batch(&pool).await?; + assert_eq!(rows.len(), 0); + + Ok(()) +} + +async fn seed_activations( + txn: &mut Transaction<'_, Postgres>, + activation_ts: DateTime, +) -> anyhow::Result<()> { + db::insert_activated_hex( + txn, + 0x8a1fb466d2dffff_u64, + &BOOSTED_HEX1_PUBKEY.to_string(), + &BOOSTED_HEX_CONFIG_PUBKEY.to_string(), + activation_ts, + ) + .await?; + db::insert_activated_hex( + txn, + 0x8a1fb49642dffff_u64, + &BOOSTED_HEX2_PUBKEY.to_string(), + &BOOSTED_HEX_CONFIG_PUBKEY.to_string(), + activation_ts, + ) + .await?; + db::insert_activated_hex( + txn, + 0x8c2681a306607ff_u64, + &BOOSTED_HEX3_PUBKEY.to_string(), + &BOOSTED_HEX_CONFIG_PUBKEY.to_string(), + activation_ts, + ) + .await?; + Ok(()) +} diff --git a/boost_manager/tests/watcher_tests.rs b/boost_manager/tests/watcher_tests.rs new file mode 100644 index 000000000..4ce37fd06 --- /dev/null +++ b/boost_manager/tests/watcher_tests.rs @@ -0,0 +1,114 @@ +mod common; +use crate::common::{MockFileSinkReceiver, MockHexBoostingClient}; +use async_trait::async_trait; +use boost_manager::watcher::{self, Watcher}; +use chrono::{DateTime, Duration as ChronoDuration, Duration, Utc}; +use futures_util::{stream, StreamExt as FuturesStreamExt}; +use helium_proto::BoostedHexInfoV1 as BoostedHexInfoProto; +use mobile_config::{ + boosted_hex_info::{BoostedHexInfo, BoostedHexInfoStream}, + client::{hex_boosting_client::HexBoostingInfoResolver, ClientError}, +}; +use sqlx::PgPool; + +const BOOST_CONFIG_PUBKEY: &str = "11hd7HoicRgBPjBGcqcT2Y9hRQovdZeff5eKFMbCSuDYQmuCiF1"; + +impl MockHexBoostingClient { + fn new(boosted_hexes: Vec) -> Self { + Self { boosted_hexes } + } +} + +#[async_trait] +impl HexBoostingInfoResolver for MockHexBoostingClient { + type Error = ClientError; + + async fn stream_boosted_hexes_info(&mut self) -> Result { + Ok(stream::iter(self.boosted_hexes.clone()).boxed()) + } + + async fn stream_modified_boosted_hexes_info( + &mut self, + _timestamp: DateTime, + ) -> Result { + Ok(stream::iter(self.boosted_hexes.clone()).boxed()) + } +} + +#[sqlx::test] +async fn test_boosted_hex_updates_to_filestore(pool: PgPool) -> anyhow::Result<()> { + let (hex_update_client, mut hex_update) = common::create_file_sink(); + + let now = Utc::now(); + let epoch = (now - ChronoDuration::hours(24))..now; + let boost_period_length = Duration::days(30); + + // setup boosted hex data to stream as updates + let multipliers1 = vec![2, 10, 15, 35]; + let start_ts_1 = epoch.start - boost_period_length; + let end_ts_1 = start_ts_1 + (boost_period_length * multipliers1.len() as i32); + let multipliers2 = vec![3, 10, 20]; + let start_ts_2 = epoch.start - (boost_period_length * 2); + let end_ts_2 = start_ts_2 + (boost_period_length * multipliers2.len() as i32); + + let boosted_hexes = vec![ + BoostedHexInfo { + location: 0x8a1fb466d2dffff_u64, + start_ts: Some(start_ts_1), + end_ts: Some(end_ts_1), + period_length: boost_period_length, + multipliers: multipliers1, + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + BoostedHexInfo { + location: 0x8a1fb49642dffff_u64, + start_ts: Some(start_ts_2), + end_ts: Some(end_ts_2), + period_length: boost_period_length, + multipliers: multipliers2, + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + ]; + + let hex_boosting_client = MockHexBoostingClient::new(boosted_hexes); + + let mut watcher = Watcher::new(pool.clone(), hex_update_client, hex_boosting_client) + .await + .unwrap(); + + let last_processed_ts = now - Duration::days(1); + watcher::save_last_processed_timestamp(&pool, &last_processed_ts).await?; + + let (_, boosted_hexes_result) = tokio::join!( + watcher.handle_tick(), + receive_expected_msgs(&mut hex_update) + ); + if let Ok(boosted_hexes) = boosted_hexes_result { + // assert the boosted hexes outputted to filestore + assert_eq!(2, boosted_hexes.len()); + assert_eq!(0x8a1fb49642dffff_u64, boosted_hexes[0].location); + assert_eq!(0x8a1fb466d2dffff_u64, boosted_hexes[1].location); + } else { + panic!("no boosted hex updates received"); + }; + Ok(()) +} + +async fn receive_expected_msgs( + hex_update: &mut MockFileSinkReceiver, +) -> anyhow::Result> { + // get the filestore outputs + // we will have 2 updates hexes + let hex_update_1 = hex_update.receive_updated_hex().await; + let hex_update_2 = hex_update.receive_updated_hex().await; + // ordering is not guaranteed, so stick the updates into a vec and sort + let mut updates = vec![hex_update_1, hex_update_2]; + updates.sort_by(|a, b| b.location.cmp(&a.location)); + // should be no further msgs + hex_update.assert_no_messages(); + Ok(updates) +} diff --git a/file_store/src/cli/dump.rs b/file_store/src/cli/dump.rs index 6f9f0e494..2ebc05755 100644 --- a/file_store/src/cli/dump.rs +++ b/file_store/src/cli/dump.rs @@ -28,7 +28,8 @@ use helium_proto::{ }, router::PacketRouterPacketReportV1, }, - BlockchainTxn, Message, PriceReportV1, RewardManifest, SubnetworkRewards, + BlockchainTxn, BoostedHexUpdateV1 as BoostedHexUpdateProto, Message, PriceReportV1, + RewardManifest, SubnetworkRewards, }; use serde_json::json; use std::io; @@ -51,6 +52,21 @@ impl Cmd { while let Some(result) = file_stream.next().await { let msg = result?; match self.file_type { + FileType::BoostedHexUpdate => { + let dec_msg = BoostedHexUpdateProto::decode(msg)?; + let update = dec_msg.update.unwrap(); + let json = json!({ + "last_update": dec_msg.timestamp, + "location": update.location, + "start_ts": update.start_ts, + "end_ts": update.end_ts, + "period_length": update.period_length, + "multipliers": update.multipliers, + "boosted_hex_pubkey": update.boosted_hex_pubkey, + "boost_config_pubkey": update.boost_config_pubkey, + }); + print_json(&json)?; + } FileType::CbrsHeartbeat => { let dec_msg = CellHeartbeatReqV1::decode(msg)?; wtr.serialize(CbrsHeartbeat::try_from(dec_msg)?)?; @@ -221,6 +237,7 @@ impl Cmd { "hotspot_key": PublicKey::try_from(reward.hotspot_key)?, "cbsd_id": reward.cbsd_id, "poc_reward": reward.poc_reward, + "boosted_hexes": reward.boosted_hexes, }))?, Some(Reward::SubscriberReward(reward)) => print_json(&json!({ "subscriber_id": reward.subscriber_id, diff --git a/file_store/src/file_info.rs b/file_store/src/file_info.rs index a88647399..4beccad44 100644 --- a/file_store/src/file_info.rs +++ b/file_store/src/file_info.rs @@ -145,6 +145,8 @@ pub const COVERAGE_OBJECT: &str = "coverage_object"; pub const COVERAGE_OBJECT_INGEST_REPORT: &str = "coverage_object_ingest_report"; pub const SENIORITY_UPDATE: &str = "seniority_update"; +pub const BOOSTED_HEX_UPDATE: &str = "boosted_hex_update"; + #[derive(Debug, PartialEq, Eq, Clone, Serialize, Copy, strum::EnumCount)] #[serde(rename_all = "snake_case")] pub enum FileType { @@ -185,6 +187,7 @@ pub enum FileType { VerifiedSpeedtest, WifiHeartbeat, WifiHeartbeatIngestReport, + BoostedHexUpdate, } impl fmt::Display for FileType { @@ -231,6 +234,7 @@ impl fmt::Display for FileType { Self::CoverageObject => COVERAGE_OBJECT, Self::CoverageObjectIngestReport => COVERAGE_OBJECT_INGEST_REPORT, Self::SeniorityUpdate => SENIORITY_UPDATE, + Self::BoostedHexUpdate => BOOSTED_HEX_UPDATE, }; f.write_str(s) } @@ -280,6 +284,7 @@ impl FileType { Self::CoverageObject => COVERAGE_OBJECT, Self::CoverageObjectIngestReport => COVERAGE_OBJECT_INGEST_REPORT, Self::SeniorityUpdate => SENIORITY_UPDATE, + Self::BoostedHexUpdate => BOOSTED_HEX_UPDATE, } } } @@ -329,6 +334,7 @@ impl FromStr for FileType { COVERAGE_OBJECT => Self::CoverageObject, COVERAGE_OBJECT_INGEST_REPORT => Self::CoverageObjectIngestReport, SENIORITY_UPDATE => Self::SeniorityUpdate, + BOOSTED_HEX_UPDATE => Self::BoostedHexUpdate, _ => return Err(Error::from(io::Error::from(io::ErrorKind::InvalidInput))), }; Ok(result) diff --git a/file_store/src/hex_boost.rs b/file_store/src/hex_boost.rs new file mode 100644 index 000000000..b91fb8040 --- /dev/null +++ b/file_store/src/hex_boost.rs @@ -0,0 +1,9 @@ +use chrono::{DateTime, Utc}; + +#[derive(Debug, Clone)] +pub struct BoostedHexActivation { + pub location: u64, + pub activation_ts: DateTime, + pub boosted_hex_pubkey: String, + pub boost_config_pubkey: String, +} diff --git a/file_store/src/lib.rs b/file_store/src/lib.rs index 019c7c7ac..83325fef4 100644 --- a/file_store/src/lib.rs +++ b/file_store/src/lib.rs @@ -9,6 +9,7 @@ pub mod file_source; pub mod file_store; pub mod file_upload; pub mod heartbeat; +pub mod hex_boost; pub mod iot_beacon_report; pub mod iot_invalid_poc; pub mod iot_packet; diff --git a/file_store/src/traits/msg_verify.rs b/file_store/src/traits/msg_verify.rs index 9eacfd810..45c6b08cc 100644 --- a/file_store/src/traits/msg_verify.rs +++ b/file_store/src/traits/msg_verify.rs @@ -86,6 +86,9 @@ impl_msg_verify!(mobile_config::GatewayInfoStreamReqV1, signature); impl_msg_verify!(mobile_config::GatewayInfoResV1, signature); impl_msg_verify!(mobile_config::GatewayInfoBatchReqV1, signature); impl_msg_verify!(mobile_config::GatewayInfoStreamResV1, signature); +impl_msg_verify!(mobile_config::BoostedHexInfoStreamReqV1, signature); +impl_msg_verify!(mobile_config::BoostedHexModifiedInfoStreamReqV1, signature); +impl_msg_verify!(mobile_config::BoostedHexInfoStreamResV1, signature); #[cfg(test)] mod test { diff --git a/iot_packet_verifier/src/balances.rs b/iot_packet_verifier/src/balances.rs index c80532837..57f559e71 100644 --- a/iot_packet_verifier/src/balances.rs +++ b/iot_packet_verifier/src/balances.rs @@ -3,7 +3,7 @@ use crate::{ verifier::Debiter, }; use helium_crypto::PublicKeyBinary; -use solana::SolanaNetwork; +use solana::burn::SolanaNetwork; use std::{ collections::{hash_map::Entry, HashMap}, sync::Arc, diff --git a/iot_packet_verifier/src/burner.rs b/iot_packet_verifier/src/burner.rs index 5284d7eb2..9c8950cdf 100644 --- a/iot_packet_verifier/src/burner.rs +++ b/iot_packet_verifier/src/burner.rs @@ -5,7 +5,7 @@ use crate::{ }, }; use futures::{future::LocalBoxFuture, TryFutureExt}; -use solana::{GetSignature, SolanaNetwork}; +use solana::{burn::SolanaNetwork, GetSignature}; use std::time::Duration; use task_manager::ManagedTask; use tokio::time::{self, MissedTickBehavior}; diff --git a/iot_packet_verifier/src/daemon.rs b/iot_packet_verifier/src/daemon.rs index f50a72fc9..7aac79942 100644 --- a/iot_packet_verifier/src/daemon.rs +++ b/iot_packet_verifier/src/daemon.rs @@ -16,7 +16,7 @@ use file_store::{ }; use futures_util::TryFutureExt; use iot_config::client::{org_client::Orgs, OrgClient}; -use solana::SolanaRpc; +use solana::burn::SolanaRpc; use sqlx::{Pool, Postgres}; use std::{sync::Arc, time::Duration}; use task_manager::{ManagedTask, TaskManager}; diff --git a/iot_packet_verifier/src/pending.rs b/iot_packet_verifier/src/pending.rs index c7ab65e38..362ef9c13 100644 --- a/iot_packet_verifier/src/pending.rs +++ b/iot_packet_verifier/src/pending.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use chrono::{DateTime, Duration, Utc}; use helium_crypto::PublicKeyBinary; -use solana::SolanaNetwork; +use solana::burn::SolanaNetwork; use solana_sdk::signature::Signature; use sqlx::{postgres::PgRow, FromRow, PgPool, Postgres, Row, Transaction}; use std::{collections::HashMap, sync::Arc}; diff --git a/iot_packet_verifier/src/settings.rs b/iot_packet_verifier/src/settings.rs index 92ce6c33d..c67aeb886 100644 --- a/iot_packet_verifier/src/settings.rs +++ b/iot_packet_verifier/src/settings.rs @@ -24,7 +24,7 @@ pub struct Settings { /// Minimum data credit balance required for a payer before we disable them #[serde(default = "default_minimum_allowed_balance")] pub minimum_allowed_balance: u64, - pub solana: Option, + pub solana: Option, #[serde(default = "default_start_after")] pub start_after: u64, /// Number of minutes we should sleep before checking to re-enable diff --git a/iot_packet_verifier/src/verifier.rs b/iot_packet_verifier/src/verifier.rs index fc2986079..894e4b3e3 100644 --- a/iot_packet_verifier/src/verifier.rs +++ b/iot_packet_verifier/src/verifier.rs @@ -10,7 +10,7 @@ use helium_proto::services::{ router::packet_router_packet_report_v1::PacketType, }; use iot_config::client::org_client::Orgs; -use solana::SolanaNetwork; +use solana::burn::SolanaNetwork; use std::{ collections::{hash_map::Entry, HashMap}, convert::Infallible, diff --git a/iot_packet_verifier/tests/integration_tests.rs b/iot_packet_verifier/tests/integration_tests.rs index 4d7faff19..461c528e1 100644 --- a/iot_packet_verifier/tests/integration_tests.rs +++ b/iot_packet_verifier/tests/integration_tests.rs @@ -16,7 +16,10 @@ use iot_packet_verifier::{ pending::{confirm_pending_txns, AddPendingBurn, Burn, MockPendingTables, PendingTables}, verifier::{payload_size_to_dc, ConfigServer, Org, Verifier, BYTES_PER_DC}, }; -use solana::{GetSignature, MockTransaction, SolanaNetwork}; +use solana::{ + burn::{MockTransaction, SolanaNetwork}, + GetSignature, +}; use solana_sdk::signature::Signature; use sqlx::PgPool; use std::{ diff --git a/iot_verifier/src/meta.rs b/iot_verifier/src/meta.rs index d6f73bc31..6021a0ada 100644 --- a/iot_verifier/src/meta.rs +++ b/iot_verifier/src/meta.rs @@ -60,10 +60,8 @@ impl Meta { .fetch_optional(executor) .await? .and_then(|v| { - v.parse::().map_or_else( - |_| None, - |ts| ts.to_timestamp_millis().map_or_else(|_| None, Some), - ) + v.parse::() + .map_or_else(|_| None, |ts| ts.to_timestamp_millis().ok()) }); Ok(last_timestamp) } diff --git a/mobile_config/src/boosted_hex_info.rs b/mobile_config/src/boosted_hex_info.rs new file mode 100644 index 000000000..916fcadcf --- /dev/null +++ b/mobile_config/src/boosted_hex_info.rs @@ -0,0 +1,244 @@ +use crate::client::{hex_boosting_client::HexBoostingInfoResolver, ClientError}; +use chrono::{DateTime, Duration, Utc}; +use file_store::traits::TimestampDecode; +use futures::stream::{BoxStream, StreamExt}; +use helium_proto::BoostedHexInfoV1 as BoostedHexInfoProto; +use std::{collections::HashMap, convert::TryFrom, str}; + +pub type BoostedHexInfoStream = BoxStream<'static, BoostedHexInfo>; + +lazy_static::lazy_static! { + static ref PERIOD_IN_SECONDS: Duration = Duration::seconds(60 * 60 * 24 * 30); +} + +#[derive(Clone, Debug)] +pub struct BoostedHexInfo { + pub location: u64, + pub start_ts: Option>, + pub end_ts: Option>, + pub period_length: Duration, + pub multipliers: Vec, + pub boosted_hex_pubkey: String, + pub boost_config_pubkey: String, + pub version: u32, +} + +impl TryFrom for BoostedHexInfo { + type Error = anyhow::Error; + fn try_from(v: BoostedHexInfoProto) -> anyhow::Result { + let period_length = Duration::seconds(v.period_length as i64); + let multipliers = v.multipliers; + let start_ts = to_start_ts(v.start_ts); + let end_ts = to_end_ts(start_ts, period_length, multipliers.len()); + Ok(Self { + location: v.location, + start_ts, + end_ts, + period_length, + multipliers, + // todo: maybe just convert to solana keys here?? + boosted_hex_pubkey: str::from_utf8(&v.boosted_hex_pubkey)?.into(), + boost_config_pubkey: str::from_utf8(&v.boost_config_pubkey)?.into(), + version: v.version, + }) + } +} + +impl TryFrom for BoostedHexInfoProto { + type Error = anyhow::Error; + + fn try_from(v: BoostedHexInfo) -> anyhow::Result { + let start_ts = v.start_ts.map_or(0, |v| v.timestamp() as u64); + let end_ts = v.end_ts.map_or(0, |v| v.timestamp() as u64); + Ok(Self { + location: v.location, + start_ts, + end_ts, + period_length: v.period_length.num_seconds() as u32, + multipliers: v.multipliers, + boosted_hex_pubkey: v.boosted_hex_pubkey.into(), + boost_config_pubkey: v.boost_config_pubkey.into(), + version: v.version, + }) + } +} + +impl BoostedHexInfo { + pub fn current_multiplier(&self, ts: DateTime) -> anyhow::Result> { + if self.end_ts.is_some() && ts >= self.end_ts.unwrap() { + // end time has been set and the current time is after the end time, so return None + // to indicate that the hex is no longer boosted + return Ok(None); + }; + if self.start_ts.is_some() { + // start time has previously been set, so we can calculate the current multiplier + // based on the period length and the current time + let boost_start_ts = self.start_ts.unwrap(); + let diff = ts - boost_start_ts; + let index = diff + .num_seconds() + .checked_div(self.period_length.num_seconds()) + .unwrap_or(0) as usize; + Ok(Some(self.multipliers[index])) + } else { + // start time has not been previously set, assume this is the first time rewarding this hex + // and use the first multiplier + Ok(Some(self.multipliers[0])) + } + } +} + +#[derive(Debug, Clone, Default)] +pub struct BoostedHexes { + pub hexes: HashMap, +} + +#[derive(PartialEq, Debug, Clone)] +pub struct BoostedHex { + pub location: u64, + pub multiplier: u32, +} + +impl BoostedHexes { + pub async fn new(hexes: Vec) -> anyhow::Result { + let mut map = HashMap::new(); + for info in hexes { + map.insert(info.location, info); + } + Ok(Self { hexes: map }) + } + + pub async fn get_all( + hex_service_client: &impl HexBoostingInfoResolver, + ) -> anyhow::Result { + tracing::info!("getting boosted hexes"); + let mut map = HashMap::new(); + let mut stream = hex_service_client + .clone() + .stream_boosted_hexes_info() + .await?; + while let Some(info) = stream.next().await { + map.insert(info.location, info); + } + Ok(Self { hexes: map }) + } + + pub async fn get_modified( + hex_service_client: &impl HexBoostingInfoResolver, + timestamp: DateTime, + ) -> anyhow::Result { + tracing::info!("getting boosted hexes"); + let mut map = HashMap::new(); + let mut stream = hex_service_client + .clone() + .stream_modified_boosted_hexes_info(timestamp) + .await?; + while let Some(info) = stream.next().await { + map.insert(info.location, info); + } + Ok(Self { hexes: map }) + } + + pub fn get_current_multiplier(&self, location: u64, ts: DateTime) -> Option { + self.hexes + .get(&location) + .and_then(|info| info.current_multiplier(ts).ok()?) + } +} + +pub(crate) mod db { + use super::{to_end_ts, to_start_ts, BoostedHexInfo}; + use chrono::{DateTime, Duration, Utc}; + use futures::stream::{Stream, StreamExt}; + use sqlx::{PgExecutor, Row}; + + const GET_BOOSTED_HEX_INFO_SQL: &str = r#" + select + CAST(hexes.location as bigint), + CAST(hexes.start_ts as bigint), + config.period_length, + hexes.boosts_by_period as multipliers, + hexes.address as boosted_hex_pubkey, + config.address as boost_config_pubkey, + hexes.version + from boosted_hexes hexes + join boost_configs config on hexes.boost_config = config.address + "#; + + // TODO: reuse with string above + const GET_MODIFIED_BOOSTED_HEX_INFO_SQL: &str = r#" + select + CAST(hexes.location as bigint), + CAST(hexes.start_ts as bigint), + config.period_length, + hexes.boosts_by_period as multipliers, + hexes.address as boosted_hex_pubkey, + config.address as boost_config_pubkey, + hexes.version + from boosted_hexes hexes + join boost_configs config on hexes.boost_config = config.address + where hexes.refreshed_at > $1 + "#; + + pub fn all_info_stream<'a>( + db: impl PgExecutor<'a> + 'a, + ) -> impl Stream + 'a { + sqlx::query_as::<_, BoostedHexInfo>(GET_BOOSTED_HEX_INFO_SQL) + .fetch(db) + .filter_map(|info| async move { info.ok() }) + .boxed() + } + + pub fn modified_info_stream<'a>( + db: impl PgExecutor<'a> + 'a, + ts: DateTime, + ) -> impl Stream + 'a { + sqlx::query_as::<_, BoostedHexInfo>(GET_MODIFIED_BOOSTED_HEX_INFO_SQL) + .bind(ts) + .fetch(db) + .filter_map(|info| async move { info.ok() }) + .boxed() + } + + impl sqlx::FromRow<'_, sqlx::postgres::PgRow> for BoostedHexInfo { + fn from_row(row: &sqlx::postgres::PgRow) -> sqlx::Result { + let period_length = Duration::seconds(row.get::("period_length") as i64); + let start_ts = to_start_ts(row.get::("start_ts") as u64); + let multipliers = row + .get::, &str>("multipliers") + .into_iter() + .map(|v| v as u32) + .collect::>(); + let end_ts = to_end_ts(start_ts, period_length, multipliers.len()); + let boost_config_pubkey = row.get::<&str, &str>("boost_config_pubkey").into(); + let boosted_hex_pubkey = row.get::<&str, &str>("boosted_hex_pubkey").into(); + let version = row.get::("version") as u32; + Ok(Self { + location: row.get::("location") as u64, + start_ts, + end_ts, + period_length, + multipliers, + boosted_hex_pubkey, + boost_config_pubkey, + version, + }) + } + } +} + +fn to_start_ts(timestamp: u64) -> Option> { + if timestamp == 0 { + None + } else { + timestamp.to_timestamp().ok() + } +} + +fn to_end_ts( + start_ts: Option>, + period_length: Duration, + num_multipliers: usize, +) -> Option> { + start_ts.map(|ts| ts + period_length * num_multipliers as i32) +} diff --git a/mobile_config/src/client/hex_boosting_client.rs b/mobile_config/src/client/hex_boosting_client.rs new file mode 100644 index 000000000..7766a94db --- /dev/null +++ b/mobile_config/src/client/hex_boosting_client.rs @@ -0,0 +1,105 @@ +use super::{call_with_retry, ClientError, Settings}; +use crate::boosted_hex_info::{self, BoostedHexInfoStream}; +use chrono::{DateTime, Utc}; +use file_store::traits::MsgVerify; +use futures::stream::{self, StreamExt}; +use helium_crypto::{Keypair, PublicKey, Sign}; +use helium_proto::{ + services::{mobile_config, Channel}, + Message, +}; +use std::{error::Error, sync::Arc, time::Duration}; + +#[derive(Clone)] +pub struct HexBoostingClient { + pub client: mobile_config::HexBoostingClient, + signing_key: Arc, + config_pubkey: PublicKey, + batch_size: u32, +} + +impl HexBoostingClient { + pub fn from_settings(settings: &Settings) -> Result> { + Ok(Self { + client: settings.connect_hex_boosting_service_client(), + signing_key: settings.signing_keypair()?, + config_pubkey: settings.config_pubkey()?, + batch_size: settings.hex_boosting_batch_size, + }) + } +} + +#[async_trait::async_trait] +pub trait HexBoostingInfoResolver: Clone + Send + Sync + 'static { + type Error: Error + Send + Sync + 'static; + + async fn stream_boosted_hexes_info(&mut self) -> Result; + + async fn stream_modified_boosted_hexes_info( + &mut self, + timestamp: DateTime, + ) -> Result; +} + +#[async_trait::async_trait] +impl HexBoostingInfoResolver for HexBoostingClient { + type Error = ClientError; + + async fn stream_boosted_hexes_info(&mut self) -> Result { + let mut req = mobile_config::BoostedHexInfoStreamReqV1 { + batch_size: self.batch_size, + signer: self.signing_key.public_key().into(), + signature: vec![], + }; + req.signature = self.signing_key.sign(&req.encode_to_vec())?; + tracing::debug!("fetching boosted hexes info stream"); + let pubkey = Arc::new(self.config_pubkey.clone()); + let res_stream = call_with_retry!(self.client.info_stream(req.clone()))? + .into_inner() + .filter_map(|res| async move { res.ok() }) + .map(move |res| (res, pubkey.clone())) + .filter_map(|(res, pubkey)| async move { + match res.verify(&pubkey) { + Ok(()) => Some(res), + Err(_) => None, + } + }) + .flat_map(|res| stream::iter(res.hexes)) + .map(boosted_hex_info::BoostedHexInfo::try_from) + .filter_map(|hex| async move { hex.ok() }) + .boxed(); + + Ok(res_stream) + } + + async fn stream_modified_boosted_hexes_info( + &mut self, + timestamp: DateTime, + ) -> Result { + let mut req = mobile_config::BoostedHexModifiedInfoStreamReqV1 { + batch_size: self.batch_size, + timestamp: timestamp.timestamp() as u64, + signer: self.signing_key.public_key().into(), + signature: vec![], + }; + req.signature = self.signing_key.sign(&req.encode_to_vec())?; + tracing::debug!("fetching modified boosted hexes info stream"); + let pubkey = Arc::new(self.config_pubkey.clone()); + let res_stream = call_with_retry!(self.client.modified_info_stream(req.clone()))? + .into_inner() + .filter_map(|res| async move { res.ok() }) + .map(move |res| (res, pubkey.clone())) + .filter_map(|(res, pubkey)| async move { + match res.verify(&pubkey) { + Ok(()) => Some(res), + Err(_) => None, + } + }) + .flat_map(|res| stream::iter(res.hexes)) + .map(boosted_hex_info::BoostedHexInfo::try_from) + .filter_map(|hex| async move { hex.ok() }) + .boxed(); + + Ok(res_stream) + } +} diff --git a/mobile_config/src/client/mod.rs b/mobile_config/src/client/mod.rs index 68699dd0d..f9ecd7d18 100644 --- a/mobile_config/src/client/mod.rs +++ b/mobile_config/src/client/mod.rs @@ -2,6 +2,7 @@ pub mod authorization_client; pub mod carrier_service_client; pub mod entity_client; pub mod gateway_client; +pub mod hex_boosting_client; mod settings; use std::time::Duration; diff --git a/mobile_config/src/client/settings.rs b/mobile_config/src/client/settings.rs index 2bebc3f0b..ff73f8dff 100644 --- a/mobile_config/src/client/settings.rs +++ b/mobile_config/src/client/settings.rs @@ -20,6 +20,9 @@ pub struct Settings { /// Batch size for hotspot metadata stream results. Default 100 #[serde(default = "default_batch_size")] pub batch_size: u32, + /// Batch size for hex boosting stream results. Default 100 + #[serde(default = "default_hex_boosting_batch_size")] + pub hex_boosting_batch_size: u32, #[serde(default = "default_cache_ttl_in_secs")] pub cache_ttl_in_secs: u64, } @@ -36,6 +39,10 @@ pub fn default_batch_size() -> u32 { 100 } +pub fn default_hex_boosting_batch_size() -> u32 { + 100 +} + pub fn default_cache_ttl_in_secs() -> u64 { 60 * 60 } @@ -61,6 +68,11 @@ impl Settings { mobile_config::CarrierServiceClient::new(channel) } + pub fn connect_hex_boosting_service_client(&self) -> mobile_config::HexBoostingClient { + let channel = connect_channel(self); + mobile_config::HexBoostingClient::new(channel) + } + pub fn signing_keypair( &self, ) -> Result, Box> { diff --git a/mobile_config/src/hex_boosting_service.rs b/mobile_config/src/hex_boosting_service.rs new file mode 100644 index 000000000..77a56ed23 --- /dev/null +++ b/mobile_config/src/hex_boosting_service.rs @@ -0,0 +1,162 @@ +use crate::{ + boosted_hex_info::{self, BoostedHexInfo}, + key_cache::KeyCache, + telemetry, verify_public_key, GrpcResult, GrpcStreamResult, +}; +use chrono::Utc; +use file_store::traits::{MsgVerify, TimestampDecode, TimestampEncode}; +use futures::{ + stream::{Stream, StreamExt, TryStreamExt}, + TryFutureExt, +}; +use helium_crypto::{Keypair, PublicKey, Sign}; +use helium_proto::{ + services::mobile_config::{ + self, BoostedHexInfoStreamReqV1, BoostedHexInfoStreamResV1, + BoostedHexModifiedInfoStreamReqV1, + }, + BoostedHexInfoV1, Message, +}; +use sqlx::{Pool, Postgres}; +use std::sync::Arc; +use tonic::{Request, Response, Status}; + +pub struct HexBoostingService { + key_cache: KeyCache, + metadata_pool: Pool, + signing_key: Arc, +} + +impl HexBoostingService { + pub fn new(key_cache: KeyCache, metadata_pool: Pool, signing_key: Keypair) -> Self { + Self { + key_cache, + metadata_pool, + signing_key: Arc::new(signing_key), + } + } + + fn verify_request_signature(&self, signer: &PublicKey, request: &R) -> Result<(), Status> + where + R: MsgVerify, + { + if self.key_cache.verify_signature(signer, request).is_ok() { + tracing::debug!(signer = signer.to_string(), "request authorized"); + return Ok(()); + } + Err(Status::permission_denied("unauthorized request signature")) + } +} + +#[tonic::async_trait] +impl mobile_config::HexBoosting for HexBoostingService { + type info_streamStream = GrpcStreamResult; + async fn info_stream( + &self, + request: Request, + ) -> GrpcResult { + let request = request.into_inner(); + telemetry::count_request("hex-boosting", "info-stream"); + + let signer = verify_public_key(&request.signer)?; + self.verify_request_signature(&signer, &request)?; + + tracing::debug!("fetching all boosted hexes' info"); + + let pool = self.metadata_pool.clone(); + let signing_key = self.signing_key.clone(); + let batch_size = request.batch_size; + + let (tx, rx) = tokio::sync::mpsc::channel(100); + + tokio::spawn(async move { + let stream = boosted_hex_info::db::all_info_stream(&pool); + stream_multi_info(stream, tx.clone(), signing_key.clone(), batch_size).await + }); + + Ok(Response::new(GrpcStreamResult::new(rx))) + } + + type modified_info_streamStream = GrpcStreamResult; + async fn modified_info_stream( + &self, + request: Request, + ) -> GrpcResult { + let request = request.into_inner(); + telemetry::count_request("hex-boosting", "modified-info-stream"); + + let signer = verify_public_key(&request.signer)?; + self.verify_request_signature(&signer, &request)?; + + tracing::debug!("fetching all modified boosted hexes' info"); + + let pool = self.metadata_pool.clone(); + let signing_key = self.signing_key.clone(); + let batch_size = request.batch_size; + let ts = request + .timestamp + .to_timestamp() + .map_err(|_| Status::invalid_argument("invalid timestamp")) + .unwrap(); + + let (tx, rx) = tokio::sync::mpsc::channel(100); + + tokio::spawn(async move { + let stream = boosted_hex_info::db::modified_info_stream(&pool, ts); + stream_multi_info(stream, tx.clone(), signing_key.clone(), batch_size).await + }); + + Ok(Response::new(GrpcStreamResult::new(rx))) + } +} + +async fn stream_multi_info( + stream: impl Stream, + tx: tokio::sync::mpsc::Sender>, + signing_key: Arc, + batch_size: u32, +) -> anyhow::Result<()> { + let timestamp = Utc::now().encode_timestamp(); + let signer: Vec = signing_key.public_key().into(); + Ok(stream + .map(Ok::) + .try_filter_map(|info| async move { + let result: Option = info.try_into().ok(); + Ok(result) + }) + .try_chunks(batch_size as usize) + .map_ok(move |batch| { + ( + BoostedHexInfoStreamResV1 { + hexes: batch, + timestamp, + signer: signer.clone(), + signature: vec![], + }, + signing_key.clone(), + ) + }) + .try_filter_map(|(res, keypair)| async move { + let result = match keypair.sign(&res.encode_to_vec()) { + Ok(signature) => Some(BoostedHexInfoStreamResV1 { + hexes: res.hexes, + timestamp: res.timestamp, + signer: res.signer, + signature, + }), + Err(_) => None, + }; + Ok(result) + }) + .map_err(|err| Status::internal(format!("info batch failed with reason: {err:?}"))) + .try_for_each(|res| { + tx.send(Ok(res)) + .map_err(|err| Status::internal(format!("info batch send failed {err:?}"))) + }) + .or_else(|err| { + tx.send(Err(Status::internal(format!( + "info batch failed with reason: {err:?}" + )))) + }) + .await?) +} diff --git a/mobile_config/src/lib.rs b/mobile_config/src/lib.rs index ebd48dd69..203c126d1 100644 --- a/mobile_config/src/lib.rs +++ b/mobile_config/src/lib.rs @@ -6,11 +6,14 @@ use tonic::{Response, Status}; pub mod admin_service; pub mod authorization_service; +pub mod boosted_hex_info; pub mod carrier_service; pub mod client; pub mod entity_service; pub mod gateway_info; pub mod gateway_service; +pub mod hex_boosting_service; + pub mod key_cache; pub mod settings; pub mod telemetry; diff --git a/mobile_config/src/main.rs b/mobile_config/src/main.rs index 4a6ff6052..af5b1d82c 100644 --- a/mobile_config/src/main.rs +++ b/mobile_config/src/main.rs @@ -4,11 +4,13 @@ use futures::future::LocalBoxFuture; use futures_util::TryFutureExt; use helium_proto::services::mobile_config::{ AdminServer, AuthorizationServer, CarrierServiceServer, EntityServer, GatewayServer, + HexBoostingServer, }; use mobile_config::{ admin_service::AdminService, authorization_service::AuthorizationService, carrier_service::CarrierService, entity_service::EntityService, - gateway_service::GatewayService, key_cache::KeyCache, settings::Settings, + gateway_service::GatewayService, hex_boosting_service::HexBoostingService, key_cache::KeyCache, + settings::Settings, }; use std::{net::SocketAddr, path::PathBuf, time::Duration}; use task_manager::{ManagedTask, TaskManager}; @@ -89,6 +91,12 @@ impl Daemon { let carrier_svc = CarrierService::new(key_cache.clone(), pool.clone(), settings.signing_keypair()?); + let hex_boosting_svc = HexBoostingService::new( + key_cache.clone(), + metadata_pool.clone(), + settings.signing_keypair()?, + ); + let grpc_server = GrpcServer { listen_addr, admin_svc, @@ -96,6 +104,7 @@ impl Daemon { auth_svc, entity_svc, carrier_svc, + hex_boosting_svc, }; TaskManager::builder().add_task(grpc_server).start().await @@ -109,6 +118,7 @@ pub struct GrpcServer { auth_svc: AuthorizationService, entity_svc: EntityService, carrier_svc: CarrierService, + hex_boosting_svc: HexBoostingService, } impl ManagedTask for GrpcServer { @@ -126,6 +136,7 @@ impl ManagedTask for GrpcServer { .add_service(AuthorizationServer::new(self.auth_svc)) .add_service(EntityServer::new(self.entity_svc)) .add_service(CarrierServiceServer::new(self.carrier_svc)) + .add_service(HexBoostingServer::new(self.hex_boosting_svc)) .serve_with_shutdown(self.listen_addr, shutdown) .map_err(Error::from) .await diff --git a/mobile_packet_verifier/src/burner.rs b/mobile_packet_verifier/src/burner.rs index df206a029..ffb6a0c0f 100644 --- a/mobile_packet_verifier/src/burner.rs +++ b/mobile_packet_verifier/src/burner.rs @@ -2,7 +2,7 @@ use chrono::{DateTime, Utc}; use file_store::{file_sink::FileSinkClient, traits::TimestampEncode}; use helium_crypto::PublicKeyBinary; use helium_proto::services::packet_verifier::ValidDataTransferSession; -use solana::SolanaNetwork; +use solana::burn::SolanaNetwork; use sqlx::{FromRow, Pool, Postgres}; use std::collections::HashMap; diff --git a/mobile_packet_verifier/src/daemon.rs b/mobile_packet_verifier/src/daemon.rs index 91919abbf..bd8a11abf 100644 --- a/mobile_packet_verifier/src/daemon.rs +++ b/mobile_packet_verifier/src/daemon.rs @@ -13,7 +13,7 @@ use mobile_config::client::{ authorization_client::AuthorizationVerifier, gateway_client::GatewayInfoResolver, AuthorizationClient, GatewayClient, }; -use solana::{SolanaNetwork, SolanaRpc}; +use solana::burn::{SolanaNetwork, SolanaRpc}; use sqlx::{Pool, Postgres}; use task_manager::{ManagedTask, TaskManager}; use tokio::{ diff --git a/mobile_packet_verifier/src/settings.rs b/mobile_packet_verifier/src/settings.rs index 0ae5e2ee4..6b0ad51bc 100644 --- a/mobile_packet_verifier/src/settings.rs +++ b/mobile_packet_verifier/src/settings.rs @@ -20,7 +20,7 @@ pub struct Settings { pub metrics: poc_metrics::Settings, #[serde(default)] pub enable_solana_integration: bool, - pub solana: Option, + pub solana: Option, pub config_client: mobile_config::ClientSettings, #[serde(default = "default_start_after")] pub start_after: u64, diff --git a/mobile_verifier/src/boosted_hexes.rs b/mobile_verifier/src/boosted_hexes.rs new file mode 100644 index 000000000..49e44fefd --- /dev/null +++ b/mobile_verifier/src/boosted_hexes.rs @@ -0,0 +1,41 @@ +use chrono::{DateTime, Utc}; +use futures::StreamExt; +use mobile_config::{ + boosted_hex_info::BoostedHexInfo, + client::{hex_boosting_client::HexBoostingInfoResolver, ClientError}, +}; +use std::collections::HashMap; + +#[derive(Debug, Clone, Default)] +pub struct BoostedHexes { + pub hexes: HashMap, +} + +#[derive(PartialEq, Debug, Clone)] +pub struct BoostedHex { + pub location: u64, + pub multiplier: u32, +} + +impl BoostedHexes { + pub async fn new( + hex_service_client: &impl HexBoostingInfoResolver, + ) -> anyhow::Result { + tracing::info!("getting boosted hexes"); + let mut map = HashMap::new(); + let mut stream = hex_service_client + .clone() + .stream_boosted_hexes_info() + .await?; + while let Some(info) = stream.next().await { + map.insert(info.location, info); + } + Ok(Self { hexes: map }) + } + + pub fn get_current_multiplier(&self, location: u64, ts: DateTime) -> Option { + self.hexes + .get(&location) + .and_then(|info| info.current_multiplier(ts).ok()?) + } +} diff --git a/mobile_verifier/src/cli/reward_from_db.rs b/mobile_verifier/src/cli/reward_from_db.rs index e484abfef..4844567ba 100644 --- a/mobile_verifier/src/cli/reward_from_db.rs +++ b/mobile_verifier/src/cli/reward_from_db.rs @@ -8,6 +8,7 @@ use anyhow::Result; use chrono::NaiveDateTime; use helium_crypto::PublicKey; use helium_proto::services::poc_mobile as proto; +use mobile_config::boosted_hex_info::BoostedHexes; use rust_decimal::Decimal; use serde_json::json; use std::collections::HashMap; @@ -38,8 +39,15 @@ impl Cmd { let heartbeats = HeartbeatReward::validated(&pool, &epoch); let speedtest_averages = SpeedtestAverages::aggregate_epoch_averages(epoch.end, &pool).await?; - let reward_shares = - CoveragePoints::aggregate_points(&pool, heartbeats, &speedtest_averages, end).await?; + let boosted_hexes = BoostedHexes::default(); + let reward_shares = CoveragePoints::aggregate_points( + &pool, + heartbeats, + &speedtest_averages, + &boosted_hexes, + &epoch, + ) + .await?; let mut total_rewards = 0_u64; let mut owner_rewards = HashMap::<_, u64>::new(); diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index 8b95201f5..9cc027ff1 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -15,7 +15,8 @@ use file_store::{ FileType, }; use mobile_config::client::{ - entity_client::EntityClient, AuthorizationClient, CarrierServiceClient, GatewayClient, + entity_client::EntityClient, hex_boosting_client::HexBoostingClient, AuthorizationClient, + CarrierServiceClient, GatewayClient, }; use price::PriceTracker; use task_manager::TaskManager; @@ -45,6 +46,7 @@ impl Cmd { let auth_client = AuthorizationClient::from_settings(&settings.config_client)?; let entity_client = EntityClient::from_settings(&settings.config_client)?; let carrier_client = CarrierServiceClient::from_settings(&settings.config_client)?; + let hex_boosting_client = HexBoostingClient::from_settings(&settings.config_client)?; // price tracker let (price_tracker, price_daemon) = PriceTracker::new_tm(&settings.price_tracker).await?; @@ -220,6 +222,7 @@ impl Cmd { let rewarder = Rewarder::new( pool.clone(), carrier_client, + hex_boosting_client, Duration::hours(reward_period_hours), Duration::minutes(settings.reward_offset_minutes), mobile_rewards, diff --git a/mobile_verifier/src/coverage.rs b/mobile_verifier/src/coverage.rs index d6ed6c2e3..e90ac1803 100644 --- a/mobile_verifier/src/coverage.rs +++ b/mobile_verifier/src/coverage.rs @@ -1,11 +1,7 @@ -use std::{ - cmp::Ordering, - collections::{BTreeMap, BinaryHeap, HashMap}, - pin::pin, - sync::Arc, - time::Instant, +use crate::{ + heartbeats::{HbType, KeyType, OwnedKeyType}, + IsAuthorized, }; - use chrono::{DateTime, Utc}; use file_store::{ coverage::{self, CoverageObjectIngestReport}, @@ -23,20 +19,25 @@ use helium_proto::services::{ mobile_config::NetworkKeyRole, poc_mobile::{self as proto, CoverageObjectValidity, SignalLevel as SignalLevelProto}, }; -use mobile_config::client::AuthorizationClient; +use mobile_config::{ + boosted_hex_info::{BoostedHex, BoostedHexes}, + client::AuthorizationClient, +}; use retainer::{entry::CacheReadGuard, Cache}; use rust_decimal::Decimal; use rust_decimal_macros::dec; use sqlx::{FromRow, PgPool, Pool, Postgres, QueryBuilder, Transaction, Type}; +use std::{ + cmp::Ordering, + collections::{BTreeMap, BinaryHeap, HashMap}, + pin::pin, + sync::Arc, + time::Instant, +}; use task_manager::ManagedTask; use tokio::sync::mpsc::Receiver; use uuid::Uuid; -use crate::{ - heartbeats::{HbType, KeyType, OwnedKeyType}, - IsAuthorized, -}; - #[derive(Copy, Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Type)] #[sqlx(type_name = "signal_level")] #[sqlx(rename_all = "lowercase")] @@ -371,6 +372,7 @@ pub struct CoverageReward { pub radio_key: OwnedKeyType, pub points: Decimal, pub hotspot: PublicKeyBinary, + pub boosted_hex_info: BoostedHex, } #[async_trait::async_trait] @@ -535,7 +537,7 @@ impl CoveredHexes { seniority_timestamp: coverage_claim_time, signal_level, hotspot: hotspot.clone(), - }); + }) } else { // If this is an outdoor Wifi radio, we adjust the signal power by -30dbm in order // to more properly reflect signal strength. @@ -544,7 +546,6 @@ impl CoveredHexes { } else { signal_power }; - self.outdoor .entry(CellIndex::try_from(hex as u64).unwrap()) .or_default() @@ -562,32 +563,54 @@ impl CoveredHexes { } /// Returns the radios that should be rewarded for giving coverage. - pub fn into_coverage_rewards(self) -> impl Iterator { - let outdoor_rewards = self.outdoor.into_values().flat_map(|radios| { + pub fn into_coverage_rewards( + self, + boosted_hexes: &BoostedHexes, + epoch_start: DateTime, + ) -> impl Iterator + '_ { + let outdoor_rewards = self.outdoor.into_iter().flat_map(move |(hex, radios)| { radios .into_sorted_vec() .into_iter() .take(MAX_OUTDOOR_RADIOS_PER_RES12_HEX) .zip(OUTDOOR_REWARD_WEIGHTS) - .map(|(cl, rank)| CoverageReward { - points: cl.coverage_points() * rank, - hotspot: cl.hotspot, - radio_key: cl.radio_key, + .map(move |(cl, rank)| { + let boost_multiplier = boosted_hexes + .get_current_multiplier(hex.into(), epoch_start) + .unwrap_or(1); + CoverageReward { + points: cl.coverage_points() * rank, + hotspot: cl.hotspot, + radio_key: cl.radio_key, + boosted_hex_info: BoostedHex { + location: hex.into(), + multiplier: boost_multiplier, + }, + } }) }); let indoor_rewards = self .indoor - .into_values() - .flat_map(|mut radios| { - radios.pop_last().map(|(_, radios)| { + .into_iter() + .flat_map(move |(hex, mut radios)| { + radios.pop_last().map(move |(_, radios)| { radios .into_sorted_vec() .into_iter() .take(MAX_INDOOR_RADIOS_PER_RES12_HEX) - .map(|cl| CoverageReward { - points: cl.coverage_points(), - hotspot: cl.hotspot, - radio_key: cl.radio_key, + .map(move |cl| { + let boost_multiplier = boosted_hexes + .get_current_multiplier(hex.into(), epoch_start) + .unwrap_or(1); + CoverageReward { + points: cl.coverage_points(), + hotspot: cl.hotspot, + radio_key: cl.radio_key, + boosted_hex_info: BoostedHex { + location: hex.into(), + multiplier: boost_multiplier, + }, + } }) }) }) @@ -794,13 +817,19 @@ mod test { ) .await .unwrap(); - let rewards: Vec<_> = covered_hexes.into_coverage_rewards().collect(); + let rewards: Vec<_> = covered_hexes + .into_coverage_rewards(&BoostedHexes::default(), Utc::now()) + .collect(); assert_eq!( rewards, vec![CoverageReward { radio_key: OwnedKeyType::Cbrs("3".to_string()), hotspot: owner, - points: dec!(400) + points: dec!(400), + boosted_hex_info: BoostedHex { + location: 0x8a1fb46622dffff_u64, + multiplier: 1, + }, }] ); } @@ -897,34 +926,56 @@ mod test { ) .await .unwrap(); - let rewards: Vec<_> = covered_hexes.into_coverage_rewards().collect(); + let rewards: Vec<_> = covered_hexes + .into_coverage_rewards(&BoostedHexes::default(), Utc::now()) + .collect(); assert_eq!( rewards, vec![ CoverageReward { radio_key: OwnedKeyType::Cbrs("10".to_string()), hotspot: owner.clone(), - points: dec!(400) + points: dec!(400), + boosted_hex_info: BoostedHex { + location: 0x8a1fb46622dffff_u64, + multiplier: 1, + }, }, CoverageReward { radio_key: OwnedKeyType::Cbrs("8".to_string()), hotspot: owner.clone(), - points: dec!(400) + points: dec!(400), + boosted_hex_info: BoostedHex { + location: 0x8a1fb46622dffff_u64, + multiplier: 1, + }, }, CoverageReward { radio_key: OwnedKeyType::Cbrs("6".to_string()), hotspot: owner.clone(), - points: dec!(400) + points: dec!(400), + boosted_hex_info: BoostedHex { + location: 0x8a1fb46622dffff_u64, + multiplier: 1, + }, }, CoverageReward { radio_key: OwnedKeyType::Cbrs("4".to_string()), hotspot: owner.clone(), - points: dec!(400) + points: dec!(400), + boosted_hex_info: BoostedHex { + location: 0x8a1fb46622dffff_u64, + multiplier: 1, + }, }, CoverageReward { radio_key: OwnedKeyType::Cbrs("2".to_string()), hotspot: owner.clone(), - points: dec!(400) + points: dec!(400), + boosted_hex_info: BoostedHex { + location: 0x8a1fb46622dffff_u64, + multiplier: 1, + }, } ] ); @@ -966,24 +1017,38 @@ mod test { ) .await .unwrap(); - let rewards: Vec<_> = covered_hexes.into_coverage_rewards().collect(); + let rewards: Vec<_> = covered_hexes + .into_coverage_rewards(&BoostedHexes::default(), Utc::now()) + .collect(); assert_eq!( rewards, vec![ CoverageReward { radio_key: OwnedKeyType::Cbrs("5".to_string()), hotspot: owner.clone(), - points: dec!(16) + points: dec!(16), + boosted_hex_info: BoostedHex { + location: 0x8a1fb46622dffff_u64, + multiplier: 1, + }, }, CoverageReward { radio_key: OwnedKeyType::Cbrs("4".to_string()), hotspot: owner.clone(), - points: dec!(12) + points: dec!(12), + boosted_hex_info: BoostedHex { + location: 0x8a1fb46622dffff_u64, + multiplier: 1, + }, }, CoverageReward { radio_key: OwnedKeyType::Cbrs("3".to_string()), hotspot: owner, - points: dec!(4) + points: dec!(4), + boosted_hex_info: BoostedHex { + location: 0x8a1fb46622dffff_u64, + multiplier: 1, + }, } ] ); @@ -1023,24 +1088,38 @@ mod test { ) .await .unwrap(); - let rewards: Vec<_> = covered_hexes.into_coverage_rewards().collect(); + let rewards: Vec<_> = covered_hexes + .into_coverage_rewards(&BoostedHexes::default(), Utc::now()) + .collect(); assert_eq!( rewards, vec![ CoverageReward { radio_key: OwnedKeyType::Cbrs("1".to_string()), hotspot: owner.clone(), - points: dec!(16) + points: dec!(16), + boosted_hex_info: BoostedHex { + location: 0x8a1fb46622dffff_u64, + multiplier: 1, + }, }, CoverageReward { radio_key: OwnedKeyType::Cbrs("2".to_string()), hotspot: owner.clone(), - points: dec!(12) + points: dec!(12), + boosted_hex_info: BoostedHex { + location: 0x8a1fb46622dffff_u64, + multiplier: 1, + }, }, CoverageReward { radio_key: OwnedKeyType::Wifi(owner.clone()), hotspot: owner, - points: dec!(4) + points: dec!(4), + boosted_hex_info: BoostedHex { + location: 0x8a1fb46622dffff_u64, + multiplier: 1, + }, }, ] ); diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index f9d53e447..da516eb65 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -1,7 +1,7 @@ use crate::{ coverage::{CoverageReward, CoveredHexStream, CoveredHexes}, data_session::{HotspotMap, ServiceProviderDataSession}, - heartbeats::HeartbeatReward, + heartbeats::{HeartbeatReward, OwnedKeyType}, speedtests_average::{SpeedtestAverage, SpeedtestAverages}, subscriber_location::SubscriberValidatedLocations, }; @@ -18,8 +18,10 @@ use helium_proto::{ }, ServiceProvider, }; - -use mobile_config::client::{carrier_service_client::CarrierServiceVerifier, ClientError}; +use mobile_config::{ + boosted_hex_info::{BoostedHex, BoostedHexes}, + client::{carrier_service_client::CarrierServiceVerifier, ClientError}, +}; use rust_decimal::prelude::*; use rust_decimal_macros::dec; use std::{collections::HashMap, ops::Range}; @@ -378,6 +380,9 @@ struct RadioPoints { coverage_object: Uuid, seniority: DateTime, points: Decimal, + // list of all hexes that have been boosted for this hotspot along with the multiplier for each hex + // this gets included in the radio reward share proto + boosted_hexes: Vec, } impl RadioPoints { @@ -391,6 +396,7 @@ impl RadioPoints { seniority, coverage_object, points: Decimal::ZERO, + boosted_hexes: vec![], } } @@ -399,6 +405,8 @@ impl RadioPoints { } } +// pub type HotspotBoostedHexes = HashMap; + #[derive(Debug, Default)] struct HotspotPoints { /// Points are multiplied by the multiplier to get shares. @@ -407,6 +415,33 @@ struct HotspotPoints { radio_points: HashMap, RadioPoints>, } +impl HotspotPoints { + pub fn add_coverage_entry( + &mut self, + radio_key: OwnedKeyType, + points: Decimal, + boosted_hex_info: BoostedHex, + ) { + let rp = self + .radio_points + .get_mut(&radio_key.clone().into_cbsd_id()) + .unwrap(); + // as per hip93, if radio is wifi & the location trust score multiplier is less than 1, + // then no boost points for you mister + let final_boost_info = + if radio_key.is_wifi() && rp.location_trust_score_multiplier < dec!(1) { + BoostedHex { + location: boosted_hex_info.location, + multiplier: 1, + } + } else { + boosted_hex_info + }; + rp.points += points * Decimal::from(final_boost_info.multiplier); + rp.boosted_hexes.push(final_boost_info); + } +} + impl HotspotPoints { pub fn new(speedtest_multiplier: Decimal) -> Self { Self { @@ -436,7 +471,8 @@ impl CoveragePoints { hex_streams: &impl CoveredHexStream, heartbeats: impl Stream>, speedtests: &SpeedtestAverages, - period_end: DateTime, + boosted_hexes: &BoostedHexes, + reward_period: &Range>, ) -> Result { let mut heartbeats = std::pin::pin!(heartbeats); let mut covered_hexes = CoveredHexes::default(); @@ -447,11 +483,12 @@ impl CoveragePoints { .as_ref() .map_or(Decimal::ZERO, SpeedtestAverage::reward_multiplier); let seniority = hex_streams - .fetch_seniority(heartbeat.key(), period_end) + .fetch_seniority(heartbeat.key(), reward_period.end) .await?; let covered_hex_stream = hex_streams .covered_hex_stream(heartbeat.key(), &heartbeat.coverage_object, &seniority) .await?; + covered_hexes .aggregate_coverage(&heartbeat.hotspot_key, covered_hex_stream) .await?; @@ -474,18 +511,15 @@ impl CoveragePoints { radio_key, points, hotspot, - } in covered_hexes.into_coverage_rewards() + boosted_hex_info, + } in covered_hexes.into_coverage_rewards(boosted_hexes, reward_period.start) { // Guaranteed that points contains the given hotspot. coverage_points .get_mut(&hotspot) .unwrap() - .radio_points - .get_mut(&radio_key.into_cbsd_id()) - .unwrap() - .points += points; + .add_coverage_entry(radio_key, points, boosted_hex_info) } - Ok(Self { coverage_points }) } @@ -555,6 +589,7 @@ fn radio_points_into_rewards( }) } +#[allow(clippy::too_many_arguments)] fn new_radio_reward( cbsd_id: Option, hotspot_key: &PublicKeyBinary, @@ -574,6 +609,15 @@ fn new_radio_reward( .round_dp_with_strategy(0, RoundingStrategy::ToZero) .to_u64() .unwrap_or(0); + let boosted_hexes = radio_points + .boosted_hexes + .iter() + .filter(|boosted_hex| boosted_hex.multiplier > 1) + .map(|boosted_hex| proto::BoostedHex { + location: boosted_hex.location, + multiplier: boosted_hex.multiplier, + }) + .collect(); ( poc_reward, proto::MobileRewardShare { @@ -594,6 +638,7 @@ fn new_radio_reward( speedtest_multiplier: (speedtest_multiplier * dec!(1000)) .to_u32() .unwrap_or_default(), + boosted_hexes, ..Default::default() }, )), @@ -1299,8 +1344,8 @@ mod test { &hex_coverage, stream::iter(heartbeat_rewards), &speedtest_avgs, - // Field isn't used: - DateTime::::MIN_UTC, + &BoostedHexes::default(), + &epoch, ) .await .unwrap() @@ -1468,7 +1513,8 @@ mod test { &hex_coverage, stream::iter(heartbeat_rewards), &speedtest_avgs, - DateTime::::MIN_UTC, + &BoostedHexes::default(), + &epoch, ) .await .unwrap() @@ -1596,7 +1642,8 @@ mod test { &hex_coverage, stream::iter(heartbeat_rewards), &speedtest_avgs, - DateTime::::MIN_UTC, + &BoostedHexes::default(), + &epoch, ) .await .unwrap() @@ -1722,7 +1769,8 @@ mod test { &hex_coverage, stream::iter(heartbeat_rewards), &speedtest_avgs, - DateTime::::MIN_UTC, + &BoostedHexes::default(), + &epoch, ) .await .unwrap() @@ -1785,6 +1833,7 @@ mod test { seniority: DateTime::default(), coverage_object: Uuid::new_v4(), points: dec!(10.0), + boosted_hexes: vec![], }, )] .into_iter() @@ -1803,6 +1852,7 @@ mod test { seniority: DateTime::default(), coverage_object: Uuid::new_v4(), points: dec!(-1.0), + boosted_hexes: vec![], }, ), ( @@ -1812,6 +1862,7 @@ mod test { points: dec!(0.0), seniority: DateTime::default(), coverage_object: Uuid::new_v4(), + boosted_hexes: vec![], }, ), ] diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index 9b381ed8b..57bc96d47 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -10,14 +10,19 @@ use anyhow::bail; use chrono::{DateTime, Duration, TimeZone, Utc}; use db_store::meta; use file_store::{file_sink::FileSinkClient, traits::TimestampEncode}; - use futures_util::TryFutureExt; use helium_proto::services::{ poc_mobile as proto, poc_mobile::mobile_reward_share::Reward as ProtoReward, poc_mobile::UnallocatedReward, poc_mobile::UnallocatedRewardType, }; use helium_proto::RewardManifest; -use mobile_config::client::{carrier_service_client::CarrierServiceVerifier, ClientError}; +use mobile_config::{ + boosted_hex_info::BoostedHexes, + client::{ + carrier_service_client::CarrierServiceVerifier, + hex_boosting_client::HexBoostingInfoResolver, ClientError, + }, +}; use price::PriceTracker; use reward_scheduler::Scheduler; use rust_decimal::{prelude::*, Decimal}; @@ -29,9 +34,10 @@ use tokio::time::sleep; const REWARDS_NOT_CURRENT_DELAY_PERIOD: i64 = 5; -pub struct Rewarder { +pub struct Rewarder { pool: Pool, carrier_client: A, + hex_service_client: B, reward_period_duration: Duration, reward_offset: Duration, pub mobile_rewards: FileSinkClient, @@ -40,14 +46,16 @@ pub struct Rewarder { speedtest_averages: FileSinkClient, } -impl Rewarder +impl Rewarder where A: CarrierServiceVerifier, + B: HexBoostingInfoResolver, { #[allow(clippy::too_many_arguments)] pub fn new( pool: Pool, carrier_client: A, + hex_service_client: B, reward_period_duration: Duration, reward_offset: Duration, mobile_rewards: FileSinkClient, @@ -58,6 +66,7 @@ where Self { pool, carrier_client, + hex_service_client, reward_period_duration, reward_offset, mobile_rewards, @@ -181,6 +190,7 @@ where // process rewards for poc and data transfer reward_poc_and_dc( &self.pool, + &self.hex_service_client, &self.mobile_rewards, &self.speedtest_averages, reward_period, @@ -239,9 +249,10 @@ where } } -impl ManagedTask for Rewarder +impl ManagedTask for Rewarder where A: CarrierServiceVerifier + Send + Sync + 'static, + B: HexBoostingInfoResolver + Send + Sync + 'static, { fn start_task( self: Box, @@ -258,6 +269,7 @@ where pub async fn reward_poc_and_dc( pool: &Pool, + hex_service_client: &impl HexBoostingInfoResolver, mobile_rewards: &FileSinkClient, speedtest_avg_sink: &FileSinkClient, reward_period: &Range>, @@ -279,6 +291,7 @@ pub async fn reward_poc_and_dc( reward_poc( pool, + hex_service_client, mobile_rewards, speedtest_avg_sink, reward_period, @@ -293,6 +306,7 @@ pub async fn reward_poc_and_dc( async fn reward_poc( pool: &Pool, + hex_service_client: &impl HexBoostingInfoResolver, mobile_rewards: &FileSinkClient, speedtest_avg_sink: &FileSinkClient, reward_period: &Range>, @@ -308,9 +322,16 @@ async fn reward_poc( speedtest_averages.write_all(speedtest_avg_sink).await?; - let coverage_points = - CoveragePoints::aggregate_points(pool, heartbeats, &speedtest_averages, reward_period.end) - .await?; + let boosted_hexes = BoostedHexes::get_all(hex_service_client).await?; + + let coverage_points = CoveragePoints::aggregate_points( + pool, + heartbeats, + &speedtest_averages, + &boosted_hexes, + reward_period, + ) + .await?; if let Some(mobile_reward_shares) = coverage_points.into_rewards(total_poc_rewards, reward_period) diff --git a/mobile_verifier/tests/common/mod.rs b/mobile_verifier/tests/common/mod.rs index b52f6307a..a6e92b421 100644 --- a/mobile_verifier/tests/common/mod.rs +++ b/mobile_verifier/tests/common/mod.rs @@ -6,6 +6,7 @@ use helium_proto::{ }, Message, }; +use mobile_config::boosted_hex_info::BoostedHexInfo; use std::collections::HashMap; use tokio::{sync::mpsc::error::TryRecvError, time::timeout}; @@ -16,6 +17,11 @@ pub struct MockCarrierServiceClient { pub valid_sps: ValidSpMap, } +#[derive(Debug, Clone)] +pub struct MockHexBoostingClient { + pub boosted_hexes: Vec, +} + pub struct MockFileSinkReceiver { pub receiver: tokio::sync::mpsc::Receiver, } diff --git a/mobile_verifier/tests/hex_boosting.rs b/mobile_verifier/tests/hex_boosting.rs new file mode 100644 index 000000000..739b96582 --- /dev/null +++ b/mobile_verifier/tests/hex_boosting.rs @@ -0,0 +1,1086 @@ +mod common; +use crate::common::{MockFileSinkReceiver, MockHexBoostingClient}; +use async_trait::async_trait; +use chrono::{DateTime, Duration as ChronoDuration, Duration, Utc}; +use file_store::{ + coverage::{CoverageObject as FSCoverageObject, KeyType, RadioHexSignalLevel}, + speedtest::CellSpeedtest, +}; +use futures_util::{stream, StreamExt as FuturesStreamExt}; +use helium_crypto::PublicKeyBinary; +use helium_proto::services::poc_mobile::{ + CoverageObjectValidity, HeartbeatValidity, RadioReward, SeniorityUpdateReason, SignalLevel, + UnallocatedReward, +}; +use mobile_config::{ + boosted_hex_info::{BoostedHexInfo, BoostedHexInfoStream}, + client::{hex_boosting_client::HexBoostingInfoResolver, ClientError}, +}; +use mobile_verifier::{ + cell_type::CellType, + coverage::CoverageObject, + heartbeats::{HbType, Heartbeat, ValidatedHeartbeat}, + reward_shares, rewarder, speedtests, +}; +use rust_decimal::prelude::*; +use rust_decimal_macros::dec; +use sqlx::{PgPool, Postgres, Transaction}; +use uuid::Uuid; + +const HOTSPOT_1: &str = "112E7TxoNHV46M6tiPA8N1MkeMeQxc9ztb4JQLXBVAAUfq1kJLoF"; +const HOTSPOT_2: &str = "112QhnxqU8QZ3jUXpoRk51quuQVft9Pf5P5zzDDvLxj7Q9QqbMh7"; +const HOTSPOT_3: &str = "11hd7HoicRgBPjBGcqcT2Y9hRQovdZeff5eKFMbCSuDYQmuCiF1"; +const BOOST_CONFIG_PUBKEY: &str = "11hd7HoicRgBPjBGcqcT2Y9hRQovdZeff5eKFMbCSuDYQmuCiF1"; + +impl MockHexBoostingClient { + fn new(boosted_hexes: Vec) -> Self { + Self { boosted_hexes } + } +} + +#[async_trait] +impl HexBoostingInfoResolver for MockHexBoostingClient { + type Error = ClientError; + + async fn stream_boosted_hexes_info(&mut self) -> Result { + Ok(stream::iter(self.boosted_hexes.clone()).boxed()) + } + + async fn stream_modified_boosted_hexes_info( + &mut self, + _timestamp: DateTime, + ) -> Result { + Ok(stream::iter(self.boosted_hexes.clone()).boxed()) + } +} +// +// TODO: add a bootstrapper to reduce boiler plate +// + +#[sqlx::test] +async fn test_poc_with_boosted_hexes(pool: PgPool) -> anyhow::Result<()> { + let (mobile_rewards_client, mut mobile_rewards) = common::create_file_sink(); + let (speedtest_avg_client, _speedtest_avg_server) = common::create_file_sink(); + let now = Utc::now(); + let epoch = (now - ChronoDuration::hours(24))..now; + let boost_period_length = Duration::days(30); + + // seed all the things + let mut txn = pool.clone().begin().await?; + // seed HBs where we have a coverage reports for a singluar hex location per radio + seed_heartbeats_v1(epoch.start, &mut txn).await?; + seed_speedtests(epoch.end, &mut txn).await?; + txn.commit().await?; + + // setup boosted hex where reward start time is in the second period length + let multipliers1 = vec![2, 10, 15, 35]; + let start_ts_1 = epoch.start - boost_period_length; + let end_ts_1 = start_ts_1 + (boost_period_length * multipliers1.len() as i32); + + // setup boosted hex where reward start time is in the third & last period length + let multipliers2 = vec![3, 10, 20]; + let start_ts_2 = epoch.start - (boost_period_length * 2); + let end_ts_2 = start_ts_2 + (boost_period_length * multipliers2.len() as i32); + + // setup boosted hex where no start or end time is set + // will default to the first multiplier + // first multiplier is 1x for easy math when comparing relative rewards + let multipliers3 = vec![1, 10, 20]; + + let boosted_hexes = vec![ + BoostedHexInfo { + // hotspot 1's location + location: 0x8a1fb466d2dffff_u64, + start_ts: Some(start_ts_1), + end_ts: Some(end_ts_1), + period_length: boost_period_length, + multipliers: multipliers1, + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + BoostedHexInfo { + // hotspot 2's location + location: 0x8a1fb49642dffff_u64, + start_ts: Some(start_ts_2), + end_ts: Some(end_ts_2), + period_length: boost_period_length, + multipliers: multipliers2, + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + BoostedHexInfo { + // hotspot 3's location + location: 0x8c2681a306607ff_u64, + start_ts: None, + end_ts: None, + period_length: boost_period_length, + multipliers: multipliers3, + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + ]; + + let hex_boosting_client = MockHexBoostingClient::new(boosted_hexes); + + let (_, rewards) = tokio::join!( + // run rewards for poc and dc + rewarder::reward_poc_and_dc( + &pool, + &hex_boosting_client, + &mobile_rewards_client, + &speedtest_avg_client, + &epoch, + dec!(0.0001) + ), + receive_expected_rewards(&mut mobile_rewards) + ); + if let Ok((poc_rewards, unallocated_reward)) = rewards { + // assert poc reward outputs + let exp_reward_1 = 31_729_243_786_356; + let exp_reward_2 = 15_864_621_893_178; + let exp_reward_3 = 1_586_462_189_317; + + assert_eq!(exp_reward_1, poc_rewards[0].poc_reward); + assert_eq!( + HOTSPOT_2.to_string(), + PublicKeyBinary::from(poc_rewards[0].hotspot_key.clone()).to_string() + ); + assert_eq!(exp_reward_2, poc_rewards[1].poc_reward); + assert_eq!( + HOTSPOT_1.to_string(), + PublicKeyBinary::from(poc_rewards[1].hotspot_key.clone()).to_string() + ); + assert_eq!(exp_reward_3, poc_rewards[2].poc_reward); + assert_eq!( + HOTSPOT_3.to_string(), + PublicKeyBinary::from(poc_rewards[2].hotspot_key.clone()).to_string() + ); + + // assert the boosted hexes in the radio rewards + // assert the number of boosted hexes for each radio + assert_eq!(1, poc_rewards[0].boosted_hexes.len()); + assert_eq!(1, poc_rewards[1].boosted_hexes.len()); + // hotspot 3 has no boosted hexes as all its hex boosts are 1x multiplier + // and those get filtered out as they dont affect points + assert_eq!(0, poc_rewards[2].boosted_hexes.len()); + + // assert the hex boost multiplier values + assert_eq!(20, poc_rewards[0].boosted_hexes[0].multiplier); + assert_eq!(10, poc_rewards[1].boosted_hexes[0].multiplier); + + // assert the hex boost location values + assert_eq!( + 0x8a1fb49642dffff_u64, + poc_rewards[0].boosted_hexes[0].location + ); + assert_eq!( + 0x8a1fb466d2dffff_u64, + poc_rewards[1].boosted_hexes[0].location + ); + + // hotspot1 should have 20x the reward of hotspot 3 + assert_eq!(poc_rewards[0].poc_reward / poc_rewards[2].poc_reward, 20); + // hotspot1 should have 10x the reward of hotspot 3 + assert_eq!(poc_rewards[1].poc_reward / poc_rewards[2].poc_reward, 10); + + // confirm the total rewards allocated matches expectations + let poc_sum: u64 = poc_rewards.iter().map(|r| r.poc_reward).sum(); + let unallocated_sum: u64 = unallocated_reward.amount; + let total = poc_sum + unallocated_sum; + + let expected_sum = reward_shares::get_scheduled_tokens_for_poc(epoch.end - epoch.start) + .to_u64() + .unwrap(); + assert_eq!(expected_sum, total); + + // confirm the rewarded percentage amount matches expectations + let daily_total = reward_shares::get_total_scheduled_tokens(epoch.end - epoch.start); + let percent = (Decimal::from(total) / daily_total) + .round_dp_with_strategy(2, RoundingStrategy::MidpointNearestEven); + assert_eq!(percent, dec!(0.6)); + } else { + panic!("no rewards received"); + }; + Ok(()) +} + +#[sqlx::test] +async fn test_poc_with_multi_coverage_boosted_hexes(pool: PgPool) -> anyhow::Result<()> { + let (mobile_rewards_client, mut mobile_rewards) = common::create_file_sink(); + let (speedtest_avg_client, _speedtest_avg_server) = common::create_file_sink(); + + let now = Utc::now(); + let epoch = (now - ChronoDuration::hours(24))..now; + let boost_period_length = Duration::days(30); + + // seed all the things + let mut txn = pool.clone().begin().await?; + // seed HBs where we have multiple coverage reports for one radio and one report for the others + seed_heartbeats_v2(epoch.start, &mut txn).await?; + seed_speedtests(epoch.end, &mut txn).await?; + txn.commit().await?; + + // setup boosted hex where reward start time is in the second period length + let multipliers1 = vec![2, 10, 15, 35]; + let start_ts_1 = epoch.start - boost_period_length; + let end_ts_1 = start_ts_1 + (boost_period_length * multipliers1.len() as i32); + + // setup boosted hex where reward start time is in the third & last period length + let multipliers2 = vec![3, 10, 20]; + let start_ts_2 = epoch.start - (boost_period_length * 2); + let end_ts_2 = start_ts_2 + (boost_period_length * multipliers2.len() as i32); + + // setup boosted hex where reward start time is in the first period length + // default to 1x multiplier for easy math when comparing relative rewards + let multipliers3 = vec![1, 10, 20]; + let start_ts_3 = epoch.start; + let end_ts_3 = start_ts_3 + (boost_period_length * multipliers3.len() as i32); + + let boosted_hexes = vec![ + BoostedHexInfo { + // hotspot 1's first covered location + location: 0x8a1fb46622dffff_u64, + start_ts: Some(start_ts_1), + end_ts: Some(end_ts_1), + period_length: boost_period_length, + multipliers: multipliers1.clone(), + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + BoostedHexInfo { + // hotspot 1's second covered location + location: 0x8a1fb46622d7fff_u64, + start_ts: Some(start_ts_1), + end_ts: Some(end_ts_1), + period_length: boost_period_length, + multipliers: multipliers1, + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + BoostedHexInfo { + // hotspot 2's location + location: 0x8a1fb49642dffff_u64, + start_ts: Some(start_ts_2), + end_ts: Some(end_ts_2), + period_length: boost_period_length, + multipliers: multipliers2, + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + BoostedHexInfo { + // hotspot 3's location + location: 0x8c2681a306607ff_u64, + start_ts: Some(start_ts_3), + end_ts: Some(end_ts_3), + period_length: boost_period_length, + multipliers: multipliers3, + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + ]; + + let hex_boosting_client = MockHexBoostingClient::new(boosted_hexes); + + let (_, rewards) = tokio::join!( + // run rewards for poc and dc + rewarder::reward_poc_and_dc( + &pool, + &hex_boosting_client, + &mobile_rewards_client, + &speedtest_avg_client, + &epoch, + dec!(0.0001) + ), + receive_expected_rewards(&mut mobile_rewards) + ); + if let Ok((poc_rewards, unallocated_reward)) = rewards { + // assert poc reward outputs + let exp_reward_1 = 23_990_403_838_464; + let exp_reward_2 = 23_990_403_838_464; + let exp_reward_3 = 1_199_520_191_923; + + assert_eq!(exp_reward_1, poc_rewards[0].poc_reward); + assert_eq!( + HOTSPOT_2.to_string(), + PublicKeyBinary::from(poc_rewards[0].hotspot_key.clone()).to_string() + ); + assert_eq!(exp_reward_2, poc_rewards[1].poc_reward); + assert_eq!( + HOTSPOT_1.to_string(), + PublicKeyBinary::from(poc_rewards[1].hotspot_key.clone()).to_string() + ); + assert_eq!(exp_reward_3, poc_rewards[2].poc_reward); + assert_eq!( + HOTSPOT_3.to_string(), + PublicKeyBinary::from(poc_rewards[2].hotspot_key.clone()).to_string() + ); + + // assert the number of boosted hexes for each radio + assert_eq!(1, poc_rewards[0].boosted_hexes.len()); + assert_eq!(2, poc_rewards[1].boosted_hexes.len()); + // hotspot 3 has no boosted hexes as all its hex boosts are 1x multiplier + // and those get filtered out as they dont affect points + assert_eq!(0, poc_rewards[2].boosted_hexes.len()); + + // assert the hex boost multiplier values + // as hotspot 3 has 2 covered hexes, it should have 2 boosted hexes + // sort order in the vec for these is not guaranteed, so sort them + let mut hotspot_1_boosted_hexes = poc_rewards[1].boosted_hexes.clone(); + hotspot_1_boosted_hexes.sort_by(|a, b| b.location.cmp(&a.location)); + + assert_eq!(20, poc_rewards[0].boosted_hexes[0].multiplier); + assert_eq!(10, hotspot_1_boosted_hexes[1].multiplier); + assert_eq!(10, hotspot_1_boosted_hexes[1].multiplier); + + // assert the hex boost location values + assert_eq!(0x8a1fb46622dffff_u64, hotspot_1_boosted_hexes[0].location); + assert_eq!(0x8a1fb46622d7fff_u64, hotspot_1_boosted_hexes[1].location); + assert_eq!( + 0x8a1fb49642dffff_u64, + poc_rewards[0].boosted_hexes[0].location + ); + + // hotspot1 should have 20x the reward of hotspot 3 + assert_eq!(poc_rewards[0].poc_reward / poc_rewards[2].poc_reward, 20); + // hotspot1 should have 20x the reward of hotspot 3 + // due to the 2 boosted hexes each with a 10x multiplier + assert_eq!(poc_rewards[1].poc_reward / poc_rewards[2].poc_reward, 20); + + // confirm the total rewards allocated matches expectations + let poc_sum: u64 = poc_rewards.iter().map(|r| r.poc_reward).sum(); + let unallocated_sum: u64 = unallocated_reward.amount; + let total = poc_sum + unallocated_sum; + + let expected_sum = reward_shares::get_scheduled_tokens_for_poc(epoch.end - epoch.start) + .to_u64() + .unwrap(); + assert_eq!(expected_sum, total); + + // confirm the rewarded percentage amount matches expectations + let daily_total = reward_shares::get_total_scheduled_tokens(epoch.end - epoch.start); + let percent = (Decimal::from(total) / daily_total) + .round_dp_with_strategy(2, RoundingStrategy::MidpointNearestEven); + assert_eq!(percent, dec!(0.6)); + } else { + panic!("no rewards received"); + }; + Ok(()) +} + +#[sqlx::test] +async fn test_expired_boosted_hex(pool: PgPool) -> anyhow::Result<()> { + let (mobile_rewards_client, mut mobile_rewards) = common::create_file_sink(); + let (speedtest_avg_client, _speedtest_avg_server) = common::create_file_sink(); + + let now = Utc::now(); + let epoch = (now - ChronoDuration::hours(24))..now; + let boost_period_length = Duration::days(30); + + // seed all the things + let mut txn = pool.clone().begin().await?; + seed_heartbeats_v1(epoch.start, &mut txn).await?; + seed_speedtests(epoch.end, &mut txn).await?; + txn.commit().await?; + + // setup boosted hex where reward start time is after the boost period ends + let multipliers1 = vec![2, 10, 15]; + let start_ts_1 = + epoch.start - (boost_period_length * multipliers1.len() as i32 + ChronoDuration::days(1)); + let end_ts_1 = start_ts_1 + (boost_period_length * multipliers1.len() as i32); + dbg!(epoch.start, start_ts_1, end_ts_1); + // setup boosted hex where reward start time is same as the boost period ends + let multipliers2 = vec![4, 12, 17]; + let start_ts_2 = epoch.start - (boost_period_length * multipliers2.len() as i32); + let end_ts_2 = start_ts_2 + (boost_period_length * multipliers2.len() as i32); + dbg!(epoch.start, start_ts_2, end_ts_2); + + let boosted_hexes = vec![ + BoostedHexInfo { + location: 0x8a1fb466d2dffff_u64, + start_ts: Some(start_ts_1), + end_ts: Some(end_ts_1), + period_length: boost_period_length, + multipliers: multipliers1, + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + BoostedHexInfo { + location: 0x8a1fb49642dffff_u64, + start_ts: Some(start_ts_2), + end_ts: Some(end_ts_2), + period_length: boost_period_length, + multipliers: multipliers2, + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + ]; + + let hex_boosting_client = MockHexBoostingClient::new(boosted_hexes); + + let (_, rewards) = tokio::join!( + // run rewards for poc and dc + rewarder::reward_poc_and_dc( + &pool, + &hex_boosting_client, + &mobile_rewards_client, + &speedtest_avg_client, + &epoch, + dec!(0.0001) + ), + receive_expected_rewards(&mut mobile_rewards) + ); + if let Ok((poc_rewards, unallocated_reward)) = rewards { + // assert poc reward outputs + let exp_reward_1 = 16_393_442_622_950; + let exp_reward_2 = 16_393_442_622_950; + let exp_reward_3 = 16_393_442_622_950; + + assert_eq!(exp_reward_1, poc_rewards[0].poc_reward); + assert_eq!( + HOTSPOT_2.to_string(), + PublicKeyBinary::from(poc_rewards[0].hotspot_key.clone()).to_string() + ); + assert_eq!(exp_reward_2, poc_rewards[1].poc_reward); + assert_eq!( + HOTSPOT_1.to_string(), + PublicKeyBinary::from(poc_rewards[1].hotspot_key.clone()).to_string() + ); + assert_eq!(exp_reward_3, poc_rewards[2].poc_reward); + assert_eq!( + HOTSPOT_3.to_string(), + PublicKeyBinary::from(poc_rewards[2].hotspot_key.clone()).to_string() + ); + + // assert the number of boosted hexes for each radio + // all will be zero as the boost period has expired for the single boosted hex + assert_eq!(0, poc_rewards[0].boosted_hexes.len()); + assert_eq!(0, poc_rewards[1].boosted_hexes.len()); + assert_eq!(0, poc_rewards[2].boosted_hexes.len()); + + // confirm the total rewards allocated matches expectations + let poc_sum: u64 = poc_rewards.iter().map(|r| r.poc_reward).sum(); + let unallocated_sum: u64 = unallocated_reward.amount; + let total = poc_sum + unallocated_sum; + + let expected_sum = reward_shares::get_scheduled_tokens_for_poc(epoch.end - epoch.start) + .to_u64() + .unwrap(); + assert_eq!(expected_sum, total); + + // confirm the rewarded percentage amount matches expectations + let daily_total = reward_shares::get_total_scheduled_tokens(epoch.end - epoch.start); + let percent = (Decimal::from(total) / daily_total) + .round_dp_with_strategy(2, RoundingStrategy::MidpointNearestEven); + assert_eq!(percent, dec!(0.6)); + } else { + panic!("no rewards received"); + }; + Ok(()) +} + +#[sqlx::test] +async fn test_reduced_location_score_with_boosted_hexes(pool: PgPool) -> anyhow::Result<()> { + let (mobile_rewards_client, mut mobile_rewards) = common::create_file_sink(); + let (speedtest_avg_client, _speedtest_avg_server) = common::create_file_sink(); + let now = Utc::now(); + let epoch = (now - ChronoDuration::hours(24))..now; + let boost_period_length = Duration::days(30); + + // seed all the things + let mut txn = pool.clone().begin().await?; + seed_heartbeats_v3(epoch.start, &mut txn).await?; + seed_speedtests(epoch.end, &mut txn).await?; + txn.commit().await?; + + // setup boosted hex where reward start time is in the second period length + let multipliers1 = vec![2]; + let start_ts_1 = epoch.start; + let end_ts_1 = start_ts_1 + (boost_period_length * multipliers1.len() as i32); + + // setup boosted hex where no start or end time is set + let multipliers2 = vec![2]; + + let boosted_hexes = vec![ + BoostedHexInfo { + // hotspot 1's location + location: 0x8a1fb466d2dffff_u64, + start_ts: Some(start_ts_1), + end_ts: Some(end_ts_1), + period_length: boost_period_length, + multipliers: multipliers1, + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + BoostedHexInfo { + // hotspot 3's location + location: 0x8c2681a306607ff_u64, + start_ts: None, + end_ts: None, + period_length: boost_period_length, + multipliers: multipliers2, + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + ]; + + let hex_boosting_client = MockHexBoostingClient::new(boosted_hexes); + + let (_, rewards) = tokio::join!( + // run rewards for poc and dc + rewarder::reward_poc_and_dc( + &pool, + &hex_boosting_client, + &mobile_rewards_client, + &speedtest_avg_client, + &epoch, + dec!(0.0001) + ), + receive_expected_rewards(&mut mobile_rewards) + ); + if let Ok((poc_rewards, unallocated_reward)) = rewards { + // HOTSPOT 1 has full location trust score and a boosted location + // HOTSPOT 2 has full location trust score and NO boosted location + // HOTSPOT 3 has reduced location trust score and a boosted location + + // assert poc reward outputs + let hotspot_1_reward = 30_264_817_150_063; + let hotspot_2_reward = 15_132_408_575_031; + let hotspot_3_reward = 3_783_102_143_757; + + assert_eq!(hotspot_1_reward, poc_rewards[1].poc_reward); + assert_eq!( + HOTSPOT_1.to_string(), + PublicKeyBinary::from(poc_rewards[1].hotspot_key.clone()).to_string() + ); + assert_eq!(hotspot_2_reward, poc_rewards[0].poc_reward); + assert_eq!( + HOTSPOT_2.to_string(), + PublicKeyBinary::from(poc_rewards[0].hotspot_key.clone()).to_string() + ); + assert_eq!(hotspot_3_reward, poc_rewards[2].poc_reward); + assert_eq!( + HOTSPOT_3.to_string(), + PublicKeyBinary::from(poc_rewards[2].hotspot_key.clone()).to_string() + ); + + // assert the boosted hexes in the radio rewards + // assert the number of boosted hexes for each radio + + //hotspot 1 has one boosted hex + assert_eq!(1, poc_rewards[1].boosted_hexes.len()); + //hotspot 2 has no boosted hexes + assert_eq!(0, poc_rewards[0].boosted_hexes.len()); + // hotspot 3 has a boosted location but as its location trust score + // is reduced the boost does not get applied + assert_eq!(0, poc_rewards[2].boosted_hexes.len()); + + // assert the hex boost multiplier values + // assert_eq!(2, poc_rewards[0].boosted_hexes[0].multiplier); + assert_eq!(2, poc_rewards[1].boosted_hexes[0].multiplier); + + assert_eq!( + 0x8a1fb466d2dffff_u64, + poc_rewards[1].boosted_hexes[0].location + ); + + // hotspot1 should have 2x the reward of hotspot 2 + // hotspot 2 has a full location trust score but no boosted hex + assert_eq!(poc_rewards[1].poc_reward / poc_rewards[0].poc_reward, 2); + // hotspot1 should have 8x the reward of hotspot 3 + // hotspot 2 has a reduced location trust score and thus gets no boost + // even tho its location is boosted + // this results in a 4x reduction in reward compared to hotspot 1's base reward + // then when you apply hotspots 2x boost you get an 8x difference + assert_eq!(poc_rewards[1].poc_reward / poc_rewards[2].poc_reward, 8); + + // confirm the total rewards allocated matches expectations + let poc_sum: u64 = poc_rewards.iter().map(|r| r.poc_reward).sum(); + let unallocated_sum: u64 = unallocated_reward.amount; + let total = poc_sum + unallocated_sum; + + let expected_sum = reward_shares::get_scheduled_tokens_for_poc(epoch.end - epoch.start) + .to_u64() + .unwrap(); + assert_eq!(expected_sum, total); + + // confirm the rewarded percentage amount matches expectations + let daily_total = reward_shares::get_total_scheduled_tokens(epoch.end - epoch.start); + let percent = (Decimal::from(total) / daily_total) + .round_dp_with_strategy(2, RoundingStrategy::MidpointNearestEven); + assert_eq!(percent, dec!(0.6)); + } else { + panic!("no rewards received"); + }; + Ok(()) +} + +async fn receive_expected_rewards( + mobile_rewards: &mut MockFileSinkReceiver, +) -> anyhow::Result<(Vec, UnallocatedReward)> { + // get the filestore outputs from rewards run + // we will have 3 radio rewards, 1 wifi radio and 2 cbrs radios + let radio_reward1 = mobile_rewards.receive_radio_reward().await; + let radio_reward2 = mobile_rewards.receive_radio_reward().await; + let radio_reward3 = mobile_rewards.receive_radio_reward().await; + // ordering is not guaranteed, so stick the rewards into a vec and sort + let mut poc_rewards = vec![radio_reward1, radio_reward2, radio_reward3]; + // after sorting reward 1 = cbrs radio1, 2 = cbrs radio2, 3 = wifi radio + poc_rewards.sort_by(|a, b| b.hotspot_key.cmp(&a.hotspot_key)); + + // expect one unallocated reward for poc + let unallocated_poc_reward = mobile_rewards.receive_unallocated_reward().await; + + // should be no further msgs + mobile_rewards.assert_no_messages(); + + Ok((poc_rewards, unallocated_poc_reward)) +} + +async fn seed_heartbeats_v1( + ts: DateTime, + txn: &mut Transaction<'_, Postgres>, +) -> anyhow::Result<()> { + for n in 0..24 { + let hotspot_key1: PublicKeyBinary = HOTSPOT_1.to_string().parse().unwrap(); + let cov_obj_1 = create_coverage_object( + ts + ChronoDuration::hours(n), + None, + hotspot_key1.clone(), + 0x8a1fb466d2dffff_u64, + true, + ); + let wifi_heartbeat1 = ValidatedHeartbeat { + heartbeat: Heartbeat { + hb_type: HbType::Wifi, + hotspot_key: hotspot_key1, + cbsd_id: None, + operation_mode: true, + lat: 0.0, + lon: 0.0, + coverage_object: Some(cov_obj_1.coverage_object.uuid), + location_validation_timestamp: Some(ts - ChronoDuration::hours(24)), + timestamp: ts + ChronoDuration::hours(n), + }, + cell_type: CellType::NovaGenericWifiIndoor, + distance_to_asserted: Some(10), + coverage_meta: None, + location_trust_score_multiplier: dec!(1.0), + validity: HeartbeatValidity::Valid, + }; + + let hotspot_key2: PublicKeyBinary = HOTSPOT_2.to_string().parse().unwrap(); + let cov_obj_2 = create_coverage_object( + ts + ChronoDuration::hours(n), + None, + hotspot_key2.clone(), + 0x8a1fb49642dffff_u64, + true, + ); + let wifi_heartbeat2 = ValidatedHeartbeat { + heartbeat: Heartbeat { + hb_type: HbType::Wifi, + hotspot_key: hotspot_key2, + cbsd_id: None, + operation_mode: true, + lat: 0.0, + lon: 0.0, + coverage_object: Some(cov_obj_2.coverage_object.uuid), + location_validation_timestamp: Some(ts - ChronoDuration::hours(24)), + timestamp: ts + ChronoDuration::hours(n), + }, + cell_type: CellType::NovaGenericWifiIndoor, + distance_to_asserted: Some(10), + coverage_meta: None, + location_trust_score_multiplier: dec!(1.0), + validity: HeartbeatValidity::Valid, + }; + + let hotspot_key3: PublicKeyBinary = HOTSPOT_3.to_string().parse().unwrap(); + let cov_obj_3 = create_coverage_object( + ts + ChronoDuration::hours(n), + None, + hotspot_key3.clone(), + 0x8c2681a306607ff_u64, + true, + ); + let wifi_heartbeat3 = ValidatedHeartbeat { + heartbeat: Heartbeat { + hb_type: HbType::Wifi, + hotspot_key: hotspot_key3, + cbsd_id: None, + operation_mode: true, + lat: 0.0, + lon: 0.0, + coverage_object: Some(cov_obj_3.coverage_object.uuid), + location_validation_timestamp: Some(ts - ChronoDuration::hours(24)), + timestamp: ts + ChronoDuration::hours(n), + }, + cell_type: CellType::NovaGenericWifiIndoor, + distance_to_asserted: Some(10), + coverage_meta: None, + location_trust_score_multiplier: dec!(1.0), + validity: HeartbeatValidity::Valid, + }; + + save_seniority_object(ts + ChronoDuration::hours(n), &wifi_heartbeat1, txn).await?; + save_seniority_object(ts + ChronoDuration::hours(n), &wifi_heartbeat2, txn).await?; + save_seniority_object(ts + ChronoDuration::hours(n), &wifi_heartbeat3, txn).await?; + + wifi_heartbeat1.save(txn).await?; + wifi_heartbeat2.save(txn).await?; + wifi_heartbeat3.save(txn).await?; + + cov_obj_1.save(txn).await?; + cov_obj_2.save(txn).await?; + cov_obj_3.save(txn).await?; + } + Ok(()) +} + +async fn seed_heartbeats_v2( + ts: DateTime, + txn: &mut Transaction<'_, Postgres>, +) -> anyhow::Result<()> { + for n in 0..24 { + let hotspot_key1: PublicKeyBinary = HOTSPOT_1.to_string().parse().unwrap(); + println!("0x8a1fb466d2dffff_u64 as u64: {}", 0x8a1fb466d2dffff_u64); + let cov_obj_1 = create_multi_coverage_object( + ts + ChronoDuration::hours(n), + None, + hotspot_key1.clone(), + vec![0x8a1fb46622dffff_u64, 0x8a1fb46622d7fff_u64], + true, + ); + let wifi_heartbeat1 = ValidatedHeartbeat { + heartbeat: Heartbeat { + hb_type: HbType::Wifi, + hotspot_key: hotspot_key1, + cbsd_id: None, + operation_mode: true, + lat: 0.0, + lon: 0.0, + coverage_object: Some(cov_obj_1.coverage_object.uuid), + location_validation_timestamp: Some(ts - ChronoDuration::hours(24)), + timestamp: ts + ChronoDuration::hours(n), + }, + cell_type: CellType::NovaGenericWifiIndoor, + distance_to_asserted: Some(10), + coverage_meta: None, + location_trust_score_multiplier: dec!(1.0), + validity: HeartbeatValidity::Valid, + }; + + let hotspot_key2: PublicKeyBinary = HOTSPOT_2.to_string().parse().unwrap(); + let cov_obj_2 = create_coverage_object( + ts + ChronoDuration::hours(n), + None, + hotspot_key2.clone(), + 0x8a1fb49642dffff_u64, + true, + ); + let wifi_heartbeat2 = ValidatedHeartbeat { + heartbeat: Heartbeat { + hb_type: HbType::Wifi, + hotspot_key: hotspot_key2, + cbsd_id: None, + operation_mode: true, + lat: 0.0, + lon: 0.0, + coverage_object: Some(cov_obj_2.coverage_object.uuid), + location_validation_timestamp: Some(ts - ChronoDuration::hours(24)), + timestamp: ts + ChronoDuration::hours(n), + }, + cell_type: CellType::NovaGenericWifiIndoor, + distance_to_asserted: Some(10), + coverage_meta: None, + location_trust_score_multiplier: dec!(1.0), + validity: HeartbeatValidity::Valid, + }; + + let hotspot_key3: PublicKeyBinary = HOTSPOT_3.to_string().parse().unwrap(); + let cov_obj_3 = create_coverage_object( + ts + ChronoDuration::hours(n), + None, + hotspot_key3.clone(), + 0x8c2681a306607ff_u64, + true, + ); + let wifi_heartbeat3 = ValidatedHeartbeat { + heartbeat: Heartbeat { + hb_type: HbType::Wifi, + hotspot_key: hotspot_key3, + cbsd_id: None, + operation_mode: true, + lat: 0.0, + lon: 0.0, + coverage_object: Some(cov_obj_3.coverage_object.uuid), + location_validation_timestamp: Some(ts - ChronoDuration::hours(24)), + timestamp: ts + ChronoDuration::hours(n), + }, + cell_type: CellType::NovaGenericWifiIndoor, + distance_to_asserted: Some(10), + coverage_meta: None, + location_trust_score_multiplier: dec!(1.0), + validity: HeartbeatValidity::Valid, + }; + + save_seniority_object(ts + ChronoDuration::hours(n), &wifi_heartbeat1, txn).await?; + save_seniority_object(ts + ChronoDuration::hours(n), &wifi_heartbeat2, txn).await?; + save_seniority_object(ts + ChronoDuration::hours(n), &wifi_heartbeat3, txn).await?; + + wifi_heartbeat1.save(txn).await?; + wifi_heartbeat2.save(txn).await?; + wifi_heartbeat3.save(txn).await?; + + cov_obj_1.save(txn).await?; + cov_obj_2.save(txn).await?; + cov_obj_3.save(txn).await?; + } + Ok(()) +} + +async fn seed_heartbeats_v3( + ts: DateTime, + txn: &mut Transaction<'_, Postgres>, +) -> anyhow::Result<()> { + // HOTSPOT 1 has full location trust score + // HOTSPOT 2 has full location trust score + // HOTSPOT 3 has reduced location trust score + for n in 0..24 { + let hotspot_key1: PublicKeyBinary = HOTSPOT_1.to_string().parse().unwrap(); + let cov_obj_1 = create_coverage_object( + ts + ChronoDuration::hours(n), + None, + hotspot_key1.clone(), + 0x8a1fb466d2dffff_u64, + true, + ); + let wifi_heartbeat1 = ValidatedHeartbeat { + heartbeat: Heartbeat { + hb_type: HbType::Wifi, + hotspot_key: hotspot_key1, + cbsd_id: None, + operation_mode: true, + lat: 0.0, + lon: 0.0, + coverage_object: Some(cov_obj_1.coverage_object.uuid), + location_validation_timestamp: Some(ts - ChronoDuration::hours(24)), + timestamp: ts + ChronoDuration::hours(n), + }, + cell_type: CellType::NovaGenericWifiIndoor, + distance_to_asserted: Some(10), + coverage_meta: None, + location_trust_score_multiplier: dec!(1.0), + validity: HeartbeatValidity::Valid, + }; + + let hotspot_key2: PublicKeyBinary = HOTSPOT_2.to_string().parse().unwrap(); + let cov_obj_2 = create_coverage_object( + ts + ChronoDuration::hours(n), + None, + hotspot_key2.clone(), + 0x8a1fb49642dffff_u64, + true, + ); + let wifi_heartbeat2 = ValidatedHeartbeat { + heartbeat: Heartbeat { + hb_type: HbType::Wifi, + hotspot_key: hotspot_key2, + cbsd_id: None, + operation_mode: true, + lat: 0.0, + lon: 0.0, + coverage_object: Some(cov_obj_2.coverage_object.uuid), + location_validation_timestamp: Some(ts - ChronoDuration::hours(24)), + timestamp: ts + ChronoDuration::hours(n), + }, + cell_type: CellType::NovaGenericWifiIndoor, + distance_to_asserted: Some(10), + coverage_meta: None, + location_trust_score_multiplier: dec!(1.0), + validity: HeartbeatValidity::Valid, + }; + + let hotspot_key3: PublicKeyBinary = HOTSPOT_3.to_string().parse().unwrap(); + let cov_obj_3 = create_coverage_object( + ts + ChronoDuration::hours(n), + None, + hotspot_key3.clone(), + 0x8c2681a306607ff_u64, + true, + ); + let wifi_heartbeat3 = ValidatedHeartbeat { + heartbeat: Heartbeat { + hb_type: HbType::Wifi, + hotspot_key: hotspot_key3, + cbsd_id: None, + operation_mode: true, + lat: 0.0, + lon: 0.0, + coverage_object: Some(cov_obj_3.coverage_object.uuid), + location_validation_timestamp: Some(ts - ChronoDuration::hours(24)), + timestamp: ts + ChronoDuration::hours(n), + }, + cell_type: CellType::NovaGenericWifiIndoor, + distance_to_asserted: Some(300), + coverage_meta: None, + location_trust_score_multiplier: dec!(0.25), + validity: HeartbeatValidity::Valid, + }; + + save_seniority_object(ts + ChronoDuration::hours(n), &wifi_heartbeat1, txn).await?; + save_seniority_object(ts + ChronoDuration::hours(n), &wifi_heartbeat2, txn).await?; + save_seniority_object(ts + ChronoDuration::hours(n), &wifi_heartbeat3, txn).await?; + + wifi_heartbeat1.save(txn).await?; + wifi_heartbeat2.save(txn).await?; + wifi_heartbeat3.save(txn).await?; + + cov_obj_1.save(txn).await?; + cov_obj_2.save(txn).await?; + cov_obj_3.save(txn).await?; + } + Ok(()) +} + +async fn seed_speedtests( + ts: DateTime, + txn: &mut Transaction<'_, Postgres>, +) -> anyhow::Result<()> { + for n in 0..24 { + let hotspot1_speedtest = CellSpeedtest { + pubkey: HOTSPOT_1.parse().unwrap(), + serial: "serial1".to_string(), + timestamp: ts - ChronoDuration::hours(n * 4), + upload_speed: 100_000_000, + download_speed: 100_000_000, + latency: 50, + }; + + let hotspot2_speedtest = CellSpeedtest { + pubkey: HOTSPOT_2.parse().unwrap(), + serial: "serial2".to_string(), + timestamp: ts - ChronoDuration::hours(n * 4), + upload_speed: 100_000_000, + download_speed: 100_000_000, + latency: 50, + }; + + let hotspot3_speedtest = CellSpeedtest { + pubkey: HOTSPOT_3.parse().unwrap(), + serial: "serial3".to_string(), + timestamp: ts - ChronoDuration::hours(n * 4), + upload_speed: 100_000_000, + download_speed: 100_000_000, + latency: 50, + }; + + speedtests::save_speedtest(&hotspot1_speedtest, txn).await?; + speedtests::save_speedtest(&hotspot2_speedtest, txn).await?; + speedtests::save_speedtest(&hotspot3_speedtest, txn).await?; + } + Ok(()) +} + +fn create_coverage_object( + ts: DateTime, + cbsd_id: Option, + pub_key: PublicKeyBinary, + hex: u64, + indoor: bool, +) -> CoverageObject { + let location = h3o::CellIndex::try_from(hex).unwrap(); + let key_type = match cbsd_id { + Some(s) => KeyType::CbsdId(s), + None => KeyType::HotspotKey(pub_key.clone()), + }; + let report = FSCoverageObject { + pub_key, + uuid: Uuid::new_v4(), + key_type, + coverage_claim_time: ts, + coverage: vec![RadioHexSignalLevel { + location, + signal_level: SignalLevel::High, + signal_power: 1000, + }], + indoor, + trust_score: 1000, + signature: Vec::new(), + }; + CoverageObject { + coverage_object: report, + validity: CoverageObjectValidity::Valid, + } +} + +fn create_multi_coverage_object( + ts: DateTime, + cbsd_id: Option, + pub_key: PublicKeyBinary, + hex: Vec, + indoor: bool, +) -> CoverageObject { + let key_type = match cbsd_id { + Some(s) => KeyType::CbsdId(s), + None => KeyType::HotspotKey(pub_key.clone()), + }; + let coverage: Vec = hex + .iter() + .map(|h| RadioHexSignalLevel { + location: h3o::CellIndex::try_from(*h).unwrap(), + signal_level: SignalLevel::High, + signal_power: 1000, + }) + .collect(); + + let report = FSCoverageObject { + pub_key, + uuid: Uuid::new_v4(), + key_type, + coverage_claim_time: ts, + coverage, + indoor, + trust_score: 1000, + signature: Vec::new(), + }; + CoverageObject { + coverage_object: report, + validity: CoverageObjectValidity::Valid, + } +} +async fn save_seniority_object( + ts: DateTime, + hb: &ValidatedHeartbeat, + exec: &mut Transaction<'_, Postgres>, +) -> anyhow::Result<()> { + sqlx::query( + r#" + INSERT INTO seniority + (radio_key, last_heartbeat, uuid, seniority_ts, inserted_at, update_reason, radio_type) + VALUES + ($1, $2, $3, $4, $5, $6, $7) + "#, + ) + .bind(hb.heartbeat.key()) + .bind(hb.heartbeat.timestamp) + .bind(hb.heartbeat.coverage_object) + .bind(ts) + .bind(ts) + .bind(SeniorityUpdateReason::NewCoverageClaimTime as i32) + .bind(hb.heartbeat.hb_type) + .execute(&mut *exec) + .await?; + Ok(()) +} diff --git a/mobile_verifier/tests/modeled_coverage.rs b/mobile_verifier/tests/modeled_coverage.rs index 281439853..a114a70ed 100644 --- a/mobile_verifier/tests/modeled_coverage.rs +++ b/mobile_verifier/tests/modeled_coverage.rs @@ -7,8 +7,11 @@ use file_store::{ }; use futures::stream::{self, StreamExt}; use helium_crypto::PublicKeyBinary; -use helium_proto::services::mobile_config::NetworkKeyRole; -use helium_proto::services::poc_mobile::{CoverageObjectValidity, SignalLevel}; +use helium_proto::services::{ + mobile_config::NetworkKeyRole, + poc_mobile::{CoverageObjectValidity, SignalLevel}, +}; +use mobile_config::boosted_hex_info::{BoostedHexInfo, BoostedHexes}; use mobile_verifier::{ coverage::{CoverageClaimTimeCache, CoverageObject, CoverageObjectCache, Seniority}, geofence::GeofenceValidator, @@ -31,6 +34,7 @@ impl GeofenceValidator for MockGeofence { true } } +const BOOST_CONFIG_PUBKEY: &str = "11hd7HoicRgBPjBGcqcT2Y9hRQovdZeff5eKFMbCSuDYQmuCiF1"; #[sqlx::test] #[ignore] @@ -473,8 +477,14 @@ async fn scenario_one(pool: PgPool) -> anyhow::Result<()> { let reward_period = start..end; let heartbeats = HeartbeatReward::validated(&pool, &reward_period); - let coverage_points = - CoveragePoints::aggregate_points(&pool, heartbeats, &speedtest_avgs, end).await?; + let coverage_points = CoveragePoints::aggregate_points( + &pool, + heartbeats, + &speedtest_avgs, + &BoostedHexes::default(), + &reward_period, + ) + .await?; assert_eq!(coverage_points.hotspot_points(&owner), dec!(1000)); @@ -566,8 +576,14 @@ async fn scenario_two(pool: PgPool) -> anyhow::Result<()> { let reward_period = start..end; let heartbeats = HeartbeatReward::validated(&pool, &reward_period); - let coverage_points = - CoveragePoints::aggregate_points(&pool, heartbeats, &speedtest_avgs, end).await?; + let coverage_points = CoveragePoints::aggregate_points( + &pool, + heartbeats, + &speedtest_avgs, + &BoostedHexes::default(), + &reward_period, + ) + .await?; assert_eq!(coverage_points.hotspot_points(&owner_1), dec!(500)); assert_eq!(coverage_points.hotspot_points(&owner_2), dec!(1000)); @@ -798,10 +814,58 @@ async fn scenario_three(pool: PgPool) -> anyhow::Result<()> { averages.insert(owner_6.clone(), SpeedtestAverage::from(speedtests_6)); let speedtest_avgs = SpeedtestAverages { averages }; + let mut boosted_hexes = BoostedHexes::default(); + boosted_hexes.hexes.insert( + 0x8a1fb466d2dffff_u64, + BoostedHexInfo { + location: 0x8a1fb466d2dffff_u64, + start_ts: None, + end_ts: None, + period_length: Duration::hours(1), + multipliers: vec![1], + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + ); + boosted_hexes.hexes.insert( + 0x8a1fb49642dffff_u64, + BoostedHexInfo { + location: 0x8a1fb49642dffff_u64, + start_ts: None, + end_ts: None, + period_length: Duration::hours(1), + multipliers: vec![2], + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + ); + boosted_hexes.hexes.insert( + 0x8c2681a306607ff_u64, + BoostedHexInfo { + // hotspot 1's location + location: 0x8c2681a306607ff_u64, + start_ts: None, + end_ts: None, + period_length: Duration::hours(1), + multipliers: vec![3], + boosted_hex_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + boost_config_pubkey: BOOST_CONFIG_PUBKEY.to_string(), + version: 0, + }, + ); + let reward_period = start..end; let heartbeats = HeartbeatReward::validated(&pool, &reward_period); - let coverage_points = - CoveragePoints::aggregate_points(&pool, heartbeats, &speedtest_avgs, end).await?; + let coverage_points = CoveragePoints::aggregate_points( + &pool, + heartbeats, + &speedtest_avgs, + &boosted_hexes, + &reward_period, + ) + .await?; assert_eq!(coverage_points.hotspot_points(&owner_1), dec!(250)); assert_eq!(coverage_points.hotspot_points(&owner_2), dec!(250)); @@ -865,8 +929,14 @@ async fn scenario_four(pool: PgPool) -> anyhow::Result<()> { let reward_period = start..end; let heartbeats = HeartbeatReward::validated(&pool, &reward_period); - let coverage_points = - CoveragePoints::aggregate_points(&pool, heartbeats, &speedtest_avgs, end).await?; + let coverage_points = CoveragePoints::aggregate_points( + &pool, + heartbeats, + &speedtest_avgs, + &BoostedHexes::default(), + &reward_period, + ) + .await?; assert_eq!(coverage_points.hotspot_points(&owner), dec!(76)); @@ -957,8 +1027,14 @@ async fn scenario_five(pool: PgPool) -> anyhow::Result<()> { let reward_period = start..end; let heartbeats = HeartbeatReward::validated(&pool, &reward_period); - let coverage_points = - CoveragePoints::aggregate_points(&pool, heartbeats, &speedtest_avgs, end).await?; + let coverage_points = CoveragePoints::aggregate_points( + &pool, + heartbeats, + &speedtest_avgs, + &BoostedHexes::default(), + &reward_period, + ) + .await?; assert_eq!( coverage_points.hotspot_points(&owner_1), @@ -1197,8 +1273,14 @@ async fn scenario_six(pool: PgPool) -> anyhow::Result<()> { let reward_period = start..end; let heartbeats = HeartbeatReward::validated(&pool, &reward_period); - let coverage_points = - CoveragePoints::aggregate_points(&pool, heartbeats, &speedtest_avgs, end).await?; + let coverage_points = CoveragePoints::aggregate_points( + &pool, + heartbeats, + &speedtest_avgs, + &BoostedHexes::default(), + &reward_period, + ) + .await?; assert_eq!(coverage_points.hotspot_points(&owner_1), dec!(250)); assert_eq!(coverage_points.hotspot_points(&owner_2), dec!(250)); diff --git a/mobile_verifier/tests/rewarder_poc_dc.rs b/mobile_verifier/tests/rewarder_poc_dc.rs index b26dc19c3..7678ddffb 100644 --- a/mobile_verifier/tests/rewarder_poc_dc.rs +++ b/mobile_verifier/tests/rewarder_poc_dc.rs @@ -1,15 +1,21 @@ mod common; -use crate::common::MockFileSinkReceiver; +use crate::common::{MockFileSinkReceiver, MockHexBoostingClient}; +use async_trait::async_trait; use chrono::{DateTime, Duration as ChronoDuration, Utc}; use file_store::{ coverage::{CoverageObject as FSCoverageObject, KeyType, RadioHexSignalLevel}, speedtest::CellSpeedtest, }; +use futures_util::{stream, StreamExt as FuturesStreamExt}; use helium_crypto::PublicKeyBinary; use helium_proto::services::poc_mobile::{ CoverageObjectValidity, GatewayReward, HeartbeatValidity, RadioReward, SeniorityUpdateReason, SignalLevel, UnallocatedReward, UnallocatedRewardType, }; +use mobile_config::{ + boosted_hex_info::{BoostedHexInfo, BoostedHexInfoStream}, + client::{hex_boosting_client::HexBoostingInfoResolver, ClientError}, +}; use mobile_verifier::{ cell_type::CellType, coverage::CoverageObject, @@ -24,9 +30,31 @@ use uuid::Uuid; const HOTSPOT_1: &str = "112NqN2WWMwtK29PMzRby62fDydBJfsCLkCAf392stdok48ovNT6"; const HOTSPOT_2: &str = "11uJHS2YaEWJqgqC7yza9uvSmpv5FWoMQXiP8WbxBGgNUmifUJf"; -const HOTSPOT_3: &str = "11sctWiP9r5wDJVuDe1Th4XSL2vaawaLLSQF8f8iokAoMAJHxqp"; +const HOTSPOT_3: &str = "112E7TxoNHV46M6tiPA8N1MkeMeQxc9ztb4JQLXBVAAUfq1kJLoF"; const PAYER_1: &str = "11eX55faMbqZB7jzN4p67m6w7ScPMH6ubnvCjCPLh72J49PaJEL"; +impl MockHexBoostingClient { + fn new(boosted_hexes: Vec) -> Self { + Self { boosted_hexes } + } +} + +#[async_trait] +impl HexBoostingInfoResolver for MockHexBoostingClient { + type Error = ClientError; + + async fn stream_boosted_hexes_info(&mut self) -> Result { + Ok(stream::iter(self.boosted_hexes.clone()).boxed()) + } + + async fn stream_modified_boosted_hexes_info( + &mut self, + _timestamp: DateTime, + ) -> Result { + Ok(stream::iter(self.boosted_hexes.clone()).boxed()) + } +} + #[sqlx::test] async fn test_poc_and_dc_rewards(pool: PgPool) -> anyhow::Result<()> { let (mobile_rewards_client, mut mobile_rewards) = common::create_file_sink(); @@ -41,10 +69,15 @@ async fn test_poc_and_dc_rewards(pool: PgPool) -> anyhow::Result<()> { seed_data_sessions(epoch.start, &mut txn).await?; txn.commit().await?; + let boosted_hexes = vec![]; + + let hex_boosting_client = MockHexBoostingClient::new(boosted_hexes); + let (_, rewards) = tokio::join!( // run rewards for poc and dc rewarder::reward_poc_and_dc( &pool, + &hex_boosting_client, &mobile_rewards_client, &speedtest_avg_client, &epoch, @@ -54,22 +87,32 @@ async fn test_poc_and_dc_rewards(pool: PgPool) -> anyhow::Result<()> { ); if let Ok((poc_rewards, dc_rewards, unallocated_poc_reward)) = rewards { // assert poc reward outputs - assert_eq!(24_108_003_121_986, poc_rewards[0].poc_reward); + let hotspot_1_reward = 24_108_003_121_986; + let hotspot_2_reward = 24_108_003_121_986; + let hotspot_3_reward = 964_320_124_879; + assert_eq!(hotspot_1_reward, poc_rewards[0].poc_reward); assert_eq!( HOTSPOT_1.to_string(), PublicKeyBinary::from(poc_rewards[0].hotspot_key.clone()).to_string() ); - assert_eq!(964_320_124_879, poc_rewards[1].poc_reward); + assert_eq!(hotspot_2_reward, poc_rewards[1].poc_reward); assert_eq!( - HOTSPOT_2.to_string(), + HOTSPOT_3.to_string(), PublicKeyBinary::from(poc_rewards[1].hotspot_key.clone()).to_string() ); - assert_eq!(24_108_003_121_986, poc_rewards[2].poc_reward); + assert_eq!(hotspot_3_reward, poc_rewards[2].poc_reward); assert_eq!( - HOTSPOT_3.to_string(), + HOTSPOT_2.to_string(), PublicKeyBinary::from(poc_rewards[2].hotspot_key.clone()).to_string() ); + // assert the boosted hexes in the radio rewards + // boosted hexes will contain the used multiplier for each boosted hex + // in this test there are no boosted hexes + assert_eq!(0, poc_rewards[0].boosted_hexes.len()); + assert_eq!(0, poc_rewards[1].boosted_hexes.len()); + assert_eq!(0, poc_rewards[2].boosted_hexes.len()); + // assert unallocated amount assert_eq!( UnallocatedRewardType::Poc as i32, @@ -85,12 +128,12 @@ async fn test_poc_and_dc_rewards(pool: PgPool) -> anyhow::Result<()> { ); assert_eq!(500_000, dc_rewards[1].dc_transfer_reward); assert_eq!( - HOTSPOT_2.to_string(), + HOTSPOT_3.to_string(), PublicKeyBinary::from(dc_rewards[1].hotspot_key.clone()).to_string() ); assert_eq!(500_000, dc_rewards[2].dc_transfer_reward); assert_eq!( - HOTSPOT_3.to_string(), + HOTSPOT_2.to_string(), PublicKeyBinary::from(dc_rewards[2].hotspot_key.clone()).to_string() ); diff --git a/price/src/cli/check.rs b/price/src/cli/check.rs index 56d76f313..24fa8a558 100644 --- a/price/src/cli/check.rs +++ b/price/src/cli/check.rs @@ -1,7 +1,6 @@ -use anchor_lang::AccountDeserialize; use anyhow::Result; use chrono::{DateTime, TimeZone, Utc}; -use helium_anchor_gen::price_oracle::PriceOracleV0; +use helium_anchor_gen::{anchor_lang::AccountDeserialize, price_oracle::PriceOracleV0}; use solana_client::nonblocking::rpc_client::RpcClient; use solana_sdk::pubkey::Pubkey as SolPubkey; use std::str::FromStr; diff --git a/price/src/price_generator.rs b/price/src/price_generator.rs index d8899958c..510f8fdb0 100644 --- a/price/src/price_generator.rs +++ b/price/src/price_generator.rs @@ -1,10 +1,12 @@ use crate::{metrics::Metrics, Settings}; -use anchor_lang::AccountDeserialize; use anyhow::{anyhow, Error, Result}; use chrono::{DateTime, Duration, TimeZone, Utc}; use file_store::file_sink; use futures::{future::LocalBoxFuture, TryFutureExt}; -use helium_anchor_gen::price_oracle::{calculate_current_price, PriceOracleV0}; +use helium_anchor_gen::{ + anchor_lang::AccountDeserialize, + price_oracle::{calculate_current_price, PriceOracleV0}, +}; use helium_proto::{BlockchainTokenTypeV1, PriceReportV1}; use serde::{Deserialize, Serialize}; use solana_client::nonblocking::rpc_client::RpcClient; diff --git a/solana/Cargo.toml b/solana/Cargo.toml index 434224d08..9ba9b48e6 100644 --- a/solana/Cargo.toml +++ b/solana/Cargo.toml @@ -12,6 +12,8 @@ async-trait = {workspace = true} anchor-lang = {workspace = true} anchor-client = {workspace = true} clap = {workspace = true} +chrono = {workspace = true} +file-store = {path = "../file_store"} futures = {workspace = true} helium-anchor-gen = {workspace = true} helium-crypto = {workspace = true, features = ["solana"]} diff --git a/solana/src/burn.rs b/solana/src/burn.rs new file mode 100644 index 000000000..1b7a58f0c --- /dev/null +++ b/solana/src/burn.rs @@ -0,0 +1,414 @@ +use crate::{send_with_retry, GetSignature, SolanaRpcError}; +use anchor_client::{RequestBuilder, RequestNamespace}; +use async_trait::async_trait; +use helium_anchor_gen::{ + anchor_lang::AccountDeserialize, + data_credits::{self, accounts, instruction}, + helium_sub_daos::{self, DaoV0, SubDaoV0}, +}; +use helium_crypto::PublicKeyBinary; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use solana_client::{nonblocking::rpc_client::RpcClient, rpc_response::Response}; +use solana_sdk::{ + commitment_config::CommitmentConfig, + program_pack::Pack, + pubkey::Pubkey, + signature::{read_keypair_file, Keypair, Signature}, + signer::Signer, + transaction::Transaction, +}; +use std::convert::Infallible; +use std::{collections::HashMap, str::FromStr}; +use std::{ + sync::Arc, + time::{Duration, SystemTime}, +}; +use tokio::sync::Mutex; + +#[async_trait] +pub trait SolanaNetwork: Send + Sync + 'static { + type Error: std::error::Error + Send + Sync + 'static; + type Transaction: GetSignature + Send + Sync + 'static; + + async fn payer_balance(&self, payer: &PublicKeyBinary) -> Result; + + async fn make_burn_transaction( + &self, + payer: &PublicKeyBinary, + amount: u64, + ) -> Result; + + async fn submit_transaction(&self, transaction: &Self::Transaction) -> Result<(), Self::Error>; + + async fn confirm_transaction(&self, txn: &Signature) -> Result; +} + +#[derive(Debug, Deserialize)] +pub struct Settings { + rpc_url: String, + cluster: String, + burn_keypair: String, + dc_mint: String, + dnt_mint: String, + #[serde(default)] + payers_to_monitor: Vec, +} + +impl Settings { + pub fn payers_to_monitor(&self) -> Result, SolanaRpcError> { + self.payers_to_monitor + .iter() + .map(|payer| PublicKeyBinary::from_str(payer)) + .collect::>() + .map_err(SolanaRpcError::from) + } +} + +pub struct SolanaRpc { + provider: RpcClient, + program_cache: BurnProgramCache, + cluster: String, + keypair: [u8; 64], + payers_to_monitor: Vec, +} + +impl SolanaRpc { + pub async fn new(settings: &Settings) -> Result, SolanaRpcError> { + let dc_mint = settings.dc_mint.parse()?; + let dnt_mint = settings.dnt_mint.parse()?; + let Ok(keypair) = read_keypair_file(&settings.burn_keypair) else { + return Err(SolanaRpcError::FailedToReadKeypairError); + }; + let provider = + RpcClient::new_with_commitment(settings.rpc_url.clone(), CommitmentConfig::finalized()); + let program_cache = BurnProgramCache::new(&provider, dc_mint, dnt_mint).await?; + if program_cache.dc_burn_authority != keypair.pubkey() { + return Err(SolanaRpcError::InvalidKeypair); + } + Ok(Arc::new(Self { + cluster: settings.cluster.clone(), + provider, + program_cache, + keypair: keypair.to_bytes(), + payers_to_monitor: settings.payers_to_monitor()?, + })) + } +} + +#[async_trait] +impl SolanaNetwork for SolanaRpc { + type Error = SolanaRpcError; + type Transaction = Transaction; + + async fn payer_balance(&self, payer: &PublicKeyBinary) -> Result { + let ddc_key = delegated_data_credits(&self.program_cache.sub_dao, payer); + let (escrow_account, _) = Pubkey::find_program_address( + &["escrow_dc_account".as_bytes(), &ddc_key.to_bytes()], + &data_credits::ID, + ); + let account_data = match self + .provider + .get_account_with_commitment(&escrow_account, CommitmentConfig::finalized()) + .await? + { + Response { value: None, .. } => { + tracing::info!(%payer, "Account not found, therefore no balance"); + return Ok(0); + } + Response { + value: Some(account), + .. + } => account.data, + }; + let account_layout = spl_token::state::Account::unpack(account_data.as_slice())?; + + if self.payers_to_monitor.contains(payer) { + metrics::gauge!( + "balance", + account_layout.amount as f64, + "payer" => payer.to_string() + ); + } + + Ok(account_layout.amount) + } + + async fn make_burn_transaction( + &self, + payer: &PublicKeyBinary, + amount: u64, + ) -> Result { + // Fetch the sub dao epoch info: + const EPOCH_LENGTH: u64 = 60 * 60 * 24; + let epoch = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH)? + .as_secs() + / EPOCH_LENGTH; + let (sub_dao_epoch_info, _) = Pubkey::find_program_address( + &[ + "sub_dao_epoch_info".as_bytes(), + self.program_cache.sub_dao.as_ref(), + &epoch.to_le_bytes(), + ], + &helium_sub_daos::ID, + ); + + // Fetch escrow account + let ddc_key = delegated_data_credits(&self.program_cache.sub_dao, payer); + let (escrow_account, _) = Pubkey::find_program_address( + &["escrow_dc_account".as_bytes(), &ddc_key.to_bytes()], + &data_credits::ID, + ); + + let instructions = { + let request = RequestBuilder::from( + data_credits::id(), + &self.cluster, + std::rc::Rc::new(Keypair::from_bytes(&self.keypair).unwrap()), + Some(CommitmentConfig::confirmed()), + RequestNamespace::Global, + ); + + let accounts = accounts::BurnDelegatedDataCreditsV0 { + sub_dao_epoch_info, + dao: self.program_cache.dao, + sub_dao: self.program_cache.sub_dao, + account_payer: self.program_cache.account_payer, + data_credits: self.program_cache.data_credits, + delegated_data_credits: delegated_data_credits(&self.program_cache.sub_dao, payer), + token_program: spl_token::id(), + helium_sub_daos_program: helium_sub_daos::id(), + system_program: solana_program::system_program::id(), + dc_burn_authority: self.program_cache.dc_burn_authority, + dc_mint: self.program_cache.dc_mint, + escrow_account, + registrar: self.program_cache.registrar, + }; + let args = instruction::BurnDelegatedDataCreditsV0 { + _args: data_credits::BurnDelegatedDataCreditsArgsV0 { amount }, + }; + + // As far as I can tell, the instructions function does not actually have any + // error paths. + request + .accounts(accounts) + .args(args) + .instructions() + .unwrap() + }; + + let blockhash = self.provider.get_latest_blockhash().await?; + let signer = Keypair::from_bytes(&self.keypair).unwrap(); + + Ok(Transaction::new_signed_with_payer( + &instructions, + Some(&signer.pubkey()), + &[&signer], + blockhash, + )) + } + + async fn submit_transaction(&self, tx: &Self::Transaction) -> Result<(), Self::Error> { + match send_with_retry!(self.provider.send_and_confirm_transaction(tx)) { + Ok(signature) => { + tracing::info!( + transaction = %signature, + "Data credit burn successful", + ); + Ok(()) + } + Err(err) => { + let signature = tx.get_signature(); + tracing::error!( + transaction = %signature, + "Data credit burn failed: {err:?}" + ); + Err(SolanaRpcError::RpcClientError(err)) + } + } + } + + async fn confirm_transaction(&self, txn: &Signature) -> Result { + Ok(matches!( + self.provider + .get_signature_status_with_commitment_and_history( + txn, + CommitmentConfig::confirmed(), + true, + ) + .await?, + Some(Ok(())) + )) + } +} + +/// Cached pubkeys for the burn program +pub struct BurnProgramCache { + pub account_payer: Pubkey, + pub data_credits: Pubkey, + pub sub_dao: Pubkey, + pub dao: Pubkey, + pub dc_mint: Pubkey, + pub dc_burn_authority: Pubkey, + pub registrar: Pubkey, +} + +impl BurnProgramCache { + pub async fn new( + provider: &RpcClient, + dc_mint: Pubkey, + dnt_mint: Pubkey, + ) -> Result { + let (account_payer, _) = + Pubkey::find_program_address(&["account_payer".as_bytes()], &data_credits::ID); + let (data_credits, _) = + Pubkey::find_program_address(&["dc".as_bytes(), dc_mint.as_ref()], &data_credits::ID); + let (sub_dao, _) = Pubkey::find_program_address( + &["sub_dao".as_bytes(), dnt_mint.as_ref()], + &helium_sub_daos::ID, + ); + let (dao, dc_burn_authority) = { + let account_data = provider.get_account_data(&sub_dao).await?; + let mut account_data = account_data.as_ref(); + let sub_dao = SubDaoV0::try_deserialize(&mut account_data)?; + (sub_dao.dao, sub_dao.dc_burn_authority) + }; + let registrar = { + let account_data = provider.get_account_data(&dao).await?; + let mut account_data = account_data.as_ref(); + DaoV0::try_deserialize(&mut account_data)?.registrar + }; + Ok(Self { + account_payer, + data_credits, + sub_dao, + dao, + dc_mint, + dc_burn_authority, + registrar, + }) + } +} + +const FIXED_BALANCE: u64 = 1_000_000_000; + +pub enum PossibleTransaction { + NoTransaction(Signature), + Transaction(Transaction), +} + +impl GetSignature for PossibleTransaction { + fn get_signature(&self) -> &Signature { + match self { + Self::NoTransaction(ref sig) => sig, + Self::Transaction(ref txn) => txn.get_signature(), + } + } +} + +#[async_trait] +impl SolanaNetwork for Option> { + type Error = SolanaRpcError; + type Transaction = PossibleTransaction; + + async fn payer_balance(&self, payer: &PublicKeyBinary) -> Result { + if let Some(ref rpc) = self { + rpc.payer_balance(payer).await + } else { + Ok(FIXED_BALANCE) + } + } + + async fn make_burn_transaction( + &self, + payer: &PublicKeyBinary, + amount: u64, + ) -> Result { + if let Some(ref rpc) = self { + Ok(PossibleTransaction::Transaction( + rpc.make_burn_transaction(payer, amount).await?, + )) + } else { + Ok(PossibleTransaction::NoTransaction(Signature::new_unique())) + } + } + + async fn submit_transaction(&self, transaction: &Self::Transaction) -> Result<(), Self::Error> { + match (self, transaction) { + (Some(ref rpc), PossibleTransaction::Transaction(ref txn)) => { + rpc.submit_transaction(txn).await? + } + (None, PossibleTransaction::NoTransaction(_)) => (), + _ => unreachable!(), + } + Ok(()) + } + + async fn confirm_transaction(&self, txn: &Signature) -> Result { + if let Some(ref rpc) = self { + rpc.confirm_transaction(txn).await + } else { + panic!("We will not confirm transactions when Solana is disabled"); + } + } +} + +pub struct MockTransaction { + pub signature: Signature, + pub payer: PublicKeyBinary, + pub amount: u64, +} + +impl GetSignature for MockTransaction { + fn get_signature(&self) -> &Signature { + &self.signature + } +} + +#[async_trait] +impl SolanaNetwork for Arc>> { + type Error = Infallible; + type Transaction = MockTransaction; + + async fn payer_balance(&self, payer: &PublicKeyBinary) -> Result { + Ok(*self.lock().await.get(payer).unwrap()) + } + + async fn make_burn_transaction( + &self, + payer: &PublicKeyBinary, + amount: u64, + ) -> Result { + Ok(MockTransaction { + signature: Signature::new_unique(), + payer: payer.clone(), + amount, + }) + } + + async fn submit_transaction(&self, txn: &MockTransaction) -> Result<(), Self::Error> { + *self.lock().await.get_mut(&txn.payer).unwrap() -= txn.amount; + Ok(()) + } + + async fn confirm_transaction(&self, _txn: &Signature) -> Result { + Ok(true) + } +} + +/// Returns the PDA for the Delegated Data Credits of the given `payer`. +pub fn delegated_data_credits(sub_dao: &Pubkey, payer: &PublicKeyBinary) -> Pubkey { + let mut hasher = Sha256::new(); + hasher.update(payer.to_string()); + let sha_digest = hasher.finalize(); + let (ddc_key, _) = Pubkey::find_program_address( + &[ + "delegated_data_credits".as_bytes(), + sub_dao.as_ref(), + &sha_digest, + ], + &data_credits::ID, + ); + ddc_key +} diff --git a/solana/src/lib.rs b/solana/src/lib.rs index b796e9c7b..ca118930d 100644 --- a/solana/src/lib.rs +++ b/solana/src/lib.rs @@ -1,65 +1,11 @@ -use anchor_client::{RequestBuilder, RequestNamespace}; -use anchor_lang::AccountDeserialize; -use async_trait::async_trait; -use helium_anchor_gen::{ - data_credits::{self, accounts, instruction}, - helium_sub_daos::{self, DaoV0, SubDaoV0}, -}; -use helium_crypto::PublicKeyBinary; -use serde::Deserialize; -use sha2::{Digest, Sha256}; -use solana_client::{ - client_error::ClientError, nonblocking::rpc_client::RpcClient, rpc_response::Response, -}; -use solana_sdk::{ - commitment_config::CommitmentConfig, - program_pack::Pack, - pubkey::{ParsePubkeyError, Pubkey}, - signature::{read_keypair_file, Keypair, Signature}, - signer::Signer, - transaction::Transaction, -}; -use std::convert::Infallible; -use std::{collections::HashMap, str::FromStr}; -use std::{ - sync::Arc, - time::{Duration, SystemTime, SystemTimeError}, -}; -use tokio::sync::Mutex; +use solana_client::client_error::ClientError; +use solana_sdk::pubkey::ParsePubkeyError; +use solana_sdk::signature::Signature; +use solana_sdk::transaction::Transaction; +use std::time::SystemTimeError; -#[async_trait] -pub trait SolanaNetwork: Send + Sync + 'static { - type Error: std::error::Error + Send + Sync + 'static; - type Transaction: GetSignature + Send + Sync + 'static; - - async fn payer_balance(&self, payer: &PublicKeyBinary) -> Result; - - async fn make_burn_transaction( - &self, - payer: &PublicKeyBinary, - amount: u64, - ) -> Result; - - async fn submit_transaction(&self, transaction: &Self::Transaction) -> Result<(), Self::Error>; - - async fn confirm_transaction(&self, txn: &Signature) -> Result; -} - -pub trait GetSignature { - fn get_signature(&self) -> &Signature; -} - -impl GetSignature for Transaction { - fn get_signature(&self) -> &Signature { - &self.signatures[0] - } -} - -impl GetSignature for Signature { - fn get_signature(&self) -> &Signature { - self - } -} +pub mod burn; +pub mod start_boost; macro_rules! send_with_retry { ($rpc:expr) => {{ @@ -80,17 +26,20 @@ macro_rules! send_with_retry { } }}; } +pub(crate) use send_with_retry; #[derive(thiserror::Error, Debug)] pub enum SolanaRpcError { #[error("Solana rpc error: {0}")] RpcClientError(#[from] ClientError), #[error("Anchor error: {0}")] - AnchorError(Box), + AnchorError(Box), #[error("Solana program error: {0}")] ProgramError(#[from] solana_sdk::program_error::ProgramError), #[error("Parse pubkey error: {0}")] ParsePubkeyError(#[from] ParsePubkeyError), + #[error("Parse signature error: {0}")] + ParseSignatureError(#[from] solana_sdk::signature::ParseSignatureError), #[error("DC burn authority does not match keypair")] InvalidKeypair, #[error("System time error: {0}")] @@ -101,377 +50,24 @@ pub enum SolanaRpcError { Crypto(#[from] helium_crypto::Error), } -impl From for SolanaRpcError { - fn from(err: anchor_lang::error::Error) -> Self { +impl From for SolanaRpcError { + fn from(err: helium_anchor_gen::anchor_lang::error::Error) -> Self { Self::AnchorError(Box::new(err)) } } -#[derive(Debug, Deserialize)] -pub struct Settings { - rpc_url: String, - cluster: String, - burn_keypair: String, - dc_mint: String, - dnt_mint: String, - #[serde(default)] - payers_to_monitor: Vec, -} - -impl Settings { - pub fn payers_to_monitor(&self) -> Result, SolanaRpcError> { - self.payers_to_monitor - .iter() - .map(|payer| PublicKeyBinary::from_str(payer)) - .collect::>() - .map_err(SolanaRpcError::from) - } -} - -pub struct SolanaRpc { - provider: RpcClient, - program_cache: BurnProgramCache, - cluster: String, - keypair: [u8; 64], - payers_to_monitor: Vec, -} - -impl SolanaRpc { - pub async fn new(settings: &Settings) -> Result, SolanaRpcError> { - let dc_mint = settings.dc_mint.parse()?; - let dnt_mint = settings.dnt_mint.parse()?; - let Ok(keypair) = read_keypair_file(&settings.burn_keypair) else { - return Err(SolanaRpcError::FailedToReadKeypairError); - }; - let provider = - RpcClient::new_with_commitment(settings.rpc_url.clone(), CommitmentConfig::finalized()); - let program_cache = BurnProgramCache::new(&provider, dc_mint, dnt_mint).await?; - if program_cache.dc_burn_authority != keypair.pubkey() { - return Err(SolanaRpcError::InvalidKeypair); - } - Ok(Arc::new(Self { - cluster: settings.cluster.clone(), - provider, - program_cache, - keypair: keypair.to_bytes(), - payers_to_monitor: settings.payers_to_monitor()?, - })) - } -} - -#[async_trait] -impl SolanaNetwork for SolanaRpc { - type Error = SolanaRpcError; - type Transaction = Transaction; - - async fn payer_balance(&self, payer: &PublicKeyBinary) -> Result { - let ddc_key = delegated_data_credits(&self.program_cache.sub_dao, payer); - let (escrow_account, _) = Pubkey::find_program_address( - &["escrow_dc_account".as_bytes(), &ddc_key.to_bytes()], - &data_credits::ID, - ); - let account_data = match self - .provider - .get_account_with_commitment(&escrow_account, CommitmentConfig::finalized()) - .await? - { - Response { value: None, .. } => { - tracing::info!(%payer, "Account not found, therefore no balance"); - return Ok(0); - } - Response { - value: Some(account), - .. - } => account.data, - }; - let account_layout = spl_token::state::Account::unpack(account_data.as_slice())?; - - if self.payers_to_monitor.contains(payer) { - metrics::gauge!( - "balance", - account_layout.amount as f64, - "payer" => payer.to_string() - ); - } - - Ok(account_layout.amount) - } - - async fn make_burn_transaction( - &self, - payer: &PublicKeyBinary, - amount: u64, - ) -> Result { - // Fetch the sub dao epoch info: - const EPOCH_LENGTH: u64 = 60 * 60 * 24; - let epoch = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH)? - .as_secs() - / EPOCH_LENGTH; - let (sub_dao_epoch_info, _) = Pubkey::find_program_address( - &[ - "sub_dao_epoch_info".as_bytes(), - self.program_cache.sub_dao.as_ref(), - &epoch.to_le_bytes(), - ], - &helium_sub_daos::ID, - ); - - // Fetch escrow account - let ddc_key = delegated_data_credits(&self.program_cache.sub_dao, payer); - let (escrow_account, _) = Pubkey::find_program_address( - &["escrow_dc_account".as_bytes(), &ddc_key.to_bytes()], - &data_credits::ID, - ); - - let instructions = { - let request = RequestBuilder::from( - data_credits::id(), - &self.cluster, - std::rc::Rc::new(Keypair::from_bytes(&self.keypair).unwrap()), - Some(CommitmentConfig::confirmed()), - RequestNamespace::Global, - ); - - let accounts = accounts::BurnDelegatedDataCreditsV0 { - sub_dao_epoch_info, - dao: self.program_cache.dao, - sub_dao: self.program_cache.sub_dao, - account_payer: self.program_cache.account_payer, - data_credits: self.program_cache.data_credits, - delegated_data_credits: delegated_data_credits(&self.program_cache.sub_dao, payer), - token_program: spl_token::id(), - helium_sub_daos_program: helium_sub_daos::id(), - system_program: solana_program::system_program::id(), - dc_burn_authority: self.program_cache.dc_burn_authority, - dc_mint: self.program_cache.dc_mint, - escrow_account, - registrar: self.program_cache.registrar, - }; - let args = instruction::BurnDelegatedDataCreditsV0 { - _args: data_credits::BurnDelegatedDataCreditsArgsV0 { amount }, - }; - - // As far as I can tell, the instructions function does not actually have any - // error paths. - request - .accounts(accounts) - .args(args) - .instructions() - .unwrap() - }; - - let blockhash = self.provider.get_latest_blockhash().await?; - let signer = Keypair::from_bytes(&self.keypair).unwrap(); - - Ok(Transaction::new_signed_with_payer( - &instructions, - Some(&signer.pubkey()), - &[&signer], - blockhash, - )) - } - - async fn submit_transaction(&self, tx: &Self::Transaction) -> Result<(), Self::Error> { - match send_with_retry!(self.provider.send_and_confirm_transaction(tx)) { - Ok(signature) => { - tracing::info!( - transaction = %signature, - "Data credit burn successful", - ); - Ok(()) - } - Err(err) => { - let signature = tx.get_signature(); - tracing::error!( - transaction = %signature, - "Data credit burn failed: {err:?}" - ); - Err(SolanaRpcError::RpcClientError(err)) - } - } - } - - async fn confirm_transaction(&self, txn: &Signature) -> Result { - Ok(matches!( - self.provider - .get_signature_status_with_commitment_and_history( - txn, - CommitmentConfig::confirmed(), - true, - ) - .await?, - Some(Ok(())) - )) - } -} - -/// Cached pubkeys for the burn program -pub struct BurnProgramCache { - pub account_payer: Pubkey, - pub data_credits: Pubkey, - pub sub_dao: Pubkey, - pub dao: Pubkey, - pub dc_mint: Pubkey, - pub dc_burn_authority: Pubkey, - pub registrar: Pubkey, -} - -impl BurnProgramCache { - pub async fn new( - provider: &RpcClient, - dc_mint: Pubkey, - dnt_mint: Pubkey, - ) -> Result { - let (account_payer, _) = - Pubkey::find_program_address(&["account_payer".as_bytes()], &data_credits::ID); - let (data_credits, _) = - Pubkey::find_program_address(&["dc".as_bytes(), dc_mint.as_ref()], &data_credits::ID); - let (sub_dao, _) = Pubkey::find_program_address( - &["sub_dao".as_bytes(), dnt_mint.as_ref()], - &helium_sub_daos::ID, - ); - let (dao, dc_burn_authority) = { - let account_data = provider.get_account_data(&sub_dao).await?; - let mut account_data = account_data.as_ref(); - let sub_dao = SubDaoV0::try_deserialize(&mut account_data)?; - (sub_dao.dao, sub_dao.dc_burn_authority) - }; - let registrar = { - let account_data = provider.get_account_data(&dao).await?; - let mut account_data = account_data.as_ref(); - DaoV0::try_deserialize(&mut account_data)?.registrar - }; - Ok(Self { - account_payer, - data_credits, - sub_dao, - dao, - dc_mint, - dc_burn_authority, - registrar, - }) - } -} - -const FIXED_BALANCE: u64 = 1_000_000_000; - -pub enum PossibleTransaction { - NoTransaction(Signature), - Transaction(Transaction), +pub trait GetSignature { + fn get_signature(&self) -> &Signature; } -impl GetSignature for PossibleTransaction { +impl GetSignature for Transaction { fn get_signature(&self) -> &Signature { - match self { - Self::NoTransaction(ref sig) => sig, - Self::Transaction(ref txn) => txn.get_signature(), - } - } -} - -#[async_trait] -impl SolanaNetwork for Option> { - type Error = SolanaRpcError; - type Transaction = PossibleTransaction; - - async fn payer_balance(&self, payer: &PublicKeyBinary) -> Result { - if let Some(ref rpc) = self { - rpc.payer_balance(payer).await - } else { - Ok(FIXED_BALANCE) - } - } - - async fn make_burn_transaction( - &self, - payer: &PublicKeyBinary, - amount: u64, - ) -> Result { - if let Some(ref rpc) = self { - Ok(PossibleTransaction::Transaction( - rpc.make_burn_transaction(payer, amount).await?, - )) - } else { - Ok(PossibleTransaction::NoTransaction(Signature::new_unique())) - } - } - - async fn submit_transaction(&self, transaction: &Self::Transaction) -> Result<(), Self::Error> { - match (self, transaction) { - (Some(ref rpc), PossibleTransaction::Transaction(ref txn)) => { - rpc.submit_transaction(txn).await? - } - (None, PossibleTransaction::NoTransaction(_)) => (), - _ => unreachable!(), - } - Ok(()) - } - - async fn confirm_transaction(&self, txn: &Signature) -> Result { - if let Some(ref rpc) = self { - rpc.confirm_transaction(txn).await - } else { - panic!("We will not confirm transactions when Solana is disabled"); - } + &self.signatures[0] } } -pub struct MockTransaction { - pub signature: Signature, - pub payer: PublicKeyBinary, - pub amount: u64, -} - -impl GetSignature for MockTransaction { +impl GetSignature for Signature { fn get_signature(&self) -> &Signature { - &self.signature - } -} - -#[async_trait] -impl SolanaNetwork for Arc>> { - type Error = Infallible; - type Transaction = MockTransaction; - - async fn payer_balance(&self, payer: &PublicKeyBinary) -> Result { - Ok(*self.lock().await.get(payer).unwrap()) - } - - async fn make_burn_transaction( - &self, - payer: &PublicKeyBinary, - amount: u64, - ) -> Result { - Ok(MockTransaction { - signature: Signature::new_unique(), - payer: payer.clone(), - amount, - }) - } - - async fn submit_transaction(&self, txn: &MockTransaction) -> Result<(), Self::Error> { - *self.lock().await.get_mut(&txn.payer).unwrap() -= txn.amount; - Ok(()) - } - - async fn confirm_transaction(&self, _txn: &Signature) -> Result { - Ok(true) + self } } - -/// Returns the PDA for the Delegated Data Credits of the given `payer`. -pub fn delegated_data_credits(sub_dao: &Pubkey, payer: &PublicKeyBinary) -> Pubkey { - let mut hasher = Sha256::new(); - hasher.update(payer.to_string()); - let sha_digest = hasher.finalize(); - let (ddc_key, _) = Pubkey::find_program_address( - &[ - "delegated_data_credits".as_bytes(), - sub_dao.as_ref(), - &sha_digest, - ], - &data_credits::ID, - ); - ddc_key -} diff --git a/solana/src/start_boost.rs b/solana/src/start_boost.rs new file mode 100644 index 000000000..d3b85db29 --- /dev/null +++ b/solana/src/start_boost.rs @@ -0,0 +1,198 @@ +use crate::{send_with_retry, GetSignature, SolanaRpcError}; +use anchor_client::{RequestBuilder, RequestNamespace}; +use anchor_lang::{InstructionData, ToAccountMetas}; +use async_trait::async_trait; +use file_store::hex_boost::BoostedHexActivation; +use helium_anchor_gen::hexboosting::{self, accounts, instruction}; +use serde::Deserialize; +use solana_client::nonblocking::rpc_client::RpcClient; +use solana_program::instruction::Instruction; +use solana_sdk::{ + commitment_config::CommitmentConfig, + pubkey::Pubkey, + signature::{read_keypair_file, Keypair, Signature}, + signer::Signer, + transaction::Transaction, +}; +use std::{sync::Arc, time::Duration}; + +#[async_trait] +pub trait SolanaNetwork: Send + Sync + 'static { + type Error: std::error::Error + Send + Sync + 'static; + type Transaction: GetSignature + Send + Sync + 'static; + + async fn make_start_boost_transaction( + &self, + batch: &[BoostedHexActivation], + ) -> Result; + + async fn submit_transaction(&self, transaction: &Self::Transaction) -> Result<(), Self::Error>; + + async fn confirm_transaction(&self, txn: &str) -> Result; +} + +#[derive(Debug, Deserialize)] +pub struct Settings { + rpc_url: String, + cluster: String, + start_authority_keypair: String, +} + +pub struct SolanaRpc { + provider: RpcClient, + cluster: String, + keypair: [u8; 64], + start_authority: Pubkey, +} + +impl SolanaRpc { + pub async fn new(settings: &Settings) -> Result, SolanaRpcError> { + let Ok(keypair) = read_keypair_file(&settings.start_authority_keypair) else { + return Err(SolanaRpcError::FailedToReadKeypairError); + }; + let provider = + RpcClient::new_with_commitment(settings.rpc_url.clone(), CommitmentConfig::finalized()); + let start_authority = keypair.pubkey(); + Ok(Arc::new(Self { + cluster: settings.cluster.clone(), + provider, + keypair: keypair.to_bytes(), + start_authority, + })) + } +} + +#[async_trait] +impl SolanaNetwork for SolanaRpc { + type Error = SolanaRpcError; + type Transaction = Transaction; + + async fn make_start_boost_transaction( + &self, + batch: &[BoostedHexActivation], + ) -> Result { + let instructions = { + let mut request = RequestBuilder::from( + hexboosting::id(), + &self.cluster, + std::rc::Rc::new(Keypair::from_bytes(&self.keypair).unwrap()), + Some(CommitmentConfig::confirmed()), + RequestNamespace::Global, + ); + for update in batch { + let account = accounts::StartBoostV0 { + start_authority: self.start_authority, + boost_config: update.boost_config_pubkey.parse()?, + boosted_hex: update.boosted_hex_pubkey.parse()?, + }; + let args = instruction::StartBoostV0 { + _args: hexboosting::StartBoostArgsV0 { + start_ts: update.activation_ts.timestamp(), + }, + }; + let instruction = Instruction { + program_id: hexboosting::id(), + accounts: account.to_account_metas(None), + data: args.data(), + }; + request = request.instruction(instruction); + } + request.instructions().unwrap() + }; + tracing::debug!("instructions: {:?}", instructions); + let blockhash = self.provider.get_latest_blockhash().await?; + let signer = Keypair::from_bytes(&self.keypair).unwrap(); + + Ok(Transaction::new_signed_with_payer( + &instructions, + Some(&signer.pubkey()), + &[&signer], + blockhash, + )) + } + + async fn submit_transaction(&self, tx: &Self::Transaction) -> Result<(), Self::Error> { + match send_with_retry!(self.provider.send_and_confirm_transaction(tx)) { + Ok(signature) => { + tracing::info!( + transaction = %signature, + "hex start boost successful", + ); + Ok(()) + } + Err(err) => { + let signature = tx.get_signature(); + tracing::error!( + transaction = %signature, + "hex start boost failed: {err:?}" + ); + Err(SolanaRpcError::RpcClientError(err)) + } + } + } + + async fn confirm_transaction(&self, signature: &str) -> Result { + let txn: Signature = signature.parse()?; + Ok(matches!( + self.provider + .get_signature_status_with_commitment_and_history( + &txn, + CommitmentConfig::confirmed(), + true, + ) + .await?, + Some(Ok(())) + )) + } +} +pub enum PossibleTransaction { + NoTransaction(Signature), + Transaction(Transaction), +} + +impl GetSignature for PossibleTransaction { + fn get_signature(&self) -> &Signature { + match self { + Self::NoTransaction(ref sig) => sig, + Self::Transaction(ref txn) => txn.get_signature(), + } + } +} + +#[async_trait] +impl SolanaNetwork for Option> { + type Error = SolanaRpcError; + type Transaction = PossibleTransaction; + + async fn make_start_boost_transaction( + &self, + batch: &[BoostedHexActivation], + ) -> Result { + if let Some(ref rpc) = self { + Ok(PossibleTransaction::Transaction( + rpc.make_start_boost_transaction(batch).await?, + )) + } else { + Ok(PossibleTransaction::NoTransaction(Signature::new_unique())) + } + } + + async fn submit_transaction(&self, transaction: &Self::Transaction) -> Result<(), Self::Error> { + match (self, transaction) { + (Some(ref rpc), PossibleTransaction::Transaction(ref txn)) => { + rpc.submit_transaction(txn).await? + } + (None, PossibleTransaction::NoTransaction(_)) => (), + _ => unreachable!(), + } + Ok(()) + } + + async fn confirm_transaction(&self, txn: &str) -> Result { + if let Some(ref rpc) = self { + rpc.confirm_transaction(txn).await + } else { + panic!("We will not confirm transactions when Solana is disabled"); + } + } +} From 308b24888b8cd776bb9e3aa576184fd21bdd737e Mon Sep 17 00:00:00 2001 From: Andrew McKenzie Date: Mon, 12 Feb 2024 14:23:17 +0000 Subject: [PATCH 2/2] bump proto --- Cargo.lock | 8 ++++---- Cargo.toml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2cc7c7a13..1a4e053a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1448,7 +1448,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=andymck/hex-boosting-support#fb07b06d55aa9bcba6facf0307a218ab63b5d6cd" +source = "git+https://github.com/helium/proto?branch=master#53373c6d7b6abe900a49a205593380787978bfb4" dependencies = [ "base64 0.21.0", "byteorder", @@ -1458,7 +1458,7 @@ dependencies = [ "rand_chacha 0.3.0", "rust_decimal", "serde", - "sha2 0.9.9", + "sha2 0.10.6", "thiserror", ] @@ -3337,7 +3337,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=andymck/hex-boosting-support#fb07b06d55aa9bcba6facf0307a218ab63b5d6cd" +source = "git+https://github.com/helium/proto?branch=master#53373c6d7b6abe900a49a205593380787978bfb4" dependencies = [ "bytes", "prost", @@ -8320,7 +8320,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "sha2 0.9.9", + "sha2 0.10.6", "thiserror", "twox-hash", "xorf", diff --git a/Cargo.toml b/Cargo.toml index f8ecfb336..771a81052 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,14 +62,14 @@ sqlx = {version = "0", features = [ ]} helium-anchor-gen = {git = "https://github.com/helium/helium-anchor-gen.git"} helium-crypto = {version = "0.8.1", features=["sqlx-postgres", "multisig"]} -helium-proto = {git = "https://github.com/helium/proto", branch = "andymck/hex-boosting-support", features = ["services"]} +helium-proto = {git = "https://github.com/helium/proto", branch = "master", features = ["services"]} hextree = "*" solana-client = "1.14" solana-sdk = "1.14" solana-program = "1.11" spl-token = "3.5.0" reqwest = {version = "0", default-features=false, features = ["gzip", "json", "rustls-tls"]} -beacon = { git = "https://github.com/helium/proto", branch = "andymck/hex-boosting-support" } +beacon = { git = "https://github.com/helium/proto", branch = "master" } humantime = "2" metrics = "0" metrics-exporter-prometheus = "0"