From 0246715c55bbcbc20d5cc8296762aeb53d8092f3 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Tue, 25 Jun 2024 13:38:47 +0800 Subject: [PATCH 1/5] feat: add udt holder allocation --- app/controllers/api/v1/udts_controller.rb | 22 ++++++- app/models/udt.rb | 1 + app/models/udt_holder_allocation.rb | 21 +++++++ app/workers/generate_Udt_holders_worker.rb | 23 ------- .../generate_udt_holder_allocation_worker.rb | 39 ++++++++++++ app/workers/update_udt_info_worker.rb | 2 +- config/routes.rb | 1 + ...625032839_create_udt_holder_allocations.rb | 12 ++++ db/structure.sql | 60 +++++++++++++++++++ test/factories/udt_holder_allocations.rb | 5 ++ test/models/udt_holder_allocation_test.rb | 7 +++ 11 files changed, 168 insertions(+), 25 deletions(-) create mode 100644 app/models/udt_holder_allocation.rb delete mode 100644 app/workers/generate_Udt_holders_worker.rb create mode 100644 app/workers/generate_udt_holder_allocation_worker.rb create mode 100644 db/migrate/20240625032839_create_udt_holder_allocations.rb create mode 100644 test/factories/udt_holder_allocations.rb create mode 100644 test/models/udt_holder_allocation_test.rb diff --git a/app/controllers/api/v1/udts_controller.rb b/app/controllers/api/v1/udts_controller.rb index c62e617e0..016f84838 100644 --- a/app/controllers/api/v1/udts_controller.rb +++ b/app/controllers/api/v1/udts_controller.rb @@ -1,7 +1,7 @@ module Api module V1 class UdtsController < ApplicationController - before_action :validate_query_params, only: :show + before_action :validate_query_params, only: %i[show holder_allocation] before_action :validate_pagination_params, :pagination_params, only: :index def index @@ -84,6 +84,26 @@ def download_csv raise Api::V1::Exceptions::UdtNotFoundError end + def holder_allocation + udt = Udt.find_by!(type_hash: params[:id], published: true) + holder_allocation = udt.udt_holder_allocations.find_by(contract_id: nil) + + ckb_holder_count = holder_allocation&.ckb_holder_count || 0 + btc_holder_count = holder_allocation&.btc_holder_count || 0 + + lock_hashes = udt.udt_holder_allocations.includes(:contract).where.not(contract_id: nil).map do |allocation| + { + name: allocation.contract.name, + code_hash: allocation.contract.code_hash, + holder_count: allocation.ckb_holder_count, + } + end + + render json: { ckb_holder_count:, btc_holder_count:, lock_hashes: } + rescue ActiveRecord::RecordNotFound + raise Api::V1::Exceptions::UdtNotFoundError + end + private def validate_query_params diff --git a/app/models/udt.rb b/app/models/udt.rb index 991015901..8c786effb 100644 --- a/app/models/udt.rb +++ b/app/models/udt.rb @@ -5,6 +5,7 @@ class Udt < ApplicationRecord has_one :udt_verification has_one :omiga_inscription_info has_one :xudt_tag + has_many :udt_holder_allocations enum udt_type: { sudt: 0, m_nft_token: 1, nrc_721_token: 2, spore_cell: 3, omiga_inscription: 4, xudt: 5, xudt_compatible: 6 } diff --git a/app/models/udt_holder_allocation.rb b/app/models/udt_holder_allocation.rb new file mode 100644 index 000000000..79b7bce1c --- /dev/null +++ b/app/models/udt_holder_allocation.rb @@ -0,0 +1,21 @@ +class UdtHolderAllocation < ApplicationRecord + belongs_to :udt + belongs_to :contract, optional: true +end + +# == Schema Information +# +# Table name: udt_holder_allocations +# +# id :bigint not null, primary key +# udt_id :bigint not null +# contract_id :bigint +# ckb_holder_count :integer default(0), not null +# btc_holder_count :integer default(0), not null +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# index_udt_holder_allocations_on_udt_id (udt_id) +# diff --git a/app/workers/generate_Udt_holders_worker.rb b/app/workers/generate_Udt_holders_worker.rb deleted file mode 100644 index ee51aa6c3..000000000 --- a/app/workers/generate_Udt_holders_worker.rb +++ /dev/null @@ -1,23 +0,0 @@ -class GenerateUdtHoldersWorker - include Sidekiq::Worker - sidekiq_options retry: 3 - - def perform(type_hash) - udt = Udt.find_by(type_hash:) - return unless udt - - type_script = TypeScript.find_by(udt.type_script) - return unless type_script - - ckb_address_ids = CellOutput.where(type_script_id: type_script.id).pluck(:address_id).uniq - bitcoin_address_ids = [] - ckb_address_ids.each_slice(1000) do |address_ids| - ids = BitcoinAddressMapping.where(ckb_address_id: address_ids).pluck(:bitcoin_address_id) - bitcoin_address_ids.concat(ids).uniq! - end - - cache_key = "udt_holders/#{type_hash}" - cache_value = { ckb_holders_count: ckb_address_ids.count, btc_holders_count: bitcoin_address_ids.count } - Rails.cache.write(cache_key, cache_value) - end -end diff --git a/app/workers/generate_udt_holder_allocation_worker.rb b/app/workers/generate_udt_holder_allocation_worker.rb new file mode 100644 index 000000000..309058faf --- /dev/null +++ b/app/workers/generate_udt_holder_allocation_worker.rb @@ -0,0 +1,39 @@ +class GenerateUdtHolderAllocationWorker + include Sidekiq::Worker + sidekiq_options retry: 3 + + def perform(type_hash) + udt = Udt.find_by(type_hash:) + return unless udt + + update_udt_holder_allocation(udt) + update_contract_holder_allocation(udt) + end + + def update_udt_holder_allocation(udt) + type_script = TypeScript.find_by(udt.type_script) + return unless type_script + + holder_allocation = UdtHolderAllocation.find_or_initialize_by(udt:, contract_id: nil) + ckb_address_ids = CellOutput.live.where(type_script:).distinct.pluck(:address_id) + btc_address_ids = [] + ckb_address_ids.each_slice(1000) do |address_ids| + ids = BitcoinAddressMapping.where(ckb_address_id: address_ids).pluck(:bitcoin_address_id) + btc_address_ids.concat(ids).uniq! + end + + holder_allocation.update!(ckb_holder_count: ckb_address_ids.count, btc_holder_count: btc_address_ids.count) + end + + def update_contract_holder_allocation(udt) + contracts = Contract.where(role: ["LockScript", "lock_script"]) + contracts.each do |contract| + holder_allocation = UdtHolderAllocation.find_or_initialize_by(udt:, contract:) + type_script = TypeScript.find_by(udt.type_script) + next unless type_script + + ckb_holder_count = contract.referring_cell_outputs.live.where(type_script:).distinct.count(:address_id) + holder_allocation.update!(ckb_holder_count:) + end + end +end diff --git a/app/workers/update_udt_info_worker.rb b/app/workers/update_udt_info_worker.rb index 06f137407..b68946863 100644 --- a/app/workers/update_udt_info_worker.rb +++ b/app/workers/update_udt_info_worker.rb @@ -44,7 +44,7 @@ def perform(block_number) end, unique_by: :type_hash ) - type_hashes.each { GenerateUdtHoldersWorker.perform_async(_1) } + type_hashes.each { GenerateUdtHolderAllocationWorker.perform_async(_1) } end end end diff --git a/config/routes.rb b/config/routes.rb index a4fd5a485..152d7fa07 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -54,6 +54,7 @@ resources :market_data, only: %i[index show] resources :udts, only: %i(index show update) do get :download_csv, on: :collection + get :holder_allocation, on: :member end resources :xudts, only: %i(index show) do get :download_csv, on: :collection diff --git a/db/migrate/20240625032839_create_udt_holder_allocations.rb b/db/migrate/20240625032839_create_udt_holder_allocations.rb new file mode 100644 index 000000000..b5107e1dc --- /dev/null +++ b/db/migrate/20240625032839_create_udt_holder_allocations.rb @@ -0,0 +1,12 @@ +class CreateUdtHolderAllocations < ActiveRecord::Migration[7.0] + def change + create_table :udt_holder_allocations do |t| + t.bigint :udt_id, null: false, index: true + t.bigint :contract_id + t.integer :ckb_holder_count, null: false, default: 0 + t.integer :btc_holder_count, null: false, default: 0 + + t.timestamps + end + end +end diff --git a/db/structure.sql b/db/structure.sql index f3527c8df..8d005ed61 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -2400,6 +2400,40 @@ CREATE SEQUENCE public.udt_accounts_id_seq ALTER SEQUENCE public.udt_accounts_id_seq OWNED BY public.udt_accounts.id; +-- +-- Name: udt_holder_allocations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.udt_holder_allocations ( + id bigint NOT NULL, + udt_id bigint NOT NULL, + contract_id bigint, + ckb_holder_count integer DEFAULT 0 NOT NULL, + btc_holder_count integer DEFAULT 0 NOT NULL, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: udt_holder_allocations_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.udt_holder_allocations_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: udt_holder_allocations_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.udt_holder_allocations_id_seq OWNED BY public.udt_holder_allocations.id; + + -- -- Name: udt_transactions; Type: TABLE; Schema: public; Owner: - -- @@ -3010,6 +3044,13 @@ ALTER TABLE ONLY public.type_scripts ALTER COLUMN id SET DEFAULT nextval('public ALTER TABLE ONLY public.udt_accounts ALTER COLUMN id SET DEFAULT nextval('public.udt_accounts_id_seq'::regclass); +-- +-- Name: udt_holder_allocations id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.udt_holder_allocations ALTER COLUMN id SET DEFAULT nextval('public.udt_holder_allocations_id_seq'::regclass); + + -- -- Name: udt_verifications id; Type: DEFAULT; Schema: public; Owner: - -- @@ -3532,6 +3573,14 @@ ALTER TABLE ONLY public.udt_accounts ADD CONSTRAINT udt_accounts_pkey PRIMARY KEY (id); +-- +-- Name: udt_holder_allocations udt_holder_allocations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.udt_holder_allocations + ADD CONSTRAINT udt_holder_allocations_pkey PRIMARY KEY (id); + + -- -- Name: udt_verifications udt_verifications_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -4563,6 +4612,13 @@ CREATE UNIQUE INDEX index_udt_accounts_on_type_hash_and_address_id ON public.udt CREATE INDEX index_udt_accounts_on_udt_id ON public.udt_accounts USING btree (udt_id); +-- +-- Name: index_udt_holder_allocations_on_udt_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_udt_holder_allocations_on_udt_id ON public.udt_holder_allocations USING btree (udt_id); + + -- -- Name: index_udt_transactions_on_ckb_transaction_id; Type: INDEX; Schema: public; Owner: - -- @@ -5181,6 +5237,10 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240507041552'), ('20240509074313'), ('20240513055849'), +<<<<<<< Updated upstream ('20240620083123'); +======= +('20240625032839'); +>>>>>>> Stashed changes diff --git a/test/factories/udt_holder_allocations.rb b/test/factories/udt_holder_allocations.rb new file mode 100644 index 000000000..5c46b7a17 --- /dev/null +++ b/test/factories/udt_holder_allocations.rb @@ -0,0 +1,5 @@ +FactoryBot.define do + factory :udt_holder_allocation do + + end +end diff --git a/test/models/udt_holder_allocation_test.rb b/test/models/udt_holder_allocation_test.rb new file mode 100644 index 000000000..4f15bcb15 --- /dev/null +++ b/test/models/udt_holder_allocation_test.rb @@ -0,0 +1,7 @@ +require "test_helper" + +class UdtHolderAllocationTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end From 40b75da97d3ac5421fdc23528457fc920a544434 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Tue, 25 Jun 2024 13:46:20 +0800 Subject: [PATCH 2/5] chore: update db structure --- db/structure.sql | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/db/structure.sql b/db/structure.sql index 8d005ed61..a2d1b67c7 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -5237,10 +5237,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240507041552'), ('20240509074313'), ('20240513055849'), -<<<<<<< Updated upstream -('20240620083123'); -======= +('20240620083123'), ('20240625032839'); ->>>>>>> Stashed changes From 9a59fc28150b8aaf9903df96192d7b16454df4c6 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Tue, 25 Jun 2024 16:17:49 +0800 Subject: [PATCH 3/5] =?UTF-8?q?chore=EF=BC=9Aadjust=20holder=20allocation?= =?UTF-8?q?=20filtering=20criteria?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../generate_udt_holder_allocation_worker.rb | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/app/workers/generate_udt_holder_allocation_worker.rb b/app/workers/generate_udt_holder_allocation_worker.rb index 309058faf..84857e8fe 100644 --- a/app/workers/generate_udt_holder_allocation_worker.rb +++ b/app/workers/generate_udt_holder_allocation_worker.rb @@ -26,14 +26,36 @@ def update_udt_holder_allocation(udt) end def update_contract_holder_allocation(udt) - contracts = Contract.where(role: ["LockScript", "lock_script"]) - contracts.each do |contract| - holder_allocation = UdtHolderAllocation.find_or_initialize_by(udt:, contract:) - type_script = TypeScript.find_by(udt.type_script) - next unless type_script - - ckb_holder_count = contract.referring_cell_outputs.live.where(type_script:).distinct.count(:address_id) - holder_allocation.update!(ckb_holder_count:) + type_script = TypeScript.find_by(udt.type_script) + return unless type_script + + unique_ckb_address_ids = [] + CellOutput.live.where(type_script:).find_in_batches(batch_size: 1000) do |batch| + batch_ckb_address_ids = batch.pluck(:address_id) + excluded_ckb_address_ids = BitcoinAddressMapping.where(ckb_address_id: batch_ckb_address_ids).pluck(:ckb_address_id) + filtered_ckb_address_ids = batch_ckb_address_ids - excluded_ckb_address_ids + unique_ckb_address_ids.concat(filtered_ckb_address_ids).uniq! + end + + allocation_data = {} + unique_ckb_address_ids.each_slice(1000) do |batch_address_ids| + holder_count = CellOutput.joins(:lock_script). + where(address_id: batch_address_ids). + group("lock_scripts.code_hash"). + count("DISTINCT cell_outputs.address_id") + + holder_count.each do |code_hash, count| + allocation_data[code_hash] ||= 0 + allocation_data[code_hash] += count + end + end + + allocation_data.each do |code_hash, _count| + contract = Contract.find_by(code_hash:, role: ["LockScript", "lock_script"]) + next unless contract + + allocation = UdtHolderAllocation.find_or_initialize_by(udt:, contract:) + allocation.update!(ckb_holder_count: count) end end end From 29c2a1d7824a16f27285eedb40b42953877ba181 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Tue, 25 Jun 2024 16:25:39 +0800 Subject: [PATCH 4/5] fix: iteration variable for allocation_data --- app/workers/generate_udt_holder_allocation_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/generate_udt_holder_allocation_worker.rb b/app/workers/generate_udt_holder_allocation_worker.rb index 84857e8fe..d1e80bda1 100644 --- a/app/workers/generate_udt_holder_allocation_worker.rb +++ b/app/workers/generate_udt_holder_allocation_worker.rb @@ -50,7 +50,7 @@ def update_contract_holder_allocation(udt) end end - allocation_data.each do |code_hash, _count| + allocation_data.each do |code_hash, count| contract = Contract.find_by(code_hash:, role: ["LockScript", "lock_script"]) next unless contract From 7782160a298c91ff79321abc9fe02582df805ecd Mon Sep 17 00:00:00 2001 From: Rabbit Date: Tue, 25 Jun 2024 17:39:28 +0800 Subject: [PATCH 5/5] chore: remove ckb_holder_count --- app/controllers/api/v1/udts_controller.rb | 4 +--- app/workers/generate_udt_holder_allocation_worker.rb | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/v1/udts_controller.rb b/app/controllers/api/v1/udts_controller.rb index 016f84838..fdac9405b 100644 --- a/app/controllers/api/v1/udts_controller.rb +++ b/app/controllers/api/v1/udts_controller.rb @@ -87,8 +87,6 @@ def download_csv def holder_allocation udt = Udt.find_by!(type_hash: params[:id], published: true) holder_allocation = udt.udt_holder_allocations.find_by(contract_id: nil) - - ckb_holder_count = holder_allocation&.ckb_holder_count || 0 btc_holder_count = holder_allocation&.btc_holder_count || 0 lock_hashes = udt.udt_holder_allocations.includes(:contract).where.not(contract_id: nil).map do |allocation| @@ -99,7 +97,7 @@ def holder_allocation } end - render json: { ckb_holder_count:, btc_holder_count:, lock_hashes: } + render json: { btc_holder_count:, lock_hashes: } rescue ActiveRecord::RecordNotFound raise Api::V1::Exceptions::UdtNotFoundError end diff --git a/app/workers/generate_udt_holder_allocation_worker.rb b/app/workers/generate_udt_holder_allocation_worker.rb index d1e80bda1..133e6fcfa 100644 --- a/app/workers/generate_udt_holder_allocation_worker.rb +++ b/app/workers/generate_udt_holder_allocation_worker.rb @@ -22,7 +22,7 @@ def update_udt_holder_allocation(udt) btc_address_ids.concat(ids).uniq! end - holder_allocation.update!(ckb_holder_count: ckb_address_ids.count, btc_holder_count: btc_address_ids.count) + holder_allocation.update!(btc_holder_count: btc_address_ids.count) end def update_contract_holder_allocation(udt)