diff --git a/db/compat.sql b/db/compat.sql index cdf1abc6..92e00d6e 100644 --- a/db/compat.sql +++ b/db/compat.sql @@ -35,6 +35,23 @@ CREATE INDEX IF NOT EXISTS data_event_module_id_event_id_event_arg_2 ON events ( -- it reads exists in `schema.graphql`. CREATE INDEX IF NOT EXISTS data_event_transfer_from ON events (trim( '"' from attributes #>> '{2,value,did}')); +-- Denormalised filter columns on `events`. These would be `@index` in schema.graphql, but +-- `@subql/node` caps an entity at 10 indexes (`indexCountLimit`, not configurable) and Event is +-- already at the cap. Kept here, as `master` had them. +CREATE INDEX IF NOT EXISTS data_event_claim_type ON events (claim_type); +CREATE INDEX IF NOT EXISTS data_event_claim_scope ON events (claim_scope); +CREATE INDEX IF NOT EXISTS data_event_claim_issuer ON events (claim_issuer); +CREATE INDEX IF NOT EXISTS data_event_corporate_action_ticker ON events (corporate_action_ticker); +CREATE INDEX IF NOT EXISTS data_event_fundraiser_offering_asset ON events (fundraiser_offering_asset); + +-- Plain indexes that would otherwise be `@index` in schema.graphql but cannot be: `@subql/node` +-- caps an entity at 10 indexes (`indexCountLimit`, not configurable), and PolyxEntry is already +-- at the cap with its foreign keys and the three `@compositeIndexes`. These three back the +-- counterparty ("movements touching X"), era (reward/slash-per-era) and day-bucket queries. +CREATE INDEX IF NOT EXISTS data_polyx_entry_counterparty_address ON polyx_entries (counterparty_address); +CREATE INDEX IF NOT EXISTS data_polyx_entry_era_index ON polyx_entries (era_index); +CREATE INDEX IF NOT EXISTS data_polyx_entry_date ON polyx_entries (date); + -- Legacy views, dropped if an older deployment left them behind. DROP VIEW IF EXISTS data_block; DROP VIEW IF EXISTS data_event; diff --git a/db/migrations/10_schema_changes_for_portfolio_kind_support.sql b/db/migrations/10_schema_changes_for_portfolio_kind_support.sql deleted file mode 100644 index 6acda9f0..00000000 --- a/db/migrations/10_schema_changes_for_portfolio_kind_support.sql +++ /dev/null @@ -1,14 +0,0 @@ -alter table "portfolio_movements" add column if not exists "to_account" text; -alter table "portfolio_movements" add column if not exists "from_account" text; - -alter table "portfolio_movements" alter column "from_id" drop not null; -alter table "portfolio_movements" alter column "to_id" drop not null; - -alter table "legs" add column if not exists "to_account" text; -alter table "legs" add column if not exists "from_account" text; - -alter table "instruction_affirmations" add column if not exists "account" text; -alter table "instruction_events" add column if not exists "account" text; - -alter table "asset_transactions" add column if not exists "from_account" text; -alter table "asset_transactions" add column if not exists "to_account" text; \ No newline at end of file diff --git a/db/migrations/11_handle_classic_ticker_claimed.sql b/db/migrations/11_handle_classic_ticker_claimed.sql deleted file mode 100644 index b44d53b8..00000000 --- a/db/migrations/11_handle_classic_ticker_claimed.sql +++ /dev/null @@ -1,176 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; - -INSERT INTO ticker_reservations -(id, ticker, identity_id, expiry, created_block_id, updated_block_id, _id, _block_range) -SELECT DISTINCT ON(event_arg_1) - event_arg_1, - event_arg_1, - event_arg_0, - case when event_arg_2 != 'null' then to_timestamp(event_arg_2::bigint / 1000.0) else NULL end, - block_id, - block_id, - uuid_generate_v4(), - int8range(block_id::bigint, NULL::bigint) -FROM events -WHERE module_id = 'asset' - AND event_id_text = 'TickerRegistered' - AND NOT EXISTS ( - SELECT 1 - FROM ticker_reservations - WHERE ticker_reservations.id = events.event_arg_1 - ) -ORDER BY event_arg_1, block_id DESC -ON CONFLICT DO NOTHING; - -DO $$ -DECLARE - transfer_event RECORD; - closed_expiry timestamp; - registered_ticker text; - closed_created_block_id text; -BEGIN - FOR transfer_event IN - SELECT event_arg_1 AS ticker, event_arg_0 AS new_identity, block_id - FROM events - WHERE module_id = 'asset' - AND event_id_text = 'TickerTransferred' - ORDER BY block_id ASC - LOOP - IF EXISTS ( - SELECT 1 - FROM ticker_reservations - WHERE id = transfer_event.ticker - AND identity_id = transfer_event.new_identity - AND created_block_id = transfer_event.block_id - ) THEN - CONTINUE; - END IF; - - SELECT id, expiry, created_block_id INTO registered_ticker, closed_expiry, closed_created_block_id - FROM ticker_reservations - WHERE id = transfer_event.ticker - AND upper(_block_range) IS NULL - LIMIT 1; - - IF registered_ticker IS NOT NULL THEN - IF closed_created_block_id::bigint <= transfer_event.block_id::bigint THEN - UPDATE ticker_reservations tr - SET _block_range = int8range(closed_created_block_id::bigint, transfer_event.block_id::bigint), - updated_block_id = transfer_event.block_id - WHERE id = registered_ticker - AND upper(tr._block_range) IS NULL; - - INSERT INTO ticker_reservations - (id, ticker, identity_id, expiry, created_block_id, updated_block_id, _id, _block_range) - VALUES ( - transfer_event.ticker, - transfer_event.ticker, - transfer_event.new_identity, - closed_expiry, - transfer_event.block_id, - transfer_event.block_id, - uuid_generate_v4(), - int8range(transfer_event.block_id::bigint, NULL::bigint) - ); - ELSE - UPDATE ticker_reservations tr - SET identity_id = transfer_event.new_identity, - updated_block_id = transfer_event.block_id - WHERE id = registered_ticker - AND upper(tr._block_range) IS NULL; - END IF; - ELSE - -- Handle the case where the ticker wasn't registered yet - INSERT INTO ticker_reservations - (id, ticker, identity_id, expiry, created_block_id, updated_block_id, _id, _block_range) - VALUES ( - transfer_event.ticker, - transfer_event.ticker, - transfer_event.new_identity, - closed_expiry, - transfer_event.block_id, - transfer_event.block_id, - uuid_generate_v4(), - int8range(transfer_event.block_id::bigint, NULL::bigint) - ); - END IF; - END LOOP; -END $$; - -DO $$ -DECLARE - asset_event RECORD; - closed_identity_id text; - closed_expiry timestamp; - closed_created_block_id bigint; - registered_ticker text; - target_asset_id text; -BEGIN - FOR asset_event IN - SELECT event_arg_1 AS ticker, event_arg_2 AS asset_id, block_id, event_id_text - FROM events - WHERE module_id = 'asset' - AND event_id_text IN ('TickerLinkedToAsset', 'TickerUnlinkedFromAsset') - ORDER BY block_id ASC - LOOP - -- Determine the target asset_id - IF asset_event.event_id_text = 'TickerUnlinkedFromAsset' THEN - target_asset_id := NULL; - ELSE - target_asset_id := asset_event.asset_id; - END IF; - - -- Idempotency check: skip if this specific link/unlink has already been processed - IF EXISTS ( - SELECT 1 - FROM ticker_reservations - WHERE id = asset_event.ticker - AND ( - (asset_id = target_asset_id) OR - (asset_id IS NULL AND target_asset_id IS NULL) - ) - AND created_block_id = asset_event.block_id - ) THEN - CONTINUE; - END IF; - - -- Retrieve details from the currently active reservation before closing it - SELECT id, identity_id, expiry, created_block_id INTO registered_ticker, closed_identity_id, closed_expiry, closed_created_block_id - FROM ticker_reservations - WHERE id = asset_event.ticker - AND upper(_block_range) IS NULL - LIMIT 1; - - IF registered_ticker IS NOT NULL THEN - IF closed_created_block_id::bigint < asset_event.block_id::bigint THEN - UPDATE ticker_reservations tr - SET _block_range = int8range(tr.created_block_id::bigint, asset_event.block_id::bigint), - updated_block_id = asset_event.block_id - WHERE id = asset_event.ticker - AND upper(tr._block_range) IS NULL; - - INSERT INTO ticker_reservations - (id, ticker, identity_id, asset_id, expiry, created_block_id, updated_block_id, _id, _block_range) - VALUES ( - asset_event.ticker, - asset_event.ticker, - closed_identity_id, - target_asset_id, - closed_expiry, - asset_event.block_id, - asset_event.block_id, - uuid_generate_v4(), - int8range(asset_event.block_id::bigint, NULL::bigint) - ); - ELSE - UPDATE ticker_reservations tr - SET asset_id = target_asset_id, - updated_block_id = asset_event.block_id - WHERE id = asset_event.ticker - AND upper(tr._block_range) IS NULL; - END IF; - END IF; - END LOOP; -END $$; - - diff --git a/db/migrations/12_add_new_8_chain_events.sql b/db/migrations/12_add_new_8_chain_events.sql deleted file mode 100644 index 9a410897..00000000 --- a/db/migrations/12_add_new_8_chain_events.sql +++ /dev/null @@ -1,50 +0,0 @@ -alter type "7a0b4cc03e" add value if not exists 'beefy' after 'validators'; -alter type "7a0b4cc03e" add value if not exists 'revive' after 'beefy'; - -alter type "0bf3c7d4ef" add value if not exists 'self_register_did' after 'unlink_child_identity'; -alter type "0bf3c7d4ef" add value if not exists 'remove_key' after 'sudo_as'; -alter type "0bf3c7d4ef" add value if not exists 'set_mandatory_receiver_affirmation' after 'lock_instruction'; -alter type "0bf3c7d4ef" add value if not exists 'transfer_funds' after 'set_mandatory_receiver_affirmation'; -alter type "0bf3c7d4ef" add value if not exists 'unlock_instruction' after 'transfer_funds'; -alter type "0bf3c7d4ef" add value if not exists 'approve_subsidy' after 'decrease_polyx_limit'; -alter type "0bf3c7d4ef" add value if not exists 'revoke_subsidy' after 'approve_subsidy'; -alter type "0bf3c7d4ef" add value if not exists 'accept_subsidy' after 'revoke_subsidy'; -alter type "0bf3c7d4ef" add value if not exists 'remove_subsidy' after 'accept_subsidy'; -alter type "0bf3c7d4ef" add value if not exists 'report_double_voting' after 'sumbit_unsigned'; -alter type "0bf3c7d4ef" add value if not exists 'report_double_voting_unsigned' after 'report_double_voting'; -alter type "0bf3c7d4ef" add value if not exists 'set_new_genesis' after 'report_double_voting_unsigned'; -alter type "0bf3c7d4ef" add value if not exists 'report_fork_voting' after 'set_new_genesis'; -alter type "0bf3c7d4ef" add value if not exists 'report_fork_voting_unsigned' after 'report_fork_voting'; -alter type "0bf3c7d4ef" add value if not exists 'report_future_block_voting' after 'report_fork_voting_unsigned'; -alter type "0bf3c7d4ef" add value if not exists 'report_future_block_voting_unsigned' after 'report_future_block_voting'; -alter type "0bf3c7d4ef" add value if not exists 'eth_transact' after 'report_future_block_voting_unsigned'; -alter type "0bf3c7d4ef" add value if not exists 'eth_instantiate_with_code' after 'eth_transact'; -alter type "0bf3c7d4ef" add value if not exists 'eth_call' after 'eth_instantiate_with_code'; -alter type "0bf3c7d4ef" add value if not exists 'eth_substrate_call' after 'eth_call'; -alter type "0bf3c7d4ef" add value if not exists 'map_account' after 'eth_substrate_call'; -alter type "0bf3c7d4ef" add value if not exists 'unmap_account' after 'map_account'; -alter type "0bf3c7d4ef" add value if not exists 'dispatch_as_fallback_account' after 'unmap_account'; - - -alter type "8f5a39c8ee" add value if not exists 'BurnedDebt' after 'Withdraw'; -alter type "8f5a39c8ee" add value if not exists 'BurnedHeld' after 'BurnedDebt'; -alter type "8f5a39c8ee" add value if not exists 'Held' after 'BurnedHeld'; -alter type "8f5a39c8ee" add value if not exists 'MintedCredit' after 'Held'; -alter type "8f5a39c8ee" add value if not exists 'Released' after 'MintedCredit'; -alter type "8f5a39c8ee" add value if not exists 'TransferAndHold' after 'Released'; -alter type "8f5a39c8ee" add value if not exists 'TransferOnHold' after 'TransferAndHold'; -alter type "8f5a39c8ee" add value if not exists 'Unexpected' after 'TransferOnHold'; -alter type "8f5a39c8ee" add value if not exists 'RootsPruned' after 'Slashed'; -alter type "8f5a39c8ee" add value if not exists 'RootStored' after 'RootsPruned'; -alter type "8f5a39c8ee" add value if not exists 'NFTHoldingsUpdated' after 'NFTPortfolioUpdated'; -alter type "8f5a39c8ee" add value if not exists 'NewQueued' after 'ValidatorReenabled'; -alter type "8f5a39c8ee" add value if not exists 'KeyRemoved' after 'KeyChanged'; -alter type "8f5a39c8ee" add value if not exists 'InstructionUnlocked' after 'VenueSignersUpdated'; -alter type "8f5a39c8ee" add value if not exists 'MandatoryReceiverAffirmationSet' after 'InstructionUnlocked'; -alter type "8f5a39c8ee" add value if not exists 'AcceptedSubsidy' after 'UpdatedPolyxLimit'; -alter type "8f5a39c8ee" add value if not exists 'ApprovedSubsidy' after 'AcceptedSubsidy'; -alter type "8f5a39c8ee" add value if not exists 'RemovedPendingSubsidy' after 'ApprovedSubsidy'; -alter type "8f5a39c8ee" add value if not exists 'RemovedSubsidy' after 'RemovedPendingSubsidy'; -alter type "8f5a39c8ee" add value if not exists 'SubsidyDebited' after 'RemovedSubsidy'; -alter type "8f5a39c8ee" add value if not exists 'EthExtrinsicRevert' after 'RootStored'; - diff --git a/db/migrations/13_add_identity_to_asset_transactions.sql b/db/migrations/13_add_identity_to_asset_transactions.sql deleted file mode 100644 index f3f17704..00000000 --- a/db/migrations/13_add_identity_to_asset_transactions.sql +++ /dev/null @@ -1,33 +0,0 @@ -alter table "asset_transactions" add column if not exists "from_identity_id" text; -alter table "asset_transactions" add column if not exists "to_identity_id" text; - --- Backfill from portfolio IDs (format: did/portfolioNumber) -update "asset_transactions" -set "from_identity_id" = split_part("from_portfolio_id", '/', 1) -where "from_portfolio_id" is not null - and "from_identity_id" is null; - -update "asset_transactions" -set "to_identity_id" = split_part("to_portfolio_id", '/', 1) -where "to_portfolio_id" is not null - and "to_identity_id" is null; - --- Backfill account-based transactions from the accounts table. --- Note: this reflects the current identity association for the account, --- not necessarily the identity at the time of the transaction. -update "asset_transactions" at -set "from_identity_id" = a."identity_id" -from "accounts" a -where at."from_account" = a."id" - and at."from_identity_id" is null - and a."identity_id" is not null; - -update "asset_transactions" at -set "to_identity_id" = a."identity_id" -from "accounts" a -where at."to_account" = a."id" - and at."to_identity_id" is null - and a."identity_id" is not null; - -create index if not exists "asset_transactions_from_identity_id" on "asset_transactions" ("from_identity_id"); -create index if not exists "asset_transactions_to_identity_id" on "asset_transactions" ("to_identity_id"); diff --git a/db/migrations/14_confidential_assets_events_8_chain.sql b/db/migrations/14_confidential_assets_events_8_chain.sql deleted file mode 100644 index c40aa5b2..00000000 --- a/db/migrations/14_confidential_assets_events_8_chain.sql +++ /dev/null @@ -1,65 +0,0 @@ -alter type "7a0b4cc03e" add value if not exists 'didregistrars' after 'revive'; -alter type "7a0b4cc03e" add value if not exists 'polymeshtransactionpayment' after 'didregistrars'; -alter type "7a0b4cc03e" add value if not exists 'multiblockmigrations' after 'polymeshtransactionpayment'; -alter type "7a0b4cc03e" add value if not exists 'confidentialassets' after 'multiblockmigrations'; - -alter type "0bf3c7d4ef" add value if not exists 'force_set_cursor' after 'dispatch_as_fallback_account'; -alter type "0bf3c7d4ef" add value if not exists 'force_set_active_cursor' after 'force_set_cursor'; -alter type "0bf3c7d4ef" add value if not exists 'force_onboard_mbms' after 'force_set_active_cursor'; -alter type "0bf3c7d4ef" add value if not exists 'clear_historic' after 'force_onboard_mbms'; -alter type "0bf3c7d4ef" add value if not exists 'register_accounts' after 'clear_historic'; -alter type "0bf3c7d4ef" add value if not exists 'register_encryption_keys' after 'register_accounts'; -alter type "0bf3c7d4ef" add value if not exists 'register_account_assets' after 'register_encryption_keys'; -alter type "0bf3c7d4ef" add value if not exists 'mint_asset' after 'register_account_assets'; -alter type "0bf3c7d4ef" add value if not exists 'create_settlement' after 'mint_asset'; -alter type "0bf3c7d4ef" add value if not exists 'sender_affirmation' after 'create_settlement'; -alter type "0bf3c7d4ef" add value if not exists 'receiver_affirmation' after 'sender_affirmation'; -alter type "0bf3c7d4ef" add value if not exists 'mediator_affirmation' after 'receiver_affirmation'; -alter type "0bf3c7d4ef" add value if not exists 'sender_update_counter' after 'mediator_affirmation'; -alter type "0bf3c7d4ef" add value if not exists 'sender_revert_affirmation' after 'sender_update_counter'; -alter type "0bf3c7d4ef" add value if not exists 'receiver_revert_affirmation' after 'sender_revert_affirmation'; -alter type "0bf3c7d4ef" add value if not exists 'receiver_claim' after 'receiver_revert_affirmation'; -alter type "0bf3c7d4ef" add value if not exists 'batched_settlement' after 'receiver_claim'; -alter type "0bf3c7d4ef" add value if not exists 'register_fee_accounts' after 'batched_settlement'; -alter type "0bf3c7d4ef" add value if not exists 'topup_fee_accounts' after 'register_fee_accounts'; -alter type "0bf3c7d4ef" add value if not exists 'submit_batched_proofs' after 'topup_fee_accounts'; -alter type "0bf3c7d4ef" add value if not exists 'relayer_submit_batched_proofs' after 'submit_batched_proofs'; -alter type "0bf3c7d4ef" add value if not exists 'execute_instant_settlement' after 'relayer_submit_batched_proofs'; -alter type "0bf3c7d4ef" add value if not exists 'instant_sender_affirmation' after 'execute_instant_settlement'; -alter type "0bf3c7d4ef" add value if not exists 'instant_receiver_affirmation' after 'instant_sender_affirmation'; - -alter type "8f5a39c8ee" add value if not exists 'AllowanceSpent' after 'CreatedAssetTransfer'; -alter type "8f5a39c8ee" add value if not exists 'FundsTransferred' after 'TickerLinkedToAsset'; -alter type "8f5a39c8ee" add value if not exists 'AccountAssetRegistered' after 'EthExtrinsicRevert'; -alter type "8f5a39c8ee" add value if not exists 'AccountCurveTreeRootUpdated' after 'AccountAssetRegistered'; -alter type "8f5a39c8ee" add value if not exists 'AccountRegistered' after 'AccountCurveTreeRootUpdated'; -alter type "8f5a39c8ee" add value if not exists 'AccountStateLeafInserted' after 'AccountRegistered'; -alter type "8f5a39c8ee" add value if not exists 'AssetCurveTreeRootUpdated' after 'AccountStateLeafInserted'; -alter type "8f5a39c8ee" add value if not exists 'AssetMinted' after 'AssetCurveTreeRootUpdated'; -alter type "8f5a39c8ee" add value if not exists 'AssetStateLeafUpdated' after 'AssetMinted'; -alter type "8f5a39c8ee" add value if not exists 'AssetUpdated' after 'AssetStateLeafUpdated'; -alter type "8f5a39c8ee" add value if not exists 'EncryptionKeyRegistered' after 'AssetUpdated'; -alter type "8f5a39c8ee" add value if not exists 'FeeAccountCurveTreeRootUpdated' after 'EncryptionKeyRegistered'; -alter type "8f5a39c8ee" add value if not exists 'FeeAccountDeposited' after 'FeeAccountCurveTreeRootUpdated'; -alter type "8f5a39c8ee" add value if not exists 'FeeAccountStateLeafInserted' after 'FeeAccountDeposited'; -alter type "8f5a39c8ee" add value if not exists 'FeeAccountUpdated' after 'FeeAccountStateLeafInserted'; -alter type "8f5a39c8ee" add value if not exists 'FeeAccountWithdrawn' after 'FeeAccountUpdated'; -alter type "8f5a39c8ee" add value if not exists 'MediatorAffirmed' after 'FeeAccountWithdrawn'; -alter type "8f5a39c8ee" add value if not exists 'MediatorRejected' after 'MediatorAffirmed'; -alter type "8f5a39c8ee" add value if not exists 'ReceiverAffirmationReverted' after 'MediatorRejected'; -alter type "8f5a39c8ee" add value if not exists 'ReceiverAffirmed' after 'ReceiverAffirmationReverted'; -alter type "8f5a39c8ee" add value if not exists 'ReceiverClaimed' after 'ReceiverAffirmed'; -alter type "8f5a39c8ee" add value if not exists 'RelayerBatchedProofs' after 'ReceiverClaimed'; -alter type "8f5a39c8ee" add value if not exists 'SenderAffirmationReverted' after 'RelayerBatchedProofs'; -alter type "8f5a39c8ee" add value if not exists 'SenderAffirmed' after 'SenderAffirmationReverted'; -alter type "8f5a39c8ee" add value if not exists 'SenderCounterUpdated' after 'SenderAffirmed'; -alter type "8f5a39c8ee" add value if not exists 'SettlementCreated' after 'SenderCounterUpdated'; -alter type "8f5a39c8ee" add value if not exists 'SettlementStatusUpdated' after 'SettlementCreated'; -alter type "8f5a39c8ee" add value if not exists 'HistoricCleared' after 'SettlementStatusUpdated'; -alter type "8f5a39c8ee" add value if not exists 'MigrationAdvanced' after 'HistoricCleared'; -alter type "8f5a39c8ee" add value if not exists 'MigrationCompleted' after 'MigrationAdvanced'; -alter type "8f5a39c8ee" add value if not exists 'MigrationFailed' after 'MigrationCompleted'; -alter type "8f5a39c8ee" add value if not exists 'MigrationSkipped' after 'MigrationFailed'; -alter type "8f5a39c8ee" add value if not exists 'UpgradeCompleted' after 'MigrationSkipped'; -alter type "8f5a39c8ee" add value if not exists 'UpgradeFailed' after 'UpgradeCompleted'; -alter type "8f5a39c8ee" add value if not exists 'UpgradeStarted' after 'UpgradeFailed'; diff --git a/db/migrations/15_add_missing_transfer_nft.sql b/db/migrations/15_add_missing_transfer_nft.sql deleted file mode 100644 index 386c8282..00000000 --- a/db/migrations/15_add_missing_transfer_nft.sql +++ /dev/null @@ -1 +0,0 @@ -alter type "0bf3c7d4ef" add value if not exists 'transfer_nft' after 'redeem_nft'; diff --git a/db/migrations/16_rename_submit_unsigned.sql b/db/migrations/16_rename_submit_unsigned.sql deleted file mode 100644 index 99d5a7b4..00000000 --- a/db/migrations/16_rename_submit_unsigned.sql +++ /dev/null @@ -1,12 +0,0 @@ -DO $$ -BEGIN - IF EXISTS ( - SELECT 1 - FROM pg_enum e - JOIN pg_type t ON e.enumtypid = t.oid - WHERE t.typname = '0bf3c7d4ef' - AND e.enumlabel = 'sumbit_unsigned' - ) THEN - ALTER TYPE "0bf3c7d4ef" RENAME VALUE 'sumbit_unsigned' TO 'submit_unsigned'; - END IF; -END $$; \ No newline at end of file diff --git a/db/migrations/17_add_identity_to_portfolio_movement.sql b/db/migrations/17_add_identity_to_portfolio_movement.sql deleted file mode 100644 index 22b4a2f0..00000000 --- a/db/migrations/17_add_identity_to_portfolio_movement.sql +++ /dev/null @@ -1,19 +0,0 @@ -alter table "portfolio_movements" add column if not exists "identity_id" text; - --- Backfill from portfolio IDs (format: did/portfolioNumber) -update "portfolio_movements" -set "identity_id" = split_part("from_id", '/', 1) -where "from_id" is not null - and "identity_id" is null; - --- Backfill account-based transactions from the accounts table. --- Note: this reflects the current identity association for the account, --- not necessarily the identity at the time of the transaction. -update "portfolio_movements" at -set "identity_id" = a."identity_id" -from "accounts" a -where at."from_account" = a."id" - and at."identity_id" is null - and a."identity_id" is not null; - -create index if not exists "portfolio_movements_identity_id" on "portfolio_movements" ("identity_id"); diff --git a/db/migrations/18_add_reward_destination_to_staking_events.sql b/db/migrations/18_add_reward_destination_to_staking_events.sql deleted file mode 100644 index 4d6d8a80..00000000 --- a/db/migrations/18_add_reward_destination_to_staking_events.sql +++ /dev/null @@ -1,4 +0,0 @@ -alter table "staking_events" add column if not exists "reward_destination" text; -alter table "staking_events" add column if not exists "reward_destination_account" text; - -update "staking_events" set reward_destination = 'LegacyUnknown' where reward_destination is null and event_id in ('Reward', 'Rewarded'); diff --git a/db/migrations/19_add_instruction_unlocked_event_type.sql b/db/migrations/19_add_instruction_unlocked_event_type.sql deleted file mode 100644 index f972d578..00000000 --- a/db/migrations/19_add_instruction_unlocked_event_type.sql +++ /dev/null @@ -1 +0,0 @@ -alter type "3e29b3f361" add value if not exists 'InstructionUnlocked' after 'InstructionLocked'; diff --git a/db/migrations/1_fix_multi_sig_proposals_data.sql b/db/migrations/1_fix_multi_sig_proposals_data.sql deleted file mode 100644 index c38ab3e7..00000000 --- a/db/migrations/1_fix_multi_sig_proposals_data.sql +++ /dev/null @@ -1,48 +0,0 @@ --- convert all module names to lowercase -UPDATE multi_sig_proposals -SET params = jsonb_set( - params, - '{proposals}', - ( - SELECT jsonb_agg( - jsonb_set( - proposal, - '{module}', - to_jsonb(lower(proposal->>'module')) - ) - ) - FROM jsonb_array_elements(params->'proposals') AS proposal - ) -) -WHERE params->'proposals' IS NOT NULL; - --- correct the data format of 'params' when proposal with batch_all or batch_atomic is made -UPDATE multi_sig_proposals -SET params = jsonb_set( - jsonb_set(params, '{isBatch}', 'true'), - '{proposals}', - ( - SELECT jsonb_agg( - ( - elem - 'method' - 'section' -- Remove old keys - ) || jsonb_build_object( - 'args', elem->>'args', -- stringify args - 'call', ltrim( - lower( - regexp_replace( - elem->>'method', - '([A-Z])', - '_\1', - 'g' - ) - ), - '_' - ), -- Convert method to snake_case and set as 'call' - 'module', elem->>'section' -- Map section to module - ) - ) - FROM jsonb_array_elements(params->'proposals') AS proposal, - jsonb_array_elements((proposal->>'args')::jsonb->'calls') AS elem - ) -) -WHERE params->'proposals'->0->>'call' in ('batch_all', 'batch_atomic'); diff --git a/db/migrations/20_evm_transactions.sql b/db/migrations/20_evm_transactions.sql deleted file mode 100644 index df653097..00000000 --- a/db/migrations/20_evm_transactions.sql +++ /dev/null @@ -1,29 +0,0 @@ --- Support for indexing Ethereum transactions submitted through `revive.ethTransact`. --- --- The `evm_transactions` and `evm_account_mappings` tables, along with the `EvmCallKindEnum` type, --- are created by the node from `schema.graphql`. Only the two pre-existing tables need altering. - -alter table "extrinsics" add column if not exists "eth_address" text; -alter table "extrinsics" add column if not exists "eth_tx_hash" text; - -create index if not exists data_extrinsic_eth_address on extrinsics (eth_address); -create index if not exists data_extrinsic_eth_tx_hash on extrinsics (eth_tx_hash); - --- `key_type` cannot be derived here: telling an Ethereum key from a substrate one means base58 --- decoding the address and checking for the `0xEE` padding, which postgres has no built in for. --- Existing rows get a provisional 'substrate' so the column can be NOT NULL. --- --- That default is wrong for any Ethereum key attributed before this change - registering a DID or --- joining an identity emits the `0xEE` padded SS58 like any other key, and those accounts were --- indexed all along. `scripts/backfill/eth-transact-senders.ts` decodes every account offline and --- reclassifies them. -alter table "accounts" add column if not exists "key_type" text; -update "accounts" set "key_type" = 'substrate' where "key_type" is null; -alter table "accounts" alter column "key_type" set not null; - --- `evm_address` needs the same decode (plus keccak256 for substrate keys), so existing rows are --- left null. Every account has one - Ethereum keys drop their `0xEE` padding, substrate keys are --- hashed and truncated - and the same backfill fills them all, matching the forward path. -alter table "accounts" add column if not exists "evm_address" text; - -create index if not exists data_account_evm_address on accounts (evm_address); diff --git a/db/migrations/2_update_instruction_affirmation_id.sql b/db/migrations/2_update_instruction_affirmation_id.sql deleted file mode 100644 index 22e65a5f..00000000 --- a/db/migrations/2_update_instruction_affirmation_id.sql +++ /dev/null @@ -1,3 +0,0 @@ -update instruction_affirmations -set id = party_id || '/' || off_chain_receipt_id -where off_chain_receipt_id is not null; \ No newline at end of file diff --git a/db/migrations/3_update_failed_instruction_status.sql b/db/migrations/3_update_failed_instruction_status.sql deleted file mode 100644 index 8a74f07e..00000000 --- a/db/migrations/3_update_failed_instruction_status.sql +++ /dev/null @@ -1,3 +0,0 @@ -update instructions -set status = 'Failed' -where failure_reason is not null and status != 'Failed'; \ No newline at end of file diff --git a/db/migrations/4_add_new_events_extrinsics.sql b/db/migrations/4_add_new_events_extrinsics.sql deleted file mode 100644 index e0e4dae1..00000000 --- a/db/migrations/4_add_new_events_extrinsics.sql +++ /dev/null @@ -1,8 +0,0 @@ - -alter type "0bf3c7d4ef" add value if not exists 'set_disable_fees' after 'burn_account_balance'; -alter type "0bf3c7d4ef" add value if not exists 'initiate_corporate_action_and_ballot' after 'initiate_corporate_action_and_distribute'; -alter type "0bf3c7d4ef" add value if not exists 'update_global_metadata_spec' after 'unlink_ticker_from_asset_id'; - -alter type "8f5a39c8ee" add value if not exists 'GlobalMetadataSpecUpdated' after 'TickerUnlinkedFromAsset'; -alter type "8f5a39c8ee" add value if not exists 'AllowIdentityToCreatePortfolios' after 'FundsMovedBetweenPortfolios'; -alter type "8f5a39c8ee" add value if not exists 'RevokeCreatePortfoliosPermission' after 'AllowIdentityToCreatePortfolios'; diff --git a/db/migrations/5_add_enum_text_columns.sql b/db/migrations/5_add_enum_text_columns.sql deleted file mode 100644 index 30d09fa8..00000000 --- a/db/migrations/5_add_enum_text_columns.sql +++ /dev/null @@ -1,31 +0,0 @@ --- add unknown to CallIdEnum -alter type "0bf3c7d4ef" add value if not exists 'unknown' after 'set_proposal_duration'; - --- add unknown to EventIdEnum -alter type "8f5a39c8ee" add value if not exists 'Unknown' after 'VoteRejectReferendum'; - --- add unknown to ModuleIdEnum -alter type "7a0b4cc03e" add value if not exists 'unknown' after 'stocapped'; - -alter table events - add column if not exists module_id_text text, - add column if not exists event_id_text text; - -alter table extrinsics - add column if not exists module_id_text text, - add column if not exists call_id_text text; - -alter table polyx_transactions - add column if not exists event_id_text text, - add column if not exists call_id_text text, - add column if not exists module_id_text text; - -update events set event_id_text = event_id where event_id_text is null; -update events set module_id_text = module_id where module_id_text is null; - -update extrinsics set call_id_text = call_id where call_id_text is null; -update extrinsics set module_id_text = module_id where module_id_text is null; - -update polyx_transactions set event_id_text = event_id where event_id_text is null; -update polyx_transactions set call_id_text = call_id where call_id_text is null; -update polyx_transactions set module_id_text = module_id where module_id_text is null; diff --git a/db/migrations/6_new_events_for_7300000_spec.sql b/db/migrations/6_new_events_for_7300000_spec.sql deleted file mode 100644 index 3b190aef..00000000 --- a/db/migrations/6_new_events_for_7300000_spec.sql +++ /dev/null @@ -1,25 +0,0 @@ -alter type "0bf3c7d4ef" add value if not exists 'lock_instruction' after 'add_and_affirm_with_mediators'; -alter type "0bf3c7d4ef" add value if not exists 'enable_offchain_funding' after 'stop'; - -alter type "8f5a39c8ee" add value if not exists 'InstructionLocked' after 'InstructionMediators'; -alter type "8f5a39c8ee" add value if not exists 'FundraiserOffchainFundingEnabled' after 'FundraiserClosed'; - -alter table "public"."stos" add column if not exists "off_chain_funding_enabled" boolean not null default false; -alter table "public"."stos" add column if not exists "off_chain_funding_token" text; -alter table "public"."stos" alter column "raising_ticker" drop not null; - -alter type "b861be9158" add value if not exists 'Locked' after 'Failed'; - -alter type "7f3c7bae24" add value if not exists 'SettleAfterLock' after 'SettleManual'; - -DO $$ -BEGIN - IF NOT EXISTS (select 1 from pg_type where typname = '867b307be0') then - create type "867b307be0" AS ENUM ('OnChain', 'OffChain'); - END IF; -END -$$; - -alter table "public"."investments" add column if not exists "raising_asset_type" "867b307be0" not null default 'OnChain'; - -alter type "3e29b3f361" add value if not exists 'InstructionLocked' after 'InstructionFailed'; diff --git a/db/migrations/7_drop_ticker_null_constraints_on_investments.sql b/db/migrations/7_drop_ticker_null_constraints_on_investments.sql deleted file mode 100644 index f2816c91..00000000 --- a/db/migrations/7_drop_ticker_null_constraints_on_investments.sql +++ /dev/null @@ -1,2 +0,0 @@ -alter table investments alter column offering_token drop not null; -alter table investments alter column raise_token drop not null; \ No newline at end of file diff --git a/db/migrations/8_events_for_8000000_spec.sql b/db/migrations/8_events_for_8000000_spec.sql deleted file mode 100644 index 391462e2..00000000 --- a/db/migrations/8_events_for_8000000_spec.sql +++ /dev/null @@ -1,54 +0,0 @@ -alter type "7a0b4cc03e" add value if not exists 'validators' after 'electionprovidermultiphase'; - -alter type "0bf3c7d4ef" add value if not exists 'apply_authorized_upgrade' after 'placeholder_fill_block'; -alter type "0bf3c7d4ef" add value if not exists 'authorize_upgrade' after 'apply_authorized_upgrade'; -alter type "0bf3c7d4ef" add value if not exists 'authorize_upgrade_without_checks' after 'authorize_upgrade'; -alter type "0bf3c7d4ef" add value if not exists 'poke_deposit' after 'freeze'; -alter type "0bf3c7d4ef" add value if not exists 'force_set_balance' after 'burn_account_balance'; -alter type "0bf3c7d4ef" add value if not exists 'force_adjust_total_issuance' after 'force_set_balance'; -alter type "0bf3c7d4ef" add value if not exists 'force_unreserve' after 'force_adjust_total_issuance'; -alter type "0bf3c7d4ef" add value if not exists 'transfer_all' after 'force_unreserve'; -alter type "0bf3c7d4ef" add value if not exists 'transfer_allow_death' after 'transfer_all'; -alter type "0bf3c7d4ef" add value if not exists 'transfer_keep_alive' after 'transfer_allow_death'; -alter type "0bf3c7d4ef" add value if not exists 'upgrade_accounts' after 'transfer_keep_alive'; -alter type "0bf3c7d4ef" add value if not exists 'update_payee' after 'set_staking_configs'; -alter type "0bf3c7d4ef" add value if not exists 'payout_stakers_by_page' after 'update_payee'; -alter type "0bf3c7d4ef" add value if not exists 'deprecate_controller_batch' after 'payout_stakers_by_page'; -alter type "0bf3c7d4ef" add value if not exists 'manual_slash' after 'deprecate_controller_batch'; -alter type "0bf3c7d4ef" add value if not exists 'migrate_currency' after 'manual_slash'; -alter type "0bf3c7d4ef" add value if not exists 'restore_ledger' after 'migrate_currency'; -alter type "0bf3c7d4ef" add value if not exists 'cancel_retry' after 'schedule_named_after'; -alter type "0bf3c7d4ef" add value if not exists 'cancel_retry_named' after 'cancel_retry'; -alter type "0bf3c7d4ef" add value if not exists 'set_retry' after 'cancel_retry_named'; -alter type "0bf3c7d4ef" add value if not exists 'set_retry_named' after 'set_retry'; -alter type "0bf3c7d4ef" add value if not exists 'migrate' after 'instantiate_old_weight'; -alter type "0bf3c7d4ef" add value if not exists 'ensure_updated' after 'unrequest_preimage'; - -alter type "8f5a39c8ee" add value if not exists 'RejectedInvalidAuthorizedUpgrade' after 'PlaceholderFillBlock'; -alter type "8f5a39c8ee" add value if not exists 'UpgradeAuthorized' after 'RejectedInvalidAuthorizedUpgrade'; -alter type "8f5a39c8ee" add value if not exists 'DepositPoked' after 'IndexFrozen'; -alter type "8f5a39c8ee" add value if not exists 'Deposit' after 'TransactionFeePaid'; -alter type "8f5a39c8ee" add value if not exists 'DustLost' after 'Deposit'; -alter type "8f5a39c8ee" add value if not exists 'Locked' after 'DustLost'; -alter type "8f5a39c8ee" add value if not exists 'Minted' after 'Locked'; -alter type "8f5a39c8ee" add value if not exists 'Rescinded' after 'Minted'; -alter type "8f5a39c8ee" add value if not exists 'Suspended' after 'Rescinded'; -alter type "8f5a39c8ee" add value if not exists 'Thawed' after 'Suspended'; -alter type "8f5a39c8ee" add value if not exists 'TotalIssuanceForced' after 'Thawed'; -alter type "8f5a39c8ee" add value if not exists 'TransferWithMemo' after 'TotalIssuanceForced'; -alter type "8f5a39c8ee" add value if not exists 'Unlocked' after 'TransferWithMemo'; -alter type "8f5a39c8ee" add value if not exists 'Upgraded' after 'Unlocked'; -alter type "8f5a39c8ee" add value if not exists 'Withdraw' after 'Upgraded'; -alter type "8f5a39c8ee" add value if not exists 'ControllerBatchDeprecated' after 'StakingElectionFailed'; -alter type "8f5a39c8ee" add value if not exists 'CurrencyMigrated' after 'ControllerBatchDeprecated'; -alter type "8f5a39c8ee" add value if not exists 'SnapshotTargetsSizeExceeded' after 'CurrencyMigrated'; -alter type "8f5a39c8ee" add value if not exists 'SnapshotVotersSizeExceeded' after 'SnapshotTargetsSizeExceeded'; -alter type "8f5a39c8ee" add value if not exists 'ValidatorDisabled' after 'NewSession'; -alter type "8f5a39c8ee" add value if not exists 'ValidatorReenabled' after 'ValidatorDisabled'; -alter type "8f5a39c8ee" add value if not exists 'AgendaIncomplete' after 'PermanentlyOverweight'; -alter type "8f5a39c8ee" add value if not exists 'RetryCancelled' after 'AgendaIncomplete'; -alter type "8f5a39c8ee" add value if not exists 'RetryFailed' after 'RetryCancelled'; -alter type "8f5a39c8ee" add value if not exists 'RetrySet' after 'RetryFailed'; -alter type "8f5a39c8ee" add value if not exists 'StorageDepositTransferredAndHeld' after 'DelegateCalled'; -alter type "8f5a39c8ee" add value if not exists 'StorageDepositTransferredAndReleased' after 'StorageDepositTransferredAndHeld'; - diff --git a/db/migrations/9_events_for_7400000_spec.sql b/db/migrations/9_events_for_7400000_spec.sql deleted file mode 100644 index 9615ac03..00000000 --- a/db/migrations/9_events_for_7400000_spec.sql +++ /dev/null @@ -1,5 +0,0 @@ -alter type "0bf3c7d4ef" add value if not exists 'receiver_affirm_asset_transfer' after 'update_global_metadata_spec'; -alter type "0bf3c7d4ef" add value if not exists 'reject_asset_transfer' after 'receiver_affirm_asset_transfer'; -alter type "0bf3c7d4ef" add value if not exists 'transfer_asset' after 'reject_asset_transfer'; - -alter type "8f5a39c8ee" add value if not exists 'CreatedAssetTransfer' after 'GlobalMetadataSpecUpdated'; diff --git a/docker-compose.yml b/docker-compose.yml index 9f4f4688..5286aadc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,9 @@ services: ports: - 5999:5432 volumes: - - db-data:/var/lib/postgresql/data + # PostgreSQL 18+ images store data in a major-version subdirectory, so the mount is the + # parent `/var/lib/postgresql` (not `…/data`). See docker-library/postgres#1259. + - db-data:/var/lib/postgresql environment: POSTGRES_PASSWORD: postgres healthcheck: diff --git a/docker/pg-Dockerfile b/docker/pg-Dockerfile index cb141e39..1d2a1aa8 100644 --- a/docker/pg-Dockerfile +++ b/docker/pg-Dockerfile @@ -1,4 +1,4 @@ -FROM postgres:12-alpine +FROM postgres:18-alpine # Variables needed at runtime to configure postgres and run the initdb scripts ENV POSTGRES_DB='postgres' diff --git a/docs/CHANGES.md b/docs/CHANGES.md index ce12febd..15222685 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -70,7 +70,9 @@ The chain's numeric instruction sequence is stored as a `String`, so `orderBy: [ ### 2.3 A15 — pre-v8 staking rewards are unattributable **[V]** -`mapStakingEvent.ts` records `rewardDestination: 'LegacyUnknown'` for every pre-8.x reward, because the event carried only the stash. Where a staker set a payee other than their stash, the index cannot say which account received the POLYX. The v8 path is correct. +`mapStakingEvent.ts` records `rewardDestination: 'LegacyUnknown'` for every pre-8.x reward, because the event carried only the stash. Where a staker set a payee other than their stash, the index cannot say which account received the POLYX. + +> **Correction (PR #350).** "The v8 path is correct" **[V]** was wrong — the v8 `getRewardDestinationDetails` matched the variant against `'Account'` while `Enum#toJSON()` camel-cases it, so the object form (`{ account: … }`, `{ staked: null }`) never resolved. Fixed and consolidated into `readRewardDestination`. `LegacyUnknown` is an honest placeholder rather than a wrong value — this is a **coverage** gap, not a correctness one. It matters because these rows are used for accounting, where "looks complete and is not" is the expensive failure. diff --git a/docs/entity-review.md b/docs/entity-review.md index cd4e184b..aabfaf9d 100644 --- a/docs/entity-review.md +++ b/docs/entity-review.md @@ -234,7 +234,7 @@ Three entities for one concept (agent membership + history) — merge candidate | Entity | Verdict | Notes | |---|---|---| | `PolyxTransaction` | ❌ | `BalanceTypeEnum` conflates pools, lock-floors and staking-ledger states; v8 `reserved` entirely unindexed. See `reference/polyx-balance-model.md`. | -| `StakingEvent` | ❌ | A log, not a position — and for pre-v8 rewards, an *incomplete* log. `rewardDestination` is `'LegacyUnknown'` for every pre-8.x `Reward`/`Rewarded` **[V]**, because the event carried only the stash. The v8 path correctly decodes the `RewardDestination` variant and resolves the account. **No `StakingPosition` / nomination entity** — current bonded amount, nominations, and validator prefs are not queryable. `staking` is **8/32 handled**; `Nominated` is handled but `Chilled`, `Kicked`, `PayoutStarted`, `EraPaid`, `ValidatorPrefsSet`, `StakersElected` are not. | +| `StakingEvent` | ❌ | A log, not a position — and for pre-v8 rewards, an *incomplete* log. `rewardDestination` is `'LegacyUnknown'` for every pre-8.x `Reward`/`Rewarded` **[V]**, because the event carried only the stash. The v8 path decodes the `RewardDestination` variant and resolves the account (a camel-case bug in it — the object form never matched — was found and fixed in PR #350). **No `StakingPosition` / nomination entity** — current bonded amount, nominations, and validator prefs are not queryable. `staking` is **8/32 handled**; `Nominated` is handled but `Chilled`, `Kicked`, `PayoutStarted`, `EraPaid`, `ValidatorPrefsSet`, `StakersElected` are not. | | `BridgeEvent` | ⚠️ | Only `Bridged` handled of 17 events. `BridgeTxScheduled`, `BridgeTxFailed`, `BridgeLimitUpdated`, `ControllerChanged`, `AdminChanged` unhandled — so bridge failures and configuration changes are invisible. Also hardcodes `/ 1_000_000` with integer division. | 🚫 **Missing: `StakingPosition`, `Nomination`, `Validator`, `Era`.** Staking is recorded purely as an event stream, so *"how much is this account staking right now, and with whom"* requires replaying all events. diff --git a/docs/implementation/02-polyx-ledger.md b/docs/implementation/02-polyx-ledger.md index 660c098e..e9fe4247 100644 --- a/docs/implementation/02-polyx-ledger.md +++ b/docs/implementation/02-polyx-ledger.md @@ -158,7 +158,7 @@ Verified emissions **[V]** (`pallets/balances/src/lib.rs` @ v7.4.0, `types-looku - **`BalanceSet`** — a **checkpoint**. Set `AccountBalance.free`/`reserved` absolutely; write one `BalanceSetAdjustment` entry recording the delta so the ledger still reconciles. Resolves A1 structurally. - **`Locked`/`Unlocked`/`Frozen`/`Thawed`** — update `AccountBalance.locks` only; recompute `frozen = MAX(active locks)`. No entry. - **`Issued`/`Rescinded`/`TotalIssuanceForced`/`MintedCredit`/`BurnedDebt`** — total-issuance only, no account side. **[I]** Consider a `TotalIssuance` entity; out of scope here. -- **`Upgraded`** — account flag migration, no amount. +- **`Upgraded`** — account flag / lock→hold migration marker, no amount of its own (the paired `Held` / `Unlocked` carry it). - **`TransferWithMemo`** — memo enrichment of the paired `Transfer`, never its own entry (resolves A2 option (b)). ### Staking — era-dependent, and inverted at v8 **[V]** @@ -166,7 +166,9 @@ Verified emissions **[V]** (`pallets/balances/src/lib.rs` @ v7.4.0, `types-looku - **≤ v7.4**: bonding is `set_lock(STAKING_ID, …)` — **no balance moves**. `Bonded`/`Unbonded`/`Withdrawn` update `locks` only. This is the correction to A6: the current `type: Bonded` rows assert movements that never happened. - **v8**: bonding is a Hold. The balance effect arrives via `balances.Held{reason:Staking}` / `Released`; the `staking.*` events become ledger state only. Recording both would double count. -**[I]** Confirm the `staking.Bonded` ↔ `balances.Held{reason:Staking}` pairing within one extrinsic against a real v8 block before relying on it. +**[V]** Pairing confirmed on testnet — `bondExtra` / `withdrawUnbonded` / `dest:Staked` payouts all emit the `Held` / `Released` in the same extrinsic (or `Initialization` phase) as the `staking.*` event; `rebond` correctly emits `Bonded` with no `Held`. See [`../reference/polyx-reconciliation.md`](../reference/polyx-reconciliation.md). + +- **v5–v7 → v8 lock→hold migration**: a no-extrinsic `pallet_balances` storage migration in two passes. Pass 1 emits `Upgraded` + `Held{Staking}` (→ `handleBalanceHeld` moves `free → reserved`); pass 2 emits `Unlocked` for the lingering `"staking "` lock. `handleBalanceUnlocked` clears that lock on a v8 `Unlocked` that covers it, so `frozen` tracks the chain across the ~400k-block window where the account carries both lock and hold. ### Genesis @@ -192,7 +194,9 @@ Because `api.query` targets the block being indexed **[V]** (and `.at` is unsupp [`mapStakingEvent.ts:110-130`](../../src/mappings/entities/events/mapStakingEvent.ts#L110) records `rewardDestination: 'LegacyUnknown'` for every pre-8.x `Reward`/`Rewarded`, because the event carries only the **stash** and the amount. Where a staker set a payee other than their stash — `Controller`, or an explicit `Account` — the index cannot say which account received the POLYX. Defect A15. -The v8 path is correct: `get8xStakingEventDetails` decodes the `RewardDestination` variant and resolves `rewardDestinationAccount` for `Account`, `Staked` and `Stash` **[V]**. +The v8 path decodes the `RewardDestination` variant and resolves `rewardDestinationAccount` for `Account`, `Staked` and `Stash`. + +> **Correction (PR #350).** This paragraph previously said the v8 path was correct **[V]**. It was not: `getRewardDestinationDetails` compared the variant key against `'Account'`, but `Enum#toJSON()` camel-cases it (`{ account: … }`), so the object form never matched — a v8 `Rewarded` with an explicit `Account` or an object-form `Staked` payee stored `rewardDestination` lower-cased with no account. Both paths now share `readRewardDestination` in `utils/staking.ts`, which normalises the bare-string and camel-cased-object forms. `LegacyUnknown` is an honest placeholder, not a wrong value — but it means a pre-v8 reward cannot be reconciled against the receiving account's balance, and it is invisible to anyone who does not know what the string means. @@ -283,7 +287,7 @@ Register the eight missing v8 events (A9) and give the two empty ones handlers: - **Unit, per row of the transition table:** fixture → expected `(fromPool, toPool, kind, amount)`. - **Unit:** `Reserved` then `Unreserved` returns `free`/`reserved` to their starting values — the property the current model cannot satisfy. -- **Unit:** v7 `Bonded` produces **no** entry and raises `frozen`; v8 `Held{Staking}` produces `Free → Reserved`. +- **Unit:** v7 `Bonded` produces **no** entry and raises `frozen`; v8 `Held{Staking}` produces `Free → Reserved`; a v8 `Unlocked` covering the `"staking "` lock clears it (the lock→hold migration) while a smaller `Unlocked` does not. - **Unit:** `BalanceSet` sets absolutely and does not corrupt subsequent totals. - **Unit:** two overlapping locks of 100 and 150 give `frozen = 150`, not 250. - **Integration:** after resync, `SUM(amount) WHERE toPool=X` minus `SUM WHERE fromPool=X` equals `AccountBalance` for a sample of accounts. diff --git a/docs/reference/polyx-balance-model.md b/docs/reference/polyx-balance-model.md index fb9218f8..3c0da479 100644 --- a/docs/reference/polyx-balance-model.md +++ b/docs/reference/polyx-balance-model.md @@ -305,8 +305,8 @@ This is the correction to audit A6: pre-v8 these must not produce `PolyxMovement | `DustLost{account,amount}` | `account/Free` | ∅ | DustLost | | `BalanceSet{who,free}` | — | — | checkpoint | | `Issued` / `Rescinded` / `TotalIssuanceForced` / `MintedCredit` / `BurnedDebt` | — | — | **total-issuance only, no account side** — track on a separate `TotalIssuance` entity, not the account ledger | -| `Locked` / `Unlocked` / `Frozen` / `Thawed` | — | — | `BalanceLock` only, no movement | -| `Upgraded{who}` | — | — | no amount; account flag migration only | +| `Locked` / `Unlocked` / `Frozen` / `Thawed` | — | — | `BalanceLock` only, no movement. A v8 `Unlocked` covering the `"staking "` lock is the lock→hold migration — it clears that lock, not the generic one | +| `Upgraded{who}` | — | — | no amount; account flag / lock→hold migration marker | ### 4.4 v8 staking @@ -314,7 +314,7 @@ This is the correction to audit A6: pre-v8 these must not produce `PolyxMovement This resolves the A6 `Withdrawn` bug structurally: it is no longer a credit to a fictional `Unbonded` pool, it is a staking-ledger transition whose balance effect is the paired `Released`. -**[I]** The exact pairing of `staking.Bonded` with `balances.Held{reason:Staking}` within one extrinsic should be confirmed against a real v8 block before relying on it for backfill. +**[V]** Pairing confirmed against real testnet v8 blocks (`bondExtra`, `withdrawUnbonded`, `dest:Staked` payouts all carry the paired `Held`/`Released`; `rebond` carries none, correctly). The v5–v7 `set_lock("staking ")` → v8 `Staking` hold conversion is a no-extrinsic two-pass `pallet_balances` storage migration: pass 1 emits `Upgraded` + `Held{Staking}`, pass 2 emits `Unlocked` for the lock. `handleBalanceHeld` covers the hold side unchanged; `handleBalanceUnlocked` clears the `"staking "` lock on the migration `Unlocked`. See [`polyx-reconciliation.md`](./polyx-reconciliation.md). --- diff --git a/docs/reference/polyx-reconciliation.md b/docs/reference/polyx-reconciliation.md new file mode 100644 index 00000000..cea9cbb6 --- /dev/null +++ b/docs/reference/polyx-reconciliation.md @@ -0,0 +1,117 @@ +# POLYX ledger — reconciliation and the accounting-fidelity findings + +Companion to [`../implementation/02-polyx-ledger.md`](../implementation/02-polyx-ledger.md). +Records what was verified against chain state while building the ledger, and how to run the +acceptance gate (D11). + +--- + +## Verified against real v8 blocks — the `staking.Bonded` ↔ `balances.Held{Staking}` pairing + +Plan 02 §"Staking" made the v8 handling conditional on confirming that bonding's balance movement +arrives as `balances.Held{reason:Staking}` in the same extrinsic as `staking.Bonded`. Checked on +Polymesh **testnet** (spec 8001000): + +| Block | Call | Events in the extrinsic | +|---|---|---| +| 25163012 | `staking.bondExtra` | `Withdraw`(fee), **`Held{Staking, 9640951}`**, **`Bonded{9640951}`**, `Deposit`(fee refund), `TransactionFeePaid` | +| 25107475 | `staking.withdrawUnbonded` | `Withdraw`(fee), **`Released{Staking, 5000000000}`**, **`Withdrawn{5000000000}`**, `Deposit`, `TransactionFeePaid` | +| 25721280 | `staking.rebond` | `Withdraw`(fee), `Bonded{5874231757}` — **no `Held`** | +| 25774711 | payout (`Initialization`) | per nominator: `Deposit{amount}`, `Held{Staking, amount}` (for `dest:Staked`), `Rewarded{stash, dest, amount}` | + +**Conclusion — the pairing holds.** The balance movement is always the `Held`/`Released` event +where there is one; `staking.Bonded`/`Unbonded`/`Withdrawn` are ledger-state events. `rebond` +correctly emits `Bonded` with no `Held` because no balance moves (funds move between the ledger's +`unlocking` and `active` within an existing hold). So on v8 the ledger writes **no `PolyxEntry`** +for the `staking.*` events — the entry comes from the paired `balances` event — and this is +correct, not an under-count. + +Pre-v8 (≤ v7.4) bonding is `set_lock(STAKING_ID, …)` and moves no balance: `Bonded`/`Withdrawn` +maintain `AccountBalance.locks` only, `frozen = MAX(active locks)`, still no `PolyxEntry`. + +### The v5–v7 lock → v8 hold storage migration + +`pallet_balances` converts every pre-v8 `set_lock("staking ", …)` into a `RuntimeHoldReason::Staking` +hold. It runs in **two passes**, each emitting events per account but in no extrinsic of the +staker's own: + +| Pass | Block (testnet) | Phase | Events per migrated account | +|---|---|---|---| +| 1 | 24,733,771 | `Initialization` | `Upgraded{who}`, `Held{Staking, ledger.total}` | +| 2 | 25,152,593 | a permissionless `balances` extrinsic | `Upgraded{who}` (if new), `Unlocked{who, lock}`, `Held{Staking, …}` (if new) | + +Between the two passes the account carries **both** the old `"staking "` lock and the new hold +on-chain — `frozen` stays at the lock amount until pass 2's `Unlocked` drops it. Probed on +`5C7kNpSv…`: + +``` +b24730485 spec7004001 free 5938e9 reserved 0 frozen 5103e9 lock 5103e9 hold — +b24733771 spec8000000 free 835e9 reserved 5103e9 frozen 5103e9 lock 5103e9 hold 5103e9 (pass 1: Held, lock kept) +b25152593 spec8000020 free 835e9 reserved 5207e9 frozen 0 lock — hold 5207e9 (pass 2: Unlocked) +``` + +The indexer needs no special handling for the hold side — `handleBalanceHeld` turns pass 1's +`Held` into the `free → reserved` movement exactly as for any other hold. The lock side is the +one addition: `handleBalanceUnlocked` treats a v8 `Unlocked` that covers the account's `"staking "` +lock as this migration and clears that lock (rather than the generic `"balances"` one), so +`frozen` tracks the chain across the two-pass window. + +--- + +## Measured — defect A15, the pre-v8 reward-destination gap + +Pre-8.x `staking.Reward`/`Rewarded` carries only the stash. `scripts/measure-a15-payees.ts` +sampled `staking.payee(stash)` at the reward block for pre-v8 reward stashes across a spread of +eras on both networks: + +- **Near the v8 boundary** (last pre-v8 payout era): ~190 distinct stashes, **100% `Staked`/`Stash`**. +- **Across earlier eras** (mainnet spec 3010 → 7004001): a **large share** paid to `Controller` + or an explicit `Account` — e.g. every sampled stash at spec 3010 used `Controller`; `Account` + payees common at spec 5003001 / 6001031 / 7003003 / 7004001. + +**Decision: not near-zero — the storage read was added.** `resolveLegacyRewardDestination` +(`src/utils/staking.ts`) reads `staking.payee(stash)` (and `staking.bonded(stash)` for the +`Controller` case) at the reward block, resolving the real recipient. It is wired into both +`mapStakingEvent` (`StakingEvent.rewardDestination` / `rewardDestinationAccount`) and the ledger +(`handleReward` credits the resolved account). `LegacyUnknown` remains only as the fallback when +the read is not possible (a pruned node), and `rewardDestinationA15.test.ts` pins that it is +never silently resolved to the stash instead. + +Cheap now — the read happens during the D5 genesis replay, which is running anyway. Awkward +later — it would need an archive node holding the pre-v8 state. That asymmetry is why it is done +in this phase rather than deferred. + +--- + +## The reconciliation harness (D11) — the acceptance gate + +### In-flight — `src/mappings/entities/identities/reconcilePolyx.ts` + +Wired into the ledger handlers. Every 500th block for accounts touched in that block, and always +after `BalanceSet` / `DustLost`, the derived `AccountBalance` is compared against `system.account` +read at the block being indexed (`api.query` targets the current block; `.at` is unsupported). +On a mismatch it writes a `BalanceReconciliationDrift` anomaly **and corrects** the derived +value, so drift from one missed or mis-signed event cannot compound into every later balance. + +### Offline — `scripts/reconcile-polyx.ts` + +Run separately from block indexing, against a **synced local database** and a public archive RPC: + +``` +DB_HOST=… DB_PORT=… DB_USER=… DB_PASS=… DB_DATABASE=… \ + yarn ts-node scripts/reconcile-polyx.ts --rpc wss://mainnet-rpc.polymesh.network +``` + +- Samples accounts stratified by activity, oversampling everyone in a `BalanceSetAdjustment`, + `DustLost`, `Slash` or pre-v8 `StakingReward` entry. +- Compares at one block before and one after each of 5_000_000, 6_000_000, 7_000_000, + 7_003_000, 7_004_001, 8_000_000. +- Compares `free`, `reserved`, `frozen` **independently**, and cross-checks `SUM(PolyxEntry)` + against `AccountBalance` (the two time-travel mechanisms in §7.4 of plan 02 must agree). +- Output is a **mismatch taxonomy**: constant drift from a block = one missed event; growing + drift = a systematically mis-signed one; drift confined to `reserved` = a pool-mapping error. +- Resumable via `.reconcile-polyx.checkpoint.json`. + +**Acceptance for the phase:** the harness reports zero unexplained mismatches across the sample, +with every explained one written up here or in the PR body. This — not "the resync completed" — +is the gate. diff --git a/project.ts b/project.ts index c2e33050..a6b4cfc6 100644 --- a/project.ts +++ b/project.ts @@ -54,28 +54,36 @@ const filters: Record> = { TransferWithData: [], }, balances: { + // POLYX ledger (docs/implementation/02-polyx-ledger.md). Every balances movement writes a + // PolyxEntry per account-side plus a running AccountBalance, from mapPolyxLedger.ts. AccountBalanceBurned: ['handleBalanceBurned'], BalanceSet: ['handleBalanceSet'], Burned: ['handleBalanceBurned'], - Deposit: ['handleBalanceDeposit'], - DustLost: [], + BurnedDebt: [], // issuance only — no account side + BurnedHeld: ['handleBalanceBurnedHeld'], + Deposit: ['handleBalanceMinted'], + DustLost: ['handleDustLost'], Endowed: ['handleBalanceEndowed'], Frozen: ['handleBalanceFrozen'], + Held: ['handleBalanceHeld'], Issued: [], Locked: ['handleBalanceLocked'], Minted: ['handleBalanceMinted'], + MintedCredit: [], // issuance only — no account side Rescinded: [], + Released: ['handleBalanceReleased'], Reserved: ['handleBalanceReserved'], ReserveRepatriated: ['handleReserveRepatriated'], Restored: ['handleBalanceMinted'], Slashed: ['handleBalanceBurned'], - // handleBalanceSuspended does not exist yet — it arrives with the POLYX ledger work - // (docs/implementation/02-polyx-ledger.md), which rewrites mapPolyxTransaction.ts wholesale - Suspended: [], - Thawed: [], + Suspended: ['handleBalanceSuspended'], // A3 — the handler did not exist before + Thawed: ['handleBalanceThawed'], TotalIssuanceForced: [], Transfer: ['handleBalanceTransfer'], - TransferWithMemo: ['handleBalanceTransfer'], + TransferAndHold: ['handleTransferAndHold'], + TransferOnHold: ['handleTransferOnHold'], + TransferWithMemo: ['handleBalanceTransferWithMemo'], // A2 — memo enrichment only, never its own entry + Unexpected: [], // anomaly marker; consider IndexerAnomaly Unlocked: ['handleBalanceUnlocked'], Unreserved: ['handleBalanceUnreserved'], Upgraded: [], @@ -238,7 +246,7 @@ const filters: Record> = { UserPortfolios: [], }, protocolFee: { - FeeCharged: ['handleFeeCharged'], + FeeCharged: ['handleTransactionFeeCharged'], }, settlement: { AffirmationWithdrawn: ['handleAffirmationWithdrawn'], @@ -285,14 +293,15 @@ const filters: Record> = { MinimumBondThresholdUpdated: [], Nominated: ['handleStakingEvent'], OldSlashingReportDiscarded: [], - PayoutStarted: [], + // supplies the `eraIndex` that `Rewarded` lacks; consumed by the POLYX ledger's handleReward + PayoutStarted: ['handlePayoutStarted'], PermissionedIdentityAdded: [], PermissionedIdentityRemoved: [], Reward: ['handleStakingEvent', 'handleReward'], Rewarded: ['handleStakingEvent', 'handleReward'], RewardPaymentSchedulingInterrupted: [], - Slash: ['handleStakingEvent'], - Slashed: ['handleStakingEvent'], + Slash: ['handleStakingEvent', 'handleStakingSlash'], + Slashed: ['handleStakingEvent', 'handleStakingSlash'], SlashReported: [], SlashingAllowedForChanged: [], SnapshotTargetsSizeExceeded: [], @@ -492,6 +501,22 @@ const project: SubstrateProject = { handlers, }, }, + { + kind: SubstrateDatasourceKind.Runtime, + startBlock, + mapping: { + file: './dist/index.js', + handlers: [ + { + kind: SubstrateHandlerKind.Block, + handler: 'handleBlock', + // Only to flush the NftHolder write buffer; a coarse cadence keeps the per-block + // overhead negligible while bounding how stale a buffered holder can get. + filter: { modulo: 100 }, + }, + ], + }, + }, ], }; diff --git a/schema.graphql b/schema.graphql index 300bc6f7..85db9c2a 100644 --- a/schema.graphql +++ b/schema.graphql @@ -26,6 +26,8 @@ enum ModuleIdEnum { authoritydiscovery grandpa historical + mmr + mmrleaf imonline randomnesscollectiveflip sudo @@ -353,6 +355,10 @@ enum EventIdEnum { SnapshotTargetsSizeExceeded SnapshotVotersSizeExceeded + ## validators ## + AutomaticPayoutFinished + ValidatorPayoutFailed + ## offences ## Offence @@ -778,6 +784,11 @@ enum EventIdEnum { UpgradeFailed UpgradeStarted + ## stateTrieMigration ## + Migrated + AutoMigrationFinished + Halted + "`Approval` is now active for assets pallet" Approval @@ -1935,12 +1946,15 @@ type Event @entity @compositeIndexes(fields: [["moduleId", "eventId"]]) { eventArg_1: String eventArg_2: String eventArg_3: String - claimType: String @index(unique: false) - claimScope: String @index(unique: false) - claimIssuer: String @index(unique: false) + # The five denormalised filter columns below are indexed in db/compat.sql, not with `@index`: + # `@subql/node` caps an entity at 10 indexes and Event is at that cap with its foreign keys, + # eventIdx/extrinsicIdx/specVersionId, moduleId/eventId and the `[moduleId, eventId]` composite. + claimType: String + claimScope: String + claimIssuer: String claimExpiry: String - corporateActionTicker: String @index(unique: false) - fundraiserOfferingAsset: String @index(unique: false) + corporateActionTicker: String + fundraiserOfferingAsset: String transferTo: String extrinsic: Extrinsic } @@ -3041,39 +3055,156 @@ type Migration @entity { processedBlock: Int! } -""" -Represents possible all possible balance types -""" -enum BalanceTypeEnum { +# --------------------------------------------------------------------------- +# POLYX ledger (plan 02). PolyxEntry + AccountBalance replace PolyxTransaction +# and BalanceTypeEnum, which are removed once the reconciliation harness is +# green. Until then both models exist side by side. +# --------------------------------------------------------------------------- + +"The two balance pools an account holds on chain. `Bonded`/`Unbonded`/`Locked` are not pools and are not represented here." +enum PolyxPool { Free Reserved - Bonded - Unbonded - Locked +} + +"Which side of a movement an entry records." +enum EntryDirection { + Debit + Credit +} + +"Why POLYX is held in `reserved` under the v8 fungible-Holds API." +enum HoldReason { + Staking + Session + Preimage + Revive + Unknown +} + +"What an entry's movement represents. Denormalised onto the entry so aggregation does not need to traverse relations." +enum MovementKind { + Transfer + Endowment + Fee + Tip + TreasuryDisbursement + TreasuryReimbursement + StakingReward + Slash + Mint + Burn + DustLost + ReserveRepatriation + Hold + Release + BalanceSetAdjustment +} + +type KindTotal @jsonField { + "a `MovementKind` value — jsonField members cannot be enum-typed, same as `Scope.type` etc." + kind: String! + totalAbs: BigInt! + net: BigInt! + count: Int! +} + +type LockEntry @jsonField { + lockId: String! + amount: BigInt! + reasons: String +} + +type HoldEntry @jsonField { + "a `HoldReason` value — jsonField members cannot be enum-typed" + reason: String! + amount: BigInt! } """ -Represents transactions involving POLYX +One row per (account, movement side). Append-only. +Sibling entries of one on-chain movement share `movementId`. """ -type PolyxTransaction @entity { +type PolyxEntry @entity @compositeIndexes(fields: [["account", "date"], ["account", "kind"], ["kind", "date"]]) { + "padId(block)/padId(eventIdx)/side — D4" id: ID! - identityId: String - address: String @index(unique: false) - toId: String - toAddress: String @index(unique: false) + movementId: String! @index + + account: Account! + identity: Identity + "indexed in db/compat.sql — the schema is at the platform's 10-index-per-entity cap" + counterpartyAddress: String + counterpartyIdentity: Identity + + pool: PolyxPool! + "signed: negative = debit, positive = credit. SUM = net delta" amount: BigInt! - type: BalanceTypeEnum! + "unsigned. SUM = gross volume" + amountAbs: BigInt! + + kind: MovementKind! + "materialised — sign(amount) is not groupable (§8b)" + direction: EntryDirection! + holdReason: HoldReason + memo: String + + "balance after this entry — powers charts with no aggregation" + freeAfter: BigInt! + reservedAfter: BigInt! + frozenAfter: BigInt! + + "denormalised — aggregation cannot traverse relations" moduleId: ModuleIdEnum - moduleIdText: String callId: CallIdEnum - callIdText: String - eventId: EventIdEnum - eventIdText: String - memo: String + eventId: EventIdEnum! + specVersionId: Int! + "materialised time buckets — no date_trunc grouping exists (§8b). Indexed via the `[kind, date]` / `[account, date]` composites and a standalone index in db/compat.sql" + date: Date! + "indexed in db/compat.sql — the schema is at the platform's 10-index-per-entity cap" + eraIndex: Int + + "relation is `createdEvent` to avoid colliding with the denormalised `eventId` enum column, same as `AssetTransaction`" + createdEvent: Event! extrinsic: Extrinsic - datetime: Date! eventIdx: Int! + datetime: Date! createdBlock: Block! +} + +""" +Materialised POLYX balance for one account. Versioned on `historical: 'height'`, +so `accountBalance(id: X, blockHeight: N)` gives the balance at block N. +""" +type AccountBalance @entity { + "address" + id: ID! + account: Account! + identity: Identity @index + + free: BigInt! + reserved: BigInt! + "MAX over active locks — maintained from `locks`, never summed" + frozen: BigInt! + total: BigInt! + "free - frozen, floored at 0" + transferable: BigInt! + bonded: BigInt! + otherReserved: BigInt! + + "lifetime totals — free, this row is already versioned on these blocks" + totalReceived: BigInt! + totalSent: BigInt! + totalFeesPaid: BigInt! + totalRewards: BigInt! + totalSlashed: BigInt! + movementCount: Int! + lifetimeByKind: [KindTotal] + + "per-lock detail — required because frozen is a MAX" + locks: [LockEntry] + "v8 only; SUM = reserved" + holds: [HoldEntry] + updatedBlock: Block! } diff --git a/scripts/measure-a15-payees.ts b/scripts/measure-a15-payees.ts new file mode 100644 index 00000000..9c2c82ac --- /dev/null +++ b/scripts/measure-a15-payees.ts @@ -0,0 +1,211 @@ +/** + * Defect A15 — measures the share of pre-v8 staking rewards paid somewhere other than the stash. + * + * Pre-8.x `staking.Reward`/`Rewarded` carries only the stash, so where a staker set a payee of + * `Controller` or an explicit `Account` the index cannot say which account received the POLYX + * (`rewardDestination: 'LegacyUnknown'`). Whether that gap is worth closing with a + * `staking.payee(stash)` read during the genesis replay is a measurement question, not a + * principle one — this is the measurement. + * + * Method: for a spread of pre-v8 blocks (anchored across the chain's history), take the stashes + * that received a reward in that block and read `staking.payee(stash)` **at that block** (chain + * storage, historical). Classify `Staked`/`Stash` as "went to the stash" and `Controller`/ + * `Account` as "went elsewhere". + * + * Usage (from the repo root, no database needed): + * + * yarn ts-node scripts/measure-a15-payees.ts \ + * --rpc wss://mainnet-rpc.polymesh.network \ + * --dictionary https://mainnet-subql-dictionary.polymesh.network/ \ + * [--anchors 4000000,8000000,12000000,16000000,20000000,23000000] [--per-anchor 5] + */ +import { ApiPromise, WsProvider } from '@polkadot/api'; +import chainTypes from '../src/chainTypes'; + +const { types, typesBundle } = chainTypes; + +const V8 = 8_000_000; + +const argOf = (name: string, fallback?: string): string | undefined => { + const i = process.argv.indexOf(`--${name}`); + + return i >= 0 ? process.argv[i + 1] : fallback; +}; + +interface DictEvent { + blockHeight: string; +} + +const dictionaryQuery = async (url: string, query: string): Promise<{ nodes: DictEvent[] }> => { + const res = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query }), + }); + + return ((await res.json()) as { data: { events: { nodes: DictEvent[] } } }).data.events; +}; + +const rewardBlocksAfter = async ( + dictionary: string, + anchor: number, + limit: number +): Promise => { + const { nodes } = await dictionaryQuery( + dictionary, + `{ events(filter: { + module: { equalTo: "staking" }, + event: { in: ["Reward", "Rewarded"] }, + blockHeight: { greaterThan: "${anchor}" } + }, orderBy: BLOCK_HEIGHT_ASC, first: ${limit * 6}) { nodes { blockHeight } } }` + ); + + return [...new Set(nodes.map(n => Number(n.blockHeight)))].slice(0, limit); +}; + +interface Tally { + toStash: number; + elsewhere: number; + none: number; + errors: number; + nonStash: string[]; + seen: Set; +} + +/** A codec-ish value: only its `.toString()` (SS58 address / hex) is read here. */ +type Stringable = { toString(): string }; + +/** + * Stashes named by the `staking.Reward`/`Rewarded` events in a block. The stash is the first + * parameter on ≤7.x (`(stash, amount)`) and the second from 7.x (`(identityId, stash, amount)`). + */ +const rewardedStashes = ( + events: { event: { section: string; method: string; data: Stringable[] } }[] +): string[] => + events + .filter( + record => + record.event.section === 'staking' && ['Reward', 'Rewarded'].includes(record.event.method) + ) + .map(record => { + const data = record.event.data; + return (data.length >= 3 ? data[1] : data[0])?.toString() ?? ''; + }); + +/** Reads one block's reward payees and folds them into the running tally. */ +const tallyBlock = async (api: ApiPromise, block: number, tally: Tally): Promise => { + const hash = await api.rpc.chain.getBlockHash(block); + const specVersion = (await api.rpc.state.getRuntimeVersion(hash)).specVersion.toNumber(); + + if (specVersion >= V8) { + return; + } + + let events; + try { + events = await api.query.system.events.at(hash); + } catch { + tally.errors += 1; + return; + } + + const at = await api.at(hash); + + for (const stash of rewardedStashes(events as never)) { + if (!stash || tally.seen.has(stash)) { + continue; + } + tally.seen.add(stash); + + try { + const payee = (await at.query.staking.payee(stash)).toString(); + + if (payee === 'Staked' || payee === 'Stash') { + tally.toStash += 1; + } else if (payee === 'None') { + tally.none += 1; + } else { + tally.elsewhere += 1; + tally.nonStash.push(` block ${block} (spec ${specVersion}) ${stash} -> ${payee}`); + } + } catch { + tally.errors += 1; + } + } +}; + +const main = async (): Promise => { + const rpc = argOf('rpc'); + const dictionary = argOf('dictionary'); + + if (!rpc || !dictionary) { + console.error('Pass --rpc and --dictionary '); + process.exit(1); + } + + const anchors = ( + argOf('anchors', '4000000,8000000,12000000,16000000,20000000,23000000') as string + ) + .split(',') + .map(Number); + const perAnchor = Number(argOf('per-anchor', '5')); + + const api = await ApiPromise.create({ + provider: new WsProvider(rpc), + noInitWarn: true, + types: types as never, + typesBundle: typesBundle as never, + }); + + const tally: Tally = { + toStash: 0, + elsewhere: 0, + none: 0, + errors: 0, + nonStash: [], + seen: new Set(), + }; + + for (const anchor of anchors) { + for (const block of await rewardBlocksAfter(dictionary, anchor, perAnchor)) { + await tallyBlock(api, block, tally); + } + } + + const { toStash, elsewhere, none, errors, nonStash } = tally; + + await api.disconnect(); + + const total = toStash + elsewhere + none; + const pct = (n: number) => (total ? ((100 * n) / total).toFixed(1) : '0.0'); + + console.log(`\n=== A15 payee measurement (${rpc}) ===`); + console.log( + `distinct pre-v8 reward stashes sampled: ${total} across ${anchors.length} anchors (errors ${errors})` + ); + console.log( + ` payee = Stash/Staked (reward went to the stash): ${toStash} (${pct(toStash)}%)` + ); + console.log( + ` payee = Controller/Account (went elsewhere): ${elsewhere} (${pct(elsewhere)}%)` + ); + console.log(` payee = None: ${none}`); + + if (nonStash.length) { + console.log('\nnon-stash payees:'); + nonStash.forEach(line => console.log(line)); + } + + console.log( + `\nverdict: ${ + total > 0 && elsewhere / total < 0.01 + ? 'near-zero — LegacyUnknown documented in the schema is defensible; no storage-read backfill needed' + : 'material — add a staking.payee(stash) read at the reward block during the genesis replay' + }` + ); +}; + +main().catch(error => { + console.error(error); + process.exit(1); +}); diff --git a/scripts/reconcile-polyx.ts b/scripts/reconcile-polyx.ts new file mode 100644 index 00000000..52f042fa --- /dev/null +++ b/scripts/reconcile-polyx.ts @@ -0,0 +1,380 @@ +/** + * The POLYX reconciliation harness (decision D11) — the acceptance gate for the POLYX ledger. + * + * The in-flight reconciliation in `src/mappings/entities/identities/reconcilePolyx.ts` only ever + * compares at the block being indexed, so it cannot answer "is the history right". This script + * does, offline, against a synced local database and a public archive RPC. + * + * Method (docs/implementation/02-polyx-ledger.md §"The reconciliation harness"): + * + * 1. Sample accounts, stratified by activity, oversampling every account that appears in a + * `BalanceSet`, `DustLost`, `Slashed` or pre-v8 `StakingReward` entry — that is where the + * known accounting gaps live. + * 2. Compare at spec-version boundary blocks: one before and one after each of 5_000_000, + * 6_000_000, 7_000_000, 7_003_000, 7_004_001 and 8_000_000. A drift that appears on only one + * side of a boundary names the runtime that caused it. + * 3. Compare `free`, `reserved` and `frozen` INDEPENDENTLY — comparing only the total lets a + * pair of offsetting pool-mapping errors cancel out and pass. + * 4. Classify every mismatch. A drift constant from a block is one missed event; a drift that + * grows is a systematically mis-signed one; a drift confined to `reserved` is a pool-mapping + * error. The taxonomy is the output — a count is not actionable. + * + * Resumable: progress is checkpointed to `.reconcile-polyx.checkpoint.json`, so a rate-limited + * public endpoint can be worked through across several runs. + * + * Usage (from the repo root, against a synced DB): + * + * DB_HOST=h DB_PORT=p DB_USER=u DB_PASS=p DB_DATABASE=d \ + * yarn ts-node scripts/reconcile-polyx.ts \ + * --rpc wss://mainnet-rpc.polymesh.network \ + * [--sample 80] [--high 30] [--typical 30] [--reset] + */ +import { ApiPromise, WsProvider } from '@polkadot/api'; +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { DataSource } from 'typeorm'; +import { getPostgresDataSource } from '../db/utils'; + +const CHECKPOINT = join(__dirname, '..', '.reconcile-polyx.checkpoint.json'); +const PAD = 10; +const padId = (n: number | string): string => String(n).padStart(PAD, '0'); + +/** Spec-version boundaries on the public chain's scale, one comparison block on each side. */ +const BOUNDARIES = [5_000_000, 6_000_000, 7_000_000, 7_003_000, 7_004_001, 8_000_000]; + +const argOf = (name: string, fallback?: string): string | undefined => { + const i = process.argv.indexOf(`--${name}`); + + return i >= 0 ? process.argv[i + 1] : fallback; +}; + +const hasFlag = (name: string): boolean => process.argv.includes(`--${name}`); + +// --------------------------------------------------------------------------------------------- +// Sampling +// --------------------------------------------------------------------------------------------- + +interface Sample { + address: string; + reason: 'high-traffic' | 'typical' | 'balance-set' | 'dust-lost' | 'slashed' | 'legacy-reward'; +} + +const oversampledKinds: Array<[Sample['reason'], string]> = [ + ['balance-set', 'BalanceSetAdjustment'], + ['dust-lost', 'DustLost'], + ['slashed', 'Slash'], +]; + +const sampleAccounts = async ( + db: DataSource, + { high, typical }: { high: number; typical: number } +): Promise => { + const byReason = new Map(); + const add = (address: string, reason: Sample['reason']) => { + if (!byReason.has(address)) { + byReason.set(address, { address, reason }); + } + }; + + const ranked: Array<{ account_id: string; n: string }> = await db.query( + `SELECT account_id, count(*) AS n FROM polyx_entries GROUP BY account_id ORDER BY n DESC LIMIT $1`, + [high + typical * 4] + ); + + ranked.slice(0, high).forEach(r => add(r.account_id, 'high-traffic')); + ranked + .slice(high) + .filter((_, i) => i % 4 === 0) + .slice(0, typical) + .forEach(r => add(r.account_id, 'typical')); + + for (const [reason, kind] of oversampledKinds) { + const rows: Array<{ account_id: string }> = await db.query( + `SELECT DISTINCT account_id FROM polyx_entries WHERE kind = $1 LIMIT 200`, + [kind] + ); + rows.forEach(r => add(r.account_id, reason)); + } + + const legacyRewards: Array<{ account_id: string }> = await db.query( + `SELECT DISTINCT account_id FROM polyx_entries + WHERE kind = 'StakingReward' AND spec_version_id < 8000000 LIMIT 200` + ); + legacyRewards.forEach(r => add(r.account_id, 'legacy-reward')); + + return [...byReason.values()]; +}; + +// --------------------------------------------------------------------------------------------- +// Derived vs on-chain +// --------------------------------------------------------------------------------------------- + +interface Triple { + free: bigint; + reserved: bigint; + frozen: bigint; +} + +const zero: Triple = { free: BigInt(0), reserved: BigInt(0), frozen: BigInt(0) }; + +/** Derived balance at block N: the historical `account_balance` row whose range covers N. */ +const derivedAt = async (db: DataSource, address: string, block: number): Promise => { + const [row]: Array<{ free: string; reserved: string; frozen: string }> = await db.query( + `SELECT free, reserved, frozen FROM account_balance + WHERE id = $1 AND _block_range @> $2::int8 LIMIT 1`, + [address, block] + ); + + if (!row) { + return zero; + } + + return { free: BigInt(row.free), reserved: BigInt(row.reserved), frozen: BigInt(row.frozen) }; +}; + +/** The independent second mechanism: SUM(PolyxEntry.amount) by pool up to block N. */ +const summedAt = async ( + db: DataSource, + address: string, + block: number +): Promise<{ free: bigint; reserved: bigint }> => { + const rows: Array<{ pool: string; s: string }> = await db.query( + `SELECT pool, COALESCE(sum(amount), 0) AS s FROM polyx_entries + WHERE account_id = $1 AND created_block_id <= $2 GROUP BY pool`, + [address, padId(block)] + ); + + const map = new Map(rows.map(r => [r.pool, BigInt(r.s)])); + + return { free: map.get('Free') ?? BigInt(0), reserved: map.get('Reserved') ?? BigInt(0) }; +}; + +const chainAt = async (api: ApiPromise, address: string, block: number): Promise => { + const hash = await api.rpc.chain.getBlockHash(block); + const at = await api.at(hash); + const info = (await at.query.system.account(address)) as unknown as { + data: Record; + }; + const d = info.data; + const big = (v?: { toString(): string }) => BigInt(v?.toString() ?? '0'); + const legacyFrozen = big(d.miscFrozen) > big(d.feeFrozen) ? big(d.miscFrozen) : big(d.feeFrozen); + + return { + free: big(d.free), + reserved: big(d.reserved), + frozen: d.frozen !== undefined ? big(d.frozen) : legacyFrozen, + }; +}; + +// --------------------------------------------------------------------------------------------- +// Comparison and taxonomy +// --------------------------------------------------------------------------------------------- + +type Field = 'free' | 'reserved' | 'frozen'; + +interface Mismatch { + address: string; + reason: Sample['reason']; + block: number; + boundary: number; + side: 'before' | 'after'; + field: Field; + derived: bigint; + onChain: bigint; + delta: bigint; + summedDisagreesWithBalance: boolean; +} + +const compare = ( + sample: Sample, + boundary: number, + side: 'before' | 'after', + block: number, + derived: Triple, + onChain: Triple, + summed: { free: bigint; reserved: bigint } +): Mismatch[] => { + const out: Mismatch[] = []; + + (['free', 'reserved', 'frozen'] as Field[]).forEach(field => { + if (derived[field] === onChain[field]) { + return; + } + + out.push({ + address: sample.address, + reason: sample.reason, + block, + boundary, + side, + field, + derived: derived[field], + onChain: onChain[field], + delta: derived[field] - onChain[field], + summedDisagreesWithBalance: + field !== 'frozen' && summed[field as 'free' | 'reserved'] !== derived[field], + }); + }); + + return out; +}; + +const classify = (mismatches: Mismatch[]): string => { + if (mismatches.length === 0) { + return 'no unexplained mismatches across the sample'; + } + + const byAccountField = new Map(); + mismatches.forEach(m => { + const key = `${m.address}|${m.field}`; + const group = byAccountField.get(key) ?? []; + group.push(m); + byAccountField.set(key, group); + }); + + const lines: string[] = []; + + for (const [key, group] of byAccountField) { + const sorted = [...group].sort((a, b) => a.block - b.block); + const deltas = sorted.map(m => m.delta); + const constant = deltas.every(d => d === deltas[0]); + const growing = + deltas.length > 1 && + deltas.every((d, i) => i === 0 || d > deltas[i - 1] === deltas[1] > deltas[0]); + const [address, field] = key.split('|'); + const firstDrift = sorted[0]; + + let verdict: string; + if (constant) { + verdict = `constant drift of ${deltas[0]} from block ${firstDrift.block} — one missed event`; + } else if (growing) { + verdict = `growing drift (${deltas[0]} → ${deltas.at( + -1 + )}) — a systematically mis-signed event`; + } else { + verdict = `irregular drift — needs manual inspection`; + } + + if (field === 'reserved' && !sorted.some(m => m.field !== 'reserved')) { + verdict += '; confined to `reserved` — a pool-mapping error'; + } + if (sorted.some(m => m.summedDisagreesWithBalance)) { + verdict += '; SUM(entries) also disagrees with AccountBalance — the two mechanisms diverged'; + } + + lines.push(` ${address} [${field}] (${firstDrift.reason}): ${verdict}`); + } + + return `${mismatches.length} field-mismatches:\n${lines.join('\n')}`; +}; + +// --------------------------------------------------------------------------------------------- +// Runner +// --------------------------------------------------------------------------------------------- + +interface Checkpoint { + rpc: string; + done: string[]; + mismatches: Mismatch[]; +} + +const loadCheckpoint = (rpc: string): Checkpoint => { + if (hasFlag('reset') || !existsSync(CHECKPOINT)) { + return { rpc, done: [], mismatches: [] }; + } + + const cp = JSON.parse(readFileSync(CHECKPOINT, 'utf-8'), (_k, v) => + typeof v === 'string' && /^-?\d+n$/.test(v) ? BigInt(v.slice(0, -1)) : v + ) as Checkpoint; + + return cp.rpc === rpc ? cp : { rpc, done: [], mismatches: [] }; +}; + +const saveCheckpoint = (cp: Checkpoint): void => + writeFileSync( + CHECKPOINT, + JSON.stringify(cp, (_k, v) => (typeof v === 'bigint' ? `${v}n` : v), 2) + ); + +interface ProbeContext { + db: DataSource; + api: ApiPromise; + head: number; + checkpoint: Checkpoint; + done: Set; +} + +/** Compares one account against chain state at one side of one boundary, checkpointing the result. */ +const probeOne = async ( + { db, api, head, checkpoint, done }: ProbeContext, + sample: Sample, + boundary: number, + side: 'before' | 'after', + block: number +): Promise => { + if (block > head) { + return; + } + + const key = `${sample.address}|${boundary}|${side}`; + if (done.has(key)) { + return; + } + + try { + const [derived, summed, onChain] = await Promise.all([ + derivedAt(db, sample.address, block), + summedAt(db, sample.address, block), + chainAt(api, sample.address, block), + ]); + + checkpoint.mismatches.push(...compare(sample, boundary, side, block, derived, onChain, summed)); + } catch (e) { + console.warn(`skip ${key}: ${(e as Error).message}`); + } + + done.add(key); + checkpoint.done = [...done]; + saveCheckpoint(checkpoint); +}; + +const main = async (): Promise => { + const rpc = argOf('rpc'); + if (!rpc) { + console.error('Pass --rpc (public archive endpoint)'); + process.exit(1); + } + + const high = Number(argOf('high', '30')); + const typical = Number(argOf('typical', '30')); + + const db = await getPostgresDataSource(); + const api = await ApiPromise.create({ provider: new WsProvider(rpc), noInitWarn: true }); + const head = (await api.rpc.chain.getHeader()).number.toNumber(); + + const checkpoint = loadCheckpoint(rpc); + const context: ProbeContext = { db, api, head, checkpoint, done: new Set(checkpoint.done) }; + + const samples = await sampleAccounts(db, { high, typical }); + console.log( + `Sampled ${samples.length} accounts; comparing at ${BOUNDARIES.length * 2} boundary blocks each` + ); + + for (const sample of samples) { + for (const boundary of BOUNDARIES) { + await probeOne(context, sample, boundary, 'before', boundary - 1); + await probeOne(context, sample, boundary, 'after', boundary + 1); + } + } + + await api.disconnect(); + await db.destroy(); + + console.log('\n=== POLYX reconciliation taxonomy ==='); + console.log(classify(checkpoint.mismatches)); +}; + +main().catch(e => { + console.error(e); + process.exit(1); +}); diff --git a/scripts/sync-metadata.ts b/scripts/sync-metadata.ts index def77cc6..cec6e0fd 100644 --- a/scripts/sync-metadata.ts +++ b/scripts/sync-metadata.ts @@ -33,16 +33,23 @@ const SCHEMA_PATH = join(ROOT, 'schema.graphql'); const ARITY_DIR = join(ROOT, 'tests', 'fixtures', 'event-arity'); /** The pallets whose event shapes the decode layer registers, and so the ones worth capturing */ -const CAPTURED_MODULES = ['asset', 'externalAgents', 'identity', 'settlement']; +const CAPTURED_MODULES = [ + 'asset', + 'balances', + 'externalAgents', + 'identity', + 'settlement', + 'staking', +]; export interface RuntimeSnapshot { specName: string; specVersion: number; /** Lowercased pallet names, as `event.section.toLowerCase()` reports them */ modules: string[]; - /** Pallet name (as the chain spells it) to event name to parameter count */ + /** Section id (`ExternalAgents` -> `externalAgents`) to event name to parameter count */ events: Record>; - /** Pallet name to snake_cased call names */ + /** Section id to snake_cased call names */ calls: Record; } @@ -56,6 +63,15 @@ export interface ArityFixture { const snakeCase = (value: string): string => value[0].toLowerCase() + value.slice(1).replace(/[A-Z]/g, letter => `_${letter.toLowerCase()}`); +/** + * A pallet name as `@polkadot/api` reports `event.section`: `ExternalAgents` -> `externalAgents`. + * + * Metadata spells pallet names in PascalCase. Every other producer and consumer of a section here + * - the arity fixtures, `CAPTURED_MODULES`, `project.ts` - uses the api spelling, so the metadata + * spelling is normalised once, on the way in, and never leaks past `snapshotFromMetadata`. + */ +export const sectionId = (name: string): string => name[0].toLowerCase() + name.slice(1); + // --------------------------------------------------------------------------------------------- // Reading metadata // --------------------------------------------------------------------------------------------- @@ -81,7 +97,7 @@ export const snapshotFromMetadata = ( }; for (const pallet of metadata.asLatest.pallets) { - const section = pallet.name.toString(); + const section = sectionId(pallet.name.toString()); snapshot.modules.push(section.toLowerCase()); diff --git a/src/decode/decode.ts b/src/decode/decode.ts index 5b2c5ea7..fa27f385 100644 --- a/src/decode/decode.ts +++ b/src/decode/decode.ts @@ -2,7 +2,7 @@ import { Codec } from '@polkadot/types/types'; import { SubstrateEvent } from '@subql/types'; import { EventIdEnum, ModuleIdEnum } from '../types'; import { recordAnomaly } from '../utils/anomaly'; -import { DecodeError, FieldNotFound } from './errors'; +import { DecodeError, FieldNotFound, NoDecoderForSpecVersion } from './errors'; import { DecodedEvent, namedFields } from './field'; import { resolveShape } from './shapes'; import { normaliseSpecVersion } from './specVersion'; @@ -78,6 +78,50 @@ const guard = (event: SubstrateEvent, decoded: Record): DecodedEv }, }); +/** One Polymesh release line on the public spec-version scale (`v6.x` is `6_000_000..6_999_999`). */ +const ONE_RELEASE_LINE = 1_000_000; + +/** + * The shape for a tuple-style event, tolerating a stale block spec version. + * + * `@subql/node` occasionally reports the *previous* runtime's spec version for the one block a + * runtime upgrade takes effect on (seen at the v5→v6 boundary: `AssetBalanceUpdated` decoded as + * spec `5004003` and crashed the worker, though the block ran `6000001`). `api.runtimeVersion` is + * read from the block's own runtime, so when the reported version resolves no decoder, retry with + * it once — but only when it is newer and within one release line, so neither a correct reported + * version nor a stale/HEAD `api.runtimeVersion` can pull in a wildly wrong shape. + */ +const resolveShapeTolerant = ( + section: string, + method: string, + reportedSpecVersion: number, + arity: number +): ReturnType => { + try { + return resolveShape(section, method, reportedSpecVersion, arity); + } catch (error) { + if (!(error instanceof NoDecoderForSpecVersion)) { + throw error; + } + + let fromRuntime: number; + try { + fromRuntime = normaliseSpecVersion(api.runtimeVersion.specVersion.toNumber()); + } catch { + throw error; + } + + if ( + fromRuntime <= reportedSpecVersion || + fromRuntime - reportedSpecVersion > ONE_RELEASE_LINE + ) { + throw error; + } + + return resolveShape(section, method, fromRuntime, arity); + } +}; + /** * An event's parameters keyed by name. * @@ -103,7 +147,7 @@ export const decodeEvent = (event: SubstrateEvent): DecodedEvent => { const { section, method, data } = event.event; try { - const shape = resolveShape( + const shape = resolveShapeTolerant( section, method, normaliseSpecVersion(event.block.specVersion), diff --git a/src/decode/shapes/balances.ts b/src/decode/shapes/balances.ts new file mode 100644 index 00000000..8fc447cc --- /dev/null +++ b/src/decode/shapes/balances.ts @@ -0,0 +1,73 @@ +import { LAST_V7 } from './consts'; +import { discontinuedAt, registerShape } from './registry'; + +/** + * `balances` pallet parameter shapes. + * + * Polymesh ran a custom `balances` pallet with tuple-style events through v7.4. v8.0.0 deleted it + * and moved to the upstream Substrate pallet, whose events are struct-style and decode straight + * from the block metadata (`namedFields`) with no entry here. So every shape below closes at the + * last 7.x spec version. + * + * Names come from the Rust event definitions (`pallets/balances/src/lib.rs` @ v7.4.0), + * cross-checked against `docs/reference/event-shape-verification.md` and the positional reads the + * pre-ledger `mapPolyxTransaction` handlers relied on. + */ + +// Endowed(IdentityId, AccountId, Balance) +registerShape('balances', 'Endowed', discontinuedAt(LAST_V7, ['identityId', 'account', 'balance'])); + +// Transfer(Option, AccountId, Option, AccountId, Balance, Option) +// The memo tail is absent on the transfers that did not carry one, so arity is 5 or 6. +registerShape('balances', 'Transfer', [ + { + from: 0, + to: LAST_V7, + fields: ['fromIdentityId', 'from', 'toIdentityId', 'to', 'amount', 'memo'], + optionalFrom: 5, + }, +]); + +// TransferWithMemo(from, to, amount, memo) — introduced v7.4.0 only, emitted alongside `Transfer` +registerShape( + 'balances', + 'TransferWithMemo', + discontinuedAt(LAST_V7, ['from', 'to', 'amount', 'memo']) +); + +// Reserved(AccountId, Balance) / Unreserved(AccountId, Balance) +registerShape('balances', 'Reserved', discontinuedAt(LAST_V7, ['account', 'amount'])); +registerShape('balances', 'Unreserved', discontinuedAt(LAST_V7, ['account', 'amount'])); + +// ReserveRepatriated(AccountId, AccountId, Balance, BalanceStatus) +registerShape( + 'balances', + 'ReserveRepatriated', + discontinuedAt(LAST_V7, ['from', 'to', 'amount', 'destinationStatus']) +); + +// BalanceSet(IdentityId, AccountId, free, reserved) — reserved is index 3 (defect A1) +registerShape( + 'balances', + 'BalanceSet', + discontinuedAt(LAST_V7, ['identityId', 'account', 'free', 'reserved']) +); + +// AccountBalanceBurned(IdentityId, AccountId, Balance) — the Polymesh burn event; no v8 equivalent +registerShape( + 'balances', + 'AccountBalanceBurned', + discontinuedAt(LAST_V7, ['identityId', 'account', 'amount']) +); + +// Deposit(AccountId, Balance) +registerShape('balances', 'Deposit', discontinuedAt(LAST_V7, ['account', 'amount'])); + +// 2-arg (AccountId, Balance) events that also existed pre-v8. Defensive: if a runtime emitted +// these as tuples the decoder covers them, and at v8 the struct-style metadata is used instead. +registerShape('balances', 'Burned', discontinuedAt(LAST_V7, ['account', 'amount'])); +registerShape('balances', 'Slashed', discontinuedAt(LAST_V7, ['account', 'amount'])); +registerShape('balances', 'Withdraw', discontinuedAt(LAST_V7, ['account', 'amount'])); +registerShape('balances', 'Minted', discontinuedAt(LAST_V7, ['account', 'amount'])); +registerShape('balances', 'Restored', discontinuedAt(LAST_V7, ['account', 'amount'])); +registerShape('balances', 'DustLost', discontinuedAt(LAST_V7, ['account', 'amount'])); diff --git a/src/decode/shapes/index.ts b/src/decode/shapes/index.ts index b40d3a87..348337aa 100644 --- a/src/decode/shapes/index.ts +++ b/src/decode/shapes/index.ts @@ -13,9 +13,11 @@ * Importing this module registers every shape, so it is imported for its side effects. */ import './asset'; +import './balances'; import './externalAgents'; import './identity'; import './settlement'; +import './staking'; export * from './consts'; export * from './registry'; diff --git a/src/decode/shapes/settlement.ts b/src/decode/shapes/settlement.ts index 3c8fae00..a327c301 100644 --- a/src/decode/shapes/settlement.ts +++ b/src/decode/shapes/settlement.ts @@ -17,20 +17,23 @@ registerShape( stable(['did', 'venueId', 'signers', 'updateType']) ); -registerShape( - 'settlement', - 'InstructionCreated', - stable([ - 'did', - 'venueId', - 'instructionId', - 'settlementType', - 'tradeDate', - 'valueDate', - 'legs', - 'memo', - ]) -); +registerShape('settlement', 'InstructionCreated', [ + { + from: 0, + fields: [ + 'did', + 'venueId', + 'instructionId', + 'settlementType', + 'tradeDate', + 'valueDate', + 'legs', + 'memo', + ], + // `memo` was added after the v3.x era; early testnet/mainnet blocks emit 7 params. + optionalFrom: 7, + }, +]); const portfolioAffirmation = ['did', 'portfolio', 'instructionId']; diff --git a/src/decode/shapes/staking.ts b/src/decode/shapes/staking.ts new file mode 100644 index 00000000..82a0d98c --- /dev/null +++ b/src/decode/shapes/staking.ts @@ -0,0 +1,20 @@ +import { LAST_V7 } from './consts'; +import { discontinuedAt, registerShape } from './registry'; + +/** + * `staking` pallet parameter shapes for the pre-v8 (Polymesh custom staking pallet) tuple events. + * + * v8.0.0 deleted the custom pallet and moved to upstream Substrate staking, whose events are + * struct-style and decode from block metadata directly. So every shape here closes at the last + * 7.x spec version. + * + * `Bonded` / `Unbonded` / `Reward` / `Rewarded` carried the `IdentityId` as their first parameter + * through v7.4.0; `Withdrawn` and `Slash` / `Slashed` never did (verified — defect log §C). + */ +registerShape('staking', 'Bonded', discontinuedAt(LAST_V7, ['identityId', 'stash', 'amount'])); +registerShape('staking', 'Unbonded', discontinuedAt(LAST_V7, ['identityId', 'stash', 'amount'])); +registerShape('staking', 'Reward', discontinuedAt(LAST_V7, ['identityId', 'stash', 'amount'])); +registerShape('staking', 'Rewarded', discontinuedAt(LAST_V7, ['identityId', 'stash', 'amount'])); +registerShape('staking', 'Withdrawn', discontinuedAt(LAST_V7, ['stash', 'amount'])); +registerShape('staking', 'Slash', discontinuedAt(LAST_V7, ['stash', 'amount'])); +registerShape('staking', 'Slashed', discontinuedAt(LAST_V7, ['stash', 'amount'])); diff --git a/src/mappings/entities/assets/mapAsset.ts b/src/mappings/entities/assets/mapAsset.ts index bbf5d80c..3de0c71e 100644 --- a/src/mappings/entities/assets/mapAsset.ts +++ b/src/mappings/entities/assets/mapAsset.ts @@ -33,12 +33,13 @@ import { getStringArrayValue, getTextValue, is7xChain, + isMigratedAssetId, rawAssetHolderToAssetHolder, serializeTicker, specVersionOf, } from '../../../utils'; import { processInstructionId } from '../settlements/mapSettlement'; -import { extractArgs, getAsset } from './../common'; +import { extractArgs, getAsset, getAssetOrAnomaly } from './../common'; export const createFunding = ( blockId: string, @@ -181,7 +182,8 @@ export const handleAssetCreated = async (event: SubstrateEvent): Promise = const ownerId = getTextValue(rawOwnerDid); - const ticker = is7xChain(block) ? undefined : serializeTicker(rawAssetId); + const ticker = + is7xChain(block) || isMigratedAssetId(rawAssetId) ? undefined : serializeTicker(rawAssetId); /** * Name isn't present on the old events so we need to query storage. @@ -622,7 +624,16 @@ export const handleAssetBalanceUpdated = async (event: SubstrateEvent): Promise< } = decodeEvent(event); const assetId = await getAssetId(rawAssetId, block); - const asset = await getAsset(assetId); + const asset = await getAssetOrAnomaly(assetId, { + block, + eventIdx, + eventId: EventIdEnum.Transfer, + }); + + if (!asset) { + return; + } + const transferAmount = getBigIntValue(rawAmount); const promises: Promise[] = []; diff --git a/src/mappings/entities/assets/mapNfts.ts b/src/mappings/entities/assets/mapNfts.ts index 44d25627..b80b9e4e 100644 --- a/src/mappings/entities/assets/mapNfts.ts +++ b/src/mappings/entities/assets/mapNfts.ts @@ -14,6 +14,43 @@ import { import { extractArgs, getAsset } from './../common'; import { createAssetTransaction } from './mapAsset'; +/** + * `NftHolder.nftIds` is a JSON array, and under historical mode every `.save()` writes a new + * versioned row carrying the whole array. A bulk mint — hundreds of `NFTPortfolioUpdated` for one + * holder in one block — turned that into Σ(1..n) array serialisations and poisoned the store + * cache with 30k-element arrays. So holder mutations are buffered per block and each holder is + * saved once, when the block changes (or `flushNftBuffer` is called from the block handler). + * Nothing inside the indexer reads `NftHolder` — it is written for external queries only — so a + * holder being at most one block-handler interval stale is acceptable. + */ +let bufferedBlock: string | undefined; +const bufferedHolders = new Map(); + +export const flushNftBuffer = async (): Promise => { + if (bufferedHolders.size === 0) { + return; + } + + await Promise.all([...bufferedHolders.values()].map(holder => holder.save())); + bufferedHolders.clear(); + bufferedBlock = undefined; +}; + +/** Test hook. */ +export const __resetNftBuffer = (): void => { + bufferedHolders.clear(); + bufferedBlock = undefined; +}; + +const bufferHolder = async (blockId: string, holder: NftHolder): Promise => { + if (blockId !== bufferedBlock) { + await flushNftBuffer(); + bufferedBlock = blockId; + } + + bufferedHolders.set(holder.id, holder); +}; + export const getNftHolder = async ( assetId: string, did: string, @@ -21,6 +58,12 @@ export const getNftHolder = async ( ): Promise => { const id = `${assetId}/${did}`; + // A holder mutated earlier in this same block lives in the buffer, not yet in the store. + const buffered = bufferedBlock === blockId ? bufferedHolders.get(id) : undefined; + if (buffered) { + return buffered; + } + let nftHolder = await NftHolder.get(id); if (!nftHolder) { @@ -88,7 +131,8 @@ export const handleNftHoldingsUpdates = async (event: SubstrateEvent): Promise !ids.includes(heldId)); nftHolder.updatedBlockId = blockId; - promises.push(nftHolder.save()); + await bufferHolder(blockId, nftHolder); } else if (reason === 'transferred' || reason === 'controllerTransfer') { const [fromHolder, toHolder] = await Promise.all([ getNftHolder(assetId, fromDid, blockId), @@ -104,8 +148,11 @@ export const handleNftHoldingsUpdates = async (event: SubstrateEvent): Promise !ids.includes(id)); toHolder.nftIds.push(...ids); + fromHolder.updatedBlockId = blockId; + toHolder.updatedBlockId = blockId; - promises.push(fromHolder.save(), toHolder.save()); + await bufferHolder(blockId, fromHolder); + await bufferHolder(blockId, toHolder); asset.totalTransfers += BigInt(1); diff --git a/src/mappings/entities/assets/mapStatistics.ts b/src/mappings/entities/assets/mapStatistics.ts index f2f2613f..ff0ecb19 100644 --- a/src/mappings/entities/assets/mapStatistics.ts +++ b/src/mappings/entities/assets/mapStatistics.ts @@ -19,6 +19,7 @@ import { getAllByFields, getTransferManagerValue, is7xChain, + isMigratedAssetId, } from '../../../utils'; import { Attributes, extractArgs } from '../common'; @@ -28,7 +29,7 @@ export const getAssetIdForStatisticsEvent = ( ): Promise => { let assetId: string; - if (is7xChain(block)) { + if (isMigratedAssetId(item) || is7xChain(block)) { assetId = item.toString(); } else { const scope = JSON.parse(item.toString()); diff --git a/src/mappings/entities/common.ts b/src/mappings/entities/common.ts index 2b53b128..a667fba7 100644 --- a/src/mappings/entities/common.ts +++ b/src/mappings/entities/common.ts @@ -35,6 +35,32 @@ export const getAsset = async (assetId: string): Promise => { return asset; }; +/** + * `getAsset`, but records a `MissingReferencedEntity` anomaly and returns `undefined` instead of + * throwing. For handlers where a missing asset is a data gap to note, not a reason to fail the + * whole block — e.g. an `AssetBalanceUpdated` whose `AssetCreated` was skipped upstream. + */ +export const getAssetOrAnomaly = async ( + assetId: string, + context: { block: SubstrateBlock; eventIdx?: number; eventId?: EventIdEnum } +): Promise => { + const asset = await Asset.get(assetId); + + if (asset) { + return asset; + } + + await recordAnomaly({ + kind: AnomalyKind.MissingReferencedEntity, + detail: `Asset ${assetId} was not found — its creation event was not indexed`, + block: context.block, + eventIdx: context.eventIdx, + eventId: context.eventId, + }); + + return undefined; +}; + /** * Context that lets an unmapped chain value be recorded as an `IndexerAnomaly` instead of * silently becoming `Unknown`. Optional so a caller with no block in hand still type checks, diff --git a/src/mappings/entities/events/mapStakingEvent.ts b/src/mappings/entities/events/mapStakingEvent.ts index 413c6666..b09ad2bc 100644 --- a/src/mappings/entities/events/mapStakingEvent.ts +++ b/src/mappings/entities/events/mapStakingEvent.ts @@ -4,6 +4,11 @@ import { SubstrateBlock, SubstrateEvent } from '@subql/types'; import { Account, EventIdEnum, StakingEvent } from '../../../types'; import { getBigIntValue, getTextValue } from '../../../utils'; import { is8xChain } from '../../../utils/common'; +import { + readRewardDestination, + resolveLegacyRewardDestination, + RewardDestinationName, +} from '../../../utils/staking'; import { extractArgs } from '../common'; const bondedUnbondedOrReward = new Set([ @@ -13,50 +18,25 @@ const bondedUnbondedOrReward = new Set([ EventIdEnum.Rewarded, // from 7.x Reward was renamed to Rewarded ]); -type RewardDestinationDetails = { - type: string; - account?: string; -}; - type StakingEventDetails = { amount?: bigint; stashAccount?: string; nominatedValidators?: string[]; identityId?: string; - rewardDestination?: string; + rewardDestination?: RewardDestinationName; rewardDestinationAccount?: string; }; -const getRewardDestinationDetails = (destParam: Codec): RewardDestinationDetails => { - const json = destParam.toJSON() as string | Record; - - if (typeof json === 'string') { - return { type: json }; - } - - const variant = Object.keys(json)[0] ?? 'Unknown'; - - const value = json[variant]; - - if (variant === 'Account') { - return { - type: variant, - account: typeof value === 'string' ? value : undefined, - }; - } - - return { type: variant }; -}; - const getRewardDestinationAccount = ( - destinationDetails: RewardDestinationDetails, + destination: RewardDestinationName, + account?: string, stashAccount?: string ): string | undefined => { - if (destinationDetails.type === 'Account') { - return destinationDetails.account; + if (destination === 'Account') { + return account; } - if (destinationDetails.type === 'Staked' || destinationDetails.type === 'Stash') { + if (destination === 'Staked' || destination === 'Stash') { return stashAccount; } @@ -87,13 +67,13 @@ const get8xStakingEventDetails = (eventId: EventIdEnum, params: Codec[]): Stakin const stashAccount = getTextValue(rawAccount); if (eventId === EventIdEnum.Rewarded) { - const destinationDetails = getRewardDestinationDetails(rawSecondParam); + const { destination, account } = readRewardDestination(rawSecondParam.toJSON()); return { stashAccount, amount: getBigIntValue(rawThirdParam), - rewardDestination: destinationDetails.type, - rewardDestinationAccount: getRewardDestinationAccount(destinationDetails, stashAccount), + rewardDestination: destination, + rewardDestinationAccount: getRewardDestinationAccount(destination, account, stashAccount), }; } @@ -107,10 +87,10 @@ const get8xStakingEventDetails = (eventId: EventIdEnum, params: Codec[]): Stakin return { stashAccount }; }; -const getLegacyStakingEventDetails = ( +const getLegacyStakingEventDetails = async ( eventId: EventIdEnum, params: Codec[] -): StakingEventDetails => { +): Promise => { const [rawDid, rawAccount] = params; const stashAccount = getTextValue(rawAccount); const details: StakingEventDetails = { @@ -121,8 +101,11 @@ const getLegacyStakingEventDetails = ( if (bondedUnbondedOrReward.has(eventId)) { details.amount = getBigIntValue(params[2]); - if (eventId === EventIdEnum.Reward || eventId === EventIdEnum.Rewarded) { - details.rewardDestination = 'LegacyUnknown'; + if ((eventId === EventIdEnum.Reward || eventId === EventIdEnum.Rewarded) && stashAccount) { + // A15 — the pre-v8 event names only the stash; read the payee from chain storage. + const resolved = await resolveLegacyRewardDestination(stashAccount); + details.rewardDestination = resolved.rewardDestination; + details.rewardDestinationAccount = resolved.rewardDestinationAccount; } } @@ -143,7 +126,7 @@ const getStakingEventDetails = async ( } else if (is8xChain(block)) { details = get8xStakingEventDetails(eventId, params); } else { - details = getLegacyStakingEventDetails(eventId, params); + details = await getLegacyStakingEventDetails(eventId, params); } if (details.stashAccount && !details.identityId) { diff --git a/src/mappings/entities/identities/mapClaim.ts b/src/mappings/entities/identities/mapClaim.ts index 7f01a1f9..8f409523 100644 --- a/src/mappings/entities/identities/mapClaim.ts +++ b/src/mappings/entities/identities/mapClaim.ts @@ -11,6 +11,7 @@ import { } from '../../../types'; import { END_OF_TIME, + emptyDid, extractClaimInfo, getAssetIdWithTicker, getTextValue, @@ -173,6 +174,13 @@ export const handleClaimRevoked = async (event: SubstrateEvent): Promise = const target = getTextValue(decodeEvent(event).did); + // Some early-chain revocations emit a stripped `ClaimRevoked` with a zero issuer and a `NoData` + // claim — there is no indexed claim these could match, and it is not a real attributable + // revocation, so it is skipped rather than recorded as a missing-entity anomaly. + if (!claimIssuer || claimIssuer === emptyDid) { + return; + } + const id = getId(target, claimIssuer, claimType, scope, jurisdiction, cddId, customClaimTypeId); const claim = await Claim.get(id); diff --git a/src/mappings/entities/identities/mapPolyxLedger.ts b/src/mappings/entities/identities/mapPolyxLedger.ts new file mode 100644 index 00000000..99ea186d --- /dev/null +++ b/src/mappings/entities/identities/mapPolyxLedger.ts @@ -0,0 +1,1442 @@ +import { Codec } from '@polkadot/types/types'; +import { SubstrateEvent } from '@subql/types'; +import { decodeEvent } from '../../../decode'; +import { + Account, + AccountBalance, + EntryDirection, + EventIdEnum, + HoldReason, + Identity, + MovementKind, + PolyxEntry, + PolyxPool, +} from '../../../types'; +import { bytesToString, getBigIntValue, getTextValue, padId } from '../../../utils'; +import { camelToSnakeCase, is8xChain, snakeToCamelCase } from '../../../utils/common'; +import { readStakingLock, resolveLegacyRewardDestination } from '../../../utils/staking'; +import { getAccountKeyType, getOrCreateAccount } from '../../../utils/accounts'; +import { getEventParams } from '../../../utils/events'; +import { extractArgs, HandlerArgs } from '../common'; +import { getAccountId, systematicIssuers } from '../../consts'; +import { reconcileAccount } from './reconcilePolyx'; + +/** + * POLYX ledger — entry-centric replacement for `mapPolyxTransaction`. + * + * Every balances-pallet movement decodes to a pool transition (the "Event → pool transition" + * table in docs/implementation/02-polyx-ledger.md), which this module turns into one `PolyxEntry` + * per account-side plus a running `AccountBalance`. `BalanceSet` (a checkpoint), locks and staking + * are layered on in the later commits of the phase. + */ + +// --------------------------------------------------------------------------------------------- +// Decoded-field helpers +// --------------------------------------------------------------------------------------------- + +/** + * A decoded field, tolerating the snake_case ⇄ camelCase difference between an upstream struct + * event (`free_balance`) and a Polymesh shape-table entry (`freeBalance`). Returns `undefined` + * rather than letting the decode proxy throw when the field is genuinely absent. + */ +const optionalField = (decoded: Record, name: string): Codec | undefined => { + for (const candidate of new Set([name, camelToSnakeCase(name), snakeToCamelCase(name)])) { + if (candidate in decoded) { + return decoded[candidate]; + } + } + + return undefined; +}; + +const firstText = (decoded: Record, names: string[]): string | undefined => { + for (const name of names) { + const value = optionalField(decoded, name); + + if (value !== undefined) { + return getTextValue(value); + } + } + + return undefined; +}; + +const holder = (decoded: Record): string | undefined => + firstText(decoded, ['who', 'account', 'stash']); + +const amountOf = (decoded: Record): bigint => { + for (const name of ['amount', 'balance', 'freeBalance', 'free', 'value', 'actualFee']) { + const value = optionalField(decoded, name); + + if (value !== undefined) { + return getBigIntValue(value); + } + } + + return BigInt(0); +}; + +const holdReasonOf = (decoded: Record): HoldReason | undefined => { + const raw = optionalField(decoded, 'reason'); + + if (raw === undefined) { + return undefined; + } + + const key = getTextValue(raw)?.toLowerCase(); + const match = Object.values(HoldReason).find(member => member.toLowerCase() === key); + + return match ?? HoldReason.Unknown; +}; + +const memoOf = (decoded: Record): string | undefined => { + const raw = optionalField(decoded, 'memo'); + + return raw !== undefined ? bytesToString(raw) : undefined; +}; + +const startOfUtcDay = (datetime: Date): Date => + new Date(Date.UTC(datetime.getUTCFullYear(), datetime.getUTCMonth(), datetime.getUTCDate())); + +const floorZero = (value: bigint): bigint => (value > BigInt(0) ? value : BigInt(0)); + +/** `Math.max` for `bigint`, which `Math.max` itself cannot take. */ +const maxBig = (a: bigint, b: bigint): bigint => (a > b ? a : b); + +/** + * `frozen` from an on-chain `AccountData`: `{ free, reserved, frozen, flags }` on v8, + * `{ free, reserved, miscFrozen, feeFrozen }` (frozen is the max of the two) on ≤ v7.4. + */ +export const accountDataFrozen = (data: Record): bigint => { + if (data.frozen !== undefined) { + return getBigIntValue(data.frozen); + } + + const misc = getBigIntValue(data.miscFrozen); + const fee = getBigIntValue(data.feeFrozen); + + return maxBig(misc, fee); +}; + +// --------------------------------------------------------------------------------------------- +// Account / balance state +// --------------------------------------------------------------------------------------------- + +/** + * The `Account` a POLYX-holding address belongs to. + * + * `getOrCreateAccount` covers every address the chain has a key record for. A pallet or system + * address (the treasury pot, the block-reward pot, …) holds POLYX without being a key, so a bare + * `Account` is created for it — `PolyxEntry.account` and `AccountBalance.account` are non-null + * relations and the account page query is keyed on them. + */ +export const ledgerAccount = async ( + address: string, + blockId: string, + datetime: Date +): Promise => { + const resolved = await getOrCreateAccount(address, blockId, datetime); + + if (resolved) { + return resolved; + } + + const account = Account.create({ + id: address, + address, + eventId: EventIdEnum.AccountCreated, + datetime, + ...getAccountKeyType(address), + createdBlockId: blockId, + updatedBlockId: blockId, + }); + + await account.save(); + + return account; +}; + +export const emptyBalance = ( + address: string, + identityId: string | undefined, + blockId: string +): AccountBalance => + AccountBalance.create({ + id: address, + accountId: address, + identityId, + free: BigInt(0), + reserved: BigInt(0), + frozen: BigInt(0), + total: BigInt(0), + transferable: BigInt(0), + bonded: BigInt(0), + otherReserved: BigInt(0), + totalReceived: BigInt(0), + totalSent: BigInt(0), + totalFeesPaid: BigInt(0), + totalRewards: BigInt(0), + totalSlashed: BigInt(0), + movementCount: 0, + lifetimeByKind: [], + locks: [], + holds: [], + updatedBlockId: blockId, + }); + +export const loadBalance = async ( + address: string, + identityId: string | undefined, + blockId: string +): Promise => { + const existing = await AccountBalance.get(address); + + if (existing) { + if (!existing.identityId && identityId) { + existing.identityId = identityId; + } + + return existing; + } + + return emptyBalance(address, identityId, blockId); +}; + +/** Pre-v8 staking bonds via a lock with this identifier; v8 bonds via a `Staking` hold. */ +export const STAKING_LOCK_ID = 'staking '; + +/** + * Recomputes every field that is a pure function of the pools, `locks` and `holds`: + * + * - `frozen` is the **MAX** over active locks, never a sum — the property the old model could not + * represent. + * - `bonded` is the staking lock (≤ v7.4) or the `Staking` hold (v8). + * - `total = free + reserved`; `transferable = free - frozen`, floored at 0. + */ +export const recomputeDerived = (balance: AccountBalance): void => { + const locks = balance.locks ?? []; + const holds = balance.holds ?? []; + + const stakingHold = holds.find(hold => hold.reason === HoldReason.Staking)?.amount ?? BigInt(0); + const stakingLock = locks.find(lock => lock.lockId === STAKING_LOCK_ID)?.amount ?? BigInt(0); + + balance.frozen = locks.reduce((max, lock) => maxBig(max, lock.amount), BigInt(0)); + balance.bonded = maxBig(stakingHold, stakingLock); + balance.otherReserved = floorZero(balance.reserved - stakingHold); + balance.total = balance.free + balance.reserved; + balance.transferable = floorZero(balance.free - balance.frozen); +}; + +const bumpLifetimeByKind = ( + balance: AccountBalance, + kind: MovementKind, + direction: EntryDirection, + amountAbs: bigint +): void => { + const totals = balance.lifetimeByKind ?? []; + const signed = direction === EntryDirection.Credit ? amountAbs : -amountAbs; + const entry = totals.find(total => total.kind === kind); + + if (entry) { + entry.totalAbs += amountAbs; + entry.net += signed; + entry.count += 1; + } else { + totals.push({ kind, totalAbs: amountAbs, net: signed, count: 1 }); + } + + balance.lifetimeByKind = totals; +}; + +// --------------------------------------------------------------------------------------------- +// Pool transition → entries + balance mutation +// --------------------------------------------------------------------------------------------- + +interface Endpoint { + address: string; + pool: PolyxPool; +} + +interface Transition { + /** debit side; absent when value entered the system (mint, endow, reward) */ + from?: Endpoint; + /** credit side; absent when value left the system (burn, slash, fee, dust) */ + to?: Endpoint; + amount: bigint; + kind: MovementKind; + holdReason?: HoldReason; + memo?: string; +} + +const poolTag = (pool: PolyxPool): string => (pool === PolyxPool.Free ? 'f' : 'r'); + +interface MovementSide { + endpoint: Endpoint; + direction: EntryDirection; + counterparty?: string; +} + +/** The one or two account-sides a transition touches, each paired with its counterparty. */ +const movementSides = (transition: Transition): MovementSide[] => { + const sides: MovementSide[] = []; + + if (transition.from?.address) { + sides.push({ + endpoint: transition.from, + direction: EntryDirection.Debit, + counterparty: transition.to?.address, + }); + } + + if (transition.to?.address) { + sides.push({ + endpoint: transition.to, + direction: EntryDirection.Credit, + counterparty: transition.from?.address, + }); + } + + return sides; +}; + +/** The lifetime aggregate a movement kind feeds, if any. */ +const LIFETIME_TOTAL: Partial< + Record +> = { + [MovementKind.Fee]: 'totalFeesPaid', + [MovementKind.StakingReward]: 'totalRewards', + [MovementKind.Slash]: 'totalSlashed', +}; + +/** Advances the running `AccountBalance` for one side of a movement (pools + aggregates). */ +const advanceBalance = ( + balance: AccountBalance, + side: MovementSide, + transition: Transition, + signed: bigint, + isInternal: boolean +): void => { + if (side.endpoint.pool === PolyxPool.Free) { + balance.free += signed; + } else { + balance.reserved += signed; + } + + if (!isInternal) { + if (side.direction === EntryDirection.Credit) { + balance.totalReceived += transition.amount; + } else { + balance.totalSent += transition.amount; + } + } + + const lifetimeTotal = LIFETIME_TOTAL[transition.kind]; + if (lifetimeTotal) { + balance[lifetimeTotal] += transition.amount; + } + + bumpLifetimeByKind(balance, transition.kind, side.direction, transition.amount); + balance.movementCount += 1; + recomputeDerived(balance); +}; + +interface TransitionContext { + args: HandlerArgs; + transition: Transition; + options: { eraIndex?: number }; + isInternal: boolean; + date: Date; + params: ReturnType; +} + +/** Advances one account, writes its `PolyxEntry`, and reconciles it. */ +const writeMovementSide = async ( + side: MovementSide, + { args, transition, options, isInternal, date, params }: TransitionContext +): Promise => { + const { blockId, block, eventIdx, blockEventId } = args; + const { address, pool } = side.endpoint; + const signed = side.direction === EntryDirection.Credit ? transition.amount : -transition.amount; + + const account = await ledgerAccount(address, blockId, block.timestamp); + const balance = await loadBalance(address, account.identityId, blockId); + + advanceBalance(balance, side, transition, signed, isInternal); + balance.updatedBlockId = blockId; + await balance.save(); + + const counterpartyAccount = side.counterparty ? await Account.get(side.counterparty) : undefined; + + await PolyxEntry.create({ + id: `${blockId}/${padId(eventIdx.toString())}/${poolTag(pool)}${ + side.direction === EntryDirection.Debit ? 'd' : 'c' + }`, + movementId: blockEventId, + accountId: address, + identityId: account.identityId, + counterpartyAddress: side.counterparty, + counterpartyIdentityId: counterpartyAccount?.identityId, + pool, + amount: signed, + amountAbs: transition.amount, + kind: transition.kind, + direction: side.direction, + holdReason: transition.holdReason, + memo: transition.memo, + freeAfter: balance.free, + reservedAfter: balance.reserved, + frozenAfter: balance.frozen, + moduleId: params.moduleId, + callId: params.callId, + eventId: params.eventId, + specVersionId: block.specVersion, + date, + eraIndex: options.eraIndex, + createdEventId: blockEventId, + extrinsicId: params.extrinsicId, + eventIdx, + datetime: block.timestamp, + createdBlockId: blockId, + }).save(); + + await reconcileAccount(address, blockId, block, { eventIdx }); +}; + +/** + * Writes the entries for one pool transition and advances every touched `AccountBalance`. + * + * Sibling entries of one movement share `movementId` (the block/event id). Each entry carries the + * balance-after snapshot, so Balance History is a pure index scan. + */ +export const postTransition = async ( + args: HandlerArgs, + transition: Transition, + options: { eraIndex?: number } = {} +): Promise => { + if (!transition.from && !transition.to) { + return; + } + + const isInternal = + transition.from?.address !== undefined && transition.from.address === transition.to?.address; + + const context: TransitionContext = { + args, + transition, + options, + isInternal, + date: startOfUtcDay(args.block.timestamp), + params: getEventParams(args), + }; + + for (const side of movementSides(transition)) { + await writeMovementSide(side, context); + } +}; + +/** + * v8-only hold tracking. `Held`/`Released`/`BurnedHeld` move the balance through `postTransition`; + * this keeps the per-reason breakdown in `AccountBalance.holds` so `bonded`/`otherReserved` stay + * derivable without a scan. On a v8 chain `SUM(holds) == reserved`. + */ +const adjustHold = async ( + address: string, + reason: HoldReason, + delta: bigint, + blockId: string +): Promise => { + const balance = await AccountBalance.get(address); + + if (!balance) { + return; + } + + const holds = balance.holds ?? []; + const entry = holds.find(hold => hold.reason === reason); + + if (entry) { + entry.amount = floorZero(entry.amount + delta); + } else if (delta > BigInt(0)) { + holds.push({ reason, amount: delta }); + } + + balance.holds = holds.filter(hold => hold.amount > BigInt(0)); + recomputeDerived(balance); + balance.updatedBlockId = blockId; + + await balance.save(); +}; + +// --------------------------------------------------------------------------------------------- +// Locks (Locked / Unlocked / Frozen / Thawed) — a floor on `free`, not a pool. No PolyxEntry. +// --------------------------------------------------------------------------------------------- + +/** + * Adjusts one lock on `address` by `delta`, then recomputes `frozen = MAX(active locks)`. + * + * Locks aggregate by maximum, not by sum: two overlapping locks of 100 and 150 leave + * `frozen = 150`. This is why each lock is tracked individually in `AccountBalance.locks` rather + * than folded into a single number. + * + * PIPs vote locks (`PIPS_LOCK_ID`) are not wired here — the pips pallet emits no lock/unlock + * event, only `Voted`, and attributing the deposit needs the proposal-deposit model. Follow-up. + */ +export const adjustLock = async ( + address: string, + lockId: string, + delta: bigint, + blockId: string, + reasons?: string +): Promise => { + const balance = await AccountBalance.get(address); + + if (!balance) { + return; + } + + const locks = balance.locks ?? []; + const entry = locks.find(lock => lock.lockId === lockId); + + if (entry) { + entry.amount = floorZero(entry.amount + delta); + if (reasons !== undefined) { + entry.reasons = reasons; + } + } else if (delta > BigInt(0)) { + locks.push({ lockId, amount: delta, reasons }); + } + + balance.locks = locks.filter(lock => lock.amount > BigInt(0)); + recomputeDerived(balance); + balance.updatedBlockId = blockId; + + await balance.save(); +}; + +/** Sets one lock on `address` to an absolute amount (0 clears it). */ +export const setLock = async ( + address: string, + lockId: string, + amount: bigint, + blockId: string, + reasons?: string +): Promise => { + const balance = await AccountBalance.get(address); + const current = balance?.locks?.find(lock => lock.lockId === lockId)?.amount ?? BigInt(0); + + await adjustLock(address, lockId, amount - current, blockId, reasons); +}; + +/** + * Sets the pre-v8 `'staking '` lock on `stash` to `staking.ledger.total` read from chain. + * + * The chain read is authoritative — it already accounts for the max-bond cap, the rounding of a + * compounded `Staked` reward, unbonding chunks, and slashes. When the ledger cannot be read the + * `fallbackDelta` keeps the old accumulator behaviour so the reconciler still has a base to work + * from. + */ +const syncStakingLock = async ( + stash: string, + fallbackDelta: bigint, + blockId: string +): Promise => { + const total = await readStakingLock(stash); + + if (total === undefined) { + await adjustLock(stash, STAKING_LOCK_ID, fallbackDelta, blockId, 'staking'); + return; + } + + await setLock(stash, STAKING_LOCK_ID, total, blockId, 'staking'); +}; + +const lockHandler = + (lockId: string, sign: bigint) => + async (event: SubstrateEvent): Promise => { + const { blockId, block } = extractArgs(event); + const decoded = decodeEvent(event); + const who = holder(decoded); + + if (!who) { + return; + } + + // Ensure the balance row exists so the lock has somewhere to live. + await ledgerAccount(who, blockId, block.timestamp); + const balance = await loadBalance(who, undefined, blockId); + await balance.save(); + + await adjustLock(who, lockId, sign * amountOf(decoded), blockId); + }; + +/** + * `balances.Locked` / `Unlocked` — the `LockableCurrency` floor on `free`. + * + * v8 adds one more source of `Unlocked`: the lock → hold storage migration. Polymesh holds a + * pre-v8 bond as `Currency::set_lock("staking ", …)`, and v8 converts it to a `Staking` hold, + * emitting per account — across two migration passes, in no extrinsic of the staker's own — + * `balances.Upgraded`, the paired `balances.Held{Staking}` (which `handleBalanceHeld` already + * turns into the `free → reserved` movement), and `balances.Unlocked{who, amount}` for the lock + * removal. Only the lock side is left to do here. Post-v8 nothing re-creates a `'staking '` + * `Currency` lock (new bonds are holds), so an account that still carries one on a v8 block has + * an un-migrated pre-v8 lock and the `Unlocked` it sees is that migration removing it — clear the + * `'staking '` lock rather than the generic `'balances'` one. Without this the pre-v8 staking lock + * lingers and `frozen` over-reports for the ~420k blocks between the hold appearing (pass 1) and + * the chain dropping the lock (pass 2). + */ +export const handleBalanceLocked = lockHandler('balances', BigInt(1)); + +const unlockGeneric = lockHandler('balances', BigInt(-1)); + +export const handleBalanceUnlocked = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + + if (is8xChain(args.block)) { + const who = holder(decodeEvent(event)); + const hasStakingLock = + !!who && + ((await AccountBalance.get(who))?.locks ?? []).some(lock => lock.lockId === STAKING_LOCK_ID); + + if (who && hasStakingLock) { + await setLock(who, STAKING_LOCK_ID, BigInt(0), args.blockId, 'staking'); + return; + } + } + + await unlockGeneric(event); +}; + +/** `balances.Frozen` / `Thawed` — the upstream `fungible` freeze, also a floor on `free`. */ +export const handleBalanceFrozen = lockHandler('freeze', BigInt(1)); +export const handleBalanceThawed = lockHandler('freeze', BigInt(-1)); + +// --------------------------------------------------------------------------------------------- +// Cross-event pairing helpers +// --------------------------------------------------------------------------------------------- + +/** Memos seen before their paired `Transfer`, keyed by extrinsic + endpoints + amount, per block. */ +let pendingMemoBlock: string | undefined; +let pendingMemos = new Map(); + +const memoKey = ( + extrinsicId: string | undefined, + from: string, + to: string, + amount: bigint +): string => `${extrinsicId ?? '-'}/${from}/${to}/${amount.toString()}`; + +const stashPendingMemo = ( + args: HandlerArgs, + from: string, + to: string, + amount: bigint, + memo: string +): void => { + if (pendingMemoBlock !== args.blockId) { + pendingMemoBlock = args.blockId; + pendingMemos = new Map(); + } + + pendingMemos.set(memoKey(args.extrinsicId, from, to, amount), memo); +}; + +const takePendingMemo = ( + args: HandlerArgs, + from: string, + to: string, + amount: bigint +): string | undefined => { + if (pendingMemoBlock !== args.blockId) { + return undefined; + } + + const key = memoKey(args.extrinsicId, from, to, amount); + const memo = pendingMemos.get(key); + + if (memo !== undefined) { + pendingMemos.delete(key); + } + + return memo; +}; + +/** + * Entries already written in this extrinsic for `(kind, account)`, narrowed to `amountAbs` here. + * + * `store.getByFields` reads the write cache before the database, so a row saved earlier in this + * block is visible. Only indexed fields can go in the filter (`amountAbs` is not one — the schema + * is at the 10-index cap), so the amount match is applied in memory. + */ +const findExtrinsicEntries = async ( + args: HandlerArgs, + kind: MovementKind, + account: string | undefined, + amountAbs: bigint +): Promise => { + if (!args.extrinsicId || !account) { + return []; + } + + const rows = await PolyxEntry.getByFields( + [ + ['extrinsicId', '=', args.extrinsicId], + ['kind', '=', kind], + ['accountId', '=', account], + ], + { limit: 50 } + ); + + return rows.filter(row => row.amountAbs === amountAbs); +}; + +/** + * Entries written earlier in this block for `account` of one of `kinds`, narrowed to `amountAbs`. + * + * Staking rewards arrive from `on_initialize`, not an extrinsic, so the reward event and any + * paired `balances` deposit can only be matched on the block. Used to keep a v8 reward from being + * counted twice — once as `Mint`, once as `StakingReward`. + */ +const findBlockEntries = async ( + blockId: string, + account: string | undefined, + amountAbs: bigint, + kinds: MovementKind[] +): Promise => { + if (!account) { + return []; + } + + const rows = await PolyxEntry.getByFields( + [ + ['createdBlockId', '=', blockId], + ['accountId', '=', account], + ], + { limit: 100 } + ); + + return rows.filter(row => kinds.includes(row.kind) && row.amountAbs === amountAbs); +}; + +// --------------------------------------------------------------------------------------------- +// Balances-pallet handlers +// --------------------------------------------------------------------------------------------- + +export const handleBalanceEndowed = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + await postTransition(args, { + to: { address: holder(decoded), pool: PolyxPool.Free }, + amount: amountOf(decoded), + kind: MovementKind.Endowment, + }); +}; + +export const handleBalanceTransfer = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + const from = firstText(decoded, ['from']); + const to = firstText(decoded, ['to']); + const amount = amountOf(decoded); + const memo = memoOf(decoded) ?? takePendingMemo(args, from, to, amount); + + const [endowment] = await findExtrinsicEntries(args, MovementKind.Endowment, to, amount); + + if (endowment) { + // `balances.transfer` to a fresh account emits `Endowed` (already crediting `to/Free`) and + // `Transfer`. Enrich the endowment with the sender and post only the debit side. + endowment.counterpartyAddress = from; + endowment.counterpartyIdentityId = (await Account.get(from))?.identityId; + + if (memo) { + endowment.memo = memo; + } + + await endowment.save(); + + await postTransition(args, { + from: { address: from, pool: PolyxPool.Free }, + amount, + kind: MovementKind.Transfer, + memo, + }); + + return; + } + + await postTransition(args, { + from: { address: from, pool: PolyxPool.Free }, + to: { address: to, pool: PolyxPool.Free }, + amount, + kind: MovementKind.Transfer, + memo, + }); +}; + +/** + * A2: `TransferWithMemo` is emitted alongside the classic `Transfer` for one `transfer_with_memo` + * call. It is never its own movement — it only supplies the memo. If the `Transfer` was already + * indexed this enriches it; otherwise the memo is stashed for the `Transfer` still to come. + */ +export const handleBalanceTransferWithMemo = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + const from = firstText(decoded, ['from']); + const to = firstText(decoded, ['to']); + const amount = amountOf(decoded); + const memo = memoOf(decoded); + + if (!memo) { + return; + } + + const existing = await findExtrinsicEntries(args, MovementKind.Transfer, to, amount); + + if (existing.length > 0) { + for (const entry of existing) { + entry.memo = memo; + await entry.save(); + } + + return; + } + + stashPendingMemo(args, from, to, amount, memo); +}; + +export const handleBalanceReserved = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + const who = holder(decoded); + + await postTransition(args, { + from: { address: who, pool: PolyxPool.Free }, + to: { address: who, pool: PolyxPool.Reserved }, + amount: amountOf(decoded), + kind: MovementKind.Hold, + }); +}; + +export const handleBalanceUnreserved = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + const who = holder(decoded); + + await postTransition(args, { + from: { address: who, pool: PolyxPool.Reserved }, + to: { address: who, pool: PolyxPool.Free }, + amount: amountOf(decoded), + kind: MovementKind.Release, + }); +}; + +export const handleReserveRepatriated = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + const from = firstText(decoded, ['from']); + const to = firstText(decoded, ['to']); + const status = firstText(decoded, ['destinationStatus'])?.toLowerCase(); + + await postTransition(args, { + from: { address: from, pool: PolyxPool.Reserved }, + to: { + address: to, + pool: status?.includes('reserved') ? PolyxPool.Reserved : PolyxPool.Free, + }, + amount: amountOf(decoded), + kind: MovementKind.ReserveRepatriation, + }); +}; + +export const handleBalanceHeld = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + const who = holder(decoded); + const amount = amountOf(decoded); + const reason = holdReasonOf(decoded) ?? HoldReason.Unknown; + + await postTransition(args, { + from: { address: who, pool: PolyxPool.Free }, + to: { address: who, pool: PolyxPool.Reserved }, + amount, + kind: MovementKind.Hold, + holdReason: reason, + }); + + await adjustHold(who, reason, amount, args.blockId); +}; + +export const handleBalanceReleased = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + const who = holder(decoded); + const amount = amountOf(decoded); + const reason = holdReasonOf(decoded) ?? HoldReason.Unknown; + + await postTransition(args, { + from: { address: who, pool: PolyxPool.Reserved }, + to: { address: who, pool: PolyxPool.Free }, + amount, + kind: MovementKind.Release, + holdReason: reason, + }); + + await adjustHold(who, reason, -amount, args.blockId); +}; + +export const handleBalanceBurnedHeld = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + const who = holder(decoded); + const amount = amountOf(decoded); + const reason = holdReasonOf(decoded) ?? HoldReason.Unknown; + + await postTransition(args, { + from: { address: who, pool: PolyxPool.Reserved }, + amount, + kind: MovementKind.Slash, + holdReason: reason, + }); + + await adjustHold(who, reason, -amount, args.blockId); +}; + +export const handleTransferOnHold = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + const from = firstText(decoded, ['source', 'from']); + const to = firstText(decoded, ['dest', 'to']); + const amount = amountOf(decoded); + const reason = holdReasonOf(decoded); + + await postTransition(args, { + from: { address: from, pool: PolyxPool.Reserved }, + to: { address: to, pool: PolyxPool.Reserved }, + amount, + kind: MovementKind.ReserveRepatriation, + holdReason: reason, + }); + + if (reason) { + await adjustHold(from, reason, -amount, args.blockId); + await adjustHold(to, reason, amount, args.blockId); + } +}; + +export const handleTransferAndHold = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + const from = firstText(decoded, ['source', 'from']); + const to = firstText(decoded, ['dest', 'to']); + const amount = amountOf(decoded); + const reason = holdReasonOf(decoded); + + await postTransition(args, { + from: { address: from, pool: PolyxPool.Free }, + to: { address: to, pool: PolyxPool.Reserved }, + amount, + kind: MovementKind.ReserveRepatriation, + holdReason: reason, + }); + + if (reason) { + await adjustHold(to, reason, amount, args.blockId); + } +}; + +/** `Burned` / `Slashed` / `Withdraw` / `AccountBalanceBurned` — value leaves `who/Free`. */ +export const handleBalanceBurned = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + await postTransition(args, { + from: { address: holder(decoded), pool: PolyxPool.Free }, + amount: amountOf(decoded), + kind: args.eventId === EventIdEnum.Slashed ? MovementKind.Slash : MovementKind.Burn, + }); +}; + +/** + * A3: `balances.Suspended` — an upstream (v8-only) event that reaps `who`'s free balance. The + * registration pointed at a `handleBalanceSuspended` that never existed. + */ +export const handleBalanceSuspended = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + await postTransition(args, { + from: { address: holder(decoded), pool: PolyxPool.Free }, + amount: amountOf(decoded), + kind: MovementKind.Burn, + }); +}; + +/** `Minted` / `Deposit` / `Restored` — value enters `who/Free`. */ +export const handleBalanceMinted = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + const who = holder(decoded); + const amount = amountOf(decoded); + + // A v8 staking payout can emit both `staking.Rewarded` and a `balances` deposit for the same + // POLYX. If the reward side already recorded it, this is not a second movement. + const reward = await findBlockEntries(args.blockId, who, amount, [MovementKind.StakingReward]); + + if (reward.length > 0) { + return; + } + + await postTransition(args, { + to: { address: who, pool: PolyxPool.Free }, + amount, + kind: MovementKind.Mint, + }); +}; + +/** + * A9: `balances.DustLost` — account reaping. The remaining free balance is destroyed; the row was + * never written (`DustLost: []`). + */ +export const handleDustLost = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + const account = firstText(decoded, ['account', 'who']); + + await postTransition(args, { + from: { address: account, pool: PolyxPool.Free }, + amount: amountOf(decoded), + kind: MovementKind.DustLost, + }); + + // account reaping — always reconcile the reaped account against chain state + await reconcileAccount(account, args.blockId, args.block, { + force: true, + eventIdx: args.eventIdx, + }); +}; + +// --------------------------------------------------------------------------------------------- +// BalanceSet — a checkpoint, not a movement (resolves A1 structurally) +// --------------------------------------------------------------------------------------------- + +interface PoolDelta { + pool: PolyxPool; + delta: bigint; +} + +const signOf = (delta: bigint): EntryDirection => + delta > BigInt(0) ? EntryDirection.Credit : EntryDirection.Debit; + +/** Sets each pool to its new absolute value, returning the non-zero deltas the set produced. */ +const applyBalanceSet = ( + balance: AccountBalance, + newFree: bigint, + newReserved: bigint | undefined +): PoolDelta[] => { + const deltas: PoolDelta[] = []; + + const freeDelta = newFree - balance.free; + balance.free = newFree; + if (freeDelta !== BigInt(0)) { + deltas.push({ pool: PolyxPool.Free, delta: freeDelta }); + } + + if (newReserved !== undefined) { + const reservedDelta = newReserved - balance.reserved; + balance.reserved = newReserved; + if (reservedDelta !== BigInt(0)) { + deltas.push({ pool: PolyxPool.Reserved, delta: reservedDelta }); + } + } + + return deltas; +}; + +/** + * `BalanceSet` *sets* `free` (and, pre-v8, `reserved`) to an absolute value — it is not a + * movement of that size. The old model recorded the set value as a delta, corrupting every + * running total after it. Here the pools are set directly and one `BalanceSetAdjustment` entry + * per changed pool records the delta so `SUM(entries)` still reconciles to the balance. + */ +export const handleBalanceSet = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const { blockId, block, eventIdx, blockEventId } = args; + const datetime = block.timestamp; + const decoded = decodeEvent(event); + + const who = holder(decoded) ?? firstText(decoded, ['account']); + const newFree = getBigIntValue(optionalField(decoded, 'free')); + const rawReserved = optionalField(decoded, 'reserved'); + const newReserved = rawReserved !== undefined ? getBigIntValue(rawReserved) : undefined; + + const account = await ledgerAccount(who, blockId, datetime); + const balance = await loadBalance(who, account.identityId, blockId); + + const deltas = applyBalanceSet(balance, newFree, newReserved); + + if (deltas.length > 0) { + balance.movementCount += 1; + for (const { delta } of deltas) { + bumpLifetimeByKind( + balance, + MovementKind.BalanceSetAdjustment, + signOf(delta), + delta > BigInt(0) ? delta : -delta + ); + } + } + + recomputeDerived(balance); + balance.updatedBlockId = blockId; + await balance.save(); + + const params = getEventParams(args); + const date = startOfUtcDay(datetime); + + for (const { pool, delta } of deltas) { + await PolyxEntry.create({ + id: `${blockId}/${padId(eventIdx.toString())}/${poolTag(pool)}s`, + movementId: blockEventId, + accountId: who, + identityId: account.identityId, + counterpartyAddress: undefined, + counterpartyIdentityId: undefined, + pool, + amount: delta, + amountAbs: delta > BigInt(0) ? delta : -delta, + kind: MovementKind.BalanceSetAdjustment, + direction: signOf(delta), + holdReason: undefined, + memo: undefined, + freeAfter: balance.free, + reservedAfter: balance.reserved, + frozenAfter: balance.frozen, + moduleId: params.moduleId, + callId: params.callId, + eventId: params.eventId, + specVersionId: block.specVersion, + date, + eraIndex: undefined, + createdEventId: blockEventId, + extrinsicId: params.extrinsicId, + eventIdx, + datetime, + createdBlockId: blockId, + }).save(); + } + + // a checkpoint should equal chain state — always reconcile right after it + await reconcileAccount(who, blockId, block, { force: true, eventIdx }); +}; + +// --------------------------------------------------------------------------------------------- +// Treasury and fees +// --------------------------------------------------------------------------------------------- + +const identityPrimaryAccount = async (did: string | undefined): Promise => + did ? (await Identity.get(did))?.primaryAccount : undefined; + +const treasuryPalletAccount = (): string => + getAccountId(systematicIssuers.treasury.accountId, api.registry.chainSS58); + +export const handleTreasuryDisbursement = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const [, rawToDid, rawTo, rawBalance] = args.params; + + // `TreasuryDisbursement(authorizingDid, targetDid, targetAccount?, amount)`. The first param is + // the committee that authorised the spend, **not** the source of funds — `treasury.disbursement` + // always moves POLYX out of the treasury pallet account (the mirror of the reimbursement + // handler's credit). Debiting `authorizingDid`'s primary key instead left the treasury drifting + // high forever and the committee account low (defect: the block-352,843 disbursements). + // + // (IdentityId, IdentityId, AccountId, Balance) from 5.0.0; (IdentityId, IdentityId, Balance) before + const hasToAddress = args.params.length >= 4; + const amount = getBigIntValue(hasToAddress ? rawBalance : rawTo); + const toAddress = + (hasToAddress ? getTextValue(rawTo) : undefined) ?? + (await identityPrimaryAccount(getTextValue(rawToDid))); + + const [existingTransfer] = await findExtrinsicEntries( + args, + MovementKind.Transfer, + toAddress, + amount + ); + + if (existingTransfer) { + // From 5.0.0 `treasury.disbursement` also emits `balances.Transfer{treasury → recipient}`; + // relabel both sides of it rather than writing a second movement. + const siblings = await PolyxEntry.getByFields( + [['movementId', '=', existingTransfer.movementId]], + { limit: 10 } + ); + + for (const sibling of siblings) { + sibling.kind = MovementKind.TreasuryDisbursement; + await sibling.save(); + } + + return; + } + + await postTransition(args, { + from: { address: treasuryPalletAccount(), pool: PolyxPool.Free }, + to: toAddress ? { address: toAddress, pool: PolyxPool.Free } : undefined, + amount, + kind: MovementKind.TreasuryDisbursement, + }); +}; + +export const handleTreasuryReimbursement = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const [, rawBalance] = args.params; + + // `TreasuryReimbursement(payerDid, amount)` — the amount routed to the treasury out of a fee. + // The payer's full fee is already debited by `protocolFee.FeeCharged` / + // `transactionPayment.TransactionFeePaid`, so this is the treasury's credit, not a refund to + // the payer. (The pre-5.4.1 author split is not emitted and stays outside the ledger.) + await postTransition(args, { + to: { address: treasuryPalletAccount(), pool: PolyxPool.Free }, + amount: getBigIntValue(rawBalance), + kind: MovementKind.TreasuryReimbursement, + }); +}; + +/** + * `protocolFee.FeeCharged` and `transactionPayment.TransactionFeePaid` — `who/Free → ∅`. + * + * Both carry `(AccountId, Balance)` as their first two parameters at every spec version they + * exist for (`TransactionFeePaid` adds a trailing `tip` that is not a POLYX movement), so these + * are read positionally rather than through the shape table. + */ +export const handleTransactionFeeCharged = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const [rawWho, rawAmount] = args.params; + + await postTransition(args, { + from: { address: getTextValue(rawWho), pool: PolyxPool.Free }, + amount: getBigIntValue(rawAmount), + kind: MovementKind.Fee, + }); +}; + +// --------------------------------------------------------------------------------------------- +// Staking — era-dependent, and inverted at v8 (defect A10, resolves A6) +// --------------------------------------------------------------------------------------------- + +/** + * ≤ v7.4: bonding is `set_lock(STAKING_ID, …)` — **no balance moves**. `Bonded`/`Unbonded`/ + * `Withdrawn` maintain the staking lock only, so `frozen` reflects it; they write **no + * `PolyxEntry`**. This is the A6 correction — the old `type: Bonded` rows asserted movements + * that never happened. + * + * v8: bonding is a Hold. The balance-side movement is the paired `balances.Held{reason:Staking}` + * / `Released` (written by the balances handlers). The `staking.*` events here become ledger + * state only, so they must **not** write a second entry or bonds/rewards double-count. + * + * NOT YET VERIFIED against a real v8 block: that `staking.Bonded` and `balances.Held{Staking}` + * are emitted within one extrinsic. If that pairing does not hold, v8 bonded POLYX is unindexed + * and this assumption must change — the reconciliation harness is designed to catch it. + */ + +/** `staking.PayoutStarted { eraIndex, validatorStash, … }` precedes the payout's `Rewarded` events. */ +let payoutEraBlock: string | undefined; +let payoutEraIndex: number | undefined; + +export const handlePayoutStarted = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + const raw = optionalField(decoded, 'eraIndex'); + + payoutEraBlock = args.blockId; + payoutEraIndex = raw !== undefined ? Number(getTextValue(raw)) : undefined; +}; + +const currentPayoutEra = (blockId: string): number | undefined => + payoutEraBlock === blockId ? payoutEraIndex : undefined; + +const stakingStash = (decoded: Record): string | undefined => + firstText(decoded, ['stash', 'account', 'staker', 'who']); + +/** + * The account a reward was actually paid to. + * + * v8: the `Rewarded` event carries the `RewardDestination`. Pre-v8 it carries only the stash, so + * `staking.payee(stash)` is read from chain storage (A15) — measured on mainnet to matter for a + * large share of pre-v8 rewards. + */ +const rewardRecipient = async ( + decoded: Record, + stash: string | undefined, + is8x: boolean +): Promise<{ recipient: string; restaked: boolean } | undefined> => { + if (!stash) { + return undefined; + } + + if (is8x) { + // v8: `dest: Staked` restakes via the paired `balances.Held{Staking}`, so no lock work here. + const dest = optionalField(decoded, 'dest'); + const json = dest?.toJSON() as string | Record | undefined; + + if (json && typeof json === 'object') { + return { + recipient: ((json.account ?? json.Account) as string | undefined) ?? stash, + restaked: false, + }; + } + + return { recipient: stash, restaked: false }; + } + + const { rewardDestination, rewardDestinationAccount } = await resolveLegacyRewardDestination( + stash + ); + + // Pre-v8 `Staked` auto-restakes: the reward lands in `free` and is immediately locked, and no + // `staking.Bonded` is emitted for it — so the lock has to be raised here. + return { + recipient: rewardDestinationAccount ?? stash, + restaked: rewardDestination === 'Staked', + }; +}; + +/** + * `staking.Reward` / `Rewarded` — `∅ → recipient/Free`, a real movement at both eras. + * + * On v8 the reward is deposited to the recipient's free balance (and, for `dest: Staked`, + * immediately held) by paired `balances` events — this relabels that `Mint` rather than writing a + * second movement. Pre-v8 there is no paired balances event, so the credit is written here. + */ +export const handleReward = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + const stash = stakingStash(decoded); + const amount = amountOf(decoded); + const eraIndex = currentPayoutEra(args.blockId); + const resolved = await rewardRecipient(decoded, stash, is8xChain(args.block)); + + if (!resolved) { + return; + } + + const { recipient, restaked } = resolved; + + // If a `balances` deposit for this reward was already recorded as a plain mint, relabel it. + const mints = await findBlockEntries(args.blockId, recipient, amount, [MovementKind.Mint]); + + if (mints.length > 0) { + for (const mint of mints) { + mint.kind = MovementKind.StakingReward; + mint.eraIndex = eraIndex; + await mint.save(); + } + + const balance = await AccountBalance.get(recipient); + if (balance) { + balance.totalRewards += amount; + await balance.save(); + } + } else { + await postTransition( + args, + { + to: { address: recipient, pool: PolyxPool.Free }, + amount, + kind: MovementKind.StakingReward, + }, + { eraIndex } + ); + } + + // Pre-v8 `Staked` payee: the reward is added to the staking lock in the same step. + if (restaked) { + await syncStakingLock(recipient, amount, args.blockId); + } +}; + +/** `staking.Slash` / `Slashed` — `staker/Free → ∅`, a real movement at both eras. */ +export const handleStakingSlash = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + const stash = stakingStash(decoded); + + await postTransition( + args, + { + from: { address: stash, pool: PolyxPool.Free }, + amount: amountOf(decoded), + kind: MovementKind.Slash, + }, + { eraIndex: currentPayoutEra(args.blockId) } + ); + + // A slash reduces `ledger.total`, and pre-v8 nothing else re-reads the lock — resync it. + if (stash && !is8xChain(args.block)) { + await syncStakingLock(stash, -amountOf(decoded), args.blockId); + } +}; + +const ensureBalanceRow = async ( + address: string, + blockId: string, + datetime: Date +): Promise => { + await ledgerAccount(address, blockId, datetime); + const balance = await loadBalance(address, undefined, blockId); + await balance.save(); +}; + +/** `staking.Bonded` — pre-v8 raises the staking lock; v8 is ledger state only (see `Held`). */ +export const handleBonded = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + + if (is8xChain(args.block)) { + return; + } + + const decoded = decodeEvent(event); + const stash = stakingStash(decoded); + + if (!stash) { + return; + } + + await ensureBalanceRow(stash, args.blockId, args.block.timestamp); + await syncStakingLock(stash, amountOf(decoded), args.blockId); +}; + +/** + * `staking.Unbonded` — the unbonding queue keeps the balance locked (`ledger.total` is unchanged + * until `withdraw_unbonded`), so the lock does not move here on either era. + */ +export const handleUnbonded = async (): Promise => { + // intentionally a no-op for the balance ledger +}; + +/** `staking.Withdrawn` — pre-v8 lowers the staking lock as matured chunks leave; v8 is state only. */ +export const handleWithdrawn = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + + if (is8xChain(args.block)) { + return; + } + + const decoded = decodeEvent(event); + const stash = stakingStash(decoded); + + if (!stash) { + return; + } + + await syncStakingLock(stash, -amountOf(decoded), args.blockId); +}; diff --git a/src/mappings/entities/identities/mapPolyxTransaction.ts b/src/mappings/entities/identities/mapPolyxTransaction.ts deleted file mode 100644 index ce0d71dd..00000000 --- a/src/mappings/entities/identities/mapPolyxTransaction.ts +++ /dev/null @@ -1,464 +0,0 @@ -import { SubstrateEvent } from '@subql/types'; -import BigNumber from 'bignumber.js'; -import { - Account, - BalanceTypeEnum, - EventIdEnum, - Identity, - ModuleIdEnum, - PolyxTransaction, -} from '../../../types'; -import { bytesToString, getBigIntValue, getEventParams, getTextValue } from '../../../utils'; -import { extract8xStakingAmount, getFirstKeyFromJson, is8xChain } from '../../../utils/common'; -import { HandlerArgs, extractArgs } from '../common'; -import { getPaginatedData } from './../../../utils/common'; - -const getBasicDetails = async ( - args: HandlerArgs -): Promise<{ address: string; amount: bigint; identityId: string | undefined }> => { - let address: string; - let amount: bigint; - let identityId: string | undefined; - if (is8xChain(args.block)) { - // On 8.x chain, staking events don't have DID as first param - // Bonded/Unbonded: [stash, amount] - 2 params - // Rewarded: [stash, dest, amount] - 3 params (dest is RewardDestination enum) - const [rawAddress, rawDest, rawAmount] = args.params; - address = getTextValue(rawAddress); - amount = extract8xStakingAmount(rawDest, rawAmount); - identityId = (await Account.get(address))?.identityId; - } else { - const [rawDid, rawAddress, rawBalance] = args.params; - identityId = getTextValue(rawDid); - address = getTextValue(rawAddress); - amount = getBigIntValue(rawBalance); - } - return { address, amount, identityId }; -}; - -export const handleTreasuryReimbursement = async (event: SubstrateEvent): Promise => { - const args = extractArgs(event); - const [rawIdentity, rawBalance] = args.params; - const did = getTextValue(rawIdentity); - const balance = getTextValue(rawBalance); - const { specVersion } = args.block; - - const identity = await Identity.get(did); - - /** - * Till chain 5.4.1, treasury reimbursement was only 80% of the actual amount deducted - * Post that the split between author/treasury was removed - */ - let amount: bigint; - if (specVersion < 5004001) { - amount = BigInt( - new BigNumber(balance || 0).multipliedBy(1.25).integerValue(BigNumber.ROUND_FLOOR).toString() - ); - } else { - amount = BigInt(balance || 0); - } - const details = getEventParams(args); - - if (details.extrinsicId) { - const transactions: PolyxTransaction[] = await getPaginatedData< - PolyxTransaction, - 'extrinsicId' - >('PolyxTransaction', 'extrinsicId', details.extrinsicId); - - const protocolFeePolyxTransaction: PolyxTransaction | undefined = transactions.find( - ({ eventId }) => eventId === EventIdEnum.FeeCharged - ); - - if (amount === protocolFeePolyxTransaction?.amount) { - // this is the case where treasury reimbursement is showing that 80% of protocol fee charged - // We ignore this case to insert in PolyxTransaction - return; - } - } - - await PolyxTransaction.create({ - ...details, - identityId: did, - address: identity?.primaryAccount, - toId: null, - toAddress: null, - amount, - type: BalanceTypeEnum.Free, - }).save(); -}; - -const processTreasuryDisbursementArgs = async (args: HandlerArgs) => { - let rawFromIdentity, rawToDid, rawTo, rawBalance; - - const specName = api.runtimeVersion.specName.toString(); - if (args.block.specVersion < 5000000 && specName !== 'polymesh_private_dev') { - [rawFromIdentity, rawToDid, rawBalance] = args.params; - } else { - [rawFromIdentity, rawToDid, rawTo, rawBalance] = args.params; - } - const identityId = getTextValue(rawFromIdentity); - const toId = getTextValue(rawToDid); - const amount = getBigIntValue(rawBalance); - - let toAddress = getTextValue(rawTo); - - if (!toAddress) { - ({ primaryAccount: toAddress } = await Identity.get(toId)); - } - - return { identityId, toId, toAddress, amount }; -}; - -export const handleTreasuryDisbursement = async (event: SubstrateEvent): Promise => { - const args = extractArgs(event); - - const { identityId, toId, toAddress, amount } = await processTreasuryDisbursementArgs(args); - - const details = getEventParams(args); - - if (details.extrinsicId) { - const transactions: PolyxTransaction[] = await getPaginatedData< - PolyxTransaction, - 'extrinsicId' - >('PolyxTransaction', 'extrinsicId', details.extrinsicId); - - const transferPolyxTransaction: PolyxTransaction | undefined = transactions.find( - ({ eventId }) => eventId === EventIdEnum.Transfer - ); - /** - * in case when `treasury.disbursement` extrinsic is used to disburse some amount to an identity, - * both `Transfer` and `TreasuryDisbursement` events are triggered, - * in this case we update the `Transfer` entry to reflect `Disbursement` - * and skip adding a separate `TreasuryDisbursement` entry - */ - if (amount === transferPolyxTransaction?.amount) { - // this is the case where treasury reimbursement is showing that 80% of protocol fee charged - // We ignore this case to insert in PolyxTransaction - transferPolyxTransaction.eventId = EventIdEnum.TreasuryDisbursement; - await PolyxTransaction.create(transferPolyxTransaction).save(); - return; - } - } - - const fromIdentity = await Identity.get(identityId); - - await PolyxTransaction.create({ - ...details, - identityId, - address: fromIdentity?.primaryAccount, - toId, - toAddress, - amount, - type: BalanceTypeEnum.Free, - }).save(); -}; - -export const handleBalanceTransfer = async (event: SubstrateEvent): Promise => { - const args = extractArgs(event); - - let address: string; - let toAddress: string; - let amount: bigint; - let memo: string | undefined; - let identityId: string | undefined; - let toId: string | undefined; - - if (is8xChain(args.block)) { - const [rawFromAddress, rawToAddress, rawBalance] = args.params; - address = getTextValue(rawFromAddress); - toAddress = getTextValue(rawToAddress); - amount = getBigIntValue(rawBalance); - if (args.params.length > 3) { - const rawMemo = args.params[3]; - memo = bytesToString(rawMemo); - } - identityId = (await Account.get(address))?.identityId; - toId = (await Account.get(toAddress))?.identityId; - } else { - const [rawFromDid, rawFrom, rawToDid, rawTo, rawBalance] = args.params; - - amount = getBigIntValue(rawBalance); - identityId = getTextValue(rawFromDid); - address = getTextValue(rawFrom); - toId = getTextValue(rawToDid); - toAddress = getTextValue(rawTo); - if (args.params.length > 5) { - const rawMemo = args.params[5]; - memo = bytesToString(rawMemo); - } - } - - const details = getEventParams(args); - - if (details.extrinsicId) { - const transactions: PolyxTransaction[] = await getPaginatedData< - PolyxTransaction, - 'extrinsicId' - >('PolyxTransaction', 'extrinsicId', details.extrinsicId); - - const endowedPolyxTransaction: PolyxTransaction | undefined = transactions.find( - ({ eventId }) => eventId === EventIdEnum.Endowed - ); - /** - * in case when `balances.transfer` extrinsic is used to transfer some balance - * to an account for the first time, both `Endowed` and `Transfer` events are triggered, - * in this case we update the `Endowed` entry to reflect details of the account from which - * transfer was initiated and skip adding a separate `Transfer` entry - */ - if (amount === endowedPolyxTransaction?.amount) { - // this is the case where treasury reimbursement is showing that 80% of protocol fee charged - // We ignore this case to insert in PolyxTransaction - endowedPolyxTransaction.identityId = identityId; - endowedPolyxTransaction.address = address; - endowedPolyxTransaction.memo = memo; - - await PolyxTransaction.create(endowedPolyxTransaction).save(); - return; - } - } - - await PolyxTransaction.create({ - ...details, - identityId, - address, - toId, - toAddress, - amount, - memo, - type: BalanceTypeEnum.Free, - }).save(); -}; - -export const handleReserveRepatriated = async (event: SubstrateEvent): Promise => { - const args = extractArgs(event); - - const [rawFromAddress, rawToAddress, rawAmount, rawType] = args.params; - - const fromAddress = getTextValue(rawFromAddress); - const toAddress = getTextValue(rawToAddress); - const amount = getBigIntValue(rawAmount); - const type = - getFirstKeyFromJson(rawType) === 'free' ? BalanceTypeEnum.Free : BalanceTypeEnum.Reserved; - - const details = getEventParams(args); - - await PolyxTransaction.create({ - ...details, - address: fromAddress, - identityId: (await Account.get(fromAddress))?.identityId, - toAddress, - toId: (await Account.get(toAddress))?.identityId, - amount, - type, - }).save(); -}; - -export const handleTransactionFeeCharged = async (event: SubstrateEvent): Promise => { - const args = extractArgs(event); - - const [rawAddress, rawActualFee] = args.params; - const address = getTextValue(rawAddress); - const amount = getBigIntValue(rawActualFee); - - const details = getEventParams(args); - if (details.extrinsicId) { - const transactions: PolyxTransaction[] = await getPaginatedData< - PolyxTransaction, - 'extrinsicId' - >('PolyxTransaction', 'extrinsicId', details.extrinsicId); - - const reimbursementTransaction: PolyxTransaction | undefined = transactions - .slice() - .reverse() - .find(({ eventId }) => eventId === EventIdEnum.TreasuryReimbursement); - /** - * From chain 5.4, with `TreasuryReimbursement` there is a `TransactionFeePaid` event as well. - * In this case, we will update the already inserted `TreasuryReimbursement` to point that it was indeed for done for `TransactionFeePaid` - * We also update the amount to get the exact value (since in treasury reimbursement, we calculate the amount as amount * 1.25 which can be off by some balance amount) - */ - if (reimbursementTransaction) { - reimbursementTransaction.address = address; - reimbursementTransaction.amount = amount; - reimbursementTransaction.moduleId = ModuleIdEnum.transactionpayment; - reimbursementTransaction.eventId = EventIdEnum.TransactionFeePaid; - await PolyxTransaction.create(reimbursementTransaction).save(); - return; - } - } - - const account = await Account.get(address); - - await PolyxTransaction.create({ - ...details, - identityId: account?.identityId, - address, - amount, - type: BalanceTypeEnum.Free, - }).save(); -}; - -// this is not affected by 8x chain -const handleBalanceAdded = async (event: SubstrateEvent, type: BalanceTypeEnum): Promise => { - const args = extractArgs(event); - - const [rawAddress, rawBalance] = args.params; - const toAddress = getTextValue(rawAddress); - const amount = getBigIntValue(rawBalance); - - const details = getEventParams(args); - const account = await Account.get(toAddress); - - await PolyxTransaction.create({ - ...details, - toAddress: toAddress, - toId: account?.identityId, - amount, - type, - }).save(); -}; - -const handleBalanceCharged = async ( - event: SubstrateEvent, - type: BalanceTypeEnum -): Promise => { - const args = extractArgs(event); - - const [rawAddress, rawBalance] = args.params; - const address = getTextValue(rawAddress); - const amount = getBigIntValue(rawBalance); - - const account = await Account.get(address); - - await PolyxTransaction.create({ - ...getEventParams(args), - address, - identityId: account?.identityId, - amount, - type, - }).save(); -}; - -const handleBalanceReceived = async ( - event: SubstrateEvent, - type: BalanceTypeEnum -): Promise => { - const args = extractArgs(event); - - const { address: toAddress, amount, identityId: toId } = await getBasicDetails(args); - - await PolyxTransaction.create({ - ...getEventParams(args), - toId, - toAddress, - amount, - type, - }).save(); -}; - -const handleBalanceSpent = async (event: SubstrateEvent, type: BalanceTypeEnum): Promise => { - const args = extractArgs(event); - - const { address, amount, identityId } = await getBasicDetails(args); - - await PolyxTransaction.create({ - ...getEventParams(args), - identityId, - address, - amount, - type, - }).save(); -}; - -export const handleBalanceSet = async (event: SubstrateEvent): Promise => { - const args = extractArgs(event); - - const { address: toAddress, amount, identityId: toId } = await getBasicDetails(args); - let reservedAmount: bigint; - - if (!is8xChain(args.block)) { - // BalanceSet(IdentityId, AccountId, free, reserved) — reserved is params[3], not params[4] - reservedAmount = getBigIntValue(args.params[3]); - } - - const details = getEventParams(args); - - // add the newly set free balance - await PolyxTransaction.create({ - ...details, - toId, - toAddress, - amount, - type: BalanceTypeEnum.Free, - }).save(); - - if (reservedAmount) { - // add the newly set reserve balance - await PolyxTransaction.create({ - ...details, - toId, - toAddress, - amount: reservedAmount, - type: BalanceTypeEnum.Reserved, - }).save(); - } -}; - -export const handleBalanceEndowed = async (event: SubstrateEvent): Promise => { - await handleBalanceReceived(event, BalanceTypeEnum.Free); -}; - -export const handleBalanceFrozen = async (event: SubstrateEvent): Promise => { - await handleBalanceCharged(event, BalanceTypeEnum.Locked); -}; - -export const handleBalanceLocked = async (event: SubstrateEvent): Promise => { - await handleBalanceCharged(event, BalanceTypeEnum.Locked); -}; - -export const handleBalanceUnlocked = async (event: SubstrateEvent): Promise => { - await handleBalanceAdded(event, BalanceTypeEnum.Free); -}; - -export const handleBalanceReserved = async (event: SubstrateEvent): Promise => { - await handleBalanceCharged(event, BalanceTypeEnum.Reserved); -}; - -export const handleBalanceMinted = async (event: SubstrateEvent): Promise => { - await handleBalanceAdded(event, BalanceTypeEnum.Free); -}; - -export const handleBalanceSlashed = async (event: SubstrateEvent): Promise => { - await handleBalanceSpent(event, BalanceTypeEnum.Free); -}; - -export const handleBalanceUnreserved = async (event: SubstrateEvent): Promise => { - await handleBalanceAdded(event, BalanceTypeEnum.Free); -}; - -export const handleBalanceBurned = async (event: SubstrateEvent): Promise => { - await handleBalanceSpent(event, BalanceTypeEnum.Free); -}; - -export const handleBonded = async (event: SubstrateEvent): Promise => { - await handleBalanceSpent(event, BalanceTypeEnum.Bonded); -}; - -export const handleUnbonded = async (event: SubstrateEvent): Promise => { - await handleBalanceReceived(event, BalanceTypeEnum.Unbonded); -}; - -export const handleReward = async (event: SubstrateEvent): Promise => { - await handleBalanceReceived(event, BalanceTypeEnum.Free); -}; - -export const handleWithdrawn = async (event: SubstrateEvent): Promise => { - await handleBalanceAdded(event, BalanceTypeEnum.Unbonded); -}; - -export const handleFeeCharged = async (event: SubstrateEvent): Promise => { - await handleBalanceCharged(event, BalanceTypeEnum.Free); -}; - -export const handleBalanceDeposit = async (event: SubstrateEvent): Promise => { - await handleBalanceAdded(event, BalanceTypeEnum.Free); -}; diff --git a/src/mappings/entities/identities/reconcilePolyx.ts b/src/mappings/entities/identities/reconcilePolyx.ts new file mode 100644 index 00000000..e9fc7152 --- /dev/null +++ b/src/mappings/entities/identities/reconcilePolyx.ts @@ -0,0 +1,156 @@ +import { Codec } from '@polkadot/types/types'; +import { SubstrateBlock } from '@subql/types'; +import { AccountBalance, AnomalyKind } from '../../../types'; +import { getBigIntValue } from '../../../utils'; +import { recordAnomaly } from '../../../utils/anomaly'; +import { accountDataFrozen, recomputeDerived, STAKING_LOCK_ID } from './mapPolyxLedger'; + +/** + * In-flight reconciliation (D11). + * + * `api.query` targets the block being indexed, and `.at` is unsupported, so authoritative state + * can only be read for the current block. This compares the derived `AccountBalance` against + * `system.account` there — every Nth block for accounts touched in that block, and always after + * a `BalanceSet` or `DustLost`. + * + * On a mismatch it records a `BalanceReconciliationDrift` anomaly **and corrects** the derived + * value, so drift from one missed or mis-signed event cannot compound into every later balance. + * The offline harness (`scripts/reconcile-polyx.ts`) is what answers "is the history right"; this + * is the going-forward safety net. + */ + +/** + * Sample rate for the routine check. `BalanceSet`/`DustLost` always reconcile regardless. + * Each sampled account costs one `system.account` RPC read; against a remote node this is the + * dominant cost of the genesis sweep, so the interval is coarse. The safety net still catches a + * mis-mapped event well before it can compound — a real defect drifts by thousands of POLYX and + * shows up at the next sample; the offline harness is what proves the history exact. + */ +const RECONCILE_EVERY_N_BLOCKS = 2000; + +/** + * Ignore drift below this (100 POLYX, 6 decimals). Two things produce sub-POLYX noise that is not + * a handler defect: the pre-v5.4 weight fee, which older Substrate charges with no event at all + * (so the ledger cannot see it and the balance runs a little high until this corrects it); and a + * sample landing mid-block on an account touched more than once, where the partial derived state + * is compared against the block's final on-chain state. A real mis-mapped or missed event drifts + * by thousands of POLYX. The offline harness sums with no threshold and catches slow accumulation. + */ +const MIN_DRIFT = BigInt(100_000_000); + +const abs = (value: bigint): bigint => (value < BigInt(0) ? -value : value); + +const blockNumber = (block: SubstrateBlock): number => Number(block.block.header.number.toString()); + +const shouldSample = (block: SubstrateBlock, force: boolean): boolean => + force || blockNumber(block) % RECONCILE_EVERY_N_BLOCKS === 0; + +interface OnChain { + free: bigint; + reserved: bigint; + frozen: bigint; +} + +/** + * `system.account` is end-of-block state, so within one block it is the same no matter how many + * times or how late it is read. An account touched N times in a sampled block would otherwise + * cost N identical RPC reads; this memoises the read for the current block (the compare and + * correct still run every call, so the last one — with the most complete derived state — wins). + */ +let onChainCacheBlock = -1; +const onChainCache = new Map(); + +/** Test hook — the cache is keyed only by block height, so a suite reusing one height must clear it. */ +export const __resetOnChainCache = (): void => { + onChainCacheBlock = -1; + onChainCache.clear(); +}; + +const readOnChain = async (address: string, blockHeight: number): Promise => { + if (blockHeight !== onChainCacheBlock) { + onChainCacheBlock = blockHeight; + onChainCache.clear(); + } + + const hit = onChainCache.get(address); + if (hit) { + return hit; + } + + const info = (await api.query.system.account(address)) as unknown as { + data: Record; + }; + + const onChain: OnChain = { + free: getBigIntValue(info.data.free), + reserved: getBigIntValue(info.data.reserved), + frozen: accountDataFrozen(info.data), + }; + + onChainCache.set(address, onChain); + + return onChain; +}; + +export const reconcileAccount = async ( + address: string, + blockId: string, + block: SubstrateBlock, + { force = false, eventIdx }: { force?: boolean; eventIdx?: number } = {} +): Promise => { + if (!address || !shouldSample(block, force)) { + return; + } + + const balance = await AccountBalance.get(address); + + if (!balance) { + return; + } + + let onChain: OnChain; + + try { + onChain = await readOnChain(address, blockNumber(block)); + } catch { + // A pruned node or a transient RPC error is not a ledger defect. + return; + } + + const drifts: string[] = []; + + if (abs(balance.free - onChain.free) >= MIN_DRIFT) { + drifts.push(`free ${balance.free} vs ${onChain.free}`); + } + if (abs(balance.reserved - onChain.reserved) >= MIN_DRIFT) { + drifts.push(`reserved ${balance.reserved} vs ${onChain.reserved}`); + } + if (abs(balance.frozen - onChain.frozen) >= MIN_DRIFT) { + drifts.push(`frozen ${balance.frozen} vs ${onChain.frozen}`); + } + + if (drifts.length === 0) { + return; + } + + await recordAnomaly({ + kind: AnomalyKind.BalanceReconciliationDrift, + detail: `${address}: ${drifts.join('; ')}`, + block, + eventIdx, + }); + + // Correct the derived value so the drift cannot compound. `frozen` is corrected by pinning the + // staking lock (which is nearly all of any pre-v8 account's frozen amount) to the on-chain + // value, so later `staking.*` events keep adjusting a realistic base rather than starting over. + balance.free = onChain.free; + balance.reserved = onChain.reserved; + balance.locks = + onChain.frozen > BigInt(0) + ? [{ lockId: STAKING_LOCK_ID, amount: onChain.frozen, reasons: 'staking' }] + : []; + recomputeDerived(balance); + balance.updatedBlockId = blockId; + + await balance.save(); +}; diff --git a/src/mappings/entities/index.ts b/src/mappings/entities/index.ts index 6136eeab..21bbe99b 100644 --- a/src/mappings/entities/index.ts +++ b/src/mappings/entities/index.ts @@ -18,7 +18,7 @@ export * from './identities/mapIdentities'; export * from './multiSig/mapMultiSig'; export * from './multiSig/mapMultiSigProposal'; export * from './assets/mapNfts'; -export * from './identities/mapPolyxTransaction'; +export * from './identities/mapPolyxLedger'; export * from './identities/mapPortfolio'; export * from './pips/mapProposal'; export * from './settlements/mapSettlement'; diff --git a/src/mappings/mappingHandlers.ts b/src/mappings/mappingHandlers.ts index 90ecfae0..2f9f7e7b 100644 --- a/src/mappings/mappingHandlers.ts +++ b/src/mappings/mappingHandlers.ts @@ -1,4 +1,4 @@ -import { SubstrateEvent } from '@subql/types'; +import { SubstrateBlock, SubstrateEvent } from '@subql/types'; import { logError } from '../utils'; import { getBlockContext } from './blockContext'; import { mapExternalAgentAction } from './entities'; @@ -7,6 +7,7 @@ import mapChainUpgrade from './entities/block/mapChainUpgrade'; import { handleExtrinsic } from './entities/block/mapExtrinsic'; import mapSubqueryVersion from './entities/block/mapSubqueryVersion'; import { handleToolingEvent } from './entities/events/mapEvent'; +import { flushNftBuffer } from './entities/assets/mapNfts'; import genesisHandler from './migrations/genesisHandler'; export async function handleGenesis(): Promise { @@ -22,6 +23,16 @@ export async function handleMigration(substrateEvent: SubstrateEvent): Promise logError(e)); } +/** + * Runs on a coarse block cadence (see `project.ts`). Its only job is to flush the per-block + * `NftHolder` write buffer so a bulk mint's last block is not left pending until the next NFT + * event — which, in a quiet period, could be a long way off. + */ +// eslint-disable-next-line @typescript-eslint/no-unused-vars +export async function handleBlock(_block: SubstrateBlock): Promise { + await flushNftBuffer().catch(e => logError(e)); +} + export async function handleStartup(): Promise { /** * This handles the insertion of new SQ version on every restart. diff --git a/src/mappings/migrations/genesisHandler.ts b/src/mappings/migrations/genesisHandler.ts index ae6c44a2..e03fa75b 100644 --- a/src/mappings/migrations/genesisHandler.ts +++ b/src/mappings/migrations/genesisHandler.ts @@ -25,6 +25,7 @@ import { createMultiSigSigner, } from '../entities/multiSig/mapMultiSig'; import { upsertEvmAccountMapping } from '../entities/revive/mapEvmAccountMapping'; +import { seedAccountBalances } from '../../seed/accountBalance'; const genesisBlock = padId('0'); type DidWithAccount = { did: string; accountId: string }; @@ -284,5 +285,9 @@ export default async (): Promise => { // runs last so that it can link to the Accounts created above await handleEvmAccountMappings(datetime); + // opening balance snapshot for the POLYX ledger — without it every derived balance is wrong by + // the genesis allocation (docs/implementation/02-polyx-ledger.md) + await seedAccountBalances({ blockId: genesisBlock, datetime }); + logger.info('Applied genesis migrations'); }; diff --git a/src/seed/accountBalance.ts b/src/seed/accountBalance.ts new file mode 100644 index 00000000..90051655 --- /dev/null +++ b/src/seed/accountBalance.ts @@ -0,0 +1,67 @@ +import { Codec } from '@polkadot/types/types'; +import { AccountBalance } from '../types'; +import { getBigIntValue } from '../utils'; +import { + accountDataFrozen, + emptyBalance, + ledgerAccount, + recomputeDerived, +} from '../mappings/entities/identities/mapPolyxLedger'; + +/** + * Snapshots `system.account` into `AccountBalance` rows. + * + * The POLYX ledger derives every balance from events, so without an opening snapshot every + * derived balance is wrong by the genesis allocation. `genesisHandler` seeds Accounts, + * Identities and Portfolios but **no balances** — this fills that gap. + * + * Written as a domain seeder rather than inline in `genesisHandler` because plan + * [10](../../docs/implementation/10-partial-index.md) needs the identical read at an arbitrary + * `START_BLOCK`: `api.query` always targets the block being indexed, so the same call seeds + * genesis when run from the genesis handler and block B when run from the partial-index seeder. + */ + +export interface SeedContext { + blockId: string; + datetime: Date; +} + +export const seedAccountBalances = async ({ + blockId, + datetime, +}: SeedContext): Promise<{ seeded: number }> => { + const entries = await api.query.system.account.entries(); + + const rows: AccountBalance[] = []; + + for (const [key, accountInfo] of entries) { + const address = key.args[0].toString(); + const data = (accountInfo as unknown as { data: Record }).data; + + const free = getBigIntValue(data.free); + const reserved = getBigIntValue(data.reserved); + const frozen = accountDataFrozen(data); + + if (free === BigInt(0) && reserved === BigInt(0) && frozen === BigInt(0)) { + continue; + } + + const account = await ledgerAccount(address, blockId, datetime); + const balance = emptyBalance(address, account.identityId, blockId); + + balance.free = free; + balance.reserved = reserved; + // A genesis freeze is recorded as a single lock so `frozen` stays a MAX going forward. + balance.locks = + frozen > BigInt(0) ? [{ lockId: 'genesis', amount: frozen, reasons: undefined }] : []; + recomputeDerived(balance); + + rows.push(balance); + } + + await Promise.all(rows.map(row => row.save())); + + logger.info(`Seeded ${rows.length} AccountBalance rows from system.account at ${blockId}`); + + return { seeded: rows.length }; +}; diff --git a/src/utils/accounts.ts b/src/utils/accounts.ts index fbe377a5..be98638b 100644 --- a/src/utils/accounts.ts +++ b/src/utils/accounts.ts @@ -3,10 +3,18 @@ import { Codec } from '@polkadot/types/types'; import { u8aToHex } from '@polkadot/util'; import { getAccountCache } from '../mappings/blockContext'; import { createIdentity, createPermissions } from '../mappings/entities/identities/mapIdentities'; +import { createPortfolio } from '../mappings/entities/identities/mapPortfolio'; import { Attributes } from '../mappings/entities/common'; import { Account, EventIdEnum, Identity } from '../types'; -import { getFirstKeyFromJson, getFirstValueFromJson } from './common'; +import { + extractString, + getFirstKeyFromJson, + getFirstValueFromJson, + getTextValue, + padId, +} from './common'; import { evmAddressFromSs58, isEthDerivedAddress } from './eth'; +import { legacyQuery } from './legacyQuery'; export const serializeAccount = (item: Codec): string | undefined => { const s = item.toString(); @@ -36,6 +44,47 @@ export const getAccountKeyType = ( }; }; +/** + * The DID an address is a key of, and whether it is the primary key. + * + * `identity.keyRecords` is a 5.x rename of `identity.keyToIdentityIds`; a genesis resync sees the + * older name on early blocks. The legacy storage is `Option` on both public chains + * (Polymesh launched at v3, so there is no `LinkedKeyInfo` enum to unwrap), and primary vs + * secondary is read from `identity.didRecords`. + */ +const resolveKeyIdentity = async ( + address: string +): Promise<{ did: string; type: 'primaryKey' | 'secondaryKey' } | undefined> => { + if (typeof api.query.identity.keyRecords === 'function') { + const raw = (await api.query.identity.keyRecords(address)) as unknown as Codec; + + if (raw.isEmpty) { + return undefined; + } + + return { + did: getFirstValueFromJson(raw), + type: getFirstKeyFromJson(raw) === 'primaryKey' ? 'primaryKey' : 'secondaryKey', + }; + } + + const raw = (await legacyQuery( + 'identity', + 'keyToIdentityIds', + [3000, 5_000_002] + )(address)) as unknown as Codec; + + if (raw.isEmpty) { + return undefined; + } + + const did = getTextValue(raw); + const record = (await api.query.identity.didRecords(did)).toJSON() as Record; + const primaryKey = extractString(record, 'primary_key'); + + return { did, type: primaryKey === address ? 'primaryKey' : 'secondaryKey' }; +}; + /** * The `Account` an address belongs to, creating it and its identity when the chain knows of one. * @@ -65,22 +114,33 @@ export const getOrCreateAccount = async ( return account; } - const rawKeyRecord = (await api.query.identity.keyRecords(address)) as unknown as Codec; + const keyIdentity = await resolveKeyIdentity(address); - if (rawKeyRecord.isEmpty) { + if (!keyIdentity) { cache.set(address, undefined); return; } - const did = getFirstValueFromJson(rawKeyRecord); - const type = getFirstKeyFromJson(rawKeyRecord); + const { did, type } = keyIdentity; const eventId = EventIdEnum.AccountCreated; const identity = await Identity.get(did); - if (!identity || (type === 'primaryKey' && identity.primaryAccount !== address)) { + if (!identity) { + await createIdentity( + { did, eventId, datetime, primaryAccount: address, secondaryKeysFrozen: false }, + blockId + ); + + // The default portfolio, so a later `identity.DidCreated` for this DID finds it — its handler + // only creates portfolio 0 when it creates the identity, and this path got there first. + await createPortfolio( + { identityId: did, number: 0, eventIdx: 0, createdEventId: `${blockId}/${padId('0')}` }, + blockId + ); + } else if (type === 'primaryKey' && identity.primaryAccount !== address) { await createIdentity( { did, eventId, datetime, primaryAccount: address, secondaryKeysFrozen: false }, blockId diff --git a/src/utils/assets.ts b/src/utils/assets.ts index fff78dd3..0048a62a 100644 --- a/src/utils/assets.ts +++ b/src/utils/assets.ts @@ -112,11 +112,29 @@ export const getAssetIdForLegacyTicker = async (ticker: Codec | string): Promise return u8aToHex(rawBytes); }; +/** + * Whether a raw asset identifier is already a migrated 16-byte asset ID, rather than a legacy + * ticker. + * + * The public chain switched `asset` events from carrying a 12-byte `Ticker` to a 16-byte + * `PolymeshPrimitivesAssetAssetId` at v7.0.0. The spec-version gate below would be enough if the + * block always reported its true runtime — but `@subql/node` has been seen serving the + * *pre-upgrade* spec for a long run of blocks after v7.0.0 actually activated (testnet: ~169k + * blocks reported as spec 6003050 instead of 7000003, block 15,978,579 onward). A byte-length + * test is immune to that: a `Ticker` is `[u8; 12]`, so any `0x`-prefixed value that decodes to + * 16 bytes is unambiguously a migrated asset ID whatever spec the block claims. + */ +export const isMigratedAssetId = (value: string | Codec): boolean => { + const hex = typeof value === 'string' ? value : value.toString(); + + return hexHasPrefix(hex) && hexStripPrefix(hex).length === 32; +}; + export const getAssetId = async ( assetId: string | Codec, block: SubstrateBlock ): Promise => { - if (is7xChain(block)) { + if (isMigratedAssetId(assetId) || is7xChain(block)) { return typeof assetId === 'string' ? assetId : assetId.toString(); } @@ -138,7 +156,7 @@ export const getAssetIdWithTicker = async ( ): Promise => { let assetId: string; let ticker: string; - if (is7xChain(block)) { + if (isMigratedAssetId(assetIdOrTicker) || is7xChain(block)) { assetId = typeof assetIdOrTicker === 'string' ? assetIdOrTicker : assetIdOrTicker.toString(); const asset = await Asset.get(assetId); diff --git a/src/utils/common.ts b/src/utils/common.ts index c4553ef0..12b5a65b 100644 --- a/src/utils/common.ts +++ b/src/utils/common.ts @@ -377,15 +377,3 @@ export const getAllByFields = async ( offset += page.length; } }; - -/** - * @deprecated Single-field adapter for `getAllByFields`. - * - * Only `mapPolyxTransaction` still calls this, and the POLYX ledger phase deletes that file - * along with the `PolyxTransaction` entity. It goes with it. - */ -export const getPaginatedData = async ( - entityName: string, - field: F, - param: T[F] -): Promise => getAllByFields(entityName, [[field, '=', param]]); diff --git a/src/utils/index.ts b/src/utils/index.ts index 769042af..653de68b 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -13,5 +13,6 @@ export * from './multisigs'; export * from './portfolios'; export * from './proposals'; export * from './settlements'; +export * from './staking'; export * from './stos'; export * from './transferManagers'; diff --git a/src/utils/staking.ts b/src/utils/staking.ts new file mode 100644 index 00000000..dc34c2da --- /dev/null +++ b/src/utils/staking.ts @@ -0,0 +1,182 @@ +import type { AnyJson } from '@polkadot/types/types'; + +/** + * The chain's own `RewardDestination` variants, plus the placeholder recorded when the payee + * cannot be read at all. + * + * Spelled out rather than derived from `PalletStakingRewardDestination['type']`: `src/index.ts` + * loads both `polymesh-types` and `@polkadot/types-augment`, and each declares that interface + * into `@polkadot/types/lookup`. The duplicate declaration collapses `type` to a bare `string` + * (the conflict is in `node_modules`, so `skipLibCheck` hides it) — `is*`/`as*` survive it, + * `type` does not — so deriving the union would silently widen it back to `string`. + */ +export type RewardDestinationName = + | 'Staked' + | 'Stash' + | 'Controller' + | 'Account' + | 'None' + | 'LegacyUnknown'; + +export interface LegacyRewardDestination { + rewardDestination: RewardDestinationName; + /** The account the reward was actually paid to, where it can be resolved */ + rewardDestinationAccount?: string; +} + +/** `.toJSON()` camel-cases the variant name; this maps it back to the name the index records. */ +const rewardDestinationByVariant: Record = { + staked: 'Staked', + stash: 'Stash', + controller: 'Controller', + account: 'Account', + none: 'None', +}; + +/** + * Reads a decoded `RewardDestination` — from storage or from an event parameter — out of its + * `.toJSON()` form. + * + * `.toJSON()` rather than the generated `Option` accessors, + * deliberately: the generated types describe one metadata snapshot — the current one — while + * both callers read blocks from runtimes that predate it, and `api` decodes against the block's + * own registry. `.unwrap()` written against today's `OptionQuery` throws on a block where the + * entry decodes as a bare `RewardDestination`, and that throw lands in a `catch` that would turn + * every reward into `LegacyUnknown`. `.toJSON()` is the one accessor whose output is the same + * either way: `null`, a bare string (`"Staked"` — when every variant of that runtime's type is a + * unit variant), or a single-key object with the variant camel-cased (`{ staked: null }`, + * `{ account: "0x…" }`). + * + * An unrecognised variant resolves to `None` — no destination account is claimed for it. + */ +export const readRewardDestination = ( + json: AnyJson +): { destination: RewardDestinationName; account?: string } => { + let variant = ''; + let value: AnyJson = null; + + if (typeof json === 'string') { + variant = json; + } else if (json && typeof json === 'object' && !Array.isArray(json)) { + [variant = ''] = Object.keys(json); + value = json[variant] ?? null; + } + + const destination = rewardDestinationByVariant[variant.toLowerCase()] ?? 'None'; + + return destination === 'Account' && typeof value === 'string' + ? { destination, account: value } + : { destination }; +}; + +/** + * Per-stash cache of a resolved payee. `staking.payee(stash)` is a chain-storage read on every + * reward, and a validator's set of ~20 stashes is re-queried every era for the whole genesis + * replay — the dominant RPC cost of the sweep against a remote node. A payee changes very rarely + * (an explicit `staking.setPayee`), and the in-flight reconciler corrects any `frozen` drift a + * stale entry could cause within ~1 era, so a plain process-lifetime cache is the right trade. + * Only successful resolutions are cached — a transient read failure must be retried, not pinned. + */ +const payeeCache = new Map(); + +/** Test hook — the cache is process-lifetime, so a suite that re-mocks `staking.payee` must clear it. */ +export const __resetPayeeCache = (): void => payeeCache.clear(); + +/** + * Resolves where a pre-v8 staking reward for `stash` was actually paid. + * + * Defect A15: the pre-8.x `Reward`/`Rewarded` event carries only the stash and the amount. A + * staker who set a payee of `Controller` or an explicit `Account` received the POLYX somewhere + * the event does not name. Measured across a spread of eras on mainnet, a large share of pre-v8 + * rewards went somewhere other than the stash, so the destination is read from + * `staking.payee(stash)` — chain storage, at the block being indexed (`api.query` targets the + * current block). Cheap during the D5 genesis replay; needs an archive node afterwards, which is + * why it is done now rather than deferred. + */ +export const resolveLegacyRewardDestination = async ( + stash: string +): Promise => { + const cached = payeeCache.get(stash); + if (cached) { + return cached; + } + + try { + const payee = await api.query.staking.payee(stash); + const { destination, account } = readRewardDestination(payee.toJSON()); + + let result: LegacyRewardDestination; + + if (destination === 'Staked' || destination === 'Stash') { + result = { + rewardDestination: destination, + rewardDestinationAccount: stash, + }; + } else if (destination === 'Controller') { + const controller = (await api.query.staking.bonded(stash)).toJSON(); + + result = { + rewardDestination: 'Controller', + rewardDestinationAccount: typeof controller === 'string' ? controller : undefined, + }; + } else if (destination === 'Account') { + result = { + rewardDestination: 'Account', + rewardDestinationAccount: account, + }; + } else { + result = { rewardDestination: 'None' }; + } + + payeeCache.set(stash, result); + + return result; + } catch { + // A pruned node, or a runtime with no `staking.payee` storage — fall back to the placeholder. + return { rewardDestination: 'LegacyUnknown' }; + } +}; + +/** + * Cache of `stash -> controller`. `staking.ledger` is keyed by the controller, not the stash, so + * a `bonded(stash)` read is needed before every ledger read. `set_controller` is very rare, so a + * process-lifetime cache is safe; a stale entry only matters if the controller changed, and the + * in-flight reconciler corrects the resulting `frozen` drift. + */ +const controllerCache = new Map(); + +/** Test hook — the cache is process-lifetime, so a suite re-mocking `staking.bonded` must clear it. */ +export const __resetControllerCache = (): void => controllerCache.clear(); + +/** + * The pre-v8 staking lock on `stash`, read from chain: `staking.ledger(controller).total` + * (bonded active + everything still unlocking), which is exactly the value `pallet-staking` + * passes to `Currency::set_lock`, so it is what `miscFrozen` reports. + * + * Read rather than accumulated from `Bonded` / `Withdrawn` / restaked-`Reward` deltas: the deltas + * do not see the max-bond cap, the rounding of a compounded `RewardDestination::Staked` reward, + * or a slash — each of which leaves the accumulator drifting from the real lock. + * + * `undefined` when the ledger cannot be read (a runtime with a different shape, a pruned node) — + * the caller keeps its delta accumulator as the fallback. A killed ledger (fully withdrawn) + * reads back as `0`. + */ +export const readStakingLock = async (stash: string): Promise => { + try { + let controller = controllerCache.get(stash); + + if (!controller) { + const bonded = (await api.query.staking.bonded(stash)).toJSON(); + controller = typeof bonded === 'string' ? bonded : stash; + controllerCache.set(stash, controller); + } + + const ledger = (await api.query.staking.ledger(controller)).toJSON() as { + total?: string | number; + } | null; + + return ledger ? BigInt(ledger.total ?? 0) : BigInt(0); + } catch { + return undefined; + } +}; diff --git a/tests/unit/assetId.test.ts b/tests/unit/assetId.test.ts new file mode 100644 index 00000000..8da9b132 --- /dev/null +++ b/tests/unit/assetId.test.ts @@ -0,0 +1,66 @@ +/** + * `getAssetId` maps a raw `asset` event identifier to the id the `Asset` row is keyed on. + * + * The public chain switched `asset` events from a 12-byte `Ticker` to a 16-byte + * `PolymeshPrimitivesAssetAssetId` at v7.0.0. The switch was gated on the block's spec version — + * but `@subql/node` was seen serving the pre-upgrade spec (6003050) for ~169k blocks after + * v7.0.0 actually activated on testnet (block 15,978,579 onward), so an already-migrated + * 16-byte asset id in one of those blocks was run through `getAssetIdForLegacyTicker` and + * blake2-hashed into a bogus id — its `Asset` lookup then missed (`MissingReferencedEntity`). + * + * The fix disambiguates by byte length: a `Ticker` is `[u8; 12]`, so a 16-byte value is a + * migrated asset id whatever spec the block claims. + */ + +import { SubstrateBlock } from '@subql/types'; +import { getAssetId, getAssetIdForLegacyTicker, isMigratedAssetId } from '../../src/utils/assets'; + +const globalAny = globalThis as any; + +const block = (specVersion: number): SubstrateBlock => + ({ specVersion } as unknown as SubstrateBlock); + +/** A 16-byte migrated asset id (ticker "PSRF" as it exists on testnet from v7.0.0). */ +const ASSET_ID = '0x8f68f310c5ea8f27a189154812efd457'; +/** "PSRF" as a 12-byte ticker, hex (0x + 24 chars). */ +const TICKER_HEX = '0x505352460000000000000000'; + +beforeEach(() => { + globalAny.chainId = '0xnotstaging'; + globalAny.api.runtimeVersion.specName = { toString: () => 'polymesh' }; +}); + +describe('isMigratedAssetId', () => { + it('is true only for a 16-byte hex value', () => { + expect(isMigratedAssetId(ASSET_ID)).toBe(true); + expect(isMigratedAssetId({ toString: () => ASSET_ID } as any)).toBe(true); + }); + + it('is false for a 12-byte ticker, a short hex, or a plain string', () => { + expect(isMigratedAssetId(TICKER_HEX)).toBe(false); + expect(isMigratedAssetId('0xdead')).toBe(false); + expect(isMigratedAssetId('PSRF')).toBe(false); + }); +}); + +describe('getAssetId', () => { + it('returns a 16-byte asset id unchanged on a 7.x block', async () => { + expect(await getAssetId(ASSET_ID, block(7_000_003))).toBe(ASSET_ID); + }); + + it('hashes a legacy ticker on a pre-7.x block', async () => { + // "PSRF" was created pre-v7 as a ticker; the chain migration derives its asset id + // deterministically, and `getAssetIdForLegacyTicker` reproduces that — it is ASSET_ID. + expect(await getAssetId('PSRF', block(6_003_040))).toBe( + await getAssetIdForLegacyTicker('PSRF') + ); + expect(await getAssetId('PSRF', block(6_003_040))).toBe(ASSET_ID); + }); + + it('returns a 16-byte asset id unchanged even when the block reports a stale pre-7.x spec', async () => { + // the regression: v7.0.0 was live and the event already carried the 16-byte asset id, but + // @subql/node still reported spec 6003050 — the old code re-hashed ASSET_ID into a bogus id + expect(await getAssetId(ASSET_ID, block(6_003_050))).toBe(ASSET_ID); + expect(await getAssetId({ toString: () => ASSET_ID } as any, block(6_003_050))).toBe(ASSET_ID); + }); +}); diff --git a/tests/unit/decode.test.ts b/tests/unit/decode.test.ts index 8e58ae81..bf477ead 100644 --- a/tests/unit/decode.test.ts +++ b/tests/unit/decode.test.ts @@ -219,6 +219,64 @@ describe('decodeEvent, tuple events', () => { NoDecoderForSpecVersion ); }); + + describe('stale block spec version at a runtime-upgrade boundary', () => { + const setRuntimeSpec = (value: number): void => { + (api.runtimeVersion.specVersion as any).toNumber = () => value; + }; + + afterEach(() => setRuntimeSpec(8_000_000)); + + it('falls back to api.runtimeVersion when the reported version resolves no decoder', () => { + // `AssetBalanceUpdated` is registered from v6.0.0; the block reports a stale v5 spec but + // the block actually ran v6.0.1. + setRuntimeSpec(6_000_001); + + const decoded = decodeEvent( + tupleEvent( + 'asset', + 'AssetBalanceUpdated', + ['0xdid', '0xasset', '100', '0xfrom', '0xto', '{"transferred":null}'], + 5_999_999 + ) + ); + + expect(decoded.assetId.toString()).toBe('0xasset'); + expect(decoded.updateReason.toString()).toBe('{"transferred":null}'); + }); + + it('still throws when api.runtimeVersion is not newer than the reported version', () => { + setRuntimeSpec(5_999_999); + + expect(() => + decodeEvent( + tupleEvent( + 'asset', + 'AssetBalanceUpdated', + ['0xdid', '0xasset', '100', '0xfrom', '0xto', '{"transferred":null}'], + 5_999_999 + ) + ) + ).toThrow(NoDecoderForSpecVersion); + }); + + it('does not reach for a shape more than one release line ahead of the reported version', () => { + // A stale boundary spec is at most one release line off; a HEAD `api.runtimeVersion` + // against an old block must not decode it with a much later shape. + setRuntimeSpec(8_000_000); + + expect(() => + decodeEvent( + tupleEvent( + 'asset', + 'AssetBalanceUpdated', + ['0xdid', '0xasset', '100', '0xfrom', '0xto', '{"transferred":null}'], + 5_999_999 + ) + ) + ).toThrow(NoDecoderForSpecVersion); + }); + }); }); /** diff --git a/tests/unit/extract8xStakingAmount.test.ts b/tests/unit/extract8xStakingAmount.test.ts new file mode 100644 index 00000000..2fd462ab --- /dev/null +++ b/tests/unit/extract8xStakingAmount.test.ts @@ -0,0 +1,44 @@ +/** + * `extract8xStakingAmount` — on a v8 chain a staking event is `[stash, amount]` (Bonded/Unbonded) + * or `[stash, dest, amount]` (Rewarded, where `dest` is a `RewardDestination` enum). The amount's + * position is decided by whether the second parameter renders as a bare number. + * + * (These tests moved here from the deleted `mapPolyxTransaction.test.ts` when the POLYX ledger + * replaced that module; the helper itself still lives in `src/utils/common.ts`.) + */ + +import { extract8xStakingAmount } from '../../src/utils/common'; + +describe('extract8xStakingAmount', () => { + it('returns the amount from the second param when it is numeric (Bonded/Unbonded)', () => { + expect(extract8xStakingAmount(createMockCodec('1000000000000'))).toBe(BigInt('1000000000000')); + }); + + it('returns the amount from the third param when the second is a RewardDestination', () => { + expect( + extract8xStakingAmount(createMockCodec('Staked'), createMockCodec('2000000000000')) + ).toBe(BigInt('2000000000000')); + }); + + it('returns 0 when the second param is non-numeric and there is no third param', () => { + expect(extract8xStakingAmount(createMockCodec('Controller'), undefined)).toBe(BigInt(0)); + }); + + it('handles every RewardDestination enum value', () => { + REWARD_DESTINATIONS.forEach(dest => { + expect(extract8xStakingAmount(createMockCodec(dest), createMockCodec('5000000000000'))).toBe( + BigInt('5000000000000') + ); + }); + }); + + it('handles a zero amount', () => { + expect(extract8xStakingAmount(createMockCodec('0'))).toBe(BigInt(0)); + }); + + it('does not treat a numeric-prefixed string as the amount', () => { + expect( + extract8xStakingAmount(createMockCodec('123abc'), createMockCodec('8000000000000')) + ).toBe(BigInt('8000000000000')); + }); +}); diff --git a/tests/unit/mapClaim.test.ts b/tests/unit/mapClaim.test.ts index 0e04fb34..219fbeda 100644 --- a/tests/unit/mapClaim.test.ts +++ b/tests/unit/mapClaim.test.ts @@ -260,4 +260,21 @@ describe('handleClaimAdded / handleClaimRevoked', () => { expect(anomalies[0].detail).toContain(ISSUER_A); expect(Object.keys(claims)).toHaveLength(0); }); + + it('silently skips a stripped ClaimRevoked with a zero issuer (no anomaly)', async () => { + await handleClaimRevoked( + mockClaimEvent('ClaimRevoked', { + issuer: '0x0000000000000000000000000000000000000000000000000000000000000000', + cddId: 'cdd-1', + dateValue: '0', + }) + ); + + const anomalies = storeSet() + .mock.calls.filter(([entity]) => entity === 'IndexerAnomaly') + .map(([, , row]) => row); + + expect(anomalies).toHaveLength(0); + expect(Object.keys(claims)).toHaveLength(0); + }); }); diff --git a/tests/unit/mapPolyxLedger.test.ts b/tests/unit/mapPolyxLedger.test.ts new file mode 100644 index 00000000..eeefe25c --- /dev/null +++ b/tests/unit/mapPolyxLedger.test.ts @@ -0,0 +1,685 @@ +/** + * The POLYX ledger — one fixture per row of the "Event → pool transition" table in + * docs/implementation/02-polyx-ledger.md, plus the properties the old `PolyxTransaction` model + * could not satisfy: a `Reserved`/`Unreserved` round-trip returns the pools to their starting + * values, and every movement writes a signed entry per account-side sharing one `movementId`. + * + * Events are built struct-style (v8), which is the surface A9 left entirely unindexed. + */ + +import { SubstrateEvent } from '@subql/types'; +import { EntryDirection, HoldReason, MovementKind, PolyxPool } from '../../src/types'; +import { + adjustLock, + handleBalanceBurned, + handleBalanceEndowed, + handleBalanceFrozen, + handleBalanceHeld, + handleBalanceReleased, + handleBalanceMinted, + handleBalanceReserved, + handleBalanceSet, + handleBalanceSuspended, + handleBalanceThawed, + handleBalanceTransfer, + handleBalanceUnlocked, + handleBalanceUnreserved, + handleBonded, + handlePayoutStarted, + handleReward, + handleWithdrawn, + handleDustLost, + handleReserveRepatriated, + handleTreasuryDisbursement, + handleTreasuryReimbursement, +} from '../../src/mappings/entities/identities/mapPolyxLedger'; +import { getAccountId, systematicIssuers } from '../../src/mappings/consts'; +import { __resetControllerCache, __resetPayeeCache } from '../../src/utils/staking'; + +const ALICE = '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY'; +const BOB = '5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty'; + +const storeGet = (): jest.Mock => (globalThis as any).store.get as jest.Mock; +const storeSet = (): jest.Mock => (globalThis as any).store.set as jest.Mock; +const storeGetByFields = (): jest.Mock => (globalThis as any).store.getByFields as jest.Mock; + +const mockCodec = (value: string) => ({ + toString: () => value, + toJSON: () => value, + toU8a: () => Buffer.from(value), +}); + +let blockHeight = 1_000_000; + +/** A struct-style event: the block metadata names every field. */ +const structEvent = ( + section: string, + method: string, + fields: Record, + { specVersion = 8_000_000, atHeight }: { specVersion?: number; atHeight?: number } = {} +): SubstrateEvent => { + if (atHeight === undefined) { + blockHeight += 1; + } + const height = atHeight ?? blockHeight; + + return { + idx: 4, + block: { + block: { header: { number: { toString: () => String(height) } } }, + timestamp: new Date('2024-06-01T12:00:00.000Z'), + specVersion, + }, + event: { + section, + method, + data: Object.values(fields).map(mockCodec), + meta: { + fields: Object.keys(fields).map(name => ({ + name: { isSome: true, unwrap: () => mockCodec(name) }, + typeName: { isSome: true, unwrap: () => mockCodec('Dummy') }, + })), + }, + }, + } as unknown as SubstrateEvent; +}; + +const balancesEvent = ( + method: string, + fields: Record, + opts: { specVersion?: number; atHeight?: number } = {} +): SubstrateEvent => structEvent('balances', method, fields, opts); + +/** A tuple-style event (pre-v8 Polymesh pallet): the block metadata carries no field names. */ +const tupleEvent = ( + section: string, + method: string, + values: string[], + specVersion: number +): SubstrateEvent => { + blockHeight += 1; + + return { + idx: 4, + block: { + block: { header: { number: { toString: () => String(blockHeight) } } }, + timestamp: new Date('2024-06-01T12:00:00.000Z'), + specVersion, + }, + event: { + section, + method, + data: values.map(mockCodec), + meta: { + fields: values.map(() => ({ + name: { isSome: false }, + typeName: { isSome: true, unwrap: () => mockCodec('Dummy') }, + })), + }, + }, + } as unknown as SubstrateEvent; +}; + +type Row = Record; + +let db: Record>; + +const clone = (value: Row): Row => { + const copy: Row = {}; + for (const [k, v] of Object.entries(value)) { + copy[k] = Array.isArray(v) + ? v.map(item => (item && typeof item === 'object' ? { ...item } : item)) + : v; + } + return copy; +}; + +beforeEach(() => { + __resetPayeeCache(); + __resetControllerCache(); + db = {}; + blockHeight = 1_000_000; + (globalThis as any).api.registry = { chainSS58: 42 }; + + storeGet().mockImplementation((entity: string, id: string) => { + if (entity === 'Account') { + // Every test address is a known key, so `getOrCreateAccount` never reaches the chain. + return Promise.resolve({ id, address: id, identityId: undefined }); + } + const row = db[entity]?.[id]; + return Promise.resolve(row ? clone(row) : undefined); + }); + + storeSet().mockImplementation((entity: string, id: string, data: Row) => { + (db[entity] ??= {})[id] = clone(data); + return Promise.resolve(); + }); + + storeGetByFields().mockResolvedValue([]); +}); + +const entries = (): Row[] => Object.values(db['PolyxEntry'] ?? {}); +const balance = (address: string): Row | undefined => db['AccountBalance']?.[address]; + +describe('Event → pool transition', () => { + it('Transfer: from/Free → to/Free, one debit + one credit sharing a movementId', async () => { + await handleBalanceTransfer( + balancesEvent('Transfer', { from: ALICE, to: BOB, amount: '1000' }) + ); + + const rows = entries(); + expect(rows).toHaveLength(2); + expect(new Set(rows.map(r => r.movementId)).size).toBe(1); + + const debit = rows.find(r => r.direction === EntryDirection.Debit); + const credit = rows.find(r => r.direction === EntryDirection.Credit); + + expect(debit).toMatchObject({ + accountId: ALICE, + counterpartyAddress: BOB, + pool: PolyxPool.Free, + kind: MovementKind.Transfer, + amount: BigInt(-1000), + amountAbs: BigInt(1000), + }); + expect(credit).toMatchObject({ + accountId: BOB, + counterpartyAddress: ALICE, + amount: BigInt(1000), + }); + + expect(balance(ALICE)?.free).toBe(BigInt(-1000)); + expect(balance(BOB)?.free).toBe(BigInt(1000)); + }); + + it('Endowed: ∅ → who/Free', async () => { + await handleBalanceEndowed(balancesEvent('Endowed', { account: BOB, freeBalance: '500' })); + + expect(entries()).toHaveLength(1); + expect(entries()[0]).toMatchObject({ + accountId: BOB, + direction: EntryDirection.Credit, + kind: MovementKind.Endowment, + pool: PolyxPool.Free, + }); + expect(balance(BOB)?.free).toBe(BigInt(500)); + }); + + it('Reserved: who/Free → who/Reserved (Hold)', async () => { + await handleBalanceReserved(balancesEvent('Reserved', { who: ALICE, amount: '300' })); + + const rows = entries(); + expect(rows.map(r => r.pool).sort()).toEqual([PolyxPool.Free, PolyxPool.Reserved].sort()); + expect(rows.every(r => r.kind === MovementKind.Hold)).toBe(true); + expect(balance(ALICE)).toMatchObject({ free: BigInt(-300), reserved: BigInt(300) }); + }); + + it('Unreserved: who/Reserved → who/Free (Release)', async () => { + await handleBalanceUnreserved(balancesEvent('Unreserved', { who: ALICE, amount: '300' })); + + expect(balance(ALICE)).toMatchObject({ free: BigInt(300), reserved: BigInt(-300) }); + expect(entries().every(r => r.kind === MovementKind.Release)).toBe(true); + }); + + it('ReserveRepatriated to Free: from/Reserved → to/Free', async () => { + await handleReserveRepatriated( + balancesEvent('ReserveRepatriated', { + from: ALICE, + to: BOB, + amount: '250', + destinationStatus: 'Free', + }) + ); + + const credit = entries().find(r => r.direction === EntryDirection.Credit); + expect(credit).toMatchObject({ accountId: BOB, pool: PolyxPool.Free }); + expect(balance(ALICE)?.reserved).toBe(BigInt(-250)); + expect(balance(BOB)?.free).toBe(BigInt(250)); + }); + + it('Held{Staking}: who/Free → who/Reserved, and bonded tracks the hold', async () => { + await handleBalanceHeld( + balancesEvent('Held', { reason: 'Staking', who: ALICE, amount: '900' }) + ); + + expect(balance(ALICE)).toMatchObject({ + free: BigInt(-900), + reserved: BigInt(900), + bonded: BigInt(900), + }); + expect(entries().every(r => r.holdReason === HoldReason.Staking)).toBe(true); + }); + + it('Released{Staking}: who/Reserved → who/Free, unwinding the hold', async () => { + await handleBalanceHeld( + balancesEvent('Held', { reason: 'Staking', who: ALICE, amount: '900' }) + ); + await handleBalanceReleased( + balancesEvent('Released', { reason: 'Staking', who: ALICE, amount: '900' }) + ); + + expect(balance(ALICE)).toMatchObject({ + free: BigInt(0), + reserved: BigInt(0), + bonded: BigInt(0), + }); + }); + + it('Burned: who/Free → ∅', async () => { + await handleBalanceBurned(balancesEvent('Burned', { who: ALICE, amount: '120' })); + + expect(entries()).toHaveLength(1); + expect(entries()[0]).toMatchObject({ + direction: EntryDirection.Debit, + kind: MovementKind.Burn, + }); + expect(balance(ALICE)?.free).toBe(BigInt(-120)); + }); + + it('Slashed: who/Free → ∅ with kind Slash and totalSlashed', async () => { + await handleBalanceBurned(balancesEvent('Slashed', { who: ALICE, amount: '75' })); + + expect(entries()[0].kind).toBe(MovementKind.Slash); + expect(balance(ALICE)).toMatchObject({ free: BigInt(-75), totalSlashed: BigInt(75) }); + }); + + it('Minted: ∅ → who/Free', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: BOB, amount: '4000' })); + + expect(entries()[0]).toMatchObject({ + direction: EntryDirection.Credit, + kind: MovementKind.Mint, + }); + expect(balance(BOB)?.free).toBe(BigInt(4000)); + }); + + it('DustLost: account/Free → ∅', async () => { + await handleDustLost(balancesEvent('DustLost', { account: ALICE, amount: '7' })); + + expect(entries()[0].kind).toBe(MovementKind.DustLost); + expect(balance(ALICE)?.free).toBe(BigInt(-7)); + }); + + it('Suspended: who/Free → ∅ (A3 — handler now exists)', async () => { + await handleBalanceSuspended(balancesEvent('Suspended', { who: ALICE, amount: '9' })); + + expect(entries()).toHaveLength(1); + expect(balance(ALICE)?.free).toBe(BigInt(-9)); + }); + + it('TreasuryReimbursement credits the treasury pallet account, not the fee payer', async () => { + const payerDid = '0x8015a1702789fedf8474a042af07ba6a37f94e8d24b4eed89414e6eb79df084e'; + const treasury = getAccountId(systematicIssuers.treasury.accountId, 42); + + await handleTreasuryReimbursement( + tupleEvent('treasury', 'TreasuryReimbursement', [payerDid, '400'], 4_000_000) + ); + + expect(entries()).toHaveLength(1); + expect(entries()[0]).toMatchObject({ + accountId: treasury, + direction: EntryDirection.Credit, + kind: MovementKind.TreasuryReimbursement, + amount: BigInt(400), + }); + expect(entries()[0].accountId).not.toBe(payerDid); + expect(balance(treasury)?.free).toBe(BigInt(400)); + }); + + it('TreasuryDisbursement debits the treasury pallet account, not the authorising committee', async () => { + const committeeDid = '0x73797374656d3a676f7665726e616e63655f636f6d6d69747465650000000000'; + const recipientDid = '0x8015a1702789fedf8474a042af07ba6a37f94e8d24b4eed89414e6eb79df084e'; + const treasury = getAccountId(systematicIssuers.treasury.accountId, 42); + + // pre-5.0.0 shape carries no recipient account and no paired balances.Transfer + await handleTreasuryDisbursement( + tupleEvent( + 'treasury', + 'TreasuryDisbursement', + [committeeDid, recipientDid, BOB, '4014'], + 3010 + ) + ); + + const debit = entries().find(r => r.direction === EntryDirection.Debit); + const credit = entries().find(r => r.direction === EntryDirection.Credit); + + expect(debit).toMatchObject({ + accountId: treasury, + kind: MovementKind.TreasuryDisbursement, + amount: BigInt(-4014), + }); + expect(credit?.accountId).toBe(BOB); + expect(entries().some(r => r.accountId === committeeDid)).toBe(false); + expect(balance(treasury)?.free).toBe(BigInt(-4014)); + expect(balance(BOB)?.free).toBe(BigInt(4014)); + }); +}); + +describe('properties the one-column model could not satisfy', () => { + it('Reserved then Unreserved returns free and reserved to their starting values', async () => { + await handleBalanceReserved(balancesEvent('Reserved', { who: ALICE, amount: '600' })); + await handleBalanceUnreserved(balancesEvent('Unreserved', { who: ALICE, amount: '600' })); + + expect(balance(ALICE)).toMatchObject({ free: BigInt(0), reserved: BigInt(0) }); + }); + + it('every movement is recorded as a signed entry: SUM(amount) is the net delta', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '1000' })); + await handleBalanceBurned(balancesEvent('Burned', { who: ALICE, amount: '250' })); + + const net = entries() + .filter(r => r.accountId === ALICE) + .reduce((sum, r) => sum + r.amount, BigInt(0)); + + expect(net).toBe(BigInt(750)); + expect(balance(ALICE)?.free).toBe(BigInt(750)); + }); + + it('BalanceSet sets the pool absolutely and does not corrupt subsequent totals', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '1000' })); + + await handleBalanceSet(balancesEvent('BalanceSet', { who: ALICE, free: '5000' })); + + // 5000, not 1000 + 5000 + expect(balance(ALICE)?.free).toBe(BigInt(5000)); + + const adjustment = entries().find(r => r.kind === MovementKind.BalanceSetAdjustment); + expect(adjustment).toMatchObject({ amount: BigInt(4000), freeAfter: BigInt(5000) }); + + await handleBalanceBurned(balancesEvent('Burned', { who: ALICE, amount: '500' })); + expect(balance(ALICE)?.free).toBe(BigInt(4500)); + }); + + it('BalanceSet writes a debit adjustment when it lowers the balance, and a per-pool entry pre-v8', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: BOB, amount: '9000' })); + + await handleBalanceSet( + balancesEvent('BalanceSet', { + identityId: '0x00', + account: BOB, + free: '8000', + reserved: '250', + }) + ); + + expect(balance(BOB)).toMatchObject({ free: BigInt(8000), reserved: BigInt(250) }); + + const adjustments = entries().filter(r => r.kind === MovementKind.BalanceSetAdjustment); + expect(adjustments.map(r => [r.pool, r.amount]).sort()).toEqual( + [ + [PolyxPool.Free, BigInt(-1000)], + [PolyxPool.Reserved, BigInt(250)], + ].sort() + ); + }); + + it('frozen is the MAX over active locks, not their sum', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '1000' })); + + await adjustLock(ALICE, 'staking ', BigInt(100), '0000009999'); + await adjustLock(ALICE, 'pips ', BigInt(150), '0000009999'); + + expect(balance(ALICE)?.frozen).toBe(BigInt(150)); // not 250 + expect(balance(ALICE)?.transferable).toBe(BigInt(850)); // free 1000 - frozen 150 + }); + + it('a lock writes no PolyxEntry and does not move the pools', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '1000' })); + const entriesAfterMint = entries().length; + + await handleBalanceFrozen(balancesEvent('Frozen', { who: ALICE, amount: '400' })); + + expect(entries()).toHaveLength(entriesAfterMint); // no new entry + expect(balance(ALICE)).toMatchObject({ + free: BigInt(1000), + reserved: BigInt(0), + frozen: BigInt(400), + }); + + await handleBalanceThawed(balancesEvent('Thawed', { who: ALICE, amount: '400' })); + expect(balance(ALICE)?.frozen).toBe(BigInt(0)); + }); + + it('carries the balance-after snapshot on each entry', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '1000' })); + await handleBalanceReserved(balancesEvent('Reserved', { who: ALICE, amount: '400' })); + + const last = entries() + .filter(r => r.accountId === ALICE) + .sort((a, b) => a.id.localeCompare(b.id)) + .at(-1); + + expect(last).toMatchObject({ freeAfter: BigInt(600), reservedAfter: BigInt(400) }); + }); +}); + +describe('staking — era-dependent, inverted at v8 (A10 / A6)', () => { + it('v7 Bonded produces no PolyxEntry and raises frozen via the staking lock', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + const beforeEntries = entries().length; + + await handleBonded(tupleEvent('staking', 'Bonded', ['0xdid', ALICE, '4000'], 7_004_001)); + + expect(entries()).toHaveLength(beforeEntries); // no movement row + expect(balance(ALICE)).toMatchObject({ + free: BigInt(10000), // unchanged — pre-v8 bonding moves nothing + frozen: BigInt(4000), + bonded: BigInt(4000), + transferable: BigInt(6000), + }); + }); + + it('v7 Withdrawn lowers the staking lock', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + await handleBonded(tupleEvent('staking', 'Bonded', ['0xdid', ALICE, '4000'], 7_004_001)); + + await handleWithdrawn(tupleEvent('staking', 'Withdrawn', [ALICE, '1500'], 7_004_001)); + + expect(balance(ALICE)?.frozen).toBe(BigInt(2500)); + }); + + it('v8 Bonded is ledger state only — no entry, no lock (the move is the paired Held)', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + const beforeEntries = entries().length; + + await handleBonded( + balancesEvent('Bonded', { stash: ALICE, amount: '4000' }, { specVersion: 8_000_000 }) + ); + + expect(entries()).toHaveLength(beforeEntries); + expect(balance(ALICE)).toMatchObject({ + free: BigInt(10000), + frozen: BigInt(0), + bonded: BigInt(0), + }); + + // the actual v8 bonding movement: + await handleBalanceHeld( + balancesEvent('Held', { reason: 'Staking', who: ALICE, amount: '4000' }) + ); + expect(balance(ALICE)).toMatchObject({ + free: BigInt(6000), + reserved: BigInt(4000), + bonded: BigInt(4000), + }); + }); + + describe('the v5–v7 lock → v8 hold storage migration', () => { + it('a v8 Held{Staking} moves the bonded amount free → reserved, lock still standing', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + await adjustLock(ALICE, 'staking ', BigInt(4000), '0000001000000', 'staking'); + + // pass 1 of the migration — Held with no paired Deposit + await handleBalanceHeld( + balancesEvent('Held', { reason: 'Staking', who: ALICE, amount: '4000' }) + ); + + expect(balance(ALICE)).toMatchObject({ + free: BigInt(6000), + reserved: BigInt(4000), + frozen: BigInt(4000), // the "staking " lock is deliberately kept — chain frozen is still 4000 + bonded: BigInt(4000), + }); + }); + + it('a v8 Unlocked that covers the staking lock clears it (pass 2)', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + await adjustLock(ALICE, 'staking ', BigInt(4000), '0000001000000', 'staking'); + await handleBalanceHeld( + balancesEvent('Held', { reason: 'Staking', who: ALICE, amount: '4000' }) + ); + + await handleBalanceUnlocked( + balancesEvent('Unlocked', { who: ALICE, amount: '4000' }, { specVersion: 8_000_000 }) + ); + + expect(balance(ALICE)).toMatchObject({ + free: BigInt(6000), + reserved: BigInt(4000), + frozen: BigInt(0), // lock gone + bonded: BigInt(4000), // still bonded, now via the hold + transferable: BigInt(6000), + }); + expect(balance(ALICE)?.locks ?? []).toHaveLength(0); + }); + + it('a pre-v8 Unlocked leaves the staking lock alone (generic "balances" lock only)', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + await adjustLock(ALICE, 'staking ', BigInt(4000), '0000001000000', 'staking'); + + await handleBalanceUnlocked( + balancesEvent('Unlocked', { who: ALICE, amount: '10' }, { specVersion: 7_004_001 }) + ); + + expect(balance(ALICE)?.locks?.find((l: any) => l.lockId === 'staking ')?.amount).toBe( + BigInt(4000) + ); + }); + }); + + it('pre-v8 Reward resolves an explicit Account payee — credited to that account, not the stash', async () => { + const PAYEE = '5DAAnrj7VHTznn2AWBemMuyBwZWs6FNFjdyVXUeYum3PTXFy'; + (globalThis as any).api.query = { + staking: { + payee: jest.fn().mockResolvedValue({ toJSON: () => ({ account: PAYEE }) }), + bonded: jest.fn().mockResolvedValue({ toJSON: () => null }), + }, + }; + + await handleReward(tupleEvent('staking', 'Reward', ['0xdid', ALICE, '900'], 7_004_001)); + + const reward = entries().find(r => r.kind === MovementKind.StakingReward); + expect(reward?.accountId).toBe(PAYEE); + expect(reward?.accountId).not.toBe(ALICE); + expect(balance(PAYEE)?.free).toBe(BigInt(900)); + + (globalThis as any).api.query = {}; + }); + + it('pre-v8 Reward with a Staked payee credits free AND raises the staking lock', async () => { + (globalThis as any).api.query = { + staking: { + payee: jest.fn().mockResolvedValue({ toJSON: () => 'Staked' }), + bonded: jest.fn().mockResolvedValue({ toJSON: () => null }), + }, + }; + + await handleReward(tupleEvent('staking', 'Reward', ['0xdid', ALICE, '500'], 7_004_001)); + + expect(balance(ALICE)).toMatchObject({ + free: BigInt(500), + frozen: BigInt(500), // restaked — locked in the same step + bonded: BigInt(500), + transferable: BigInt(0), + }); + + (globalThis as any).api.query = {}; + }); + + describe('the staking lock is read from staking.ledger.total, not accumulated', () => { + const mockLedger = (total: string, bonded: string | null = null) => { + (globalThis as any).api.query = { + staking: { + payee: jest.fn().mockResolvedValue({ toJSON: () => 'Staked' }), + bonded: jest.fn().mockResolvedValue({ toJSON: () => bonded }), + ledger: jest.fn().mockResolvedValue({ toJSON: () => ({ total, active: total }) }), + }, + }; + }; + + afterEach(() => { + (globalThis as any).api.query = {}; + }); + + it('v7 Bonded pins the lock to ledger.total, ignoring the event amount', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + mockLedger('4200'); // chain: 4200 bonded (4000 + a compounded 200 the event never carried) + + await handleBonded(tupleEvent('staking', 'Bonded', ['0xdid', ALICE, '4000'], 7_004_001)); + + expect(balance(ALICE)).toMatchObject({ frozen: BigInt(4200), bonded: BigInt(4200) }); + }); + + it('a restaked Staked reward pins the lock to ledger.total (capped below the gross reward)', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + mockLedger('4000000000000'); // at the max-bond cap + + await handleReward(tupleEvent('staking', 'Reward', ['0xdid', ALICE, '900'], 7_004_001)); + + // free still takes the whole reward; the lock is whatever the ledger says, not += 900 + expect(balance(ALICE)).toMatchObject({ + free: BigInt(10900), + frozen: BigInt('4000000000000'), + }); + }); + + it('v7 Withdrawn pins the lock to the reduced ledger.total', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + mockLedger('4000'); + await handleBonded(tupleEvent('staking', 'Bonded', ['0xdid', ALICE, '4000'], 7_004_001)); + + mockLedger('2500'); + await handleWithdrawn(tupleEvent('staking', 'Withdrawn', [ALICE, '1500'], 7_004_001)); + + expect(balance(ALICE)?.frozen).toBe(BigInt(2500)); + }); + + it('falls back to the delta accumulator when the ledger cannot be read', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + (globalThis as any).api.query = {}; // no staking.ledger + + await handleBonded(tupleEvent('staking', 'Bonded', ['0xdid', ALICE, '4000'], 7_004_001)); + + expect(balance(ALICE)?.frozen).toBe(BigInt(4000)); + }); + }); + + it('staking Reward is ∅ → stash/Free with the era from the preceding PayoutStarted', async () => { + await handlePayoutStarted( + structEvent( + 'staking', + 'PayoutStarted', + { eraIndex: '742', validatorStash: BOB, page: '0', next: '0' }, + { atHeight: 5_000_000 } + ) + ); + await handleReward( + structEvent( + 'staking', + 'Rewarded', + { stash: ALICE, dest: 'Staked', amount: '333' }, + { atHeight: 5_000_000 } + ) + ); + + const reward = entries().find(r => r.kind === MovementKind.StakingReward); + expect(reward).toMatchObject({ + accountId: ALICE, + direction: EntryDirection.Credit, + amount: BigInt(333), + eraIndex: 742, + }); + expect(balance(ALICE)).toMatchObject({ free: BigInt(333), totalRewards: BigInt(333) }); + }); +}); diff --git a/tests/unit/mapPolyxTransaction.test.ts b/tests/unit/mapPolyxTransaction.test.ts deleted file mode 100644 index fc8f36a6..00000000 --- a/tests/unit/mapPolyxTransaction.test.ts +++ /dev/null @@ -1,192 +0,0 @@ -/** - * Unit tests for mapPolyxTransaction parameter extraction logic. - * - * These tests verify the core logic for handling 8.x chain staking event parameters - * using the extract8xStakingAmount utility function from common.ts. - * - * The actual parameter extraction logic tested here: - * - On 8.x chain, Rewarded event has [stash, dest, amount] where dest is RewardDestination enum - * - On 8.x chain, Bonded/Unbonded have [stash, amount] - * - The code detects if second param is numeric to determine which param holds the amount - */ - -import { Codec } from '@polkadot/types/types'; -import { extract8xStakingAmount } from '../../src/utils/common'; - -/** - * Helper that wraps extract8xStakingAmount to return address too (for test compatibility) - */ -const extractAmountFrom8xParams = (params: Codec[]): { address: string; amount: bigint } => { - const [rawAddress, rawSecondParam, rawThirdParam] = params; - return { - address: getTextValue(rawAddress), - amount: extract8xStakingAmount(rawSecondParam, rawThirdParam), - }; -}; - -/** - * Pre-8.x parameter extraction logic - */ -const extractAmountFromPre8xParams = ( - params: Codec[] -): { identityId: string; address: string; amount: bigint } => { - const [rawDid, rawAddress, rawBalance] = params; - return { - identityId: getTextValue(rawDid), - address: getTextValue(rawAddress), - amount: getBigIntValue(rawBalance), - }; -}; - -describe('mapPolyxTransaction parameter extraction logic', () => { - describe('8.x chain - Bonded/Unbonded events (2 params)', () => { - it('should extract amount from second param when it is numeric', () => { - const params = [createMockCodec(TEST_ADDRESS), createMockCodec('1000000000000')]; - const result = extractAmountFrom8xParams(params); - - expect(result.address).toBe(TEST_ADDRESS); - expect(result.amount).toBe(BigInt('1000000000000')); - }); - - it.each([ - ['zero', '0', BigInt(0)], - ['small', '1', BigInt(1)], - ['large', '999999999999999999999999', BigInt('999999999999999999999999')], - ])('should handle %s amount', (_name, amountStr, expected) => { - const params = [createMockCodec(TEST_ADDRESS), createMockCodec(amountStr)]; - const result = extractAmountFrom8xParams(params); - expect(result.amount).toBe(expected); - }); - }); - - describe('8.x chain - Rewarded event with RewardDestination (3 params)', () => { - it.each(REWARD_DESTINATIONS)( - 'should extract amount from third param when second param is "%s"', - rewardDest => { - const params = [ - createMockCodec(TEST_ADDRESS), - createMockCodec(rewardDest), - createMockCodec('5000000000000'), - ]; - const result = extractAmountFrom8xParams(params); - - expect(result.address).toBe(TEST_ADDRESS); - expect(result.amount).toBe(BigInt('5000000000000')); - } - ); - }); - - describe('8.x chain - edge cases', () => { - it('should return 0 when second param is non-numeric and third param is missing', () => { - const params = [createMockCodec(TEST_ADDRESS), createMockCodec('UnknownEnum')]; - const result = extractAmountFrom8xParams(params); - expect(result.amount).toBe(BigInt(0)); - }); - - it('should not match strings with numeric prefix but non-numeric suffix', () => { - const params = [ - createMockCodec(TEST_ADDRESS), - createMockCodec('123abc'), - createMockCodec('8000000000000'), - ]; - const result = extractAmountFrom8xParams(params); - expect(result.amount).toBe(BigInt('8000000000000')); - }); - }); - - describe('pre-8.x chain parameter extraction', () => { - it('should extract identityId from first param and amount from third param', () => { - const params = [ - createMockCodec(TEST_DID), - createMockCodec(TEST_ADDRESS), - createMockCodec('9000000000000'), - ]; - const result = extractAmountFromPre8xParams(params); - - expect(result.identityId).toBe(TEST_DID); - expect(result.address).toBe(TEST_ADDRESS); - expect(result.amount).toBe(BigInt('9000000000000')); - }); - }); - - describe('pre-8.x BalanceSet layout (defect A1)', () => { - /** - * The chain emits `BalanceSet(IdentityId, AccountId, free, reserved)` — four params, with - * reserved at index 3 — identically at v5.4.3 / v6.3.5 / v7.0.0 / v7.4.0. `handleBalanceSet` - * read `params[4]`, which is out of bounds, so `getBigIntValue` returned `BigInt(0)` and the - * `if (reservedAmount)` guard silently skipped the Reserved row. It now reads `params[3]`. - */ - const extractBalanceSetReserved = (params: Codec[]): bigint => getBigIntValue(params[3]); - - it('reads the reserved balance from index 3', () => { - const params = [ - createMockCodec(TEST_DID), - createMockCodec(TEST_ADDRESS), - createMockCodec('1000'), // free - createMockCodec('250'), // reserved - ]; - - expect(extractBalanceSetReserved(params)).toBe(BigInt('250')); - }); - - it('index 4 is out of bounds for this event and would yield 0', () => { - const params = [ - createMockCodec(TEST_DID), - createMockCodec(TEST_ADDRESS), - createMockCodec('1000'), - createMockCodec('250'), - ]; - - expect(getBigIntValue(params[4])).toBe(BigInt(0)); - }); - }); - - describe('Numeric detection regex', () => { - it.each([ - ['0', true], - ['1', true], - ['123456789', true], - ['999999999999999999999999', true], - ...REWARD_DESTINATIONS.map(v => [v, false] as [string, boolean]), - ['123abc', false], - ['abc123', false], - ['', false], - [' 123', false], - ['123 ', false], - ])('isNumericString("%s") should be %s', (value, expected) => { - expect(isNumericString(value)).toBe(expected); - }); - }); - - describe('extract8xStakingAmount utility function', () => { - it('should return amount from second param when numeric (Bonded/Unbonded)', () => { - const rawSecondParam = createMockCodec('1000000000000'); - expect(extract8xStakingAmount(rawSecondParam)).toBe(BigInt('1000000000000')); - }); - - it('should return amount from third param when second is non-numeric (Rewarded)', () => { - const rawSecondParam = createMockCodec('Staked'); - const rawThirdParam = createMockCodec('2000000000000'); - expect(extract8xStakingAmount(rawSecondParam, rawThirdParam)).toBe(BigInt('2000000000000')); - }); - - it('should return 0 when second is non-numeric and third is undefined', () => { - const rawSecondParam = createMockCodec('Controller'); - expect(extract8xStakingAmount(rawSecondParam, undefined)).toBe(BigInt(0)); - }); - - it('should handle all RewardDestination enum values', () => { - const expectedAmount = BigInt('5000000000000'); - REWARD_DESTINATIONS.forEach(enumValue => { - const rawSecondParam = createMockCodec(enumValue); - const rawThirdParam = createMockCodec('5000000000000'); - expect(extract8xStakingAmount(rawSecondParam, rawThirdParam)).toBe(expectedAmount); - }); - }); - - it('should handle zero balance', () => { - const rawSecondParam = createMockCodec('0'); - expect(extract8xStakingAmount(rawSecondParam)).toBe(BigInt(0)); - }); - }); -}); diff --git a/tests/unit/reconcilePolyx.test.ts b/tests/unit/reconcilePolyx.test.ts new file mode 100644 index 00000000..58db7756 --- /dev/null +++ b/tests/unit/reconcilePolyx.test.ts @@ -0,0 +1,143 @@ +/** + * In-flight POLYX reconciliation (D11). Every Nth block for touched accounts, and always after a + * `BalanceSet` / `DustLost`, the derived `AccountBalance` is checked against `system.account` at + * the block being indexed. On a mismatch it records a `BalanceReconciliationDrift` anomaly and + * corrects the derived value so the drift cannot compound. + */ + +import { SubstrateBlock } from '@subql/types'; +import { + __resetOnChainCache, + reconcileAccount, +} from '../../src/mappings/entities/identities/reconcilePolyx'; + +const ADDR = '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY'; + +const storeGet = (): jest.Mock => (globalThis as any).store.get as jest.Mock; +const storeSet = (): jest.Mock => (globalThis as any).store.set as jest.Mock; + +const codec = (v: string) => ({ toString: () => v }); + +const block = (height: number): SubstrateBlock => + ({ + block: { header: { number: { toString: () => String(height) } } }, + timestamp: new Date('2024-01-01T00:00:00Z'), + specVersion: 8_000_000, + } as unknown as SubstrateBlock); + +let db: Record>; + +const setDerived = (row: Partial>) => { + db['AccountBalance'] = { + [ADDR]: { + id: ADDR, + accountId: ADDR, + free: BigInt(0), + reserved: BigInt(0), + frozen: BigInt(0), + total: BigInt(0), + transferable: BigInt(0), + bonded: BigInt(0), + otherReserved: BigInt(0), + totalReceived: BigInt(0), + totalSent: BigInt(0), + totalFeesPaid: BigInt(0), + totalRewards: BigInt(0), + totalSlashed: BigInt(0), + movementCount: 0, + locks: [], + holds: [], + updatedBlockId: '0', + ...row, + }, + }; +}; + +const setChain = (free: string, reserved: string, frozen: string) => { + (globalThis as any).api.query = { + system: { + account: jest.fn().mockResolvedValue({ + data: { free: codec(free), reserved: codec(reserved), frozen: codec(frozen) }, + }), + }, + }; +}; + +const anomalies = () => + storeSet() + .mock.calls.filter(([e]) => e === 'IndexerAnomaly') + .map(([, , row]) => row); + +beforeEach(() => { + __resetOnChainCache(); + db = {}; + storeGet().mockImplementation((entity: string, id: string) => Promise.resolve(db[entity]?.[id])); + storeSet().mockImplementation((entity: string, id: string, data: any) => { + (db[entity] ??= {})[id] = { ...data }; + return Promise.resolve(); + }); +}); + +// Values are in base units (6 decimals); drifts here are far above the MIN_DRIFT (100 POLYX) floor. +const P = (polyx: number): bigint => BigInt(polyx) * BigInt(1_000_000); + +describe('reconcileAccount', () => { + it('does nothing when the derived balance agrees with chain state', async () => { + setDerived({ free: P(1000), total: P(1000), transferable: P(1000) }); + setChain(P(1000).toString(), '0', '0'); + + await reconcileAccount(ADDR, '0000009000', block(9000), { force: true }); + + expect(anomalies()).toHaveLength(0); + }); + + it('ignores sub-100-POLYX drift (weight-fee gap / mid-block sample noise)', async () => { + setDerived({ free: P(1000) + BigInt(50_000_000), total: P(1000) }); + setChain(P(1000).toString(), '0', '0'); + + await reconcileAccount(ADDR, '0000009000', block(9000), { force: true }); + + expect(anomalies()).toHaveLength(0); + }); + + it('records a drift anomaly and corrects each pool independently', async () => { + setDerived({ free: P(900), reserved: P(100), total: P(1000) }); + setChain(P(1000).toString(), P(50).toString(), '0'); + + await reconcileAccount(ADDR, '0000009000', block(9000), { force: true, eventIdx: 3 }); + + expect(anomalies()).toHaveLength(1); + expect(anomalies()[0]).toMatchObject({ kind: 'BalanceReconciliationDrift' }); + expect(anomalies()[0].detail).toContain(`free ${P(900)} vs ${P(1000)}`); + + expect(db['AccountBalance'][ADDR]).toMatchObject({ + free: P(1000), + reserved: P(50), + total: P(1050), + }); + }); + + it('corrects frozen by pinning the staking lock, so later staking events adjust a real base', async () => { + setDerived({ free: P(1000), frozen: BigInt(0), transferable: P(1000) }); + setChain(P(1000).toString(), '0', P(400).toString()); + + await reconcileAccount(ADDR, '0000009000', block(9000), { force: true }); + + expect(db['AccountBalance'][ADDR]).toMatchObject({ + frozen: P(400), + transferable: P(600), + locks: [{ lockId: 'staking ', amount: P(400), reasons: 'staking' }], + }); + }); + + it('only samples every Nth block unless forced', async () => { + setDerived({ free: BigInt(1) }); + setChain(P(999).toString(), '0', '0'); + + await reconcileAccount(ADDR, '0000009001', block(9001)); // 9001 % 2000 != 0 + expect(anomalies()).toHaveLength(0); + + await reconcileAccount(ADDR, '0000008000', block(8000)); // 8000 % 2000 == 0 + expect(anomalies()).toHaveLength(1); + }); +}); diff --git a/tests/unit/rewardDestinationA15.test.ts b/tests/unit/rewardDestinationA15.test.ts new file mode 100644 index 00000000..a2fda21b --- /dev/null +++ b/tests/unit/rewardDestinationA15.test.ts @@ -0,0 +1,170 @@ +/** + * Defect A15 — pre-v8 staking rewards and the account that received them. + * + * Measured across a spread of eras (`scripts/measure-a15-payees.ts`): a large share of pre-v8 + * mainnet rewards were paid to a `Controller` or an explicit `Account`, not the stash. So + * `handleStakingEvent` now reads `staking.payee(stash)` from chain storage at the reward block + * (`resolveLegacyRewardDestination`) rather than recording `LegacyUnknown`. + * + * These tests pin: the storage read resolves `Controller`/`Account`/`Staked`; and the + * `LegacyUnknown` placeholder still stands when the read is not possible (a pruned node), so it + * is never silently resolved to something wrong. + */ + +import { SubstrateEvent } from '@subql/types'; +import { handleStakingEvent } from '../../src/mappings/entities/events/mapStakingEvent'; +import { __resetPayeeCache } from '../../src/utils/staking'; + +const STASH = '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY'; +const PAYEE = '5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty'; +const CONTROLLER = '5DAAnrj7VHTznn2AWBemMuyBwZWs6FNFjdyVXUeYum3PTXFy'; + +const storeSet = (): jest.Mock => (globalThis as any).store.set as jest.Mock; + +const mockPayee = (payee: any, bonded?: string) => { + (globalThis as any).api.query = { + staking: { + payee: jest.fn().mockResolvedValue({ toJSON: () => payee }), + bonded: jest.fn().mockResolvedValue({ toJSON: () => bonded ?? null }), + }, + }; +}; + +const codec = (value: string) => ({ toString: () => value, toJSON: () => value }); + +const rewardEvent = ( + method: 'Reward' | 'Rewarded', + data: any[], + specVersion: number +): SubstrateEvent => + ({ + idx: 1, + block: { + block: { header: { number: { toString: () => '5000' } } }, + timestamp: new Date('2022-01-01T00:00:00Z'), + specVersion, + }, + event: { + section: 'staking', + method, + data, + meta: { fields: data.map(() => ({ name: { isSome: false }, typeName: { isSome: false } })) }, + }, + extrinsic: undefined, + } as unknown as SubstrateEvent); + +const savedStakingEvent = () => + storeSet() + .mock.calls.filter(([entity]) => entity === 'StakingEvent') + .map(([, , row]) => row) + .at(-1); + +beforeEach(() => { + __resetPayeeCache(); + (globalThis as any).api.runtimeVersion.specName = { toString: () => 'polymesh' }; + (globalThis as any).api.query = {}; +}); + +describe('A15 — pre-v8 reward destination', () => { + it('reads staking.payee and resolves a Controller payee to the controller account', async () => { + mockPayee('Controller', CONTROLLER); + + await handleStakingEvent( + rewardEvent('Reward', [codec('0x00'), codec(STASH), codec('1000')], 7_004_001) + ); + + const row = savedStakingEvent(); + expect(row.rewardDestination).toBe('Controller'); + expect(row.rewardDestinationAccount).toBe(CONTROLLER); + expect(row.stashAccount).toBe(STASH); + }); + + it('resolves an explicit Account payee', async () => { + mockPayee({ account: PAYEE }); + + await handleStakingEvent( + rewardEvent('Rewarded', [codec('0x00'), codec(STASH), codec('2000')], 7_000_000) + ); + + const row = savedStakingEvent(); + expect(row.rewardDestination).toBe('Account'); + expect(row.rewardDestinationAccount).toBe(PAYEE); + }); + + it('resolves Staked/Stash to the stash itself', async () => { + mockPayee('Staked'); + + await handleStakingEvent( + rewardEvent('Reward', [codec('0x00'), codec(STASH), codec('1500')], 7_004_001) + ); + + expect(savedStakingEvent()).toMatchObject({ + rewardDestination: 'Staked', + rewardDestinationAccount: STASH, + }); + }); + + it('resolves the object form staking.payee returns via .toJSON() ({ staked: null }) to the stash', async () => { + // `RewardDestination::Staked` renders as the lower-cased single-key object `{ staked: null }` + // through `.toJSON()`, not the bare string `"Staked"`. + mockPayee({ staked: null }); + + await handleStakingEvent( + rewardEvent('Reward', [codec('0x00'), codec(STASH), codec('1500')], 7_004_001) + ); + + expect(savedStakingEvent()).toMatchObject({ + rewardDestination: 'Staked', + rewardDestinationAccount: STASH, + }); + }); + + it('falls back to LegacyUnknown when the payee read is not possible, never to the stash', async () => { + // api.query.staking absent — a pruned node or a runtime with no such storage + await handleStakingEvent( + rewardEvent('Reward', [codec('0x00'), codec(STASH), codec('1000')], 7_004_001) + ); + + const row = savedStakingEvent(); + expect(row.rewardDestination).toBe('LegacyUnknown'); + expect(row.rewardDestinationAccount).toBeUndefined(); + }); + + it('v8 resolves the destination account for an explicit Account payee', async () => { + await handleStakingEvent( + rewardEvent( + 'Rewarded', + [codec(STASH), { toJSON: () => ({ account: PAYEE }) }, codec('3000')], + 8_000_000 + ) + ); + + const row = savedStakingEvent(); + expect(row.rewardDestination).toBe('Account'); + expect(row.rewardDestinationAccount).toBe(PAYEE); + }); + + it('v8 resolves Staked/Stash to the stash itself', async () => { + await handleStakingEvent( + rewardEvent('Rewarded', [codec(STASH), { toJSON: () => 'Staked' }, codec('4000')], 8_000_000) + ); + + const row = savedStakingEvent(); + expect(row.rewardDestination).toBe('Staked'); + expect(row.rewardDestinationAccount).toBe(STASH); + }); + + it('v8 resolves the object form of a Staked payee to the stash', async () => { + await handleStakingEvent( + rewardEvent( + 'Rewarded', + [codec(STASH), { toJSON: () => ({ staked: null }) }, codec('4500')], + 8_000_000 + ) + ); + + const row = savedStakingEvent(); + expect(row.rewardDestination).toBe('Staked'); + expect(row.rewardDestinationAccount).toBe(STASH); + }); +}); diff --git a/tests/unit/seedAccountBalances.test.ts b/tests/unit/seedAccountBalances.test.ts new file mode 100644 index 00000000..4c6677aa --- /dev/null +++ b/tests/unit/seedAccountBalances.test.ts @@ -0,0 +1,98 @@ +/** + * The genesis balance seeder (`src/seed/accountBalance.ts`). Without an opening snapshot every + * balance the POLYX ledger derives is wrong by the genesis allocation, so this is a hard + * prerequisite for the ledger, not an optimisation. + */ + +import { seedAccountBalances } from '../../src/seed/accountBalance'; + +const A = '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY'; +const B = '5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty'; + +const storeGet = (): jest.Mock => (globalThis as any).store.get as jest.Mock; +const storeSet = (): jest.Mock => (globalThis as any).store.set as jest.Mock; + +const codec = (value: string) => ({ toString: () => value, toJSON: () => value }); + +/** `[storageKey, accountInfo]` pairs as `api.query.system.account.entries()` yields them. */ +const accountEntry = ( + address: string, + data: { + free: string; + reserved?: string; + miscFrozen?: string; + feeFrozen?: string; + frozen?: string; + } +) => [ + { args: [codec(address)] }, + { + data: Object.fromEntries(Object.entries(data).map(([k, v]) => [k, codec(v as string)])), + }, +]; + +describe('seedAccountBalances', () => { + let db: Record>; + + beforeEach(() => { + db = {}; + + storeGet().mockImplementation((entity: string, id: string) => { + if (entity === 'Account') { + return Promise.resolve({ id, address: id, identityId: undefined }); + } + return Promise.resolve(db[entity]?.[id]); + }); + storeSet().mockImplementation((entity: string, id: string, data: any) => { + (db[entity] ??= {})[id] = { ...data }; + return Promise.resolve(); + }); + + (globalThis as any).api.query = { + system: { + account: { + entries: jest + .fn() + .mockResolvedValue([ + accountEntry(A, { + free: '1000000', + reserved: '250', + miscFrozen: '400', + feeFrozen: '100', + }), + accountEntry(B, { free: '5000', reserved: '0' }), + accountEntry('5zeroBalance', { free: '0', reserved: '0' }), + ]), + }, + }, + }; + }); + + it('creates one AccountBalance per funded account, skipping empty ones', async () => { + const { seeded } = await seedAccountBalances({ blockId: '0000000000', datetime: new Date(0) }); + + expect(seeded).toBe(2); + expect(Object.keys(db['AccountBalance'])).toEqual([A, B]); + }); + + it('takes free/reserved verbatim and frozen as MAX(miscFrozen, feeFrozen) pre-v8', async () => { + await seedAccountBalances({ blockId: '0000000000', datetime: new Date(0) }); + + expect(db['AccountBalance'][A]).toMatchObject({ + free: BigInt(1000000), + reserved: BigInt(250), + frozen: BigInt(400), // max(400, 100), not the sum + total: BigInt(1000250), + transferable: BigInt(999600), // free - frozen + }); + }); + + it('records a genesis freeze as a single lock so frozen stays a MAX going forward', async () => { + await seedAccountBalances({ blockId: '0000000000', datetime: new Date(0) }); + + expect(db['AccountBalance'][A].locks).toEqual([ + { lockId: 'genesis', amount: BigInt(400), reasons: undefined }, + ]); + expect(db['AccountBalance'][B].locks).toEqual([]); + }); +}); diff --git a/tests/unit/stakingEventHandlers.test.ts b/tests/unit/stakingEventHandlers.test.ts index b4512df9..0873a918 100644 --- a/tests/unit/stakingEventHandlers.test.ts +++ b/tests/unit/stakingEventHandlers.test.ts @@ -8,8 +8,9 @@ * Pre-8.x chain: * - Bonded/Unbonded/Rewarded: [did, account, amount] (3 params with DID) * - * Note: The extract8xStakingAmount utility function is comprehensively tested in - * mapPolyxTransaction.test.ts. This file focuses on is8xChain detection logic. + * Note: `extract8xStakingAmount` is covered in `extract8xStakingAmount.test.ts` and the pre-v8 + * reward-destination (A15) resolution in `rewardDestinationA15.test.ts`. This file focuses on + * `is8xChain` detection logic. */ import { is8xChain } from '../../src/utils/common'; diff --git a/tests/unit/syncMetadata.test.ts b/tests/unit/syncMetadata.test.ts index 79a671a8..e529383d 100644 --- a/tests/unit/syncMetadata.test.ts +++ b/tests/unit/syncMetadata.test.ts @@ -1,11 +1,16 @@ +import { Metadata, TypeRegistry } from '@polkadot/types'; +import metadataHex from '@polkadot/types-support/metadata/static-substrate'; import { applyEnumUpdates, + arityFixtureFor, ArityFixture, enumMembers, eventDrift, findEnumBlock, planEnumUpdates, RuntimeSnapshot, + sectionId, + snapshotFromMetadata, unhandledEvents, withAddedMembers, } from '../../scripts/sync-metadata'; @@ -36,11 +41,67 @@ const snapshot = (overrides: Partial = {}): RuntimeSnapshot => specName: 'polymesh', specVersion: 8_000_000, modules: ['system', 'balances'], - events: { Balances: { BalanceSet: 2, TransferWithMemo: 4 } }, - calls: { Balances: ['set_balance'] }, + events: { balances: { BalanceSet: 2, TransferWithMemo: 4 } }, + calls: { balances: ['set_balance'] }, ...overrides, }); +describe('snapshotFromMetadata', () => { + const registry = new TypeRegistry(); + const metadata = new Metadata(registry, metadataHex); + + registry.setMetadata(metadata); + + const real = snapshotFromMetadata(registry, metadata, 'substrate', 1); + + it('keys a section the way the arity fixtures spell it, not the way metadata spells it', () => { + expect(real.events.balances).toBeDefined(); + expect(real.events.Balances).toBeUndefined(); + }); + + it('lowercases the first letter only, so a multi-word pallet keeps its camelCase', () => { + expect(Object.keys(real.events)).toContain('transactionPayment'); + expect(Object.keys(real.calls)).toContain('electionProviderMultiPhase'); + }); + + it('still spells modules the way ModuleIdEnum does, fully lowercased', () => { + expect(real.modules).toContain('transactionpayment'); + }); + + /** + * The regression this file exists for: a fixture captured from a runtime has to read back + * against that same runtime as no drift at all. Keyed by the metadata spelling instead, every + * event in the fixture reads as removed - 81 of them, against mainnet. + */ + it('produces a fixture that reads back against its own runtime as no drift', () => { + const fixture: ArityFixture = { + specVersion: 1, + source: 'substrate static metadata', + modules: { balances: real.events.balances }, + }; + const drift = eventDrift(fixture, real); + + expect([drift.added, drift.removed, drift.reshaped]).toEqual([[], [], []]); + }); +}); + +describe('sectionId', () => { + it('maps a metadata pallet name onto the api section name', () => { + expect(sectionId('ExternalAgents')).toBe('externalAgents'); + expect(sectionId('Asset')).toBe('asset'); + }); +}); + +describe('arityFixtureFor', () => { + it('captures a pallet it was asked for, rather than writing an empty fixture', () => { + const captured = arityFixtureFor( + snapshot({ events: { asset: { AssetCreated: 8 }, notCaptured: { Whatever: 1 } } }) + ); + + expect(captured.modules).toEqual({ asset: { AssetCreated: 8 } }); + }); +}); + describe('enum parsing', () => { it('reads member names past comments, docstrings and directives', () => { expect(enumMembers(findEnumBlock(SCHEMA, 'ModuleIdEnum').body)).toEqual([ @@ -72,7 +133,7 @@ describe('planEnumUpdates', () => { it('sorts additions so two runs over the same runtime produce the same file', () => { const updates = planEnumUpdates( SCHEMA, - snapshot({ events: { Balances: { Zebra: 1, Apple: 1, BalanceSet: 2 } } }) + snapshot({ events: { balances: { Zebra: 1, Apple: 1, BalanceSet: 2 } } }) ); expect(updates.EventIdEnum.added).toEqual(['Apple', 'Zebra']); @@ -118,23 +179,23 @@ describe('eventDrift', () => { const fixture: ArityFixture = { specVersion: 7_004_001, source: 'test', - modules: { Balances: { BalanceSet: 4, Gone: 1 } }, + modules: { balances: { BalanceSet: 4, Gone: 1 } }, }; it('names an event whose parameter count changed, which positional decoding cannot see', () => { - expect(eventDrift(fixture, snapshot()).reshaped).toEqual(['Balances.BalanceSet: 4 -> 2']); + expect(eventDrift(fixture, snapshot()).reshaped).toEqual(['balances.BalanceSet: 4 -> 2']); }); it('names an event the runtime added since the fixture was captured', () => { - expect(eventDrift(fixture, snapshot()).added).toEqual(['Balances.TransferWithMemo']); + expect(eventDrift(fixture, snapshot()).added).toEqual(['balances.TransferWithMemo']); }); it('names an event the runtime no longer has', () => { - expect(eventDrift(fixture, snapshot()).removed).toEqual(['Balances.Gone']); + expect(eventDrift(fixture, snapshot()).removed).toEqual(['balances.Gone']); }); it('reports nothing when the runtime matches the fixture', () => { - const same = snapshot({ events: { Balances: { BalanceSet: 4, Gone: 1 } } }); + const same = snapshot({ events: { balances: { BalanceSet: 4, Gone: 1 } } }); const drift = eventDrift(fixture, same); expect([drift.added, drift.removed, drift.reshaped]).toEqual([[], [], []]);