From bc9f4198ba5cfab7b3a255d3cd0dd7aafea1f658 Mon Sep 17 00:00:00 2001 From: Charlie Chen <34498985+ws4charlie@users.noreply.github.com> Date: Tue, 27 Aug 2024 05:45:04 -0500 Subject: [PATCH] feat: enable parsing Bitcoin deposit memo with inscription; deposit fee improvement (#2768) * allow parsing Bitcoin memo iwth inscription; improve Bitcoin depositor fee * add changelog entry * corrected PR# and function comment typos * replace GetRecentBlockhash with GetLatestBlockhash; add check on Vsize; misc * test: cherry pick E2E bitcoin test for inscription for 19.1.2 (#2769) * test: cherry pick E2E bitcoin test for inscription * fix dockerfile --------- Co-authored-by: Lucas Bertrand --- changelog.md | 19 + contrib/localnet/bitcoin-sidecar/Dockerfile | 14 + .../localnet/bitcoin-sidecar/js/package.json | 23 ++ .../localnet/bitcoin-sidecar/js/src/client.ts | 183 +++++++++ .../localnet/bitcoin-sidecar/js/src/index.ts | 38 ++ .../localnet/bitcoin-sidecar/js/src/script.ts | 52 +++ .../bitcoin-sidecar/js/src/tsconfig.json | 11 + .../localnet/bitcoin-sidecar/js/src/util.ts | 1 + contrib/localnet/docker-compose.yml | 13 + e2e/e2etests/e2etests.go | 8 + .../test_extract_bitcoin_inscription_memo.go | 60 +++ e2e/runner/bitcoin.go | 55 ++- e2e/runner/inscription.go | 119 ++++++ e2e/runner/solana.go | 2 +- zetaclient/chains/bitcoin/fee.go | 119 +++++- zetaclient/chains/bitcoin/observer/inbound.go | 17 +- .../chains/bitcoin/observer/observer.go | 3 +- zetaclient/chains/bitcoin/observer/witness.go | 29 +- zetaclient/chains/bitcoin/rpc/rpc.go | 92 +++-- .../chains/bitcoin/rpc/rpc_live_test.go | 360 +++++++++++------- zetaclient/chains/interfaces/interfaces.go | 2 +- zetaclient/chains/solana/signer/withdraw.go | 4 +- zetaclient/testutils/mocks/solana_rpc.go | 16 +- 23 files changed, 1019 insertions(+), 221 deletions(-) create mode 100644 contrib/localnet/bitcoin-sidecar/Dockerfile create mode 100644 contrib/localnet/bitcoin-sidecar/js/package.json create mode 100644 contrib/localnet/bitcoin-sidecar/js/src/client.ts create mode 100644 contrib/localnet/bitcoin-sidecar/js/src/index.ts create mode 100644 contrib/localnet/bitcoin-sidecar/js/src/script.ts create mode 100644 contrib/localnet/bitcoin-sidecar/js/src/tsconfig.json create mode 100644 contrib/localnet/bitcoin-sidecar/js/src/util.ts create mode 100644 e2e/e2etests/test_extract_bitcoin_inscription_memo.go create mode 100644 e2e/runner/inscription.go diff --git a/changelog.md b/changelog.md index 68ed22496f..a97efce164 100644 --- a/changelog.md +++ b/changelog.md @@ -1,7 +1,26 @@ # CHANGELOG +## v19.2.0 + +### Features + +* [2768](https://github.com/zeta-chain/node/pull/2768) - enable parsing Bitcoin deposit memo with inscription; deposit fee improvement + +## v19.1.1 + +Fix release CI build + ## v19.1.0 +### Breaking Changes + +* `zetaclientd` now prompts for a "Solana Relayer Key" password. If you have not configured a solana relayer key, you should enter an empty password. + +### Fixes +* [2628](https://github.com/zeta-chain/node/pull/2628) - avoid submitting invalid hashes to outbound tracker + +### Features + * [2524](https://github.com/zeta-chain/node/pull/2524) - add inscription envolop parsing * [2533](https://github.com/zeta-chain/node/pull/2533) - parse memo from both OP_RETURN and inscription * [2568](https://github.com/zeta-chain/node/pull/2568) - improve AppContext by converging chains, chainParams, enabledChains, and additionalChains into a single zctx.Chain diff --git a/contrib/localnet/bitcoin-sidecar/Dockerfile b/contrib/localnet/bitcoin-sidecar/Dockerfile new file mode 100644 index 0000000000..aef54cf56d --- /dev/null +++ b/contrib/localnet/bitcoin-sidecar/Dockerfile @@ -0,0 +1,14 @@ +FROM node:18.20.4 as builder + +WORKDIR /home/zeta/node + +COPY bitcoin-sidecar/js/* . + +RUN npm install && npm install typescript -g && tsc + +FROM node:alpine + +COPY --from=builder /home/zeta/node/dist ./dist +COPY --from=builder /home/zeta/node/node_modules ./node_modules + +CMD ["node", "dist/index.js"] \ No newline at end of file diff --git a/contrib/localnet/bitcoin-sidecar/js/package.json b/contrib/localnet/bitcoin-sidecar/js/package.json new file mode 100644 index 0000000000..1a4dd4b90a --- /dev/null +++ b/contrib/localnet/bitcoin-sidecar/js/package.json @@ -0,0 +1,23 @@ +{ + "name": "zeta-btc-client", + "version": "0.0.1", + "description": "The Zetachain BTC client", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "bip32": "^4.0.0", + "bitcoinjs-lib": "^6.1.6", + "ecpair": "^2.1.0", + "express": "^4.19.2", + "randombytes": "^2.1.0", + "tiny-secp256k1": "^2.2.3" + }, + "devDependencies": { + "@types/node": "^20.14.11", + "typescript": "^5.5.3" + } +} \ No newline at end of file diff --git a/contrib/localnet/bitcoin-sidecar/js/src/client.ts b/contrib/localnet/bitcoin-sidecar/js/src/client.ts new file mode 100644 index 0000000000..678b90f6da --- /dev/null +++ b/contrib/localnet/bitcoin-sidecar/js/src/client.ts @@ -0,0 +1,183 @@ +import { initEccLib, payments, Psbt } from "bitcoinjs-lib"; +import { bitcoin, Network, regtest } from "bitcoinjs-lib/src/networks"; +import BIP32Factory, { BIP32Interface } from 'bip32'; +import * as ecc from 'tiny-secp256k1'; +import randomBytes from "randombytes"; +import { ScriptBuilder } from "./script"; +import { Taptree } from "bitcoinjs-lib/src/types"; +import { toXOnly } from "./util"; + +const LEAF_VERSION_TAPSCRIPT = 0xc0; + +initEccLib(ecc); +const bip32 = BIP32Factory(ecc); +const rng = randomBytes; + +/// The evm address type, a 20 bytes hex string +export type Address = String; +export type BtcAddress = String; + +/// The BTC transaction hash returned +export type BtcTxnHash = String; +export interface BtcInput { + txn: BtcTxnHash, + idx: number, +} + +/** + * The example client for interacting with ZetaChain in BTC. There are currently two ways + * of calling a smart contract on ZetaChain from BTC: + * + * - Using OP_RETURN + * - Using Witness + * + * The method used is now based on the data size. Within 80 bytes, `OP_RETURN` is used, else + * the data is written to Witness. + * + * This class handles only the case where data is more than 80 bytes. + */ +export class ZetaBtcClient { + /** The BTC network interracting with */ + readonly network: Network; + + private reveal: RevealTxnBuilder | null; + + private constructor(network: Network) { + this.network = network; + } + + public static regtest(): ZetaBtcClient { + return new ZetaBtcClient(regtest); + } + + public static mainnet(): ZetaBtcClient { + return new ZetaBtcClient(bitcoin); + } + + /** + * Call a target address and passing the data call. + * + * @param address The target zetachain evm address + * @param calldata The calldata that will be invoked on Zetachain + */ + public call( + address: Address, + calldata: Buffer, + ): Address { + if (calldata.length <= 80) { + throw Error("Use op return instead"); + } + + if (address.startsWith("0x")) { + address = address.substring(2); + } + + return this.callWithWitness(Buffer.concat([Buffer.from(address, "hex"), calldata])); + } + + private callWithWitness( + data: Buffer, + ): Address { + const internalKey = bip32.fromSeed(rng(64), this.network); + + const leafScript = this.genLeafScript(internalKey.publicKey, data); + + const scriptTree: Taptree = { output: leafScript }; + + const { address: commitAddress } = payments.p2tr({ + internalPubkey: toXOnly(internalKey.publicKey), + scriptTree, + network: this.network, + }); + + this.reveal = new RevealTxnBuilder(internalKey, leafScript, this.network); + + return commitAddress; + } + + public buildRevealTxn(to: string, commitTxn: BtcInput, commitAmount: number, feeRate: number): Buffer { + if (this.reveal === null) { + throw new Error("commit txn not built yet"); + } + + this.reveal.with_commit_tx(to, commitTxn, commitAmount, feeRate); + return this.reveal.dump(); + } + + private genLeafScript(publicKey: Buffer, data: Buffer,): Buffer { + const builder = ScriptBuilder.new(publicKey); + builder.pushData(data); + return builder.build(); + } +} + +class RevealTxnBuilder { + private psbt: Psbt; + private key: BIP32Interface; + private leafScript: Buffer; + private network: Network + + constructor(key: BIP32Interface, leafScript: Buffer, network: Network) { + this.psbt = new Psbt({ network });; + this.key = key; + this.leafScript = leafScript; + this.network = network; + } + + public with_commit_tx(to: string, commitTxn: BtcInput, commitAmount: number, feeRate: number): RevealTxnBuilder { + const scriptTree: Taptree = { output: this.leafScript }; + + const { output, witness } = payments.p2tr({ + internalPubkey: toXOnly(this.key.publicKey), + scriptTree, + redeem: { + output: this.leafScript, + redeemVersion: LEAF_VERSION_TAPSCRIPT, + }, + network: this.network, + }); + + this.psbt.addInput({ + hash: commitTxn.txn.toString(), + index: commitTxn.idx, + witnessUtxo: { value: commitAmount, script: output! }, + tapLeafScript: [ + { + leafVersion: LEAF_VERSION_TAPSCRIPT, + script: this.leafScript, + controlBlock: witness![witness!.length - 1], + }, + ], + }); + + this.psbt.addOutput({ + value: commitAmount - this.estimateFee(to, commitAmount, feeRate), + address: to, + }); + + this.psbt.signAllInputs(this.key); + this.psbt.finalizeAllInputs(); + + return this; + } + + public dump(): Buffer { + return this.psbt.extractTransaction(true).toBuffer(); + } + + private estimateFee(to: string, amount: number, feeRate: number): number { + const cloned = this.psbt.clone(); + + cloned.addOutput({ + value: amount, + address: to, + }); + + // should have a way to avoid signing but just providing mocked signautre + cloned.signAllInputs(this.key); + cloned.finalizeAllInputs(); + + const size = cloned.extractTransaction().virtualSize(); + return size * feeRate; + } +} \ No newline at end of file diff --git a/contrib/localnet/bitcoin-sidecar/js/src/index.ts b/contrib/localnet/bitcoin-sidecar/js/src/index.ts new file mode 100644 index 0000000000..5164a6f148 --- /dev/null +++ b/contrib/localnet/bitcoin-sidecar/js/src/index.ts @@ -0,0 +1,38 @@ +import { ZetaBtcClient } from "./client"; +import express, { Request, Response } from 'express'; + +const app = express(); +const PORT = process.env.PORT || 3000; +let zetaClient = ZetaBtcClient.regtest(); + +app.use(express.json()); + +// Middleware to parse URL-encoded bodies +app.use(express.urlencoded({ extended: true })); + +// Route to handle JSON POST requests +app.post('/commit', (req: Request, res: Response) => { + const memo: string = req.body.memo; + const address = zetaClient.call("", Buffer.from(memo, "hex")); + res.json({ address }); +}); + +// Route to handle URL-encoded POST requests +app.post('/reveal', (req: Request, res: Response) => { + const { txn, idx, amount, feeRate, to } = req.body; + console.log(txn, idx, amount, feeRate); + + const rawHex = zetaClient.buildRevealTxn(to,{ txn, idx }, Number(amount), feeRate).toString("hex"); + zetaClient = ZetaBtcClient.regtest(); + res.json({ rawHex }); +}); + +// Start the server +app.listen(PORT, () => { + console.log(`Server is running on http://localhost:${PORT}`); +}); + +/** + * curl --request POST --header "Content-Type: application/json" --data '{"memo":"72f080c854647755d0d9e6f6821f6931f855b9acffd53d87433395672756d58822fd143360762109ab898626556b1c3b8d3096d2361f1297df4a41c1b429471a9aa2fc9be5f27c13b3863d6ac269e4b587d8389f8fd9649859935b0d48dea88cdb40f20c"}' http://localhost:3000/commit + * curl --request POST --header "Content-Type: application/json" --data '{"txn": "7a57f987a3cb605896a5909d9ef2bf7afbf0c78f21e4118b85d00d9e4cce0c2c", "idx": 0, "amount": 1000, "feeRate": 10}' http://localhost:3000/reveal + */ \ No newline at end of file diff --git a/contrib/localnet/bitcoin-sidecar/js/src/script.ts b/contrib/localnet/bitcoin-sidecar/js/src/script.ts new file mode 100644 index 0000000000..f282e39f01 --- /dev/null +++ b/contrib/localnet/bitcoin-sidecar/js/src/script.ts @@ -0,0 +1,52 @@ +import { opcodes, script, Stack } from "bitcoinjs-lib"; +import { toXOnly } from "./util"; + +const MAX_SCRIPT_ELEMENT_SIZE = 520; + +/** The tapscript builder for zetaclient spending script */ +export class ScriptBuilder { + private script: Stack; + + private constructor(initialScript: Stack) { + this.script = initialScript; + } + + public static new(publicKey: Buffer): ScriptBuilder { + const stack = [ + toXOnly(publicKey), + opcodes.OP_CHECKSIG, + ]; + return new ScriptBuilder(stack); + } + + public pushData(data: Buffer) { + if (data.length <= 80) { + throw new Error("data length should be more than 80 bytes"); + } + + this.script.push( + opcodes.OP_FALSE, + opcodes.OP_IF + ); + + const chunks = chunkBuffer(data, MAX_SCRIPT_ELEMENT_SIZE); + for (const chunk of chunks) { + this.script.push(chunk); + } + + this.script.push(opcodes.OP_ENDIF); + } + + public build(): Buffer { + return script.compile(this.script); + } +} + +function chunkBuffer(buffer: Buffer, chunkSize: number): Buffer[] { + const chunks = []; + for (let i = 0; i < buffer.length; i += chunkSize) { + const chunk = buffer.slice(i, i + chunkSize); + chunks.push(chunk); + } + return chunks; +} \ No newline at end of file diff --git a/contrib/localnet/bitcoin-sidecar/js/src/tsconfig.json b/contrib/localnet/bitcoin-sidecar/js/src/tsconfig.json new file mode 100644 index 0000000000..4033670b3d --- /dev/null +++ b/contrib/localnet/bitcoin-sidecar/js/src/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "module": "commonjs", + "esModuleInterop": true, + "target": "es6", + "moduleResolution": "node", + "sourceMap": true, + "outDir": "dist" + }, + "lib": ["es2015"] +} \ No newline at end of file diff --git a/contrib/localnet/bitcoin-sidecar/js/src/util.ts b/contrib/localnet/bitcoin-sidecar/js/src/util.ts new file mode 100644 index 0000000000..87c4d36d0f --- /dev/null +++ b/contrib/localnet/bitcoin-sidecar/js/src/util.ts @@ -0,0 +1 @@ +export const toXOnly = pubKey => (pubKey.length === 32 ? pubKey : pubKey.slice(1, 33)); \ No newline at end of file diff --git a/contrib/localnet/docker-compose.yml b/contrib/localnet/docker-compose.yml index 6eece58fce..2f57daa6c1 100644 --- a/contrib/localnet/docker-compose.yml +++ b/contrib/localnet/docker-compose.yml @@ -223,6 +223,19 @@ services: -rpcauth=smoketest:63acf9b8dccecce914d85ff8c044b78b$$5892f9bbc84f4364e79f0970039f88bdd823f168d4acc76099ab97b14a766a99 -txindex=1 + bitcoin-node-sidecar: + build: + dockerfile: ./bitcoin-sidecar/Dockerfile + container_name: bitcoin-node-sidecar + hostname: bitcoin-node-sidecar + networks: + mynetwork: + ipv4_address: 172.20.0.111 + environment: + - PORT=8000 + ports: + - "8000:8000" + solana: image: solana-local:latest container_name: solana diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index 402eed367e..7e95ccb2e4 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -71,6 +71,7 @@ const ( TestBitcoinWithdrawP2SHName = "bitcoin_withdraw_p2sh" TestBitcoinWithdrawInvalidAddressName = "bitcoin_withdraw_invalid" TestBitcoinWithdrawRestrictedName = "bitcoin_withdraw_restricted" + TestExtractBitcoinInscriptionMemoName = "bitcoin_memo_from_inscription" /* Application tests @@ -355,6 +356,13 @@ var AllE2ETests = []runner.E2ETest{ /* Bitcoin tests */ + runner.NewE2ETest( + TestExtractBitcoinInscriptionMemoName, + "extract memo from BTC inscription", []runner.ArgDefinition{ + {Description: "amount in btc", DefaultValue: "0.1"}, + }, + TestExtractBitcoinInscriptionMemo, + ), runner.NewE2ETest( TestBitcoinDepositName, "deposit Bitcoin into ZEVM", diff --git a/e2e/e2etests/test_extract_bitcoin_inscription_memo.go b/e2e/e2etests/test_extract_bitcoin_inscription_memo.go new file mode 100644 index 0000000000..2713326ba7 --- /dev/null +++ b/e2e/e2etests/test_extract_bitcoin_inscription_memo.go @@ -0,0 +1,60 @@ +package e2etests + +import ( + "encoding/hex" + + "github.com/btcsuite/btcd/btcjson" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/require" + + "github.com/zeta-chain/zetacore/e2e/runner" + zetabitcoin "github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin" + btcobserver "github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin/observer" +) + +func TestExtractBitcoinInscriptionMemo(r *runner.E2ERunner, args []string) { + r.SetBtcAddress(r.Name, false) + + // obtain some initial fund + stop := r.MineBlocksIfLocalBitcoin() + defer stop() + r.Logger.Info("Mined blocks") + + // list deployer utxos + utxos, err := r.ListDeployerUTXOs() + require.NoError(r, err) + + amount := parseFloat(r, args[0]) + // this is just some random test memo for inscription + memo, err := hex.DecodeString( + "72f080c854647755d0d9e6f6821f6931f855b9acffd53d87433395672756d58822fd143360762109ab898626556b1c3b8d3096d2361f1297df4a41c1b429471a9aa2fc9be5f27c13b3863d6ac269e4b587d8389f8fd9649859935b0d48dea88cdb40f20c", + ) + require.NoError(r, err) + + txid := r.InscribeToTSSFromDeployerWithMemo(amount, utxos, memo) + + _, err = r.GenerateToAddressIfLocalBitcoin(6, r.BTCDeployerAddress) + require.NoError(r, err) + + rawtx, err := r.BtcRPCClient.GetRawTransactionVerbose(txid) + require.NoError(r, err) + r.Logger.Info("obtained reveal txn id %s", txid) + + dummyCoinbaseTxn := rawtx + depositorFee := zetabitcoin.DefaultDepositorFee + events, err := btcobserver.FilterAndParseIncomingTx( + r.BtcRPCClient, + []btcjson.TxRawResult{*dummyCoinbaseTxn, *rawtx}, + 0, + r.BTCTSSAddress.String(), + log.Logger, + r.BitcoinParams, + depositorFee, + ) + require.NoError(r, err) + + require.Equal(r, 1, len(events)) + event := events[0] + + require.Equal(r, event.MemoBytes, memo) +} diff --git a/e2e/runner/bitcoin.go b/e2e/runner/bitcoin.go index 3a4dad583e..b7c601a21e 100644 --- a/e2e/runner/bitcoin.go +++ b/e2e/runner/bitcoin.go @@ -2,7 +2,9 @@ package runner import ( "bytes" + "encoding/hex" "fmt" + "net/http" "sort" "time" @@ -176,9 +178,17 @@ func (r *E2ERunner) SendToTSSFromDeployerWithMemo( amount float64, inputUTXOs []btcjson.ListUnspentResult, memo []byte, +) (*chainhash.Hash, error) { + return r.sendToAddrFromDeployerWithMemo(amount, r.BTCTSSAddress, inputUTXOs, memo) +} + +func (r *E2ERunner) sendToAddrFromDeployerWithMemo( + amount float64, + to btcutil.Address, + inputUTXOs []btcjson.ListUnspentResult, + memo []byte, ) (*chainhash.Hash, error) { btcRPC := r.BtcRPCClient - to := r.BTCTSSAddress btcDeployerAddress := r.BTCDeployerAddress require.NotNil(r, r.BTCDeployerAddress, "btcDeployerAddress is nil") @@ -286,6 +296,49 @@ func (r *E2ERunner) SendToTSSFromDeployerWithMemo( return txid, nil } +// InscribeToTSSFromDeployerWithMemo creates an inscription that is sent to the tss address with the corresponding memo +func (r *E2ERunner) InscribeToTSSFromDeployerWithMemo( + amount float64, + inputUTXOs []btcjson.ListUnspentResult, + memo []byte, +) *chainhash.Hash { + // TODO: replace builder with Go function to enable instructions + // https://github.com/zeta-chain/node/issues/2759 + builder := InscriptionBuilder{sidecarURL: "http://bitcoin-node-sidecar:8000", client: http.Client{}} + + address, err := builder.GenerateCommitAddress(memo) + require.NoError(r, err) + r.Logger.Info("received inscription commit address %s", address) + + receiver, err := chains.DecodeBtcAddress(address, r.GetBitcoinChainID()) + require.NoError(r, err) + + txnHash, err := r.sendToAddrFromDeployerWithMemo(amount, receiver, inputUTXOs, []byte(constant.DonationMessage)) + require.NoError(r, err) + r.Logger.Info("obtained inscription commit txn hash %s", txnHash.String()) + + // sendToAddrFromDeployerWithMemo makes sure index is 0 + outpointIdx := 0 + hexTx, err := builder.GenerateRevealTxn(r.BTCTSSAddress.String(), txnHash.String(), outpointIdx, amount) + require.NoError(r, err) + + // Decode the hex string into raw bytes + rawTxBytes, err := hex.DecodeString(hexTx) + require.NoError(r, err) + + // Deserialize the raw bytes into a wire.MsgTx structure + msgTx := wire.NewMsgTx(wire.TxVersion) + err = msgTx.Deserialize(bytes.NewReader(rawTxBytes)) + require.NoError(r, err) + r.Logger.Info("recovered inscription reveal txn %s", hexTx) + + txid, err := r.BtcRPCClient.SendRawTransaction(msgTx, true) + require.NoError(r, err) + r.Logger.Info("txid: %+v", txid) + + return txid +} + // GetBitcoinChainID gets the bitcoin chain ID from the network params func (r *E2ERunner) GetBitcoinChainID() int64 { chainID, err := chains.BitcoinChainIDFromNetworkName(r.BitcoinParams.Name) diff --git a/e2e/runner/inscription.go b/e2e/runner/inscription.go new file mode 100644 index 0000000000..6f90068905 --- /dev/null +++ b/e2e/runner/inscription.go @@ -0,0 +1,119 @@ +package runner + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/pkg/errors" +) + +type commitResponse struct { + Address string `json:"address"` +} + +type revealResponse struct { + RawHex string `json:"rawHex"` +} + +type revealRequest struct { + Txn string `json:"txn"` + Idx int `json:"idx"` + Amount int `json:"amount"` + FeeRate int `json:"feeRate"` + To string `json:"to"` +} + +// InscriptionBuilder is a util struct that help create inscription commit and reveal transactions +type InscriptionBuilder struct { + sidecarURL string + client http.Client +} + +// GenerateCommitAddress generates a commit p2tr address that one can send funds to this address +func (r *InscriptionBuilder) GenerateCommitAddress(memo []byte) (string, error) { + // Create the payload + postData := map[string]string{ + "memo": hex.EncodeToString(memo), + } + + // Convert the payload to JSON + jsonData, err := json.Marshal(postData) + if err != nil { + return "", err + } + + postURL := r.sidecarURL + "/commit" + req, err := http.NewRequest("POST", postURL, bytes.NewBuffer(jsonData)) + if err != nil { + return "", errors.Wrap(err, "cannot create commit request") + } + req.Header.Set("Content-Type", "application/json") + + // Send the request + resp, err := r.client.Do(req) + if err != nil { + return "", errors.Wrap(err, "cannot send to sidecar") + } + defer resp.Body.Close() + + // Read the response body + var response commitResponse + err = json.NewDecoder(resp.Body).Decode(&response) + if err != nil { + return "", err + } + + fmt.Print("raw commit response ", response.Address) + + return response.Address, nil +} + +// GenerateRevealTxn creates the corresponding reveal txn to the commit txn. +func (r *InscriptionBuilder) GenerateRevealTxn(to string, txnHash string, idx int, amount float64) (string, error) { + postData := revealRequest{ + Txn: txnHash, + Idx: idx, + Amount: int(amount * 100000000), + FeeRate: 10, + To: to, + } + + // Convert the payload to JSON + jsonData, err := json.Marshal(postData) + if err != nil { + return "", err + } + + postURL := r.sidecarURL + "/reveal" + req, err := http.NewRequest("POST", postURL, bytes.NewBuffer(jsonData)) + if err != nil { + return "", errors.Wrap(err, "cannot create reveal request") + } + req.Header.Set("Content-Type", "application/json") + + // Send the request + resp, err := r.client.Do(req) + if err != nil { + return "", errors.Wrap(err, "cannot send reveal to sidecar") + } + defer resp.Body.Close() + + // Read the response body + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", errors.Wrap(err, "cannot read reveal response body") + } + + // Parse the JSON response + var response revealResponse + if err := json.Unmarshal(body, &response); err != nil { + return "", errors.Wrap(err, "cannot parse reveal response body") + } + + // Access the "address" field + return response.RawHex, nil +} diff --git a/e2e/runner/solana.go b/e2e/runner/solana.go index 71d2cfb629..7854e62716 100644 --- a/e2e/runner/solana.go +++ b/e2e/runner/solana.go @@ -64,7 +64,7 @@ func (r *E2ERunner) CreateSignedTransaction( privateKey solana.PrivateKey, ) *solana.Transaction { // get a recent blockhash - recent, err := r.SolanaClient.GetRecentBlockhash(r.Ctx, rpc.CommitmentFinalized) + recent, err := r.SolanaClient.GetLatestBlockhash(r.Ctx, rpc.CommitmentFinalized) require.NoError(r, err) // create the initialize transaction diff --git a/zetaclient/chains/bitcoin/fee.go b/zetaclient/chains/bitcoin/fee.go index aec41407d9..3293727b47 100644 --- a/zetaclient/chains/bitcoin/fee.go +++ b/zetaclient/chains/bitcoin/fee.go @@ -15,27 +15,42 @@ import ( "github.com/rs/zerolog" "github.com/zeta-chain/zetacore/pkg/chains" + "github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin/rpc" + "github.com/zeta-chain/zetacore/zetaclient/chains/interfaces" clientcommon "github.com/zeta-chain/zetacore/zetaclient/common" ) const ( - bytesPerKB = 1000 - bytesPerInput = 41 // each input is 41 bytes - bytesPerOutputP2TR = 43 // each P2TR output is 43 bytes - bytesPerOutputP2WSH = 43 // each P2WSH output is 43 bytes - bytesPerOutputP2WPKH = 31 // each P2WPKH output is 31 bytes - bytesPerOutputP2SH = 32 // each P2SH output is 32 bytes - bytesPerOutputP2PKH = 34 // each P2PKH output is 34 bytes - bytesPerOutputAvg = 37 // average size of all above types of outputs (36.6 bytes) - bytes1stWitness = 110 // the 1st witness incurs about 110 bytes and it may vary - bytesPerWitness = 108 // each additional witness incurs about 108 bytes and it may vary - defaultDepositorFeeRate = 20 // 20 sat/byte is the default depositor fee rate - - OutboundBytesMin = uint64(239) // 239vB == EstimateSegWitTxSize(2, 2, toP2WPKH) - OutboundBytesMax = uint64(1543) // 1543v == EstimateSegWitTxSize(21, 2, toP2TR) - OutboundBytesAvg = uint64(245) // 245vB is a suggested gas limit for zetacore - - DynamicDepositorFeeHeight = 834500 // DynamicDepositorFeeHeight contains the starting height (Bitcoin mainnet) from which dynamic depositor fee will take effect + // constants related to transaction size calculations + bytesPerKB = 1000 + bytesPerInput = 41 // each input is 41 bytes + bytesPerOutputP2TR = 43 // each P2TR output is 43 bytes + bytesPerOutputP2WSH = 43 // each P2WSH output is 43 bytes + bytesPerOutputP2WPKH = 31 // each P2WPKH output is 31 bytes + bytesPerOutputP2SH = 32 // each P2SH output is 32 bytes + bytesPerOutputP2PKH = 34 // each P2PKH output is 34 bytes + bytesPerOutputAvg = 37 // average size of all above types of outputs (36.6 bytes) + bytes1stWitness = 110 // the 1st witness incurs about 110 bytes and it may vary + bytesPerWitness = 108 // each additional witness incurs about 108 bytes and it may vary + OutboundBytesMin = uint64(239) // 239vB == EstimateSegWitTxSize(2, 2, toP2WPKH) + OutboundBytesMax = uint64(1543) // 1543v == EstimateSegWitTxSize(21, 2, toP2TR) + OutboundBytesAvg = uint64(245) // 245vB is a suggested gas limit for zetacore + + // defaultDepositorFeeRate is the default fee rate for depositor fee, 20 sat/vB + defaultDepositorFeeRate = 20 + + // defaultTestnetFeeRate is the default fee rate for testnet, 10 sat/vB + defaultTestnetFeeRate = 10 + + // feeRateCountBackBlocks is the default number of blocks to look back for fee rate estimation + feeRateCountBackBlocks = 2 + + // DynamicDepositorFeeHeight is the mainnet height from which dynamic depositor fee V1 is applied + DynamicDepositorFeeHeight = 834500 + + // DynamicDepositorFeeHeightV2 is the mainnet height from which dynamic depositor fee V2 is applied + // Height 863400 is approximately a month away (2024-09-28) from the time of writing, allowing enough time for the upgrade + DynamicDepositorFeeHeightV2 = 863400 ) var ( @@ -239,3 +254,73 @@ func CalcDepositorFee( return DepositorFee(feeRate) } + +// CalcDepositorFeeV2 calculates the depositor fee for a given tx result +func CalcDepositorFeeV2( + rpcClient interfaces.BTCRPCClient, + rawResult *btcjson.TxRawResult, + netParams *chaincfg.Params, +) (float64, error) { + // use default fee for regnet + if netParams.Name == chaincfg.RegressionNetParams.Name { + return DefaultDepositorFee, nil + } + + // get fee rate of the transaction + _, feeRate, err := rpc.GetTransactionFeeAndRate(rpcClient, rawResult) + if err != nil { + return 0, errors.Wrapf(err, "error getting fee rate for tx %s", rawResult.Txid) + } + + // apply gas price multiplier + // #nosec G115 always in range + feeRate = int64(float64(feeRate) * clientcommon.BTCOutboundGasPriceMultiplier) + + return DepositorFee(feeRate), nil +} + +// GetRecentFeeRate gets the highest fee rate from recent blocks +// Note: this method should be used for testnet ONLY +func GetRecentFeeRate(rpcClient interfaces.BTCRPCClient, netParams *chaincfg.Params) (uint64, error) { + // should avoid using this method for mainnet + if netParams.Name == chaincfg.MainNetParams.Name { + return 0, errors.New("GetRecentFeeRate should not be used for mainnet") + } + + // get the current block number + blockNumber, err := rpcClient.GetBlockCount() + if err != nil { + return 0, err + } + + // get the highest fee rate among recent 'countBack' blocks to avoid underestimation + highestRate := int64(0) + for i := int64(0); i < feeRateCountBackBlocks; i++ { + // get the block + hash, err := rpcClient.GetBlockHash(blockNumber - i) + if err != nil { + return 0, err + } + block, err := rpcClient.GetBlockVerboseTx(hash) + if err != nil { + return 0, err + } + + // computes the average fee rate of the block and take the higher rate + avgFeeRate, err := CalcBlockAvgFeeRate(block, netParams) + if err != nil { + return 0, err + } + if avgFeeRate > highestRate { + highestRate = avgFeeRate + } + } + + // use 10 sat/byte as default estimation if recent fee rate drops to 0 + if highestRate == 0 { + highestRate = defaultTestnetFeeRate + } + + // #nosec G115 always in range + return uint64(highestRate), nil +} diff --git a/zetaclient/chains/bitcoin/observer/inbound.go b/zetaclient/chains/bitcoin/observer/inbound.go index a7dc5afe3d..546d23277f 100644 --- a/zetaclient/chains/bitcoin/observer/inbound.go +++ b/zetaclient/chains/bitcoin/observer/inbound.go @@ -266,7 +266,7 @@ func (ob *Observer) CheckReceiptForBtcTxHash(ctx context.Context, txHash string, } // #nosec G115 always positive - event, err := GetBtcEvent( + event, err := GetBtcEventWithWitness( ob.btcClient, *tx, tss, @@ -328,7 +328,7 @@ func FilterAndParseIncomingTx( continue // the first tx is coinbase; we do not process coinbase tx } - inbound, err := GetBtcEvent(rpcClient, tx, tssAddress, blockNumber, logger, netParams, depositorFee) + inbound, err := GetBtcEventWithWitness(rpcClient, tx, tssAddress, blockNumber, logger, netParams, depositorFee) if err != nil { // unable to parse the tx, the caller should retry return nil, errors.Wrapf(err, "error getting btc event for tx %s in block %d", tx.Txid, blockNumber) @@ -421,6 +421,19 @@ func GetBtcEvent( return nil, nil } + // switch to depositor fee V2 if + // 1. it is bitcoin testnet, or + // 2. it is bitcoin mainnet and upgrade height is reached + // TODO: remove CalcDepositorFeeV1 and below conditions after the upgrade height + // https://github.com/zeta-chain/node/issues/2766 + if netParams.Name == chaincfg.TestNet3Params.Name || + (netParams.Name == chaincfg.MainNetParams.Name && blockNumber >= bitcoin.DynamicDepositorFeeHeightV2) { + depositorFee, err = bitcoin.CalcDepositorFeeV2(rpcClient, &tx, netParams) + if err != nil { + return nil, errors.Wrapf(err, "error calculating depositor fee V2 for inbound: %s", tx.Txid) + } + } + // deposit amount has to be no less than the minimum depositor fee if vout0.Value < depositorFee { logger.Info(). diff --git a/zetaclient/chains/bitcoin/observer/observer.go b/zetaclient/chains/bitcoin/observer/observer.go index 6a15173c33..d8b7378b58 100644 --- a/zetaclient/chains/bitcoin/observer/observer.go +++ b/zetaclient/chains/bitcoin/observer/observer.go @@ -23,7 +23,6 @@ import ( observertypes "github.com/zeta-chain/zetacore/x/observer/types" "github.com/zeta-chain/zetacore/zetaclient/chains/base" "github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin" - "github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin/rpc" "github.com/zeta-chain/zetacore/zetaclient/chains/interfaces" "github.com/zeta-chain/zetacore/zetaclient/db" "github.com/zeta-chain/zetacore/zetaclient/metrics" @@ -628,7 +627,7 @@ func (ob *Observer) specialHandleFeeRate() (uint64, error) { // hardcode gas price for regnet return 1, nil case chains.NetworkType_testnet: - feeRateEstimated, err := rpc.GetRecentFeeRate(ob.btcClient, ob.netParams) + feeRateEstimated, err := bitcoin.GetRecentFeeRate(ob.btcClient, ob.netParams) if err != nil { return 0, errors.Wrapf(err, "error GetRecentFeeRate") } diff --git a/zetaclient/chains/bitcoin/observer/witness.go b/zetaclient/chains/bitcoin/observer/witness.go index 0af55c62a9..b0996895ba 100644 --- a/zetaclient/chains/bitcoin/observer/witness.go +++ b/zetaclient/chains/bitcoin/observer/witness.go @@ -16,6 +16,8 @@ import ( // GetBtcEventWithWitness either returns a valid BTCInboundEvent or nil. // This method supports data with more than 80 bytes by scanning the witness for possible presence of a tapscript. // It will first prioritize OP_RETURN over tapscript. +// +// Note: the caller should rescan the tx on error (e.g., GetSenderAddressByVin RPC failed) func GetBtcEventWithWitness( client interfaces.BTCRPCClient, tx btcjson.TxRawResult, @@ -34,11 +36,26 @@ func GetBtcEventWithWitness( return nil, nil } - if err := isValidRecipient(tx.Vout[0].ScriptPubKey.Hex, tssAddress, netParams); err != nil { + // check if the recipient is the tss address + err := isValidRecipient(tx.Vout[0].ScriptPubKey.Hex, tssAddress, netParams) + if err != nil { logger.Debug().Msgf("irrelevant recipient %s for tx %s, err: %s", tx.Vout[0].ScriptPubKey.Hex, tx.Txid, err) return nil, nil } + // switch to depositor fee V2 if + // 1. it is bitcoin testnet, or + // 2. it is bitcoin mainnet and upgrade height is reached + // TODO: remove CalcDepositorFeeV1 and below conditions after the upgrade height + // https://github.com/zeta-chain/node/issues/2766 + if netParams.Name == chaincfg.TestNet3Params.Name || + (netParams.Name == chaincfg.MainNetParams.Name && blockNumber >= bitcoin.DynamicDepositorFeeHeightV2) { + depositorFee, err = bitcoin.CalcDepositorFeeV2(client, &tx, netParams) + if err != nil { + return nil, errors.Wrapf(err, "error calculating depositor fee V2 for inbound: %s", tx.Txid) + } + } + isAmountValid, amount := isValidAmount(tx.Vout[0].Value, depositorFee) if !isAmountValid { logger.Info(). @@ -52,13 +69,13 @@ func GetBtcEventWithWitness( var memo []byte if candidate := tryExtractOpRet(tx, logger); candidate != nil { memo = candidate - logger.Debug(). + logger.Info(). Msgf("GetBtcEventWithWitness: found OP_RETURN memo %s in tx %s", hex.EncodeToString(memo), tx.Txid) } else if candidate = tryExtractInscription(tx, logger); candidate != nil { memo = candidate - logger.Debug().Msgf("GetBtcEventWithWitness: found inscription memo %s in tx %s", hex.EncodeToString(memo), tx.Txid) + logger.Info().Msgf("GetBtcEventWithWitness: found inscription memo %s in tx %s", hex.EncodeToString(memo), tx.Txid) } else { - return nil, errors.Errorf("error getting memo for inbound: %s", tx.Txid) + return nil, nil } // event found, get sender address @@ -116,7 +133,7 @@ func ParseScriptFromWitness(witness []string, logger zerolog.Logger) []byte { return script } -// / Try to extract the memo from the OP_RETURN +// Try to extract the memo from the OP_RETURN func tryExtractOpRet(tx btcjson.TxRawResult, logger zerolog.Logger) []byte { if len(tx.Vout) < 2 { logger.Debug().Msgf("txn %s has fewer than 2 outputs, not target OP_RETURN txn", tx.Txid) @@ -135,7 +152,7 @@ func tryExtractOpRet(tx btcjson.TxRawResult, logger zerolog.Logger) []byte { return nil } -// / Try to extract the memo from inscription +// Try to extract the memo from inscription func tryExtractInscription(tx btcjson.TxRawResult, logger zerolog.Logger) []byte { for i, input := range tx.Vin { script := ParseScriptFromWitness(input.Witness, logger) diff --git a/zetaclient/chains/bitcoin/rpc/rpc.go b/zetaclient/chains/bitcoin/rpc/rpc.go index 079b6fa298..5449c915e5 100644 --- a/zetaclient/chains/bitcoin/rpc/rpc.go +++ b/zetaclient/chains/bitcoin/rpc/rpc.go @@ -4,24 +4,15 @@ import ( "fmt" "github.com/btcsuite/btcd/btcjson" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/rpcclient" + "github.com/btcsuite/btcutil" "github.com/pkg/errors" - "github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin" "github.com/zeta-chain/zetacore/zetaclient/chains/interfaces" "github.com/zeta-chain/zetacore/zetaclient/config" ) -const ( - // feeRateCountBackBlocks is the default number of blocks to look back for fee rate estimation - feeRateCountBackBlocks = 2 - - // defaultTestnetFeeRate is the default fee rate for testnet, 10 sat/byte - defaultTestnetFeeRate = 10 -) - // NewRPCClient creates a new RPC client by the given config. func NewRPCClient(btcConfig config.BTCConfig) (*rpcclient.Client, error) { connCfg := &rpcclient.ConnConfig{ @@ -63,6 +54,20 @@ func GetTxResultByHash( return hash, txResult, nil } +// GetRawTxByHash gets the raw transaction by hash +func GetRawTxByHash(rpcClient interfaces.BTCRPCClient, txID string) (*btcutil.Tx, error) { + hash, err := chainhash.NewHashFromStr(txID) + if err != nil { + return nil, errors.Wrapf(err, "GetRawTxByHash: error NewHashFromStr: %s", txID) + } + + tx, err := rpcClient.GetRawTransaction(hash) + if err != nil { + return nil, errors.Wrapf(err, "GetRawTxByHash: error GetRawTransaction %s", txID) + } + return tx, nil +} + // GetBlockHeightByHash gets the block height by block hash func GetBlockHeightByHash( rpcClient interfaces.BTCRPCClient, @@ -118,42 +123,53 @@ func GetRawTxResult( return btcjson.TxRawResult{}, fmt.Errorf("GetRawTxResult: tx %s not included yet", hash) } -// GetRecentFeeRate gets the highest fee rate from recent blocks -// Note: this method is only used for testnet -func GetRecentFeeRate(rpcClient interfaces.BTCRPCClient, netParams *chaincfg.Params) (uint64, error) { - blockNumber, err := rpcClient.GetBlockCount() - if err != nil { - return 0, err +// GetTransactionFeeAndRate gets the transaction fee and rate for a given tx result +func GetTransactionFeeAndRate(rpcClient interfaces.BTCRPCClient, rawResult *btcjson.TxRawResult) (int64, int64, error) { + var ( + totalInputValue int64 + totalOutputValue int64 + ) + + // make sure the tx Vsize is not zero (should not happen) + if rawResult.Vsize <= 0 { + return 0, 0, fmt.Errorf("tx %s has non-positive Vsize: %d", rawResult.Txid, rawResult.Vsize) } - // get the highest fee rate among recent 'countBack' blocks to avoid underestimation - highestRate := int64(0) - for i := int64(0); i < feeRateCountBackBlocks; i++ { - // get the block - hash, err := rpcClient.GetBlockHash(blockNumber - i) - if err != nil { - return 0, err - } - block, err := rpcClient.GetBlockVerboseTx(hash) + // sum up total input value + for _, vin := range rawResult.Vin { + prevTx, err := GetRawTxByHash(rpcClient, vin.Txid) if err != nil { - return 0, err + return 0, 0, errors.Wrapf(err, "failed to get previous tx: %s", vin.Txid) } + totalInputValue += prevTx.MsgTx().TxOut[vin.Vout].Value + } - // computes the average fee rate of the block and take the higher rate - avgFeeRate, err := bitcoin.CalcBlockAvgFeeRate(block, netParams) - if err != nil { - return 0, err - } - if avgFeeRate > highestRate { - highestRate = avgFeeRate - } + // query the raw tx + tx, err := GetRawTxByHash(rpcClient, rawResult.Txid) + if err != nil { + return 0, 0, errors.Wrapf(err, "failed to get tx: %s", rawResult.Txid) } - // use 10 sat/byte as default estimation if recent fee rate drops to 0 - if highestRate == 0 { - highestRate = defaultTestnetFeeRate + // sum up total output value + for _, vout := range tx.MsgTx().TxOut { + totalOutputValue += vout.Value } + // calculate the transaction fee in satoshis + fee := totalInputValue - totalOutputValue + if fee < 0 { // never happens + return 0, 0, fmt.Errorf("got negative fee: %d", fee) + } + + // Note: the calculation uses 'Vsize' returned by RPC to simplify dev experience: + // - 1. the devs could use the same value returned by their RPC endpoints to estimate deposit fee. + // - 2. the devs don't have to bother 'Vsize' calculation, even though there is more accurate formula. + // Moreoever, the accurate 'Vsize' is usually an adjusted size (float value) by Bitcoin Core. + // - 3. the 'Vsize' calculation could depend on program language and the library used. + // + // calculate the fee rate in satoshis/vByte // #nosec G115 always in range - return uint64(highestRate), nil + feeRate := fee / int64(rawResult.Vsize) + + return fee, feeRate, nil } diff --git a/zetaclient/chains/bitcoin/rpc/rpc_live_test.go b/zetaclient/chains/bitcoin/rpc/rpc_live_test.go index 54964d7403..f8320c97a2 100644 --- a/zetaclient/chains/bitcoin/rpc/rpc_live_test.go +++ b/zetaclient/chains/bitcoin/rpc/rpc_live_test.go @@ -10,91 +10,24 @@ import ( "testing" "time" + "github.com/btcsuite/btcd/blockchain" "github.com/btcsuite/btcd/btcjson" "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/rpcclient" - "github.com/ethereum/go-ethereum/crypto" "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - "github.com/zeta-chain/zetacore/zetaclient/db" "github.com/zeta-chain/zetacore/pkg/chains" - "github.com/zeta-chain/zetacore/zetaclient/chains/base" "github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin" "github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin/observer" "github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin/rpc" + "github.com/zeta-chain/zetacore/zetaclient/common" "github.com/zeta-chain/zetacore/zetaclient/config" "github.com/zeta-chain/zetacore/zetaclient/testutils" - "github.com/zeta-chain/zetacore/zetaclient/testutils/mocks" ) -type BitcoinObserverTestSuite struct { - suite.Suite - rpcClient *rpcclient.Client -} - -func (suite *BitcoinObserverTestSuite) SetupTest() { - // test private key with EVM address - //// EVM: 0x236C7f53a90493Bb423411fe4117Cb4c2De71DfB - // BTC testnet3: muGe9prUBjQwEnX19zG26fVRHNi8z7kSPo - skHex := "7b8507ba117e069f4a3f456f505276084f8c92aee86ac78ae37b4d1801d35fa8" - privateKey, err := crypto.HexToECDSA(skHex) - suite.Require().NoError(err) - pkBytes := crypto.FromECDSAPub(&privateKey.PublicKey) - suite.T().Logf("pubkey: %d", len(pkBytes)) - - tss := &mocks.TSS{ - PrivKey: privateKey, - } - - // create mock arguments for constructor - chain := chains.BitcoinMainnet - params := mocks.MockChainParams(chain.ChainId, 10) - btcClient := mocks.NewMockBTCRPCClient() - - database, err := db.NewFromSqliteInMemory(true) - suite.Require().NoError(err) - - // create observer - ob, err := observer.NewObserver(chain, btcClient, params, nil, tss, database, base.DefaultLogger(), nil) - - suite.Require().NoError(err) - suite.Require().NotNil(ob) - suite.rpcClient, err = createRPCClient(18332) - suite.Require().NoError(err) - skBytes, err := hex.DecodeString(skHex) - suite.Require().NoError(err) - suite.T().Logf("skBytes: %d", len(skBytes)) - - _, err = btcClient.CreateWallet("e2e") - suite.Require().NoError(err) - addr, err := btcClient.GetNewAddress("test") - suite.Require().NoError(err) - suite.T().Logf("deployer address: %s", addr) - //err = btc.ImportPrivKey(privkeyWIF) - //suite.Require().NoError(err) - - btcClient.GenerateToAddress(101, addr, nil) - suite.Require().NoError(err) - - bal, err := btcClient.GetBalance("*") - suite.Require().NoError(err) - suite.T().Logf("balance: %f", bal.ToBTC()) - - utxo, err := btcClient.ListUnspent() - suite.Require().NoError(err) - suite.T().Logf("utxo: %d", len(utxo)) - for _, u := range utxo { - suite.T().Logf("utxo: %s %f", u.Address, u.Amount) - } -} - -func (suite *BitcoinObserverTestSuite) TearDownSuite() { -} - // createRPCClient creates a new Bitcoin RPC client for given chainID func createRPCClient(chainID int64) (*rpcclient.Client, error) { var connCfg *rpcclient.ConnConfig @@ -126,6 +59,7 @@ func createRPCClient(chainID int64) (*rpcclient.Client, error) { return rpcclient.New(connCfg, nil) } +// getFeeRate is a helper function to get fee rate for a given confirmation target func getFeeRate( client *rpcclient.Client, confTarget int64, @@ -144,28 +78,60 @@ func getFeeRate( return new(big.Int).SetInt64(int64(*feeResult.FeeRate * 1e8)), nil } -// All methods that begin with "Test" are run as tests within a -// suite. -func (suite *BitcoinObserverTestSuite) Test1() { - feeResult, err := suite.rpcClient.EstimateSmartFee(1, nil) - suite.Require().NoError(err) - suite.T().Logf("fee result: %f", *feeResult.FeeRate) - bn, err := suite.rpcClient.GetBlockCount() - suite.Require().NoError(err) - suite.T().Logf("block %d", bn) +// getMempoolSpaceTxsByBlock gets mempool.space txs for a given block +func getMempoolSpaceTxsByBlock( + t *testing.T, + client *rpcclient.Client, + blkNumber int64, + testnet bool, +) (*chainhash.Hash, []testutils.MempoolTx, error) { + blkHash, err := client.GetBlockHash(blkNumber) + if err != nil { + t.Logf("error GetBlockHash for block %d: %s\n", blkNumber, err) + return nil, nil, err + } + + // get mempool.space txs for the block + mempoolTxs, err := testutils.GetBlockTxs(context.Background(), blkHash.String(), testnet) + if err != nil { + t.Logf("error GetBlockTxs %d: %s\n", blkNumber, err) + return nil, nil, err + } + + return blkHash, mempoolTxs, nil +} + +// Test_BitcoinLive is a phony test to run each live test individually +func Test_BitcoinLive(t *testing.T) { + // LiveTest_FilterAndParseIncomingTx(t) + // LiveTest_FilterAndParseIncomingTx_Nop(t) + // LiveTest_NewRPCClient(t) + // LiveTest_GetBlockHeightByHash(t) + // LiveTest_BitcoinFeeRate(t) + // LiveTest_AvgFeeRateMainnetMempoolSpace(t) + // LiveTest_AvgFeeRateTestnetMempoolSpace(t) + // LiveTest_GetRecentFeeRate(t) + // LiveTest_GetSenderByVin(t) + // LiveTest_GetTransactionFeeAndRate(t) + // LiveTest_CalcDepositorFeeV2(t) +} +func LiveTest_FilterAndParseIncomingTx(t *testing.T) { + // setup Bitcoin client + client, err := createRPCClient(chains.BitcoinTestnet.ChainId) + require.NoError(t, err) + + // get the block that contains the incoming tx hashStr := "0000000000000032cb372f5d5d99c1ebf4430a3059b67c47a54dd626550fb50d" - var hash chainhash.Hash - err = chainhash.Decode(&hash, hashStr) - suite.Require().NoError(err) + hash, err := chainhash.NewHashFromStr(hashStr) + require.NoError(t, err) - block, err := suite.rpcClient.GetBlockVerboseTx(&hash) - suite.Require().NoError(err) - suite.T().Logf("block confirmation %d", block.Confirmations) - suite.T().Logf("block txs len %d", len(block.Tx)) + block, err := client.GetBlockVerboseTx(hash) + require.NoError(t, err) + // filter incoming tx inbounds, err := observer.FilterAndParseIncomingTx( - suite.rpcClient, + client, block.Tx, uint64(block.Height), "tb1qsa222mn2rhdq9cruxkz8p2teutvxuextx3ees2", @@ -173,36 +139,37 @@ func (suite *BitcoinObserverTestSuite) Test1() { &chaincfg.TestNet3Params, 0.0, ) - suite.Require().NoError(err) - suite.Require().Equal(1, len(inbounds)) - suite.Require().Equal(inbounds[0].Value, 0.0001) - suite.Require().Equal(inbounds[0].ToAddress, "tb1qsa222mn2rhdq9cruxkz8p2teutvxuextx3ees2") + require.NoError(t, err) + require.Len(t, inbounds, 1) + require.Equal(t, inbounds[0].Value, 0.0001) + require.Equal(t, inbounds[0].ToAddress, "tb1qsa222mn2rhdq9cruxkz8p2teutvxuextx3ees2") + // the text memo is base64 std encoded string:DSRR1RmDCwWmxqY201/TMtsJdmA= // see https://blockstream.info/testnet/tx/889bfa69eaff80a826286d42ec3f725fd97c3338357ddc3a1f543c2d6266f797 - memo, err := hex.DecodeString("0d2451D519830B05a6C6a636d35fd332dB097660") - suite.Require().NoError(err) - suite.Require().Equal((inbounds[0].MemoBytes), memo) - suite.Require().Equal(inbounds[0].FromAddress, "tb1qyslx2s8evalx67n88wf42yv7236303ezj3tm2l") - suite.T().Logf("from: %s", inbounds[0].FromAddress) - suite.Require().Equal(inbounds[0].BlockNumber, uint64(2406185)) - suite.Require().Equal(inbounds[0].TxHash, "889bfa69eaff80a826286d42ec3f725fd97c3338357ddc3a1f543c2d6266f797") + memo, err := hex.DecodeString("4453525231526d444377576d7871593230312f544d74734a646d413d") + require.NoError(t, err) + require.Equal(t, inbounds[0].MemoBytes, memo) + require.Equal(t, inbounds[0].FromAddress, "tb1qyslx2s8evalx67n88wf42yv7236303ezj3tm2l") + require.Equal(t, inbounds[0].BlockNumber, uint64(2406185)) + require.Equal(t, inbounds[0].TxHash, "889bfa69eaff80a826286d42ec3f725fd97c3338357ddc3a1f543c2d6266f797") } -// a tx with memo around 81B (is this allowed1?) -func (suite *BitcoinObserverTestSuite) Test2() { +func LiveTest_FilterAndParseIncomingTx_Nop(t *testing.T) { + // setup Bitcoin client + client, err := createRPCClient(chains.BitcoinTestnet.ChainId) + require.NoError(t, err) + + // get a block that contains no incoming tx hashStr := "000000000000002fd8136dbf91708898da9d6ae61d7c354065a052568e2f2888" - var hash chainhash.Hash - err := chainhash.Decode(&hash, hashStr) - suite.Require().NoError(err) + hash, err := chainhash.NewHashFromStr(hashStr) + require.NoError(t, err) - block, err := suite.rpcClient.GetBlockVerboseTx(&hash) - suite.Require().NoError(err) - suite.T().Logf("block confirmation %d", block.Confirmations) - suite.T().Logf("block height %d", block.Height) - suite.T().Logf("block txs len %d", len(block.Tx)) + block, err := client.GetBlockVerboseTx(hash) + require.NoError(t, err) + // filter incoming tx inbounds, err := observer.FilterAndParseIncomingTx( - suite.rpcClient, + client, block.Tx, uint64(block.Height), "tb1qsa222mn2rhdq9cruxkz8p2teutvxuextx3ees2", @@ -210,25 +177,12 @@ func (suite *BitcoinObserverTestSuite) Test2() { &chaincfg.TestNet3Params, 0.0, ) - suite.Require().NoError(err) - suite.Require().Equal(0, len(inbounds)) -} - -// TestBitcoinObserverLive is a phony test to run each live test individually -func TestBitcoinObserverLive(t *testing.T) { - // suite.Run(t, new(BitcoinClientTestSuite)) - - // LiveTestNewRPCClient(t) - // LiveTestGetBlockHeightByHash(t) - // LiveTestBitcoinFeeRate(t) - // LiveTestAvgFeeRateMainnetMempoolSpace(t) - // LiveTestAvgFeeRateTestnetMempoolSpace(t) - // LiveTestGetRecentFeeRate(t) - // LiveTestGetSenderByVin(t) + require.NoError(t, err) + require.Empty(t, inbounds) } // LiveTestNewRPCClient creates a new Bitcoin RPC client -func LiveTestNewRPCClient(t *testing.T) { +func LiveTest_NewRPCClient(t *testing.T) { btcConfig := config.BTCConfig{ RPCUsername: "user", RPCPassword: "pass", @@ -247,7 +201,7 @@ func LiveTestNewRPCClient(t *testing.T) { } // LiveTestGetBlockHeightByHash queries Bitcoin block height by hash -func LiveTestGetBlockHeightByHash(t *testing.T) { +func LiveTest_GetBlockHeightByHash(t *testing.T) { // setup Bitcoin client client, err := createRPCClient(chains.BitcoinMainnet.ChainId) require.NoError(t, err) @@ -269,7 +223,7 @@ func LiveTestGetBlockHeightByHash(t *testing.T) { // LiveTestBitcoinFeeRate query Bitcoin mainnet fee rate every 5 minutes // and compares Conservative and Economical fee rates for different block targets (1 and 2) -func LiveTestBitcoinFeeRate(t *testing.T) { +func LiveTest_BitcoinFeeRate(t *testing.T) { // setup Bitcoin client client, err := createRPCClient(chains.BitcoinMainnet.ChainId) require.NoError(t, err) @@ -394,7 +348,7 @@ func compareAvgFeeRate(t *testing.T, client *rpcclient.Client, startBlock int, e } // LiveTestAvgFeeRateMainnetMempoolSpace compares calculated fee rate with mempool.space fee rate for mainnet -func LiveTestAvgFeeRateMainnetMempoolSpace(t *testing.T) { +func LiveTest_AvgFeeRateMainnetMempoolSpace(t *testing.T) { // setup Bitcoin client client, err := createRPCClient(chains.BitcoinMainnet.ChainId) require.NoError(t, err) @@ -408,7 +362,7 @@ func LiveTestAvgFeeRateMainnetMempoolSpace(t *testing.T) { } // LiveTestAvgFeeRateTestnetMempoolSpace compares calculated fee rate with mempool.space fee rate for testnet -func LiveTestAvgFeeRateTestnetMempoolSpace(t *testing.T) { +func LiveTest_AvgFeeRateTestnetMempoolSpace(t *testing.T) { // setup Bitcoin client client, err := createRPCClient(chains.BitcoinTestnet.ChainId) require.NoError(t, err) @@ -422,13 +376,13 @@ func LiveTestAvgFeeRateTestnetMempoolSpace(t *testing.T) { } // LiveTestGetRecentFeeRate gets the highest fee rate from recent blocks -func LiveTestGetRecentFeeRate(t *testing.T) { +func LiveTest_GetRecentFeeRate(t *testing.T) { // setup Bitcoin testnet client client, err := createRPCClient(chains.BitcoinTestnet.ChainId) require.NoError(t, err) // get fee rate from recent blocks - feeRate, err := rpc.GetRecentFeeRate(client, &chaincfg.TestNet3Params) + feeRate, err := bitcoin.GetRecentFeeRate(client, &chaincfg.TestNet3Params) require.NoError(t, err) require.Greater(t, feeRate, uint64(0)) } @@ -453,22 +407,13 @@ func LiveTestGetSenderByVin(t *testing.T) { require.NoError(t, err) endBlock := startBlock - 5000 - // loop through mempool.space blocks in descending order + // loop through mempool.space blocks backwards BLOCKLOOP: for bn := startBlock; bn >= endBlock; { - // get block hash - blkHash, err := client.GetBlockHash(int64(bn)) - if err != nil { - fmt.Printf("error GetBlockHash for block %d: %s\n", bn, err) - time.Sleep(3 * time.Second) - continue - } - // get mempool.space txs for the block - mempoolTxs, err := testutils.GetBlockTxs(context.Background(), blkHash.String(), testnet) + _, mempoolTxs, err := getMempoolSpaceTxsByBlock(t, client, bn, testnet) if err != nil { - fmt.Printf("error GetBlockTxs %d: %s\n", bn, err) - time.Sleep(10 * time.Second) + time.Sleep(3 * time.Second) continue } @@ -502,6 +447,135 @@ BLOCKLOOP: } } bn-- - time.Sleep(500 * time.Millisecond) + time.Sleep(100 * time.Millisecond) } } + +// LiveTestGetTransactionFeeAndRate gets the transaction fee and rate for each tx and compares with mempool.space fee rate +func LiveTest_GetTransactionFeeAndRate(t *testing.T) { + // setup Bitcoin client + chainID := chains.BitcoinTestnet.ChainId + client, err := createRPCClient(chainID) + require.NoError(t, err) + + // testnet or mainnet + testnet := false + if chainID == chains.BitcoinTestnet.ChainId { + testnet = true + } + + // calculates block range to test + startBlock, err := client.GetBlockCount() + require.NoError(t, err) + endBlock := startBlock - 10 // go back whatever blocks as needed + + // loop through mempool.space blocks backwards + for bn := startBlock; bn >= endBlock; { + // get mempool.space txs for the block + blkHash, mempoolTxs, err := getMempoolSpaceTxsByBlock(t, client, bn, testnet) + if err != nil { + time.Sleep(3 * time.Second) + continue + } + + // get the block from rpc client + block, err := client.GetBlockVerboseTx(blkHash) + if err != nil { + time.Sleep(3 * time.Second) + continue + } + + // loop through each tx in the block (skip coinbase tx) + for i := 1; i < len(block.Tx); { + // sample 20 txs per block + if i >= 20 { + break + } + + // the two txs from two different sources + tx := block.Tx[i] + mpTx := mempoolTxs[i] + require.Equal(t, tx.Txid, mpTx.TxID) + + // get transaction fee rate for the raw result + fee, feeRate, err := rpc.GetTransactionFeeAndRate(client, &tx) + if err != nil { + t.Logf("error GetTransactionFeeRate %s: %s\n", mpTx.TxID, err) + continue + } + require.EqualValues(t, mpTx.Fee, fee) + require.EqualValues(t, mpTx.Weight, tx.Weight) + + // calculate mempool.space fee rate + vBytes := mpTx.Weight / blockchain.WitnessScaleFactor + mpFeeRate := int64(mpTx.Fee / vBytes) + + // compare our fee rate with mempool.space fee rate + var diff int64 + var diffPercent float64 + if feeRate == mpFeeRate { + fmt.Printf("tx %s: [our rate] %5d == %5d [mempool.space]", mpTx.TxID, feeRate, mpFeeRate) + } else if feeRate > mpFeeRate { + diff = feeRate - mpFeeRate + fmt.Printf("tx %s: [our rate] %5d > %5d [mempool.space]", mpTx.TxID, feeRate, mpFeeRate) + } else { + diff = mpFeeRate - feeRate + fmt.Printf("tx %s: [our rate] %5d < %5d [mempool.space]", mpTx.TxID, feeRate, mpFeeRate) + } + + // print the diff percentage + diffPercent = float64(diff) / float64(mpFeeRate) * 100 + if diff > 0 { + fmt.Printf(", diff: %f%%\n", diffPercent) + } else { + fmt.Printf("\n") + } + + // the expected diff percentage should be within 5% + if mpFeeRate >= 20 { + require.LessOrEqual(t, diffPercent, 5.0) + } else { + // for small fee rate, the absolute diff should be within 1 satoshi/vByte + require.LessOrEqual(t, diff, int64(1)) + } + + // next tx + i++ + } + + bn-- + time.Sleep(100 * time.Millisecond) + } +} + +func LiveTest_CalcDepositorFeeV2(t *testing.T) { + // setup Bitcoin client + client, err := createRPCClient(chains.BitcoinMainnet.ChainId) + require.NoError(t, err) + + // test tx hash + // https://mempool.space/tx/8dc0d51f83810cec7fcb5b194caebfc5fc64b10f9fe21845dfecc621d2a28538 + hash, err := chainhash.NewHashFromStr("8dc0d51f83810cec7fcb5b194caebfc5fc64b10f9fe21845dfecc621d2a28538") + require.NoError(t, err) + + // get the raw transaction result + rawResult, err := client.GetRawTransactionVerbose(hash) + require.NoError(t, err) + + t.Run("should return default depositor fee", func(t *testing.T) { + depositorFee, err := bitcoin.CalcDepositorFeeV2(client, rawResult, &chaincfg.RegressionNetParams) + require.NoError(t, err) + require.Equal(t, bitcoin.DefaultDepositorFee, depositorFee) + }) + + t.Run("should return correct depositor fee for a given tx", func(t *testing.T) { + depositorFee, err := bitcoin.CalcDepositorFeeV2(client, rawResult, &chaincfg.MainNetParams) + require.NoError(t, err) + + // the actual fee rate is 860 sat/vByte + // #nosec G115 always in range + expectedRate := int64(float64(860) * common.BTCOutboundGasPriceMultiplier) + expectedFee := bitcoin.DepositorFee(expectedRate) + require.Equal(t, expectedFee, depositorFee) + }) +} diff --git a/zetaclient/chains/interfaces/interfaces.go b/zetaclient/chains/interfaces/interfaces.go index edd656979b..ed26019f8c 100644 --- a/zetaclient/chains/interfaces/interfaces.go +++ b/zetaclient/chains/interfaces/interfaces.go @@ -198,7 +198,7 @@ type SolanaRPCClient interface { account solana.PublicKey, commitment solrpc.CommitmentType, ) (*solrpc.GetBalanceResult, error) - GetRecentBlockhash(ctx context.Context, commitment solrpc.CommitmentType) (*solrpc.GetRecentBlockhashResult, error) + GetLatestBlockhash(ctx context.Context, commitment solrpc.CommitmentType) (*solrpc.GetLatestBlockhashResult, error) GetRecentPrioritizationFees( ctx context.Context, accounts solana.PublicKeySlice, diff --git a/zetaclient/chains/solana/signer/withdraw.go b/zetaclient/chains/solana/signer/withdraw.go index f44dc3fc30..8b91dc8f23 100644 --- a/zetaclient/chains/solana/signer/withdraw.go +++ b/zetaclient/chains/solana/signer/withdraw.go @@ -69,9 +69,9 @@ func (signer *Signer) SignWithdrawTx(ctx context.Context, msg contracts.MsgWithd attachWithdrawAccounts(&inst, privkey.PublicKey(), signer.pda, msg.To(), signer.gatewayID) // get a recent blockhash - recent, err := signer.client.GetRecentBlockhash(ctx, rpc.CommitmentFinalized) + recent, err := signer.client.GetLatestBlockhash(ctx, rpc.CommitmentFinalized) if err != nil { - return nil, errors.Wrap(err, "GetRecentBlockhash error") + return nil, errors.Wrap(err, "GetLatestBlockhash error") } // create a transaction that wraps the instruction diff --git a/zetaclient/testutils/mocks/solana_rpc.go b/zetaclient/testutils/mocks/solana_rpc.go index fad147037c..26b923abb3 100644 --- a/zetaclient/testutils/mocks/solana_rpc.go +++ b/zetaclient/testutils/mocks/solana_rpc.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.38.0. DO NOT EDIT. +// Code generated by mockery v2.42.2. DO NOT EDIT. package mocks @@ -135,24 +135,24 @@ func (_m *SolanaRPCClient) GetHealth(ctx context.Context) (string, error) { return r0, r1 } -// GetRecentBlockhash provides a mock function with given fields: ctx, commitment -func (_m *SolanaRPCClient) GetRecentBlockhash(ctx context.Context, commitment rpc.CommitmentType) (*rpc.GetRecentBlockhashResult, error) { +// GetLatestBlockhash provides a mock function with given fields: ctx, commitment +func (_m *SolanaRPCClient) GetLatestBlockhash(ctx context.Context, commitment rpc.CommitmentType) (*rpc.GetLatestBlockhashResult, error) { ret := _m.Called(ctx, commitment) if len(ret) == 0 { - panic("no return value specified for GetRecentBlockhash") + panic("no return value specified for GetLatestBlockhash") } - var r0 *rpc.GetRecentBlockhashResult + var r0 *rpc.GetLatestBlockhashResult var r1 error - if rf, ok := ret.Get(0).(func(context.Context, rpc.CommitmentType) (*rpc.GetRecentBlockhashResult, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, rpc.CommitmentType) (*rpc.GetLatestBlockhashResult, error)); ok { return rf(ctx, commitment) } - if rf, ok := ret.Get(0).(func(context.Context, rpc.CommitmentType) *rpc.GetRecentBlockhashResult); ok { + if rf, ok := ret.Get(0).(func(context.Context, rpc.CommitmentType) *rpc.GetLatestBlockhashResult); ok { r0 = rf(ctx, commitment) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*rpc.GetRecentBlockhashResult) + r0 = ret.Get(0).(*rpc.GetLatestBlockhashResult) } }