From 0a6e247163f737926b582c12c8ceb1b8161e7a4a Mon Sep 17 00:00:00 2001 From: Craig Raw Date: Mon, 13 Jul 2020 14:27:54 +0200 Subject: [PATCH] support more accurate virtual size calculations --- .../drongo/protocol/ScriptType.java | 51 ++++- .../drongo/protocol/Transaction.java | 10 +- .../drongo/wallet/BnBUtxoSelector.java | 207 ++++++++++++++++++ .../sparrowwallet/drongo/wallet/Wallet.java | 86 +++++++- 4 files changed, 338 insertions(+), 16 deletions(-) create mode 100644 src/main/java/com/sparrowwallet/drongo/wallet/BnBUtxoSelector.java diff --git a/src/main/java/com/sparrowwallet/drongo/protocol/ScriptType.java b/src/main/java/com/sparrowwallet/drongo/protocol/ScriptType.java index a979314..c60d83c 100644 --- a/src/main/java/com/sparrowwallet/drongo/protocol/ScriptType.java +++ b/src/main/java/com/sparrowwallet/drongo/protocol/ScriptType.java @@ -1054,27 +1054,56 @@ public enum ScriptType { } /** - * Determines the dust threshold for this script type. + * Determines the dust threshold for the given output for this script type. * * @param output The output under consideration - * @param feeRate The fee rate at which the fee required will be calculated * @return the minimum viable value than the provided output must have in order to not be dust */ public long getDustThreshold(TransactionOutput output, Double feeRate) { + return getFee(output, feeRate, Transaction.DUST_RELAY_TX_FEE); + } + + /** + * Determines the minimum incremental fee necessary to pay for added the provided output to a transaction + * This is done by calculating the sum of multiplying the size of the output at the current fee rate, + * and the size of the input needed to spend it in future at the long term fee rate + * + * @param output The output to be added + * @param feeRate The transaction's fee rate + * @param longTermFeeRate The long term minimum fee rate + * @return The fee that adding this output would add + */ + public long getFee(TransactionOutput output, Double feeRate, Double longTermFeeRate) { //Start with length of output - int totalLength = output.getLength(); - if(Arrays.asList(WITNESS_TYPES).contains(this)) { - //Add length of spending input with 75% discount to script size - totalLength += (32 + 4 + 1 + (107 / WITNESS_SCALE_FACTOR) + 4); + int outputVbytes = output.getLength(); + //Add length of spending input (with or without discount depending on script type) + int inputVbytes = getInputVbytes(); + + //Return fee rate in sats/vByte multiplied by the calculated output and input vByte lengths + return (long)(feeRate * outputVbytes + longTermFeeRate * inputVbytes); + } + + /** + * Return a coarse estimation of the minimum number of vBytes required to spend an input of this script type. + * Because we don't know the nature of the scriptSig/witnessScript required, pay to script inputs will likely be underestimated. + * Use Wallet.getInputVbytes() for an accurate value to spend a wallet UTXO. + * + * @return The number of vBytes required for an input of this script type + */ + public int getInputVbytes() { + if(P2SH_P2WPKH.equals(this)) { + return (32 + 4 + 1 + 13 + (107 / WITNESS_SCALE_FACTOR) + 4); + } else if(P2SH_P2WSH.equals(this)) { + return (32 + 4 + 1 + 35 + (107 / WITNESS_SCALE_FACTOR) + 4); + } else if(Arrays.asList(WITNESS_TYPES).contains(this)) { + //Return length of spending input with 75% discount to script size + return (32 + 4 + 1 + (107 / WITNESS_SCALE_FACTOR) + 4); } else if(Arrays.asList(NON_WITNESS_TYPES).contains(this)) { - //Add length of spending input with no discount - totalLength += (32 + 4 + 1 + 107 + 4); + //Return length of spending input with no discount + return (32 + 4 + 1 + 107 + 4); } else { throw new UnsupportedOperationException("Cannot determine dust threshold for script type " + this.getName()); } - - //Return fee rate in sats/vbyte multiplied by the calculated total byte length - return (long)(feeRate * totalLength); } @Override diff --git a/src/main/java/com/sparrowwallet/drongo/protocol/Transaction.java b/src/main/java/com/sparrowwallet/drongo/protocol/Transaction.java index b492a39..972a7e6 100644 --- a/src/main/java/com/sparrowwallet/drongo/protocol/Transaction.java +++ b/src/main/java/com/sparrowwallet/drongo/protocol/Transaction.java @@ -267,6 +267,10 @@ public class Transaction extends ChildMessage { } public int getVirtualSize() { + return (int)Math.ceil((double)getWeightUnits() / (double)WITNESS_SCALE_FACTOR); + } + + public int getWeightUnits() { int wu = 0; // version @@ -294,7 +298,7 @@ public class Transaction extends ChildMessage { // lock_time wu += 4 * WITNESS_SCALE_FACTOR; - return (int)Math.ceil((double)wu / (double)WITNESS_SCALE_FACTOR); + return wu; } public List getInputs() { @@ -306,6 +310,10 @@ public class Transaction extends ChildMessage { } public TransactionInput addInput(Sha256Hash spendTxHash, long outputIndex, Script script, TransactionWitness witness) { + if(!isSegwit()) { + setSegwitVersion(0); + } + return addInput(new TransactionInput(this, new TransactionOutPoint(spendTxHash, outputIndex), script.getProgram(), witness)); } diff --git a/src/main/java/com/sparrowwallet/drongo/wallet/BnBUtxoSelector.java b/src/main/java/com/sparrowwallet/drongo/wallet/BnBUtxoSelector.java new file mode 100644 index 0000000..975c4cb --- /dev/null +++ b/src/main/java/com/sparrowwallet/drongo/wallet/BnBUtxoSelector.java @@ -0,0 +1,207 @@ +package com.sparrowwallet.drongo.wallet; + +import com.sparrowwallet.drongo.KeyPurpose; +import com.sparrowwallet.drongo.protocol.Transaction; +import com.sparrowwallet.drongo.protocol.TransactionOutput; + +import java.util.*; +import java.util.stream.Collectors; + +import static com.sparrowwallet.drongo.protocol.Transaction.WITNESS_SCALE_FACTOR; + +public class BnBUtxoSelector implements UtxoSelector { + private static final int TOTAL_TRIES = 100000; + + private final Wallet wallet; + private final int noInputsWeightUnits; + private final Double feeRate; + private final Double longTermFeeRate; + private final int inputWeightUnits; + private final long costOfChangeValue; + + public BnBUtxoSelector(Wallet wallet, int noInputsWeightUnits, Double feeRate, Double longTermFeeRate) { + this.wallet = wallet; + this.noInputsWeightUnits = noInputsWeightUnits; + this.feeRate = feeRate; + this.longTermFeeRate = longTermFeeRate; + this.inputWeightUnits = wallet.getInputWeightUnits(); + this.costOfChangeValue = getCostOfChange(); + } + + @Override + public Collection select(long targetValue, Collection candidates) { + List utxoPool = candidates.stream().map(OutputGroup::new).collect(Collectors.toList()); + + long currentValue = 0; + + ArrayDeque currentSelection = new ArrayDeque<>(utxoPool.size()); + long actualTargetValue = targetValue + (long)(noInputsWeightUnits * feeRate / WITNESS_SCALE_FACTOR); + System.out.println("Actual target: " + actualTargetValue); + System.out.println("Cost of change: " + costOfChangeValue); + System.out.println("Selected must be less than: " + (actualTargetValue + costOfChangeValue)); + + long currentAvailableValue = utxoPool.stream().mapToLong(OutputGroup::getEffectiveValue).sum(); + if(currentAvailableValue < targetValue) { + return Collections.emptyList(); + } + + utxoPool.sort((a, b) -> (int)(b.getEffectiveValue() - a.getEffectiveValue())); + + long currentWasteValue = 0; + ArrayDeque bestSelection = null; + long bestWasteValue = Transaction.MAX_BITCOIN; + + // Depth First search loop for choosing the UTXOs + for(int i = 0; i < TOTAL_TRIES; i++) { + boolean backtrack = false; + if(currentValue + currentAvailableValue < actualTargetValue || // Cannot possibly reach target with the amount remaining in the currentAvailableValue + currentValue > actualTargetValue + costOfChangeValue || // Selected value is out of range, go back and try other branch + (currentWasteValue > bestWasteValue && !utxoPool.isEmpty() && (utxoPool.get(0).getFee() - utxoPool.get(0).getLongTermFee() > 0))) { + backtrack = true; + } else if(currentValue >= actualTargetValue) { // Selected value is within range + currentWasteValue += (currentValue - actualTargetValue); // This is the excess value which is added to the waste for the below comparison + // Adding another UTXO after this check could bring the waste down if the long term fee is higher than the current fee. + // However we are not going to explore that because this optimization for the waste is only done when we have hit our target + // value. Adding any more UTXOs will be just burning the UTXO; it will go entirely to fees. Thus we aren't going to + // explore any more UTXOs to avoid burning money like that. + if(currentWasteValue <= bestWasteValue) { + bestSelection = currentSelection; + bestSelection = resize(bestSelection, utxoPool.size()); + bestWasteValue = currentWasteValue; + } + currentWasteValue -= (currentValue - actualTargetValue); // Remove the excess value as we will be selecting different coins now + backtrack = true; + } + + if(backtrack) { + System.out.println("Backtracking"); + // Walk backwards to find the last included UTXO that still needs to have its omission branch traversed + while(!currentSelection.isEmpty() && !currentSelection.getLast()) { + currentSelection.removeLast(); + currentAvailableValue += utxoPool.get(currentSelection.size()).getEffectiveValue(); + } + + if(currentSelection.isEmpty()) { // We have walked back to the first utxo and no branch is untraversed. All solutions searched + break; + } + + // Output was included on previous iterations, try excluding now + currentSelection.removeLast(); + currentSelection.add(Boolean.FALSE); + + OutputGroup utxo = utxoPool.get(currentSelection.size() - 1); + currentValue -= utxo.getEffectiveValue(); + currentWasteValue -= (utxo.getFee() - utxo.getLongTermFee()); + } else { // Moving forwards, continuing down this branch + OutputGroup utxo = utxoPool.get(currentSelection.size()); + + // Remove this utxo from the currentAvailableValue utxo amount + currentAvailableValue -= utxo.getEffectiveValue(); + + // Avoid searching a branch if the previous UTXO has the same value and same waste and was excluded. Since the ratio of fee to + // long term fee is the same, we only need to check if one of those values match in order to know that the waste is the same. + if(!currentSelection.isEmpty() && !currentSelection.getLast() && + utxo.getEffectiveValue() == utxoPool.get(currentSelection.size() - 1).getEffectiveValue() && + utxo.getFee() == utxoPool.get(currentSelection.size() - 1).getFee()) { + currentSelection.add(Boolean.FALSE); + } else { + // Inclusion branch first (Largest First Exploration) + currentSelection.add(Boolean.TRUE); + currentValue += utxo.getEffectiveValue(); + currentWasteValue += (utxo.getFee() - utxo.getLongTermFee()); + printCurrentUtxoSet(utxoPool, currentSelection, currentValue); + } + } + } + + // Check for solution + if(bestSelection == null || bestSelection.isEmpty()) { + System.out.println("No result found"); + return Collections.emptyList(); + } + + // Create output list of UTXOs + List outList = new ArrayList<>(); + int i = 0; + for(Iterator iter = bestSelection.iterator(); iter.hasNext(); i++) { + if(iter.next()) { + outList.addAll(utxoPool.get(i).getUtxos()); + } + } + + return outList; + } + + private long getCostOfChange() { + WalletNode changeNode = wallet.getFreshNode(KeyPurpose.CHANGE); + TransactionOutput changeOutput = new TransactionOutput(new Transaction(), 1L, wallet.getOutputScript(changeNode)); + return wallet.getFee(changeOutput, feeRate, longTermFeeRate); + } + + private ArrayDeque resize(ArrayDeque deque, int size) { + Boolean[] arr = new Boolean[size]; + Arrays.fill(arr, Boolean.FALSE); + + Boolean[] dequeArr = deque.toArray(new Boolean[deque.size()]); + System.arraycopy(dequeArr, 0, arr, 0, Math.min(arr.length, dequeArr.length)); + + return new ArrayDeque<>(Arrays.asList(arr)); + } + + private void printCurrentUtxoSet(List utxoPool, ArrayDeque currentSelection, long currentValue) { + long noInputsFee = (long)(noInputsWeightUnits * feeRate / WITNESS_SCALE_FACTOR); + long inputsFee = 0; + StringJoiner joiner = new StringJoiner(" + "); + int i = 0; + for(Iterator iter = currentSelection.iterator(); iter.hasNext(); i++) { + if(iter.next()) { + joiner.add(Long.toString(utxoPool.get(i).getEffectiveValue())); + inputsFee += utxoPool.get(i).getFee(); + } + } + long noChangeFeeRequiredAmt = noInputsFee + inputsFee; + System.out.println(joiner.toString() + " = " + currentValue + " (plus fee of " + noChangeFeeRequiredAmt + ")"); + } + + private class OutputGroup { + private final List utxos = new ArrayList<>(); + private long effectiveValue = 0; + private long fee = 0; + private long longTermFee = 0; + + public OutputGroup(BlockTransactionHashIndex utxo) { + add(utxo); + } + + public void add(BlockTransactionHashIndex utxo) { + utxos.add(utxo); + effectiveValue += utxo.getValue() - (long)(inputWeightUnits * feeRate / WITNESS_SCALE_FACTOR); + fee += (long)(inputWeightUnits * feeRate / WITNESS_SCALE_FACTOR); + longTermFee += (long)(inputWeightUnits * longTermFeeRate / WITNESS_SCALE_FACTOR); + } + + public void remove(BlockTransactionHashIndex utxo) { + if(utxos.remove(utxo)) { + effectiveValue -= (utxo.getValue() - (long)(inputWeightUnits * feeRate / WITNESS_SCALE_FACTOR)); + fee -= (long)(inputWeightUnits * feeRate / WITNESS_SCALE_FACTOR); + longTermFee -= (long)(inputWeightUnits * longTermFeeRate / WITNESS_SCALE_FACTOR); + } + } + + public List getUtxos() { + return utxos; + } + + public long getEffectiveValue() { + return effectiveValue; + } + + public long getFee() { + return fee; + } + + public long getLongTermFee() { + return longTermFee; + } + } +} diff --git a/src/main/java/com/sparrowwallet/drongo/wallet/Wallet.java b/src/main/java/com/sparrowwallet/drongo/wallet/Wallet.java index 0577d85..e29084d 100644 --- a/src/main/java/com/sparrowwallet/drongo/wallet/Wallet.java +++ b/src/main/java/com/sparrowwallet/drongo/wallet/Wallet.java @@ -12,6 +12,8 @@ import com.sparrowwallet.drongo.protocol.*; import java.util.*; import java.util.stream.Collectors; +import static com.sparrowwallet.drongo.protocol.Transaction.WITNESS_SCALE_FACTOR; + public class Wallet { public static final int DEFAULT_LOOKAHEAD = 20; @@ -248,7 +250,83 @@ public class Wallet { } } - public WalletTransaction createWalletTransaction(List utxoSelectors, Address recipientAddress, long recipientAmount, double feeRate, Long fee, boolean sendAll) throws InsufficientFundsException { + /** + * Determines the dust threshold for creating a new change output in this wallet. + * + * @param output The output under consideration + * @param feeRate The fee rate for the transaction creating the change UTXO + * @return the minimum viable value than the provided change output must have in order to not be dust + */ + public long getDustThreshold(TransactionOutput output, Double feeRate) { + return getFee(output, feeRate, Transaction.DUST_RELAY_TX_FEE); + } + + /** + * Determines the minimum incremental fee necessary to pay for added the provided output to a transaction + * This is done by calculating the sum of multiplying the size of the output at the current fee rate, + * and the size of the input needed to spend it in future at the long term fee rate + * + * @param output The output to be added + * @param feeRate The transaction's fee rate + * @param longTermFeeRate The long term minimum fee rate + * @return The fee that adding this output would add + */ + public long getFee(TransactionOutput output, Double feeRate, Double longTermFeeRate) { + //Start with length of output + int outputVbytes = output.getLength(); + //Add length of spending input (with or without discount depending on script type) + int inputVbytes = getInputVbytes(); + + //Return fee rate in sats/vbyte multiplied by the calculated output and input vByte lengths + return (long)(feeRate * outputVbytes + longTermFeeRate * inputVbytes); + } + + /** + * Return the number of vBytes required for an input created by this wallet. + * + * @return the number of vBytes + */ + public int getInputVbytes() { + return (int)Math.ceil((double)getInputWeightUnits() / (double)WITNESS_SCALE_FACTOR); + } + + /** + * Return the number of vBytes required for an input created by this wallet. + * + * @return the number of vBytes + */ + public int getInputWeightUnits() { + //Estimate assuming an input spending from a fresh receive node - it does not matter this node has no real utxos + WalletNode receiveNode = getFreshNode(KeyPurpose.RECEIVE); + + Transaction transaction = new Transaction(); + TransactionOutput prevTxOut = transaction.addOutput(1L, getAddress(receiveNode)); + + TransactionInput txInput = null; + if(getPolicyType().equals(PolicyType.SINGLE)) { + ECKey pubKey = getPubKey(receiveNode); + TransactionSignature signature = TransactionSignature.dummy(); + txInput = getScriptType().addSpendingInput(transaction, prevTxOut, pubKey, signature); + } else if(getPolicyType().equals(PolicyType.MULTI)) { + List pubKeys = getPubKeys(receiveNode); + int threshold = getDefaultPolicy().getNumSignaturesRequired(); + List signatures = new ArrayList<>(threshold); + for(int i = 0; i < threshold; i++) { + signatures.add(TransactionSignature.dummy()); + } + txInput = getScriptType().addMultisigSpendingInput(transaction, prevTxOut, threshold, pubKeys, signatures); + } + + assert txInput != null; + int wu = txInput.getLength() * WITNESS_SCALE_FACTOR; + if(txInput.hasWitness()) { + wu += txInput.getWitness().getLength(); + } + + return wu; + } + + public WalletTransaction createWalletTransaction(List utxoSelectors, Address recipientAddress, long recipientAmount, double feeRate, double longTermFeeRate, Long fee, boolean sendAll) throws InsufficientFundsException { long valueRequiredAmt = recipientAmount; while(true) { @@ -301,15 +379,15 @@ public class Wallet { long changeAmt = differenceAmt - noChangeFeeRequiredAmt; WalletNode changeNode = getFreshNode(KeyPurpose.CHANGE); TransactionOutput changeOutput = new TransactionOutput(transaction, changeAmt, getOutputScript(changeNode)); - long dustThreshold = getScriptType().getDustThreshold(changeOutput, Transaction.DUST_RELAY_TX_FEE); - if(changeAmt > dustThreshold) { + long costOfChangeAmt = getFee(changeOutput, feeRate, longTermFeeRate); + if(changeAmt > costOfChangeAmt) { //Change output is required, determine new fee once change output has been added int changeVSize = noChangeVSize + changeOutput.getLength(); long changeFeeRequiredAmt = (fee == null ? (long)(feeRate * changeVSize) : fee); //Recalculate the change amount with the new fee changeAmt = differenceAmt - changeFeeRequiredAmt; - if(changeAmt < dustThreshold) { + if(changeAmt < costOfChangeAmt) { //The new fee has meant that the change output is now dust. We pay too high a fee without change, but change is dust when added. Increase value required from inputs and try again valueRequiredAmt = totalSelectedAmt + 1; continue;