support more accurate virtual size calculations

src/main/java/com/sparrowwallet/drongo/protocol/ScriptType.java

@@ -1054,27 +1054,56 @@ public enum ScriptType {
     }
 
     /**
-     * Determines the dust threshold for this script type.
+     * Determines the dust threshold for the given output for this script type.
      *
      * @param output The output under consideration
-     * @param feeRate The fee rate at which the fee required will be calculated
      * @return the minimum viable value than the provided output must have in order to not be dust
      */
     public long getDustThreshold(TransactionOutput output, Double feeRate) {
+        return getFee(output, feeRate, Transaction.DUST_RELAY_TX_FEE);
+    }
+
+    /**
+     * Determines the minimum incremental fee necessary to pay for added the provided output to a transaction
+     * This is done by calculating the sum of multiplying the size of the output at the current fee rate,
+     * and the size of the input needed to spend it in future at the long term fee rate
+     *
+     * @param output The output to be added
+     * @param feeRate The transaction's fee rate
+     * @param longTermFeeRate The long term minimum fee rate
+     * @return The fee that adding this output would add
+     */
+    public long getFee(TransactionOutput output, Double feeRate, Double longTermFeeRate) {
         //Start with length of output
-        int totalLength = output.getLength();
+        int outputVbytes = output.getLength();
-        if(Arrays.asList(WITNESS_TYPES).contains(this)) {
+        //Add length of spending input (with or without discount depending on script type)
-            //Add length of spending input with 75% discount to script size
+        int inputVbytes = getInputVbytes();
-            totalLength += (32 + 4 + 1 + (107 / WITNESS_SCALE_FACTOR) + 4);
+
+        //Return fee rate in sats/vByte multiplied by the calculated output and input vByte lengths
+        return (long)(feeRate * outputVbytes + longTermFeeRate * inputVbytes);
+    }
+
+    /**
+     * Return a coarse estimation of the minimum number of vBytes required to spend an input of this script type.
+     * Because we don't know the nature of the scriptSig/witnessScript required, pay to script inputs will likely be underestimated.
+     * Use Wallet.getInputVbytes() for an accurate value to spend a wallet UTXO.
+     *
+     * @return The number of vBytes required for an input of this script type
+     */
+    public int getInputVbytes() {
+        if(P2SH_P2WPKH.equals(this)) {
+            return (32 + 4 + 1 + 13 + (107 / WITNESS_SCALE_FACTOR) + 4);
+        } else if(P2SH_P2WSH.equals(this)) {
+            return (32 + 4 + 1 + 35 + (107 / WITNESS_SCALE_FACTOR) + 4);
+        } else if(Arrays.asList(WITNESS_TYPES).contains(this)) {
+            //Return length of spending input with 75% discount to script size
+            return (32 + 4 + 1 + (107 / WITNESS_SCALE_FACTOR) + 4);
         } else if(Arrays.asList(NON_WITNESS_TYPES).contains(this)) {
-            //Add length of spending input with no discount
+            //Return length of spending input with no discount
-            totalLength += (32 + 4 + 1 + 107 + 4);
+            return (32 + 4 + 1 + 107 + 4);
         } else {
             throw new UnsupportedOperationException("Cannot determine dust threshold for script type " + this.getName());
         }
-
-        //Return fee rate in sats/vbyte multiplied by the calculated total byte length
-        return (long)(feeRate * totalLength);
     }
 
     @Override

src/main/java/com/sparrowwallet/drongo/protocol/Transaction.java

@@ -267,6 +267,10 @@ public class Transaction extends ChildMessage {
     }
 
     public int getVirtualSize() {
+        return (int)Math.ceil((double)getWeightUnits() / (double)WITNESS_SCALE_FACTOR);
+    }
+
+    public int getWeightUnits() {
         int wu = 0;
 
         // version
@@ -294,7 +298,7 @@ public class Transaction extends ChildMessage {
         // lock_time
         wu += 4 * WITNESS_SCALE_FACTOR;
 
-        return (int)Math.ceil((double)wu / (double)WITNESS_SCALE_FACTOR);
+        return wu;
     }
 
     public List<TransactionInput> getInputs() {
@@ -306,6 +310,10 @@ public class Transaction extends ChildMessage {
     }
 
     public TransactionInput addInput(Sha256Hash spendTxHash, long outputIndex, Script script, TransactionWitness witness) {
+        if(!isSegwit()) {
+            setSegwitVersion(0);
+        }
+
         return addInput(new TransactionInput(this, new TransactionOutPoint(spendTxHash, outputIndex), script.getProgram(), witness));
     }
 

src/main/java/com/sparrowwallet/drongo/wallet/BnBUtxoSelector.java

@@ -0,0 +1,207 @@
+package com.sparrowwallet.drongo.wallet;
+
+import com.sparrowwallet.drongo.KeyPurpose;
+import com.sparrowwallet.drongo.protocol.Transaction;
+import com.sparrowwallet.drongo.protocol.TransactionOutput;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static com.sparrowwallet.drongo.protocol.Transaction.WITNESS_SCALE_FACTOR;
+
+public class BnBUtxoSelector implements UtxoSelector {
+    private static final int TOTAL_TRIES = 100000;
+
+    private final Wallet wallet;
+    private final int noInputsWeightUnits;
+    private final Double feeRate;
+    private final Double longTermFeeRate;
+    private final int inputWeightUnits;
+    private final long costOfChangeValue;
+
+    public BnBUtxoSelector(Wallet wallet, int noInputsWeightUnits, Double feeRate, Double longTermFeeRate) {
+        this.wallet = wallet;
+        this.noInputsWeightUnits = noInputsWeightUnits;
+        this.feeRate = feeRate;
+        this.longTermFeeRate = longTermFeeRate;
+        this.inputWeightUnits = wallet.getInputWeightUnits();
+        this.costOfChangeValue = getCostOfChange();
+    }
+
+    @Override
+    public Collection<BlockTransactionHashIndex> select(long targetValue, Collection<BlockTransactionHashIndex> candidates) {
+        List<OutputGroup> utxoPool = candidates.stream().map(OutputGroup::new).collect(Collectors.toList());
+
+        long currentValue = 0;
+
+        ArrayDeque<Boolean> currentSelection = new ArrayDeque<>(utxoPool.size());
+        long actualTargetValue = targetValue + (long)(noInputsWeightUnits * feeRate / WITNESS_SCALE_FACTOR);
+        System.out.println("Actual target: " + actualTargetValue);
+        System.out.println("Cost of change: " + costOfChangeValue);
+        System.out.println("Selected must be less than: " + (actualTargetValue + costOfChangeValue));
+
+        long currentAvailableValue = utxoPool.stream().mapToLong(OutputGroup::getEffectiveValue).sum();
+        if(currentAvailableValue < targetValue) {
+            return Collections.emptyList();
+        }
+
+        utxoPool.sort((a, b) -> (int)(b.getEffectiveValue() - a.getEffectiveValue()));
+
+        long currentWasteValue = 0;
+        ArrayDeque<Boolean> bestSelection = null;
+        long bestWasteValue = Transaction.MAX_BITCOIN;
+
+        // Depth First search loop for choosing the UTXOs
+        for(int i = 0; i < TOTAL_TRIES; i++) {
+            boolean backtrack = false;
+            if(currentValue + currentAvailableValue < actualTargetValue ||  // Cannot possibly reach target with the amount remaining in the currentAvailableValue
+                currentValue > actualTargetValue + costOfChangeValue ||     //  Selected value is out of range, go back and try other branch
+                (currentWasteValue > bestWasteValue && !utxoPool.isEmpty() && (utxoPool.get(0).getFee() - utxoPool.get(0).getLongTermFee() > 0))) {
+                backtrack = true;
+            } else if(currentValue >= actualTargetValue) {                  // Selected value is within range
+                currentWasteValue += (currentValue - actualTargetValue);    // This is the excess value which is added to the waste for the below comparison
+                // Adding another UTXO after this check could bring the waste down if the long term fee is higher than the current fee.
+                // However we are not going to explore that because this optimization for the waste is only done when we have hit our target
+                // value. Adding any more UTXOs will be just burning the UTXO; it will go entirely to fees. Thus we aren't going to
+                // explore any more UTXOs to avoid burning money like that.
+                if(currentWasteValue <= bestWasteValue) {
+                    bestSelection = currentSelection;
+                    bestSelection = resize(bestSelection, utxoPool.size());
+                    bestWasteValue = currentWasteValue;
+                }
+                currentWasteValue -= (currentValue - actualTargetValue);    // Remove the excess value as we will be selecting different coins now
+                backtrack = true;
+            }
+
+            if(backtrack) {
+                System.out.println("Backtracking");
+                // Walk backwards to find the last included UTXO that still needs to have its omission branch traversed
+                while(!currentSelection.isEmpty() && !currentSelection.getLast()) {
+                    currentSelection.removeLast();
+                    currentAvailableValue += utxoPool.get(currentSelection.size()).getEffectiveValue();
+                }
+
+                if(currentSelection.isEmpty()) {                            // We have walked back to the first utxo and no branch is untraversed. All solutions searched
+                    break;
+                }
+
+                // Output was included on previous iterations, try excluding now
+                currentSelection.removeLast();
+                currentSelection.add(Boolean.FALSE);
+
+                OutputGroup utxo = utxoPool.get(currentSelection.size() - 1);
+                currentValue -= utxo.getEffectiveValue();
+                currentWasteValue -= (utxo.getFee() - utxo.getLongTermFee());
+            } else {                                                        // Moving forwards, continuing down this branch
+                OutputGroup utxo = utxoPool.get(currentSelection.size());
+
+                // Remove this utxo from the currentAvailableValue utxo amount
+                currentAvailableValue -= utxo.getEffectiveValue();
+
+                // Avoid searching a branch if the previous UTXO has the same value and same waste and was excluded. Since the ratio of fee to
+                // long term fee is the same, we only need to check if one of those values match in order to know that the waste is the same.
+                if(!currentSelection.isEmpty() && !currentSelection.getLast() &&
+                    utxo.getEffectiveValue() == utxoPool.get(currentSelection.size() - 1).getEffectiveValue() &&
+                    utxo.getFee() == utxoPool.get(currentSelection.size() - 1).getFee()) {
+                    currentSelection.add(Boolean.FALSE);
+                } else {
+                    // Inclusion branch first (Largest First Exploration)
+                    currentSelection.add(Boolean.TRUE);
+                    currentValue += utxo.getEffectiveValue();
+                    currentWasteValue += (utxo.getFee() - utxo.getLongTermFee());
+                    printCurrentUtxoSet(utxoPool, currentSelection, currentValue);
+                }
+            }
+        }
+
+        // Check for solution
+        if(bestSelection == null || bestSelection.isEmpty()) {
+            System.out.println("No result found");
+            return Collections.emptyList();
+        }
+
+        // Create output list of UTXOs
+        List<BlockTransactionHashIndex> outList = new ArrayList<>();
+        int i = 0;
+        for(Iterator<Boolean> iter = bestSelection.iterator(); iter.hasNext(); i++) {
+            if(iter.next()) {
+                outList.addAll(utxoPool.get(i).getUtxos());
+            }
+        }
+
+        return outList;
+    }
+
+    private long getCostOfChange() {
+        WalletNode changeNode = wallet.getFreshNode(KeyPurpose.CHANGE);
+        TransactionOutput changeOutput = new TransactionOutput(new Transaction(), 1L, wallet.getOutputScript(changeNode));
+        return wallet.getFee(changeOutput, feeRate, longTermFeeRate);
+    }
+
+    private ArrayDeque<Boolean> resize(ArrayDeque<Boolean> deque, int size) {
+        Boolean[] arr = new Boolean[size];
+        Arrays.fill(arr, Boolean.FALSE);
+
+        Boolean[] dequeArr = deque.toArray(new Boolean[deque.size()]);
+        System.arraycopy(dequeArr, 0, arr, 0, Math.min(arr.length, dequeArr.length));
+
+        return new ArrayDeque<>(Arrays.asList(arr));
+    }
+
+    private void printCurrentUtxoSet(List<OutputGroup> utxoPool, ArrayDeque<Boolean> currentSelection, long currentValue) {
+        long noInputsFee = (long)(noInputsWeightUnits * feeRate / WITNESS_SCALE_FACTOR);
+        long inputsFee = 0;
+        StringJoiner joiner = new StringJoiner(" + ");
+        int i = 0;
+        for(Iterator<Boolean> iter = currentSelection.iterator(); iter.hasNext(); i++) {
+            if(iter.next()) {
+                joiner.add(Long.toString(utxoPool.get(i).getEffectiveValue()));
+                inputsFee += utxoPool.get(i).getFee();
+            }
+        }
+        long noChangeFeeRequiredAmt = noInputsFee + inputsFee;
+        System.out.println(joiner.toString() + " = " + currentValue + " (plus fee of " + noChangeFeeRequiredAmt + ")");
+    }
+
+    private class OutputGroup {
+        private final List<BlockTransactionHashIndex> utxos = new ArrayList<>();
+        private long effectiveValue = 0;
+        private long fee = 0;
+        private long longTermFee = 0;
+
+        public OutputGroup(BlockTransactionHashIndex utxo) {
+            add(utxo);
+        }
+
+        public void add(BlockTransactionHashIndex utxo) {
+            utxos.add(utxo);
+            effectiveValue += utxo.getValue() - (long)(inputWeightUnits * feeRate / WITNESS_SCALE_FACTOR);
+            fee += (long)(inputWeightUnits * feeRate / WITNESS_SCALE_FACTOR);
+            longTermFee += (long)(inputWeightUnits * longTermFeeRate / WITNESS_SCALE_FACTOR);
+        }
+
+        public void remove(BlockTransactionHashIndex utxo) {
+            if(utxos.remove(utxo)) {
+                effectiveValue -= (utxo.getValue() - (long)(inputWeightUnits * feeRate / WITNESS_SCALE_FACTOR));
+                fee -= (long)(inputWeightUnits * feeRate / WITNESS_SCALE_FACTOR);
+                longTermFee -= (long)(inputWeightUnits * longTermFeeRate / WITNESS_SCALE_FACTOR);
+            }
+        }
+
+        public List<BlockTransactionHashIndex> getUtxos() {
+            return utxos;
+        }
+
+        public long getEffectiveValue() {
+            return effectiveValue;
+        }
+
+        public long getFee() {
+            return fee;
+        }
+
+        public long getLongTermFee() {
+            return longTermFee;
+        }
+    }
+}

src/main/java/com/sparrowwallet/drongo/wallet/Wallet.java

@@ -12,6 +12,8 @@ import com.sparrowwallet.drongo.protocol.*;
 import java.util.*;
 import java.util.stream.Collectors;
 
+import static com.sparrowwallet.drongo.protocol.Transaction.WITNESS_SCALE_FACTOR;
+
 public class Wallet {
     public static final int DEFAULT_LOOKAHEAD = 20;
 
@@ -248,7 +250,83 @@ public class Wallet {
         }
     }
 
-    public WalletTransaction createWalletTransaction(List<UtxoSelector> utxoSelectors, Address recipientAddress, long recipientAmount, double feeRate, Long fee, boolean sendAll) throws InsufficientFundsException {
+    /**
+     * Determines the dust threshold for creating a new change output in this wallet.
+     *
+     * @param output The output under consideration
+     * @param feeRate The fee rate for the transaction creating the change UTXO
+     * @return the minimum viable value than the provided change output must have in order to not be dust
+     */
+    public long getDustThreshold(TransactionOutput output, Double feeRate) {
+        return getFee(output, feeRate, Transaction.DUST_RELAY_TX_FEE);
+    }
+
+    /**
+     * Determines the minimum incremental fee necessary to pay for added the provided output to a transaction
+     * This is done by calculating the sum of multiplying the size of the output at the current fee rate,
+     * and the size of the input needed to spend it in future at the long term fee rate
+     *
+     * @param output The output to be added
+     * @param feeRate The transaction's fee rate
+     * @param longTermFeeRate The long term minimum fee rate
+     * @return The fee that adding this output would add
+     */
+    public long getFee(TransactionOutput output, Double feeRate, Double longTermFeeRate) {
+        //Start with length of output
+        int outputVbytes = output.getLength();
+        //Add length of spending input (with or without discount depending on script type)
+        int inputVbytes = getInputVbytes();
+
+        //Return fee rate in sats/vbyte multiplied by the calculated output and input vByte lengths
+        return (long)(feeRate * outputVbytes + longTermFeeRate * inputVbytes);
+    }
+
+    /**
+     * Return the number of vBytes required for an input created by this wallet.
+     *
+     * @return the number of vBytes
+     */
+    public int getInputVbytes() {
+        return (int)Math.ceil((double)getInputWeightUnits() / (double)WITNESS_SCALE_FACTOR);
+    }
+
+    /**
+     * Return the number of vBytes required for an input created by this wallet.
+     *
+     * @return the number of vBytes
+     */
+    public int getInputWeightUnits() {
+        //Estimate assuming an input spending from a fresh receive node - it does not matter this node has no real utxos
+        WalletNode receiveNode = getFreshNode(KeyPurpose.RECEIVE);
+
+        Transaction transaction = new Transaction();
+        TransactionOutput prevTxOut = transaction.addOutput(1L, getAddress(receiveNode));
+
+        TransactionInput txInput = null;
+        if(getPolicyType().equals(PolicyType.SINGLE)) {
+            ECKey pubKey = getPubKey(receiveNode);
+            TransactionSignature signature = TransactionSignature.dummy();
+            txInput = getScriptType().addSpendingInput(transaction, prevTxOut, pubKey, signature);
+        } else if(getPolicyType().equals(PolicyType.MULTI)) {
+            List<ECKey> pubKeys = getPubKeys(receiveNode);
+            int threshold = getDefaultPolicy().getNumSignaturesRequired();
+            List<TransactionSignature> signatures = new ArrayList<>(threshold);
+            for(int i = 0; i < threshold; i++) {
+                signatures.add(TransactionSignature.dummy());
+            }
+            txInput = getScriptType().addMultisigSpendingInput(transaction, prevTxOut, threshold, pubKeys, signatures);
+        }
+
+        assert txInput != null;
+        int wu = txInput.getLength() * WITNESS_SCALE_FACTOR;
+        if(txInput.hasWitness()) {
+            wu += txInput.getWitness().getLength();
+        }
+
+        return wu;
+    }
+
+    public WalletTransaction createWalletTransaction(List<UtxoSelector> utxoSelectors, Address recipientAddress, long recipientAmount, double feeRate, double longTermFeeRate, Long fee, boolean sendAll) throws InsufficientFundsException {
         long valueRequiredAmt = recipientAmount;
 
         while(true) {
@@ -301,15 +379,15 @@ public class Wallet {
             long changeAmt = differenceAmt - noChangeFeeRequiredAmt;
             WalletNode changeNode = getFreshNode(KeyPurpose.CHANGE);
             TransactionOutput changeOutput = new TransactionOutput(transaction, changeAmt, getOutputScript(changeNode));
-            long dustThreshold = getScriptType().getDustThreshold(changeOutput, Transaction.DUST_RELAY_TX_FEE);
+            long costOfChangeAmt = getFee(changeOutput, feeRate, longTermFeeRate);
-            if(changeAmt > dustThreshold) {
+            if(changeAmt > costOfChangeAmt) {
                 //Change output is required, determine new fee once change output has been added
                 int changeVSize = noChangeVSize + changeOutput.getLength();
                 long changeFeeRequiredAmt = (fee == null ? (long)(feeRate * changeVSize) : fee);
 
                 //Recalculate the change amount with the new fee
                 changeAmt = differenceAmt - changeFeeRequiredAmt;
-                if(changeAmt < dustThreshold) {
+                if(changeAmt < costOfChangeAmt) {
                     //The new fee has meant that the change output is now dust. We pay too high a fee without change, but change is dust when added. Increase value required from inputs and try again
                     valueRequiredAmt = totalSelectedAmt + 1;
                     continue;