From 8484dd397b95d200cf3c363cd48e5751550b3bcb Mon Sep 17 00:00:00 2001 From: Craig Raw Date: Tue, 11 Jul 2023 09:07:22 +0200 Subject: [PATCH] use txo filters for all wallet transaction output filtering --- ...UtxoFilter.java => CoinbaseTxoFilter.java} | 4 +- .../drongo/wallet/ExcludeTxoFilter.java | 31 +++++++++ .../drongo/wallet/ExcludeUtxoFilter.java | 31 --------- ...enUtxoFilter.java => FrozenTxoFilter.java} | 2 +- .../drongo/wallet/PresetUtxoSelector.java | 14 ++-- .../drongo/wallet/SpentTxoFilter.java | 24 +++++++ .../{UtxoFilter.java => TxoFilter.java} | 2 +- .../sparrowwallet/drongo/wallet/Wallet.java | 67 ++++++++++--------- .../drongo/wallet/WalletNode.java | 6 +- 9 files changed, 106 insertions(+), 75 deletions(-) rename src/main/java/com/sparrowwallet/drongo/wallet/{CoinbaseUtxoFilter.java => CoinbaseTxoFilter.java} (87%) create mode 100644 src/main/java/com/sparrowwallet/drongo/wallet/ExcludeTxoFilter.java delete mode 100644 src/main/java/com/sparrowwallet/drongo/wallet/ExcludeUtxoFilter.java rename src/main/java/com/sparrowwallet/drongo/wallet/{FrozenUtxoFilter.java => FrozenTxoFilter.java} (80%) create mode 100644 src/main/java/com/sparrowwallet/drongo/wallet/SpentTxoFilter.java rename src/main/java/com/sparrowwallet/drongo/wallet/{UtxoFilter.java => TxoFilter.java} (77%) diff --git a/src/main/java/com/sparrowwallet/drongo/wallet/CoinbaseUtxoFilter.java b/src/main/java/com/sparrowwallet/drongo/wallet/CoinbaseTxoFilter.java similarity index 87% rename from src/main/java/com/sparrowwallet/drongo/wallet/CoinbaseUtxoFilter.java rename to src/main/java/com/sparrowwallet/drongo/wallet/CoinbaseTxoFilter.java index 8f079f6..6c0e03c 100644 --- a/src/main/java/com/sparrowwallet/drongo/wallet/CoinbaseUtxoFilter.java +++ b/src/main/java/com/sparrowwallet/drongo/wallet/CoinbaseTxoFilter.java @@ -2,10 +2,10 @@ package com.sparrowwallet.drongo.wallet; import com.sparrowwallet.drongo.protocol.Transaction; -public class CoinbaseUtxoFilter implements UtxoFilter { +public class CoinbaseTxoFilter implements TxoFilter { private final Wallet wallet; - public CoinbaseUtxoFilter(Wallet wallet) { + public CoinbaseTxoFilter(Wallet wallet) { this.wallet = wallet; } diff --git a/src/main/java/com/sparrowwallet/drongo/wallet/ExcludeTxoFilter.java b/src/main/java/com/sparrowwallet/drongo/wallet/ExcludeTxoFilter.java new file mode 100644 index 0000000..e92837c --- /dev/null +++ b/src/main/java/com/sparrowwallet/drongo/wallet/ExcludeTxoFilter.java @@ -0,0 +1,31 @@ +package com.sparrowwallet.drongo.wallet; + +import java.util.ArrayList; +import java.util.Collection; + +public class ExcludeTxoFilter implements TxoFilter { + private final Collection excludedTxos; + + public ExcludeTxoFilter() { + this.excludedTxos = new ArrayList<>(); + } + + public ExcludeTxoFilter(Collection excludedTxos) { + this.excludedTxos = new ArrayList<>(excludedTxos); + } + + @Override + public boolean isEligible(BlockTransactionHashIndex candidate) { + for(BlockTransactionHashIndex excludedTxo : excludedTxos) { + if(candidate.getHash().equals(excludedTxo.getHash()) && candidate.getIndex() == excludedTxo.getIndex()) { + return false; + } + } + + return true; + } + + public Collection getExcludedTxos() { + return excludedTxos; + } +} diff --git a/src/main/java/com/sparrowwallet/drongo/wallet/ExcludeUtxoFilter.java b/src/main/java/com/sparrowwallet/drongo/wallet/ExcludeUtxoFilter.java deleted file mode 100644 index f87ae7d..0000000 --- a/src/main/java/com/sparrowwallet/drongo/wallet/ExcludeUtxoFilter.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.sparrowwallet.drongo.wallet; - -import java.util.ArrayList; -import java.util.Collection; - -public class ExcludeUtxoFilter implements UtxoFilter { - private final Collection excludedUtxos; - - public ExcludeUtxoFilter() { - this.excludedUtxos = new ArrayList<>(); - } - - public ExcludeUtxoFilter(Collection excludedUtxos) { - this.excludedUtxos = new ArrayList<>(excludedUtxos); - } - - @Override - public boolean isEligible(BlockTransactionHashIndex candidate) { - for(BlockTransactionHashIndex excludedUtxo : excludedUtxos) { - if(candidate.getHash().equals(excludedUtxo.getHash()) && candidate.getIndex() == excludedUtxo.getIndex()) { - return false; - } - } - - return true; - } - - public Collection getExcludedUtxos() { - return excludedUtxos; - } -} diff --git a/src/main/java/com/sparrowwallet/drongo/wallet/FrozenUtxoFilter.java b/src/main/java/com/sparrowwallet/drongo/wallet/FrozenTxoFilter.java similarity index 80% rename from src/main/java/com/sparrowwallet/drongo/wallet/FrozenUtxoFilter.java rename to src/main/java/com/sparrowwallet/drongo/wallet/FrozenTxoFilter.java index 6b61dbc..3b50b74 100644 --- a/src/main/java/com/sparrowwallet/drongo/wallet/FrozenUtxoFilter.java +++ b/src/main/java/com/sparrowwallet/drongo/wallet/FrozenTxoFilter.java @@ -1,6 +1,6 @@ package com.sparrowwallet.drongo.wallet; -public class FrozenUtxoFilter implements UtxoFilter { +public class FrozenTxoFilter implements TxoFilter { @Override public boolean isEligible(BlockTransactionHashIndex candidate) { return candidate.getStatus() == null || candidate.getStatus() != Status.FROZEN; diff --git a/src/main/java/com/sparrowwallet/drongo/wallet/PresetUtxoSelector.java b/src/main/java/com/sparrowwallet/drongo/wallet/PresetUtxoSelector.java index 996fd17..85f4608 100644 --- a/src/main/java/com/sparrowwallet/drongo/wallet/PresetUtxoSelector.java +++ b/src/main/java/com/sparrowwallet/drongo/wallet/PresetUtxoSelector.java @@ -1,14 +1,13 @@ package com.sparrowwallet.drongo.wallet; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; +import java.util.*; import java.util.stream.Collectors; public class PresetUtxoSelector extends SingleSetUtxoSelector { private final Collection presetUtxos; private final Collection excludedUtxos; private final boolean maintainOrder; + private final boolean requireAll; public PresetUtxoSelector(Collection presetUtxos) { this(presetUtxos, new ArrayList<>()); @@ -18,12 +17,14 @@ public class PresetUtxoSelector extends SingleSetUtxoSelector { this.presetUtxos = presetUtxos; this.excludedUtxos = excludedUtxos; this.maintainOrder = false; + this.requireAll = false; } - public PresetUtxoSelector(Collection presetUtxos, boolean maintainOrder) { + public PresetUtxoSelector(Collection presetUtxos, boolean maintainOrder, boolean requireAll) { this.presetUtxos = presetUtxos; this.excludedUtxos = new ArrayList<>(); this.maintainOrder = maintainOrder; + this.requireAll = requireAll; } @Override @@ -40,8 +41,11 @@ public class PresetUtxoSelector extends SingleSetUtxoSelector { } } - if(maintainOrder && utxos.containsAll(presetUtxos)) { + Set utxosSet = new HashSet<>(utxos); + if(maintainOrder && utxosSet.containsAll(presetUtxos)) { return presetUtxos; + } else if(requireAll && !utxosSet.containsAll(presetUtxos)) { + return Collections.emptyList(); } return utxos; diff --git a/src/main/java/com/sparrowwallet/drongo/wallet/SpentTxoFilter.java b/src/main/java/com/sparrowwallet/drongo/wallet/SpentTxoFilter.java new file mode 100644 index 0000000..6d882e1 --- /dev/null +++ b/src/main/java/com/sparrowwallet/drongo/wallet/SpentTxoFilter.java @@ -0,0 +1,24 @@ +package com.sparrowwallet.drongo.wallet; + +import com.sparrowwallet.drongo.protocol.Sha256Hash; + +public class SpentTxoFilter implements TxoFilter { + private final Sha256Hash replacedTxid; + + public SpentTxoFilter() { + replacedTxid = null; + } + + public SpentTxoFilter(Sha256Hash replacedTxid) { + this.replacedTxid = replacedTxid; + } + + @Override + public boolean isEligible(BlockTransactionHashIndex candidate) { + return !isSpentOrReplaced(candidate); + } + + private boolean isSpentOrReplaced(BlockTransactionHashIndex candidate) { + return candidate.getHash().equals(replacedTxid) || (candidate.isSpent() && !candidate.getSpentBy().getHash().equals(replacedTxid)); + } +} diff --git a/src/main/java/com/sparrowwallet/drongo/wallet/UtxoFilter.java b/src/main/java/com/sparrowwallet/drongo/wallet/TxoFilter.java similarity index 77% rename from src/main/java/com/sparrowwallet/drongo/wallet/UtxoFilter.java rename to src/main/java/com/sparrowwallet/drongo/wallet/TxoFilter.java index 5b2a6f8..36bd051 100644 --- a/src/main/java/com/sparrowwallet/drongo/wallet/UtxoFilter.java +++ b/src/main/java/com/sparrowwallet/drongo/wallet/TxoFilter.java @@ -1,5 +1,5 @@ package com.sparrowwallet.drongo.wallet; -public interface UtxoFilter { +public interface TxoFilter { boolean isEligible(BlockTransactionHashIndex candidate); } diff --git a/src/main/java/com/sparrowwallet/drongo/wallet/Wallet.java b/src/main/java/com/sparrowwallet/drongo/wallet/Wallet.java index 267d493..8551ca4 100644 --- a/src/main/java/com/sparrowwallet/drongo/wallet/Wallet.java +++ b/src/main/java/com/sparrowwallet/drongo/wallet/Wallet.java @@ -799,30 +799,38 @@ public class Wallet extends Persistable implements Comparable { } public Map getWalletUtxos() { - return getWalletUtxos(false); + return getWalletTxos(List.of(new SpentTxoFilter())); } - public Map getWalletUtxos(boolean includeSpentMempoolOutputs) { - Map walletUtxos = new TreeMap<>(); + public Map getSpendableUtxos() { + return getWalletTxos(List.of(new SpentTxoFilter(), new FrozenTxoFilter(), new CoinbaseTxoFilter(this))); + } + + public Map getSpendableUtxos(BlockTransaction replacedTransaction) { + return getWalletTxos(List.of(new SpentTxoFilter(replacedTransaction == null ? null : replacedTransaction.getHash()), new FrozenTxoFilter(), new CoinbaseTxoFilter(this))); + } + + public Map getWalletTxos(Collection txoFilters) { + Map walletTxos = new TreeMap<>(); for(KeyPurpose keyPurpose : getWalletKeyPurposes()) { - getWalletUtxos(walletUtxos, getNode(keyPurpose), includeSpentMempoolOutputs); + getWalletTxos(walletTxos, getNode(keyPurpose), txoFilters); } for(Wallet childWallet : getChildWallets()) { if(childWallet.isNested()) { for(KeyPurpose keyPurpose : childWallet.getWalletKeyPurposes()) { - getWalletUtxos(walletUtxos, childWallet.getNode(keyPurpose), includeSpentMempoolOutputs); + getWalletTxos(walletTxos, childWallet.getNode(keyPurpose), txoFilters); } } } - return walletUtxos; + return walletTxos; } - private void getWalletUtxos(Map walletUtxos, WalletNode purposeNode, boolean includeSpentMempoolOutputs) { + private void getWalletTxos(Map walletTxos, WalletNode purposeNode, Collection txoFilters) { for(WalletNode addressNode : purposeNode.getChildren()) { - for(BlockTransactionHashIndex utxo : addressNode.getUnspentTransactionOutputs(includeSpentMempoolOutputs)) { - walletUtxos.put(utxo, addressNode); + for(BlockTransactionHashIndex utxo : addressNode.getTransactionOutputs(txoFilters)) { + walletTxos.put(utxo, addressNode); } } } @@ -981,29 +989,30 @@ public class Wallet extends Persistable implements Comparable { return getFee(changeOutput, feeRate, longTermFeeRate); } - public WalletTransaction createWalletTransaction(List utxoSelectors, List utxoFilters, List payments, List opReturns, Set excludedChangeNodes, double feeRate, double longTermFeeRate, Long fee, Integer currentBlockHeight, boolean groupByAddress, boolean includeMempoolOutputs, boolean includeSpentMempoolOutputs) throws InsufficientFundsException { + public WalletTransaction createWalletTransaction(List utxoSelectors, List txoFilters, List payments, List opReturns, Set excludedChangeNodes, double feeRate, double longTermFeeRate, Long fee, Integer currentBlockHeight, boolean groupByAddress, boolean includeMempoolOutputs) throws InsufficientFundsException { boolean sendMax = payments.stream().anyMatch(Payment::isSendMax); long totalPaymentAmount = payments.stream().map(Payment::getAmount).mapToLong(v -> v).sum(); - long totalUtxoValue = getWalletUtxos().keySet().stream().mapToLong(BlockTransactionHashIndex::getValue).sum(); + Map availableTxos = getWalletTxos(txoFilters); + long totalAvailableValue = availableTxos.keySet().stream().mapToLong(BlockTransactionHashIndex::getValue).sum(); if(fee != null && feeRate != Transaction.DEFAULT_MIN_RELAY_FEE) { throw new IllegalArgumentException("Use an input fee rate of 1 sat/vB when using a defined fee amount so UTXO selectors overestimate effective value"); } - long maxSpendableAmt = getMaxSpendable(payments.stream().map(Payment::getAddress).collect(Collectors.toList()), feeRate, includeSpentMempoolOutputs); + long maxSpendableAmt = getMaxSpendable(payments.stream().map(Payment::getAddress).collect(Collectors.toList()), feeRate, availableTxos); if(maxSpendableAmt < 0) { throw new InsufficientFundsException("Not enough combined value in all available UTXOs to send a transaction to the provided addresses at this fee rate"); } //When a user fee is set, we can calculate the fees to spend all UTXOs because we assume all UTXOs are spendable at a fee rate of 1 sat/vB //We can then add the user set fee less this amount as a "phantom payment amount" to the value required to find (which cannot include transaction fees) - long valueRequiredAmt = totalPaymentAmount + (fee != null ? fee - (totalUtxoValue - maxSpendableAmt) : 0); + long valueRequiredAmt = totalPaymentAmount + (fee != null ? fee - (totalAvailableValue - maxSpendableAmt) : 0); if(maxSpendableAmt < valueRequiredAmt) { throw new InsufficientFundsException("Not enough combined value in all available UTXOs to send a transaction to send the provided payments at the user set fee" + (fee == null ? " rate" : "")); } while(true) { - List> selectedUtxoSets = selectInputSets(utxoSelectors, utxoFilters, valueRequiredAmt, feeRate, longTermFeeRate, groupByAddress, includeMempoolOutputs, includeSpentMempoolOutputs, sendMax); + List> selectedUtxoSets = selectInputSets(availableTxos, utxoSelectors, txoFilters, valueRequiredAmt, feeRate, longTermFeeRate, groupByAddress, includeMempoolOutputs, sendMax); Map selectedUtxos = new LinkedHashMap<>(); selectedUtxoSets.forEach(selectedUtxos::putAll); long totalSelectedAmt = selectedUtxos.keySet().stream().mapToLong(BlockTransactionHashIndex::getValue).sum(); @@ -1076,8 +1085,8 @@ public class Wallet extends Persistable implements Comparable { if(differenceAmt < noChangeFeeRequiredAmt) { valueRequiredAmt = totalSelectedAmt + 1; //If we haven't selected all UTXOs yet, don't require more than the max spendable amount - if(valueRequiredAmt > maxSpendableAmt && transaction.getInputs().size() < getWalletUtxos().size()) { - valueRequiredAmt = maxSpendableAmt; + if(valueRequiredAmt > maxSpendableAmt && transaction.getInputs().size() < availableTxos.size()) { + valueRequiredAmt = maxSpendableAmt; } continue; @@ -1180,8 +1189,8 @@ public class Wallet extends Persistable implements Comparable { } } - private List> selectInputSets(List utxoSelectors, List utxoFilters, Long targetValue, double feeRate, double longTermFeeRate, boolean groupByAddress, boolean includeMempoolOutputs, boolean includeSpentMempoolOutputs, boolean sendMax) throws InsufficientFundsException { - List utxoPool = getGroupedUtxos(utxoFilters, feeRate, longTermFeeRate, groupByAddress, includeSpentMempoolOutputs); + private List> selectInputSets(Map availableTxos, List utxoSelectors, List txoFilters, Long targetValue, double feeRate, double longTermFeeRate, boolean groupByAddress, boolean includeMempoolOutputs, boolean sendMax) throws InsufficientFundsException { + List utxoPool = getGroupedUtxos(txoFilters, feeRate, longTermFeeRate, groupByAddress); List filters = new ArrayList<>(); filters.add(new OutputGroup.Filter(1, 6, false)); @@ -1204,7 +1213,6 @@ public class Wallet extends Persistable implements Comparable { List> selectedInputSets = utxoSelector.selectSets(targetValue, filteredPool); List> selectedInputSetsList = new ArrayList<>(); long total = 0; - Map utxos = getWalletUtxos(includeSpentMempoolOutputs); for(Collection selectedInputs : selectedInputSets) { total += selectedInputs.stream().mapToLong(BlockTransactionHashIndex::getValue).sum(); Map selectedInputsMap = new LinkedHashMap<>(); @@ -1213,7 +1221,7 @@ public class Wallet extends Persistable implements Comparable { Collections.shuffle(shuffledInputs); } for(BlockTransactionHashIndex shuffledInput : shuffledInputs) { - selectedInputsMap.put(shuffledInput, utxos.get(shuffledInput)); + selectedInputsMap.put(shuffledInput, availableTxos.get(shuffledInput)); } selectedInputSetsList.add(selectedInputsMap); } @@ -1227,18 +1235,18 @@ public class Wallet extends Persistable implements Comparable { throw new InsufficientFundsException("Not enough combined value in UTXOs for output value " + targetValue, targetValue); } - private List getGroupedUtxos(List utxoFilters, double feeRate, double longTermFeeRate, boolean groupByAddress, boolean includeSpentMempoolOutputs) { + public List getGroupedUtxos(List txoFilters, double feeRate, double longTermFeeRate, boolean groupByAddress) { List outputGroups = new ArrayList<>(); Map walletTransactions = getWalletTransactions(); Map walletTxos = getWalletTxos(); for(KeyPurpose keyPurpose : getWalletKeyPurposes()) { - getGroupedUtxos(outputGroups, getNode(keyPurpose), utxoFilters, walletTransactions, walletTxos, feeRate, longTermFeeRate, groupByAddress, includeSpentMempoolOutputs); + getGroupedUtxos(outputGroups, getNode(keyPurpose), txoFilters, walletTransactions, walletTxos, feeRate, longTermFeeRate, groupByAddress); } for(Wallet childWallet : getChildWallets()) { if(childWallet.isNested()) { for(KeyPurpose keyPurpose : childWallet.getWalletKeyPurposes()) { - childWallet.getGroupedUtxos(outputGroups, childWallet.getNode(keyPurpose), utxoFilters, walletTransactions, walletTxos, feeRate, longTermFeeRate, groupByAddress, includeSpentMempoolOutputs); + childWallet.getGroupedUtxos(outputGroups, childWallet.getNode(keyPurpose), txoFilters, walletTransactions, walletTxos, feeRate, longTermFeeRate, groupByAddress); } } } @@ -1246,16 +1254,11 @@ public class Wallet extends Persistable implements Comparable { return outputGroups; } - private void getGroupedUtxos(List outputGroups, WalletNode purposeNode, List utxoFilters, Map walletTransactions, Map walletTxos, double feeRate, double longTermFeeRate, boolean groupByAddress, boolean includeSpentMempoolOutputs) { + private void getGroupedUtxos(List outputGroups, WalletNode purposeNode, List txoFilters, Map walletTransactions, Map walletTxos, double feeRate, double longTermFeeRate, boolean groupByAddress) { int inputWeightUnits = getInputWeightUnits(); for(WalletNode addressNode : purposeNode.getChildren()) { OutputGroup outputGroup = null; - for(BlockTransactionHashIndex utxo : addressNode.getUnspentTransactionOutputs(includeSpentMempoolOutputs)) { - Optional matchedFilter = utxoFilters.stream().filter(utxoFilter -> !utxoFilter.isEligible(utxo)).findAny(); - if(matchedFilter.isPresent()) { - continue; - } - + for(BlockTransactionHashIndex utxo : addressNode.getTransactionOutputs(txoFilters)) { if(outputGroup == null || !groupByAddress) { outputGroup = new OutputGroup(addressNode.getWallet().getScriptType(), getStoredBlockHeight(), inputWeightUnits, feeRate, longTermFeeRate); outputGroups.add(outputGroup); @@ -1323,12 +1326,12 @@ public class Wallet extends Persistable implements Comparable { * @param feeRate the fee rate in sats/vB * @return the maximum spendable amount (can be negative if the fee is higher than the combined UTXO value) */ - public long getMaxSpendable(List
paymentAddresses, double feeRate, boolean includeSpentMempoolOutputs) { + public long getMaxSpendable(List
paymentAddresses, double feeRate, Map availableTxos) { long maxInputValue = 0; Map cachedInputWeightUnits = new HashMap<>(); Transaction transaction = new Transaction(); - for(Map.Entry utxo : getWalletUtxos(includeSpentMempoolOutputs).entrySet()) { + for(Map.Entry utxo : availableTxos.entrySet()) { int inputWeightUnits = cachedInputWeightUnits.computeIfAbsent(utxo.getValue().getWallet(), Wallet::getInputWeightUnits); long minInputValue = (long)Math.ceil(feeRate * inputWeightUnits / WITNESS_SCALE_FACTOR); if(utxo.getKey().getValue() > minInputValue) { diff --git a/src/main/java/com/sparrowwallet/drongo/wallet/WalletNode.java b/src/main/java/com/sparrowwallet/drongo/wallet/WalletNode.java index 4882606..9f05f47 100644 --- a/src/main/java/com/sparrowwallet/drongo/wallet/WalletNode.java +++ b/src/main/java/com/sparrowwallet/drongo/wallet/WalletNode.java @@ -168,16 +168,16 @@ public class WalletNode extends Persistable implements Comparable { } public Set getUnspentTransactionOutputs() { - return getUnspentTransactionOutputs(false); + return getTransactionOutputs(List.of(new SpentTxoFilter())); } - public Set getUnspentTransactionOutputs(boolean includeSpentMempoolOutputs) { + public Set getTransactionOutputs(Collection txoFilters) { if(transactionOutputs.isEmpty()) { return Collections.emptySet(); } Set unspentTXOs = new TreeSet<>(transactionOutputs); - unspentTXOs.removeIf(txo -> txo.isSpent() && (!includeSpentMempoolOutputs || txo.getSpentBy().getHeight() > 0)); + unspentTXOs.removeIf(txo -> !txoFilters.stream().allMatch(txoFilter -> txoFilter.isEligible(txo))); return unspentTXOs; }