support multiple utxo sets and change outputs

This commit is contained in:
Craig Raw 2021-08-27 15:58:23 +02:00
parent 81c202198e
commit 7ac4bce14f
12 changed files with 180 additions and 50 deletions

View file

@ -132,8 +132,8 @@ public class PSBT {
Address address = txOutput.getScript().getToAddresses()[0];
if(walletTransaction.getPayments().stream().anyMatch(payment -> payment.getAddress().equals(address))) {
outputNodes.add(wallet.getWalletAddresses().getOrDefault(address, null));
} else if(address.equals(wallet.getAddress(walletTransaction.getChangeNode()))) {
outputNodes.add(walletTransaction.getChangeNode());
} else if(walletTransaction.getChangeMap().keySet().stream().anyMatch(changeNode -> wallet.getAddress(changeNode).equals(address))) {
outputNodes.add(wallet.getWalletAddresses().getOrDefault(address, null));
}
} catch(NonStandardScriptException e) {
//Should never happen

View file

@ -6,7 +6,7 @@ import org.slf4j.LoggerFactory;
import java.util.*;
public class BnBUtxoSelector implements UtxoSelector {
public class BnBUtxoSelector extends SingleSetUtxoSelector {
private static final Logger log = LoggerFactory.getLogger(BnBUtxoSelector.class);
private static final int TOTAL_TRIES = 100000;

View file

@ -5,7 +5,7 @@ import com.sparrowwallet.drongo.protocol.Transaction;
import java.util.*;
import java.util.stream.Collectors;
public class KnapsackUtxoSelector implements UtxoSelector {
public class KnapsackUtxoSelector extends SingleSetUtxoSelector {
private static final long MIN_CHANGE = Transaction.SATOSHIS_PER_BITCOIN / 1000;
private final long noInputsFee;

View file

@ -3,7 +3,7 @@ package com.sparrowwallet.drongo.wallet;
import java.util.Collection;
import java.util.stream.Collectors;
public class MaxUtxoSelector implements UtxoSelector {
public class MaxUtxoSelector extends SingleSetUtxoSelector {
@Override
public Collection<BlockTransactionHashIndex> select(long targetValue, Collection<OutputGroup> candidates) {
return candidates.stream().filter(outputGroup -> outputGroup.getEffectiveValue() >= 0).flatMap(outputGroup -> outputGroup.getUtxos().stream()).collect(Collectors.toUnmodifiableList());

View file

@ -57,6 +57,6 @@ public class Payment {
}
public enum Type {
DEFAULT, WHIRLPOOL_FEE;
DEFAULT, WHIRLPOOL_FEE, FAKE_MIX;
}
}

View file

@ -5,7 +5,7 @@ import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
public class PresetUtxoSelector implements UtxoSelector {
public class PresetUtxoSelector extends SingleSetUtxoSelector {
private final Collection<BlockTransactionHashIndex> presetUtxos;
public PresetUtxoSelector(Collection<BlockTransactionHashIndex> presetUtxos) {

View file

@ -4,7 +4,7 @@ import java.math.BigInteger;
import java.util.*;
import java.util.stream.Collectors;
public class PriorityUtxoSelector implements UtxoSelector {
public class PriorityUtxoSelector extends SingleSetUtxoSelector {
private final int currentBlockHeight;
public PriorityUtxoSelector(int currentBlockHeight) {

View file

@ -0,0 +1,13 @@
package com.sparrowwallet.drongo.wallet;
import java.util.Collection;
import java.util.List;
public abstract class SingleSetUtxoSelector implements UtxoSelector {
@Override
public List<Collection<BlockTransactionHashIndex>> selectSets(long targetValue, Collection<OutputGroup> candidates) {
return List.of(select(targetValue, candidates));
}
public abstract Collection<BlockTransactionHashIndex> select(long targetValue, Collection<OutputGroup> candidates);
}

View file

@ -0,0 +1,76 @@
package com.sparrowwallet.drongo.wallet;
import java.util.*;
import java.util.stream.Collectors;
public class StonewallUtxoSelector implements UtxoSelector {
private final long noInputsFee;
//Use the same seed so the UTXO selection is deterministic
private final Random random = new Random(42);
public StonewallUtxoSelector(long noInputsFee) {
this.noInputsFee = noInputsFee;
}
@Override
public List<Collection<BlockTransactionHashIndex>> selectSets(long targetValue, Collection<OutputGroup> candidates) {
long actualTargetValue = targetValue + noInputsFee;
for(int i = 0; i < 10; i++) {
List<OutputGroup> randomized = new ArrayList<>(candidates);
Collections.shuffle(randomized, random);
List<OutputGroup> set1 = new ArrayList<>();
long selectedValue1 = getUtxoSet(actualTargetValue, set1, randomized);
List<OutputGroup> set2 = new ArrayList<>();
long selectedValue2 = getUtxoSet(actualTargetValue, set2, randomized);
if(selectedValue1 >= targetValue && selectedValue2 >= targetValue) {
return List.of(getUtxos(set1), getUtxos(set2));
}
}
return Collections.emptyList();
}
private long getUtxoSet(long targetValue, List<OutputGroup> selectedSet, List<OutputGroup> randomized) {
long selectedValue = 0;
while(selectedValue <= targetValue && !randomized.isEmpty()) {
OutputGroup candidate = randomized.remove(0);
OutputGroup existingTxGroup = getTransactionAlreadySelected(selectedSet, candidate);
if(existingTxGroup != null) {
if(candidate.getValue() > existingTxGroup.getValue()) {
selectedSet.remove(existingTxGroup);
selectedSet.add(candidate);
}
} else {
selectedSet.add(candidate);
}
selectedValue = selectedSet.stream().mapToLong(OutputGroup::getEffectiveValue).sum();
}
return selectedValue;
}
private OutputGroup getTransactionAlreadySelected(List<OutputGroup> selected, OutputGroup candidateGroup) {
for(OutputGroup selectedGroup : selected) {
for(BlockTransactionHashIndex selectedUtxo : selectedGroup.getUtxos()) {
for(BlockTransactionHashIndex candidateUtxo : candidateGroup.getUtxos()) {
if(selectedUtxo.getHash().equals(candidateUtxo.getHash())) {
return selectedGroup;
}
}
}
}
return null;
}
private Collection<BlockTransactionHashIndex> getUtxos(List<OutputGroup> set) {
return set.stream().flatMap(outputGroup -> outputGroup.getUtxos().stream()).collect(Collectors.toList());
}
}

View file

@ -1,7 +1,8 @@
package com.sparrowwallet.drongo.wallet;
import java.util.Collection;
import java.util.List;
public interface UtxoSelector {
Collection<BlockTransactionHashIndex> select(long targetValue, Collection<OutputGroup> candidates);
List<Collection<BlockTransactionHashIndex>> selectSets(long targetValue, Collection<OutputGroup> candidates);
}

View file

@ -622,7 +622,7 @@ public class Wallet extends Persistable {
return getFee(changeOutput, feeRate, longTermFeeRate);
}
public WalletTransaction createWalletTransaction(List<UtxoSelector> utxoSelectors, List<UtxoFilter> utxoFilters, List<Payment> payments, List<WalletNode> excludedChangeNodes, double feeRate, double longTermFeeRate, Long fee, Integer currentBlockHeight, boolean groupByAddress, boolean includeMempoolOutputs, boolean includeSpentMempoolOutputs) throws InsufficientFundsException {
public WalletTransaction createWalletTransaction(List<UtxoSelector> utxoSelectors, List<UtxoFilter> utxoFilters, List<Payment> payments, Set<WalletNode> excludedChangeNodes, double feeRate, double longTermFeeRate, Long fee, Integer currentBlockHeight, boolean groupByAddress, boolean includeMempoolOutputs, boolean includeSpentMempoolOutputs) throws InsufficientFundsException {
boolean sendMax = payments.stream().anyMatch(Payment::isSendMax);
long totalPaymentAmount = payments.stream().map(Payment::getAmount).mapToLong(v -> v).sum();
long totalUtxoValue = getWalletUtxos().keySet().stream().mapToLong(BlockTransactionHashIndex::getValue).sum();
@ -644,8 +644,13 @@ public class Wallet extends Persistable {
}
while(true) {
Map<BlockTransactionHashIndex, WalletNode> selectedUtxos = selectInputs(utxoSelectors, utxoFilters, valueRequiredAmt, feeRate, longTermFeeRate, groupByAddress, includeMempoolOutputs, includeSpentMempoolOutputs, sendMax);
List<Map<BlockTransactionHashIndex, WalletNode>> selectedUtxoSets = selectInputSets(utxoSelectors, utxoFilters, valueRequiredAmt, feeRate, longTermFeeRate, groupByAddress, includeMempoolOutputs, includeSpentMempoolOutputs, sendMax);
Map<BlockTransactionHashIndex, WalletNode> selectedUtxos = new HashMap<>();
selectedUtxoSets.forEach(selectedUtxos::putAll);
long totalSelectedAmt = selectedUtxos.keySet().stream().mapToLong(BlockTransactionHashIndex::getValue).sum();
int numSets = selectedUtxoSets.size();
List<Payment> txPayments = new ArrayList<>(payments);
Set<WalletNode> txExcludedChangeNodes = new HashSet<>(excludedChangeNodes);
Transaction transaction = new Transaction();
transaction.setVersion(2);
@ -663,8 +668,16 @@ public class Wallet extends Persistable {
txInput.setSequenceNumber(TransactionInput.SEQUENCE_RBF_ENABLED);
}
for(int i = 1; i < numSets; i+=2) {
WalletNode mixNode = getFreshNode(KeyPurpose.CHANGE);
txExcludedChangeNodes.add(mixNode);
Payment fakeMixPayment = new Payment(getAddress(mixNode), ".." + mixNode + " (Fake Mix)", totalPaymentAmount, false);
fakeMixPayment.setType(Payment.Type.FAKE_MIX);
txPayments.add(fakeMixPayment);
}
//Add recipient outputs
for(Payment payment : payments) {
for(Payment payment : txPayments) {
transaction.addOutput(payment.getAmount(), payment.getAddress());
}
@ -689,7 +702,7 @@ public class Wallet extends Persistable {
}
//Calculate what is left over from selected utxos after paying recipient
long differenceAmt = totalSelectedAmt - totalPaymentAmount;
long differenceAmt = totalSelectedAmt - totalPaymentAmount * numSets;
//If insufficient fee, increase value required from inputs to include the fee and try again
if(differenceAmt < noChangeFeeRequiredAmt) {
@ -703,38 +716,63 @@ public class Wallet extends Persistable {
}
//Determine if a change output is required by checking if its value is greater than its dust threshold
long changeAmt = differenceAmt - noChangeFeeRequiredAmt;
List<Long> setChangeAmts = getSetChangeAmounts(selectedUtxoSets, totalPaymentAmount, noChangeFeeRequiredAmt);
double noChangeFeeRate = (fee == null ? feeRate : noChangeFeeRequiredAmt / transaction.getVirtualSize());
long costOfChangeAmt = getCostOfChange(noChangeFeeRate, longTermFeeRate);
if(changeAmt > costOfChangeAmt) {
if(setChangeAmts.stream().allMatch(amt -> amt > costOfChangeAmt)) {
//Change output is required, determine new fee once change output has been added
WalletNode changeNode = getFreshNode(KeyPurpose.CHANGE);
while(excludedChangeNodes.contains(changeNode)) {
while(txExcludedChangeNodes.contains(changeNode)) {
changeNode = getFreshNode(KeyPurpose.CHANGE, changeNode);
}
TransactionOutput changeOutput = new TransactionOutput(transaction, changeAmt, getOutputScript(changeNode));
double changeVSize = noChangeVSize + changeOutput.getLength();
TransactionOutput changeOutput = new TransactionOutput(transaction, setChangeAmts.iterator().next(), getOutputScript(changeNode));
double changeVSize = noChangeVSize + changeOutput.getLength() * numSets;
long changeFeeRequiredAmt = (fee == null ? (long)Math.floor(feeRate * changeVSize) : fee);
changeFeeRequiredAmt = (fee == null && feeRate == Transaction.DEFAULT_MIN_RELAY_FEE ? changeFeeRequiredAmt + 1 : changeFeeRequiredAmt);
//Recalculate the change amount with the new fee
changeAmt = differenceAmt - changeFeeRequiredAmt;
if(changeAmt < costOfChangeAmt) {
//The new fee has meant that the change output is now dust. We pay too high a fee without change, but change is dust when added. Increase value required from inputs and try again
valueRequiredAmt = totalSelectedAmt + 1;
continue;
while(changeFeeRequiredAmt % numSets > 0) {
changeFeeRequiredAmt++;
}
//Add change output
transaction.addOutput(changeAmt, getOutputScript(changeNode));
//Add change output(s)
Map<WalletNode, Long> changeMap = new LinkedHashMap<>();
setChangeAmts = getSetChangeAmounts(selectedUtxoSets, totalPaymentAmount, changeFeeRequiredAmt);
for(Long setChangeAmt : setChangeAmts) {
transaction.addOutput(setChangeAmt, getOutputScript(changeNode));
changeMap.put(changeNode, setChangeAmt);
changeNode = getFreshNode(KeyPurpose.CHANGE, changeNode);
}
return new WalletTransaction(this, transaction, utxoSelectors, selectedUtxos, payments, changeNode, changeAmt, changeFeeRequiredAmt);
if(setChangeAmts.stream().anyMatch(amt -> amt < costOfChangeAmt)) {
//The new fee has meant that one of the change outputs is now dust. We pay too high a fee without change, but change is dust when added.
if(numSets > 1) {
//Maximize privacy. Pay a higher fee to keep multiple output sets.
return new WalletTransaction(this, transaction, utxoSelectors, selectedUtxos, txPayments, differenceAmt);
} else {
//Maxmize efficiency. Increase value required from inputs and try again.
valueRequiredAmt = totalSelectedAmt + 1;
continue;
}
}
return new WalletTransaction(this, transaction, utxoSelectors, selectedUtxos, txPayments, changeMap, changeFeeRequiredAmt);
}
return new WalletTransaction(this, transaction, utxoSelectors, selectedUtxos, payments, differenceAmt);
return new WalletTransaction(this, transaction, utxoSelectors, selectedUtxos, txPayments, differenceAmt);
}
}
private List<Long> getSetChangeAmounts(List<Map<BlockTransactionHashIndex, WalletNode>> selectedUtxoSets, long totalPaymentAmount, long feeRequiredAmt) {
List<Long> changeAmts = new ArrayList<>();
int numSets = selectedUtxoSets.size();
for(Map<BlockTransactionHashIndex, WalletNode> selectedUtxoSet : selectedUtxoSets) {
long setAmt = selectedUtxoSet.keySet().stream().mapToLong(BlockTransactionHashIndex::getValue).sum();
long setChangeAmt = setAmt - (totalPaymentAmount + feeRequiredAmt / numSets);
changeAmts.add(setChangeAmt);
}
return changeAmts;
}
public TransactionInput addDummySpendingInput(Transaction transaction, WalletNode walletNode, TransactionOutput prevTxOut) {
if(getPolicyType().equals(PolicyType.SINGLE)) {
ECKey pubKey = getPubKey(walletNode);
@ -752,7 +790,7 @@ public class Wallet extends Persistable {
}
}
private Map<BlockTransactionHashIndex, WalletNode> selectInputs(List<UtxoSelector> utxoSelectors, List<UtxoFilter> utxoFilters, Long targetValue, double feeRate, double longTermFeeRate, boolean groupByAddress, boolean includeMempoolOutputs, boolean includeSpentMempoolOutputs, boolean sendMax) throws InsufficientFundsException {
private List<Map<BlockTransactionHashIndex, WalletNode>> selectInputSets(List<UtxoSelector> utxoSelectors, List<UtxoFilter> utxoFilters, Long targetValue, double feeRate, double longTermFeeRate, boolean groupByAddress, boolean includeMempoolOutputs, boolean includeSpentMempoolOutputs, boolean sendMax) throws InsufficientFundsException {
List<OutputGroup> utxoPool = getGroupedUtxos(utxoFilters, feeRate, longTermFeeRate, groupByAddress, includeSpentMempoolOutputs);
List<OutputGroup.Filter> filters = new ArrayList<>();
@ -770,12 +808,19 @@ public class Wallet extends Persistable {
List<OutputGroup> filteredPool = utxoPool.stream().filter(filter::isEligible).collect(Collectors.toList());
for(UtxoSelector utxoSelector : utxoSelectors) {
Collection<BlockTransactionHashIndex> selectedInputs = utxoSelector.select(targetValue, filteredPool);
long total = selectedInputs.stream().mapToLong(BlockTransactionHashIndex::getValue).sum();
if(total > targetValue) {
Map<BlockTransactionHashIndex, WalletNode> utxos = getWalletUtxos(includeSpentMempoolOutputs);
utxos.keySet().retainAll(selectedInputs);
return utxos;
List<Collection<BlockTransactionHashIndex>> selectedInputSets = utxoSelector.selectSets(targetValue, filteredPool);
List<Map<BlockTransactionHashIndex, WalletNode>> selectedInputSetsList = new ArrayList<>();
long total = 0;
Map<BlockTransactionHashIndex, WalletNode> utxos = getWalletUtxos(includeSpentMempoolOutputs);
for(Collection<BlockTransactionHashIndex> selectedInputs : selectedInputSets) {
total += selectedInputs.stream().mapToLong(BlockTransactionHashIndex::getValue).sum();
Map<BlockTransactionHashIndex, WalletNode> selectedInputsMap = new HashMap<>(utxos);
selectedInputsMap.keySet().retainAll(selectedInputs);
selectedInputSetsList.add(selectedInputsMap);
}
if(total > targetValue * selectedInputSetsList.size()) {
return selectedInputSetsList;
}
}
}

View file

@ -5,6 +5,7 @@ import com.sparrowwallet.drongo.protocol.Transaction;
import com.sparrowwallet.drongo.psbt.PSBT;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@ -18,22 +19,20 @@ public class WalletTransaction {
private final List<UtxoSelector> utxoSelectors;
private final Map<BlockTransactionHashIndex, WalletNode> selectedUtxos;
private final List<Payment> payments;
private final WalletNode changeNode;
private final long changeAmount;
private final Map<WalletNode, Long> changeMap;
private final long fee;
public WalletTransaction(Wallet wallet, Transaction transaction, List<UtxoSelector> utxoSelectors, Map<BlockTransactionHashIndex, WalletNode> selectedUtxos, List<Payment> payments, long fee) {
this(wallet, transaction, utxoSelectors, selectedUtxos, payments, null, 0L, fee);
this(wallet, transaction, utxoSelectors, selectedUtxos, payments, Collections.emptyMap(), fee);
}
public WalletTransaction(Wallet wallet, Transaction transaction, List<UtxoSelector> utxoSelectors, Map<BlockTransactionHashIndex, WalletNode> selectedUtxos, List<Payment> payments, WalletNode changeNode, long changeAmount, long fee) {
public WalletTransaction(Wallet wallet, Transaction transaction, List<UtxoSelector> utxoSelectors, Map<BlockTransactionHashIndex, WalletNode> selectedUtxos, List<Payment> payments, Map<WalletNode, Long> changeMap, long fee) {
this.wallet = wallet;
this.transaction = transaction;
this.utxoSelectors = utxoSelectors;
this.selectedUtxos = selectedUtxos;
this.payments = payments;
this.changeNode = changeNode;
this.changeAmount = changeAmount;
this.changeMap = changeMap;
this.fee = fee;
}
@ -61,16 +60,12 @@ public class WalletTransaction {
return payments;
}
public WalletNode getChangeNode() {
return changeNode;
public Map<WalletNode, Long> getChangeMap() {
return changeMap;
}
public Address getChangeAddress() {
return getWallet().getAddress(getChangeNode());
}
public long getChangeAmount() {
return changeAmount;
public Address getChangeAddress(WalletNode changeNode) {
return getWallet().getAddress(changeNode);
}
public long getFee() {