From bad209ea5b1f25a174ff850079040f98557bb3b3 Mon Sep 17 00:00:00 2001 From: Craig Raw Date: Wed, 13 Oct 2021 15:21:14 +0200 Subject: [PATCH] automatically increase gap limit if required by postmix handler --- drongo | 2 +- .../sparrowwallet/sparrow/AppController.java | 2 +- .../event/WalletGapLimitChangedEvent.java | 17 ++++ .../event/WalletHistoryStartedEvent.java | 14 ++-- .../sparrow/io/db/DbPersistence.java | 13 +++ .../sparrow/io/db/WalletDao.java | 3 + .../sparrow/net/BatchedElectrumServerRpc.java | 81 +++---------------- .../sparrow/wallet/WalletForm.java | 49 ++++++++--- .../sparrow/whirlpool/Whirlpool.java | 10 ++- .../dataSource/SparrowIndexHandler.java | 11 +++ 10 files changed, 110 insertions(+), 92 deletions(-) create mode 100644 src/main/java/com/sparrowwallet/sparrow/event/WalletGapLimitChangedEvent.java diff --git a/drongo b/drongo index 025af057..61b2fd21 160000 --- a/drongo +++ b/drongo @@ -1 +1 @@ -Subproject commit 025af0575899d6e255cc0a1f875cad25ab4811b8 +Subproject commit 61b2fd21e6f623850ad8b5df9f0cec4a0c0908cc diff --git a/src/main/java/com/sparrowwallet/sparrow/AppController.java b/src/main/java/com/sparrowwallet/sparrow/AppController.java index e14bbd18..f4913377 100644 --- a/src/main/java/com/sparrowwallet/sparrow/AppController.java +++ b/src/main/java/com/sparrowwallet/sparrow/AppController.java @@ -2029,7 +2029,7 @@ public class AppController implements Initializable { @Subscribe public void walletHistoryStarted(WalletHistoryStartedEvent event) { if(AppServices.isConnected() && getOpenWallets().containsKey(event.getWallet())) { - if(event.getWalletNode() == null && event.getWallet().getTransactions().isEmpty()) { + if(event.getWalletNodes() == null && event.getWallet().getTransactions().isEmpty()) { statusUpdated(new StatusEvent(LOADING_TRANSACTIONS_MESSAGE, 120)); if(statusTimeline == null || statusTimeline.getStatus() != Animation.Status.RUNNING) { statusBar.setProgress(-1); diff --git a/src/main/java/com/sparrowwallet/sparrow/event/WalletGapLimitChangedEvent.java b/src/main/java/com/sparrowwallet/sparrow/event/WalletGapLimitChangedEvent.java new file mode 100644 index 00000000..be54412f --- /dev/null +++ b/src/main/java/com/sparrowwallet/sparrow/event/WalletGapLimitChangedEvent.java @@ -0,0 +1,17 @@ +package com.sparrowwallet.sparrow.event; + +import com.sparrowwallet.drongo.wallet.Wallet; + +/** + * This event is posted if the wallet's gap limit has changed, and triggers a history fetch for the new nodes. + * + */ +public class WalletGapLimitChangedEvent extends WalletChangedEvent { + public WalletGapLimitChangedEvent(Wallet wallet) { + super(wallet); + } + + public int getGapLimit() { + return getWallet().getGapLimit(); + } +} diff --git a/src/main/java/com/sparrowwallet/sparrow/event/WalletHistoryStartedEvent.java b/src/main/java/com/sparrowwallet/sparrow/event/WalletHistoryStartedEvent.java index 6fc3c0ce..a1dbfad9 100644 --- a/src/main/java/com/sparrowwallet/sparrow/event/WalletHistoryStartedEvent.java +++ b/src/main/java/com/sparrowwallet/sparrow/event/WalletHistoryStartedEvent.java @@ -3,15 +3,17 @@ package com.sparrowwallet.sparrow.event; import com.sparrowwallet.drongo.wallet.Wallet; import com.sparrowwallet.drongo.wallet.WalletNode; -public class WalletHistoryStartedEvent extends WalletHistoryStatusEvent { - private final WalletNode walletNode; +import java.util.Set; - public WalletHistoryStartedEvent(Wallet wallet, WalletNode walletNode) { +public class WalletHistoryStartedEvent extends WalletHistoryStatusEvent { + private final Set walletNodes; + + public WalletHistoryStartedEvent(Wallet wallet, Set walletNodes) { super(wallet, true); - this.walletNode = walletNode; + this.walletNodes = walletNodes; } - public WalletNode getWalletNode() { - return walletNode; + public Set getWalletNodes() { + return walletNodes; } } diff --git a/src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java b/src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java index 1dfea6af..67f2bbce 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java @@ -240,6 +240,10 @@ public class DbPersistence implements Persistence { walletDao.updateStoredBlockHeight(wallet.getId(), dirtyPersistables.blockHeight); } + if(dirtyPersistables.gapLimit != null) { + walletDao.updateGapLimit(wallet.getId(), dirtyPersistables.gapLimit); + } + if(!dirtyPersistables.labelEntries.isEmpty()) { BlockTransactionDao blockTransactionDao = handle.attach(BlockTransactionDao.class); WalletNodeDao walletNodeDao = handle.attach(WalletNodeDao.class); @@ -644,6 +648,13 @@ public class DbPersistence implements Persistence { } } + @Subscribe + public void walletGapLimitChanged(WalletGapLimitChangedEvent event) { + if(persistsFor(event.getWallet())) { + dirtyPersistablesMap.computeIfAbsent(event.getWallet(), key -> new DirtyPersistables()).gapLimit = event.getGapLimit(); + } + } + @Subscribe public void walletEntryLabelsChanged(WalletEntryLabelsChangedEvent event) { if(persistsFor(event.getWallet())) { @@ -691,6 +702,7 @@ public class DbPersistence implements Persistence { public boolean clearHistory; public final List historyNodes = new ArrayList<>(); public Integer blockHeight = null; + public Integer gapLimit = null; public final List labelEntries = new ArrayList<>(); public final List utxoStatuses = new ArrayList<>(); public boolean mixConfig; @@ -704,6 +716,7 @@ public class DbPersistence implements Persistence { "\nClear history:" + clearHistory + "\nNodes:" + historyNodes + "\nBlockHeight:" + blockHeight + + "\nGap limit:" + gapLimit + "\nTx labels:" + labelEntries.stream().filter(entry -> entry instanceof TransactionEntry).map(entry -> ((TransactionEntry)entry).getBlockTransaction().getHash().toString()).collect(Collectors.toList()) + "\nAddress labels:" + labelEntries.stream().filter(entry -> entry instanceof NodeEntry).map(entry -> ((NodeEntry)entry).getNode().toString() + " " + entry.getLabel()).collect(Collectors.toList()) + "\nUTXO labels:" + labelEntries.stream().filter(entry -> entry instanceof HashIndexEntry).map(entry -> ((HashIndexEntry)entry).getHashIndex().toString()).collect(Collectors.toList()) + diff --git a/src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java b/src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java index 206bc0a8..c2d3fe5f 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java @@ -55,6 +55,9 @@ public interface WalletDao { @SqlUpdate("update wallet set storedBlockHeight = :blockHeight where id = :id") void updateStoredBlockHeight(@Bind("id") long id, @Bind("blockHeight") Integer blockHeight); + @SqlUpdate("update wallet set gapLimit = :gapLimit where id = :id") + void updateGapLimit(@Bind("id") long id, @Bind("gapLimit") Integer gapLimit); + @SqlUpdate("set schema ?") int setSchema(String schema); diff --git a/src/main/java/com/sparrowwallet/sparrow/net/BatchedElectrumServerRpc.java b/src/main/java/com/sparrowwallet/sparrow/net/BatchedElectrumServerRpc.java index 4374ab67..5029eab6 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/BatchedElectrumServerRpc.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/BatchedElectrumServerRpc.java @@ -5,8 +5,6 @@ import com.github.arteam.simplejsonrpc.client.Transport; import com.github.arteam.simplejsonrpc.client.builder.BatchRequestBuilder; import com.github.arteam.simplejsonrpc.client.exception.JsonRpcBatchException; import com.github.arteam.simplejsonrpc.client.exception.JsonRpcException; -import com.sparrowwallet.drongo.KeyDerivation; -import com.sparrowwallet.drongo.crypto.ChildNumber; import com.sparrowwallet.drongo.protocol.Sha256Hash; import com.sparrowwallet.drongo.wallet.Wallet; import com.sparrowwallet.sparrow.EventManager; @@ -19,6 +17,8 @@ import java.util.*; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; +import static com.sparrowwallet.drongo.wallet.WalletNode.nodeRangesToString; + public class BatchedElectrumServerRpc implements ElectrumServerRpc { private static final Logger log = LoggerFactory.getLogger(BatchedElectrumServerRpc.class); private static final int MAX_RETRIES = 5; @@ -75,7 +75,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc { public Map getScriptHashHistory(Transport transport, Wallet wallet, Map pathScriptHashes, boolean failOnError) { JsonRpcClient client = new JsonRpcClient(transport); BatchRequestBuilder batchRequest = client.createBatchRequest().keysType(String.class).returnType(ScriptHashTx[].class); - EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Loading transactions for " + getScriptHashesAbbreviation(pathScriptHashes.keySet()))); + EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Loading transactions for " + nodeRangesToString(pathScriptHashes.keySet()))); for(String path : pathScriptHashes.keySet()) { batchRequest.add(path, "blockchain.scripthash.get_history", pathScriptHashes.get(path)); @@ -85,7 +85,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc { return new RetryLogic>(MAX_RETRIES, RETRY_DELAY, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(batchRequest::execute); } catch (JsonRpcBatchException e) { if(failOnError) { - throw new ElectrumServerRpcException("Failed to retrieve transaction history for paths: " + getScriptHashesAbbreviation((Collection)e.getErrors().keySet()), e); + throw new ElectrumServerRpcException("Failed to retrieve transaction history for paths: " + nodeRangesToString((Collection)e.getErrors().keySet()), e); } Map result = (Map)e.getSuccesses(); @@ -95,7 +95,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc { return result; } catch(Exception e) { - throw new ElectrumServerRpcException("Failed to retrieve transaction history for paths: " + getScriptHashesAbbreviation(pathScriptHashes.keySet()), e); + throw new ElectrumServerRpcException("Failed to retrieve transaction history for paths: " + nodeRangesToString(pathScriptHashes.keySet()), e); } } @@ -113,7 +113,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc { return new RetryLogic>(MAX_RETRIES, RETRY_DELAY, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(batchRequest::execute); } catch(JsonRpcBatchException e) { if(failOnError) { - throw new ElectrumServerRpcException("Failed to retrieve mempool transactions for paths: " + getScriptHashesAbbreviation((Collection)e.getErrors().keySet()), e); + throw new ElectrumServerRpcException("Failed to retrieve mempool transactions for paths: " + nodeRangesToString((Collection)e.getErrors().keySet()), e); } Map result = (Map)e.getSuccesses(); @@ -123,7 +123,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc { return result; } catch(Exception e) { - throw new ElectrumServerRpcException("Failed to retrieve mempool transactions for paths: " + getScriptHashesAbbreviation(pathScriptHashes.keySet()), e); + throw new ElectrumServerRpcException("Failed to retrieve mempool transactions for paths: " + nodeRangesToString(pathScriptHashes.keySet()), e); } } @@ -132,7 +132,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc { public Map subscribeScriptHashes(Transport transport, Wallet wallet, Map pathScriptHashes) { JsonRpcClient client = new JsonRpcClient(transport); BatchRequestBuilder batchRequest = client.createBatchRequest().keysType(String.class).returnType(String.class); - EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Finding transactions for " + getScriptHashesAbbreviation(pathScriptHashes.keySet()))); + EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Finding transactions for " + nodeRangesToString(pathScriptHashes.keySet()))); for(String path : pathScriptHashes.keySet()) { batchRequest.add(path, "blockchain.scripthash.subscribe", pathScriptHashes.get(path)); @@ -142,9 +142,9 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc { return new RetryLogic>(MAX_RETRIES, RETRY_DELAY, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(batchRequest::execute); } catch(JsonRpcBatchException e) { //Even if we have some successes, failure to subscribe for all script hashes will result in outdated wallet view. Don't proceed. - throw new ElectrumServerRpcException("Failed to subscribe to paths: " + getScriptHashesAbbreviation((Collection)e.getErrors().keySet()), e); + throw new ElectrumServerRpcException("Failed to subscribe to paths: " + nodeRangesToString((Collection)e.getErrors().keySet()), e); } catch(Exception e) { - throw new ElectrumServerRpcException("Failed to subscribe to paths: " + getScriptHashesAbbreviation(pathScriptHashes.keySet()), e); + throw new ElectrumServerRpcException("Failed to subscribe to paths: " + nodeRangesToString(pathScriptHashes.keySet()), e); } } @@ -276,65 +276,4 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc { throw new ElectrumServerRpcException("Error broadcasting transaction", e); } } - - private static String getScriptHashesAbbreviation(Collection scriptHashes) { - List sortedHashes = new ArrayList<>(scriptHashes); - - if(scriptHashes.isEmpty()) { - return "[]"; - } - - List> contiguous = splitToContiguous(sortedHashes); - - String abbrev = "["; - for(Iterator> iter = contiguous.iterator(); iter.hasNext(); ) { - List range = iter.next(); - abbrev += range.get(0); - if(range.size() > 1) { - abbrev += "-" + range.get(range.size() - 1); - } - if(iter.hasNext()) { - abbrev += ", "; - } - } - abbrev += "]"; - - return abbrev; - } - - static List> splitToContiguous(List input) { - List> result = new ArrayList<>(); - int prev = 0; - - int keyPurpose = getKeyPurpose(input.get(0)); - int index = getIndex(input.get(0)); - - for (int cur = 0; cur < input.size(); cur++) { - if(getKeyPurpose(input.get(cur)) != keyPurpose || getIndex(input.get(cur)) != index) { - result.add(input.subList(prev, cur)); - prev = cur; - } - index = getIndex(input.get(cur)) + 1; - keyPurpose = getKeyPurpose(input.get(cur)); - } - result.add(input.subList(prev, input.size())); - - return result; - } - - private static int getKeyPurpose(String path) { - List childNumbers = KeyDerivation.parsePath(path); - if(childNumbers.isEmpty()) { - return -1; - } - return childNumbers.get(0).num(); - } - - private static int getIndex(String path) { - List childNumbers = KeyDerivation.parsePath(path); - if(childNumbers.isEmpty()) { - return -1; - } - return childNumbers.get(childNumbers.size() - 1).num(); - } } diff --git a/src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java b/src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java index 23bce179..b382343e 100644 --- a/src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java +++ b/src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java @@ -25,6 +25,8 @@ import java.io.IOException; import java.util.*; import java.util.stream.Collectors; +import static com.sparrowwallet.drongo.wallet.WalletNode.nodeRangesToString; + public class WalletForm { private static final Logger log = LoggerFactory.getLogger(WalletForm.class); @@ -123,11 +125,14 @@ public class WalletForm { refreshHistory(blockHeight, pastWallet, null); } - public void refreshHistory(Integer blockHeight, Wallet pastWallet, WalletNode node) { + public void refreshHistory(Integer blockHeight, Wallet pastWallet, Set nodes) { Wallet previousWallet = wallet.copy(); if(wallet.isValid() && AppServices.isConnected()) { - log.debug(node == null ? wallet.getFullName() + " refreshing full wallet history" : wallet.getFullName() + " requesting node wallet history for " + node.getDerivationPath()); - ElectrumServer.TransactionHistoryService historyService = new ElectrumServer.TransactionHistoryService(wallet, getWalletTransactionNodes(node)); + if(log.isDebugEnabled()) { + log.debug(nodes == null ? wallet.getFullName() + " refreshing full wallet history" : wallet.getFullName() + " requesting node wallet history for " + nodeRangesToString(nodes)); + } + + ElectrumServer.TransactionHistoryService historyService = new ElectrumServer.TransactionHistoryService(wallet, getWalletTransactionNodes(nodes)); historyService.setOnSucceeded(workerStateEvent -> { if(historyService.getValue()) { EventManager.get().post(new WalletHistoryFinishedEvent(wallet)); @@ -144,7 +149,7 @@ public class WalletForm { EventManager.get().post(new WalletHistoryFailedEvent(wallet, workerStateEvent.getSource().getException())); }); - EventManager.get().post(new WalletHistoryStartedEvent(wallet, node)); + EventManager.get().post(new WalletHistoryStartedEvent(wallet, nodes)); historyService.start(); } } @@ -249,19 +254,21 @@ public class WalletForm { walletTransactionNodes.add(transactionNodes); } - private Set getWalletTransactionNodes(WalletNode walletNode) { - if(walletNode == null) { + private Set getWalletTransactionNodes(Set walletNodes) { + if(walletNodes == null) { return null; } Set allNodes = new LinkedHashSet<>(); - for(Set nodes : walletTransactionNodes) { - if(nodes.contains(walletNode)) { - allNodes.addAll(nodes); + for(WalletNode walletNode : walletNodes) { + for(Set nodes : walletTransactionNodes) { + if(nodes.contains(walletNode)) { + allNodes.addAll(nodes); + } } } - return allNodes.isEmpty() ? Set.of(walletNode) : allNodes; + return allNodes.isEmpty() ? walletNodes : allNodes; } public NodeEntry getNodeEntry(KeyPurpose keyPurpose) { @@ -392,7 +399,7 @@ public class WalletForm { WalletNode walletNode = event.getWalletNode(wallet); if(walletNode != null) { log.debug(wallet.getFullName() + " history event for node " + walletNode + " (" + event.getScriptHash() + ")"); - refreshHistory(AppServices.getCurrentBlockHeight(), null, walletNode); + refreshHistory(AppServices.getCurrentBlockHeight(), null, Set.of(walletNode)); } } } @@ -503,6 +510,26 @@ public class WalletForm { } } + @Subscribe + public void walletGapLimitChanged(WalletGapLimitChangedEvent event) { + if(event.getWallet() == wallet) { + Platform.runLater(() -> EventManager.get().post(new WalletDataChangedEvent(wallet))); + + Set newNodes = new LinkedHashSet<>(); + for(KeyPurpose keyPurpose : KeyPurpose.DEFAULT_PURPOSES) { + Optional optPurposeNode = wallet.getPurposeNodes().stream().filter(node -> node.getKeyPurpose() == keyPurpose).findFirst(); + if(optPurposeNode.isPresent()) { + WalletNode purposeNode = optPurposeNode.get(); + newNodes.addAll(purposeNode.fillToIndex(wallet.getLookAheadIndex(purposeNode))); + } + } + + if(!newNodes.isEmpty()) { + Platform.runLater(() -> refreshHistory(AppServices.getCurrentBlockHeight(), null, newNodes)); + } + } + } + @Subscribe public void whirlpoolMixSuccess(WhirlpoolMixSuccessEvent event) { if(event.getWallet() == wallet && event.getWalletNode() != null) { diff --git a/src/main/java/com/sparrowwallet/sparrow/whirlpool/Whirlpool.java b/src/main/java/com/sparrowwallet/sparrow/whirlpool/Whirlpool.java index 7394110a..7015221b 100644 --- a/src/main/java/com/sparrowwallet/sparrow/whirlpool/Whirlpool.java +++ b/src/main/java/com/sparrowwallet/sparrow/whirlpool/Whirlpool.java @@ -294,9 +294,15 @@ public class Whirlpool { public static Wallet getStandardAccountWallet(WhirlpoolAccount whirlpoolAccount, Wallet wallet) { StandardAccount standardAccount = getStandardAccount(whirlpoolAccount); - if(StandardAccount.WHIRLPOOL_ACCOUNTS.contains(standardAccount)) { - wallet = wallet.getChildWallet(standardAccount); + if(StandardAccount.WHIRLPOOL_ACCOUNTS.contains(standardAccount) || wallet.getStandardAccountType() != standardAccount) { + Wallet standardWallet = wallet.getChildWallet(standardAccount); + if(standardWallet == null) { + throw new IllegalStateException("Cannot find " + standardAccount + " wallet"); + } + + return standardWallet; } + return wallet; } diff --git a/src/main/java/com/sparrowwallet/sparrow/whirlpool/dataSource/SparrowIndexHandler.java b/src/main/java/com/sparrowwallet/sparrow/whirlpool/dataSource/SparrowIndexHandler.java index d68f8140..d1cc7032 100644 --- a/src/main/java/com/sparrowwallet/sparrow/whirlpool/dataSource/SparrowIndexHandler.java +++ b/src/main/java/com/sparrowwallet/sparrow/whirlpool/dataSource/SparrowIndexHandler.java @@ -5,6 +5,7 @@ import com.sparrowwallet.drongo.KeyPurpose; import com.sparrowwallet.drongo.wallet.Wallet; import com.sparrowwallet.drongo.wallet.WalletNode; import com.sparrowwallet.sparrow.EventManager; +import com.sparrowwallet.sparrow.event.WalletGapLimitChangedEvent; import com.sparrowwallet.sparrow.event.WalletMixConfigChangedEvent; public class SparrowIndexHandler extends AbstractIndexHandler { @@ -37,6 +38,7 @@ public class SparrowIndexHandler extends AbstractIndexHandler { @Override public synchronized void set(int value) { setStoredIndex(value); + ensureSufficientGapLimit(value); } private int getCurrentIndex() { @@ -67,4 +69,13 @@ public class SparrowIndexHandler extends AbstractIndexHandler { EventManager.get().post(new WalletMixConfigChangedEvent(wallet)); } } + + private void ensureSufficientGapLimit(int index) { + int highestUsedIndex = getCurrentIndex() - 1; + int existingGapLimit = wallet.getGapLimit(); + if(index > highestUsedIndex + existingGapLimit) { + wallet.setGapLimit(Math.max(wallet.getGapLimit(), index - highestUsedIndex)); + EventManager.get().post(new WalletGapLimitChangedEvent(wallet)); + } + } }