automatically increase gap limit if required by postmix handler

drongo

@@ -1 +1 @@
-Subproject commit 025af0575899d6e255cc0a1f875cad25ab4811b8
+Subproject commit 61b2fd21e6f623850ad8b5df9f0cec4a0c0908cc

src/main/java/com/sparrowwallet/sparrow/AppController.java

@@ -2029,7 +2029,7 @@ public class AppController implements Initializable {
     @Subscribe
     public void walletHistoryStarted(WalletHistoryStartedEvent event) {
         if(AppServices.isConnected() && getOpenWallets().containsKey(event.getWallet())) {
-            if(event.getWalletNode() == null && event.getWallet().getTransactions().isEmpty()) {
+            if(event.getWalletNodes() == null && event.getWallet().getTransactions().isEmpty()) {
                 statusUpdated(new StatusEvent(LOADING_TRANSACTIONS_MESSAGE, 120));
                 if(statusTimeline == null || statusTimeline.getStatus() != Animation.Status.RUNNING) {
                     statusBar.setProgress(-1);

src/main/java/com/sparrowwallet/sparrow/event/WalletGapLimitChangedEvent.java

@@ -0,0 +1,17 @@
+package com.sparrowwallet.sparrow.event;
+
+import com.sparrowwallet.drongo.wallet.Wallet;
+
+/**
+ * This event is posted if the wallet's gap limit has changed, and triggers a history fetch for the new nodes.
+ *
+ */
+public class WalletGapLimitChangedEvent extends WalletChangedEvent {
+    public WalletGapLimitChangedEvent(Wallet wallet) {
+        super(wallet);
+    }
+
+    public int getGapLimit() {
+        return getWallet().getGapLimit();
+    }
+}

src/main/java/com/sparrowwallet/sparrow/event/WalletHistoryStartedEvent.java

@@ -3,15 +3,17 @@ package com.sparrowwallet.sparrow.event;
 import com.sparrowwallet.drongo.wallet.Wallet;
 import com.sparrowwallet.drongo.wallet.WalletNode;
 
-public class WalletHistoryStartedEvent extends WalletHistoryStatusEvent {
+import java.util.Set;
-    private final WalletNode walletNode;
 
-    public WalletHistoryStartedEvent(Wallet wallet, WalletNode walletNode) {
+public class WalletHistoryStartedEvent extends WalletHistoryStatusEvent {
+    private final Set<WalletNode> walletNodes;
+
+    public WalletHistoryStartedEvent(Wallet wallet, Set<WalletNode> walletNodes) {
         super(wallet, true);
-        this.walletNode = walletNode;
+        this.walletNodes = walletNodes;
     }
 
-    public WalletNode getWalletNode() {
+    public Set<WalletNode> getWalletNodes() {
-        return walletNode;
+        return walletNodes;
     }
 }

src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java

@@ -240,6 +240,10 @@ public class DbPersistence implements Persistence {
                     walletDao.updateStoredBlockHeight(wallet.getId(), dirtyPersistables.blockHeight);
                 }
 
+                if(dirtyPersistables.gapLimit != null) {
+                    walletDao.updateGapLimit(wallet.getId(), dirtyPersistables.gapLimit);
+                }
+
                 if(!dirtyPersistables.labelEntries.isEmpty()) {
                     BlockTransactionDao blockTransactionDao = handle.attach(BlockTransactionDao.class);
                     WalletNodeDao walletNodeDao = handle.attach(WalletNodeDao.class);
@@ -644,6 +648,13 @@ public class DbPersistence implements Persistence {
         }
     }
 
+    @Subscribe
+    public void walletGapLimitChanged(WalletGapLimitChangedEvent event) {
+        if(persistsFor(event.getWallet())) {
+            dirtyPersistablesMap.computeIfAbsent(event.getWallet(), key -> new DirtyPersistables()).gapLimit = event.getGapLimit();
+        }
+    }
+
     @Subscribe
     public void walletEntryLabelsChanged(WalletEntryLabelsChangedEvent event) {
         if(persistsFor(event.getWallet())) {
@@ -691,6 +702,7 @@ public class DbPersistence implements Persistence {
         public boolean clearHistory;
         public final List<WalletNode> historyNodes = new ArrayList<>();
         public Integer blockHeight = null;
+        public Integer gapLimit = null;
         public final List<Entry> labelEntries = new ArrayList<>();
         public final List<BlockTransactionHashIndex> utxoStatuses = new ArrayList<>();
         public boolean mixConfig;
@@ -704,6 +716,7 @@ public class DbPersistence implements Persistence {
                     "\nClear history:" + clearHistory +
                     "\nNodes:" + historyNodes +
                     "\nBlockHeight:" + blockHeight +
+                    "\nGap limit:" + gapLimit +
                     "\nTx labels:" + labelEntries.stream().filter(entry -> entry instanceof TransactionEntry).map(entry -> ((TransactionEntry)entry).getBlockTransaction().getHash().toString()).collect(Collectors.toList()) +
                     "\nAddress labels:" + labelEntries.stream().filter(entry -> entry instanceof NodeEntry).map(entry -> ((NodeEntry)entry).getNode().toString() + " " + entry.getLabel()).collect(Collectors.toList()) +
                     "\nUTXO labels:" + labelEntries.stream().filter(entry -> entry instanceof HashIndexEntry).map(entry -> ((HashIndexEntry)entry).getHashIndex().toString()).collect(Collectors.toList()) +

src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java

@@ -55,6 +55,9 @@ public interface WalletDao {
     @SqlUpdate("update wallet set storedBlockHeight = :blockHeight where id = :id")
     void updateStoredBlockHeight(@Bind("id") long id, @Bind("blockHeight") Integer blockHeight);
 
+    @SqlUpdate("update wallet set gapLimit = :gapLimit where id = :id")
+    void updateGapLimit(@Bind("id") long id, @Bind("gapLimit") Integer gapLimit);
+
     @SqlUpdate("set schema ?")
     int setSchema(String schema);
 

src/main/java/com/sparrowwallet/sparrow/net/BatchedElectrumServerRpc.java

@@ -5,8 +5,6 @@ import com.github.arteam.simplejsonrpc.client.Transport;
 import com.github.arteam.simplejsonrpc.client.builder.BatchRequestBuilder;
 import com.github.arteam.simplejsonrpc.client.exception.JsonRpcBatchException;
 import com.github.arteam.simplejsonrpc.client.exception.JsonRpcException;
-import com.sparrowwallet.drongo.KeyDerivation;
-import com.sparrowwallet.drongo.crypto.ChildNumber;
 import com.sparrowwallet.drongo.protocol.Sha256Hash;
 import com.sparrowwallet.drongo.wallet.Wallet;
 import com.sparrowwallet.sparrow.EventManager;
@@ -19,6 +17,8 @@ import java.util.*;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 
+import static com.sparrowwallet.drongo.wallet.WalletNode.nodeRangesToString;
+
 public class BatchedElectrumServerRpc implements ElectrumServerRpc {
     private static final Logger log = LoggerFactory.getLogger(BatchedElectrumServerRpc.class);
     private static final int MAX_RETRIES = 5;
@@ -75,7 +75,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc {
     public Map<String, ScriptHashTx[]> getScriptHashHistory(Transport transport, Wallet wallet, Map<String, String> pathScriptHashes, boolean failOnError) {
         JsonRpcClient client = new JsonRpcClient(transport);
         BatchRequestBuilder<String, ScriptHashTx[]> batchRequest = client.createBatchRequest().keysType(String.class).returnType(ScriptHashTx[].class);
-        EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Loading transactions for " + getScriptHashesAbbreviation(pathScriptHashes.keySet())));
+        EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Loading transactions for " + nodeRangesToString(pathScriptHashes.keySet())));
 
         for(String path : pathScriptHashes.keySet()) {
             batchRequest.add(path, "blockchain.scripthash.get_history", pathScriptHashes.get(path));
@@ -85,7 +85,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc {
             return new RetryLogic<Map<String, ScriptHashTx[]>>(MAX_RETRIES, RETRY_DELAY, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(batchRequest::execute);
         } catch (JsonRpcBatchException e) {
             if(failOnError) {
-                throw new ElectrumServerRpcException("Failed to retrieve transaction history for paths: " + getScriptHashesAbbreviation((Collection<String>)e.getErrors().keySet()), e);
+                throw new ElectrumServerRpcException("Failed to retrieve transaction history for paths: " + nodeRangesToString((Collection<String>)e.getErrors().keySet()), e);
             }
 
             Map<String, ScriptHashTx[]> result = (Map<String, ScriptHashTx[]>)e.getSuccesses();
@@ -95,7 +95,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc {
 
             return result;
         } catch(Exception e) {
-            throw new ElectrumServerRpcException("Failed to retrieve transaction history for paths: " + getScriptHashesAbbreviation(pathScriptHashes.keySet()), e);
+            throw new ElectrumServerRpcException("Failed to retrieve transaction history for paths: " + nodeRangesToString(pathScriptHashes.keySet()), e);
         }
     }
 
@@ -113,7 +113,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc {
             return new RetryLogic<Map<String, ScriptHashTx[]>>(MAX_RETRIES, RETRY_DELAY, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(batchRequest::execute);
         } catch(JsonRpcBatchException e) {
             if(failOnError) {
-                throw new ElectrumServerRpcException("Failed to retrieve mempool transactions for paths: " + getScriptHashesAbbreviation((Collection<String>)e.getErrors().keySet()), e);
+                throw new ElectrumServerRpcException("Failed to retrieve mempool transactions for paths: " + nodeRangesToString((Collection<String>)e.getErrors().keySet()), e);
             }
 
             Map<String, ScriptHashTx[]> result = (Map<String, ScriptHashTx[]>)e.getSuccesses();
@@ -123,7 +123,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc {
 
             return result;
         } catch(Exception e) {
-            throw new ElectrumServerRpcException("Failed to retrieve mempool transactions for paths: " + getScriptHashesAbbreviation(pathScriptHashes.keySet()), e);
+            throw new ElectrumServerRpcException("Failed to retrieve mempool transactions for paths: " + nodeRangesToString(pathScriptHashes.keySet()), e);
         }
     }
 
@@ -132,7 +132,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc {
     public Map<String, String> subscribeScriptHashes(Transport transport, Wallet wallet, Map<String, String> pathScriptHashes) {
         JsonRpcClient client = new JsonRpcClient(transport);
         BatchRequestBuilder<String, String> batchRequest = client.createBatchRequest().keysType(String.class).returnType(String.class);
-        EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Finding transactions for " + getScriptHashesAbbreviation(pathScriptHashes.keySet())));
+        EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Finding transactions for " + nodeRangesToString(pathScriptHashes.keySet())));
 
         for(String path : pathScriptHashes.keySet()) {
             batchRequest.add(path, "blockchain.scripthash.subscribe", pathScriptHashes.get(path));
@@ -142,9 +142,9 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc {
             return new RetryLogic<Map<String, String>>(MAX_RETRIES, RETRY_DELAY, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(batchRequest::execute);
         } catch(JsonRpcBatchException e) {
             //Even if we have some successes, failure to subscribe for all script hashes will result in outdated wallet view. Don't proceed.
-            throw new ElectrumServerRpcException("Failed to subscribe to paths: " + getScriptHashesAbbreviation((Collection<String>)e.getErrors().keySet()), e);
+            throw new ElectrumServerRpcException("Failed to subscribe to paths: " + nodeRangesToString((Collection<String>)e.getErrors().keySet()), e);
         } catch(Exception e) {
-            throw new ElectrumServerRpcException("Failed to subscribe to paths: " + getScriptHashesAbbreviation(pathScriptHashes.keySet()), e);
+            throw new ElectrumServerRpcException("Failed to subscribe to paths: " + nodeRangesToString(pathScriptHashes.keySet()), e);
         }
     }
 
@@ -276,65 +276,4 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc {
             throw new ElectrumServerRpcException("Error broadcasting transaction", e);
         }
     }
-
-    private static String getScriptHashesAbbreviation(Collection<String> scriptHashes) {
-        List<String> sortedHashes = new ArrayList<>(scriptHashes);
-
-        if(scriptHashes.isEmpty()) {
-            return "[]";
-        }
-
-        List<List<String>> contiguous = splitToContiguous(sortedHashes);
-
-        String abbrev = "[";
-        for(Iterator<List<String>> iter = contiguous.iterator(); iter.hasNext(); ) {
-            List<String> range = iter.next();
-            abbrev += range.get(0);
-            if(range.size() > 1) {
-                abbrev += "-" + range.get(range.size() - 1);
-            }
-            if(iter.hasNext()) {
-                abbrev += ", ";
-            }
-        }
-        abbrev += "]";
-
-        return abbrev;
-    }
-
-    static List<List<String>> splitToContiguous(List<String> input) {
-        List<List<String>> result = new ArrayList<>();
-        int prev = 0;
-
-        int keyPurpose = getKeyPurpose(input.get(0));
-        int index = getIndex(input.get(0));
-
-        for (int cur = 0; cur < input.size(); cur++) {
-            if(getKeyPurpose(input.get(cur)) != keyPurpose || getIndex(input.get(cur)) != index) {
-                result.add(input.subList(prev, cur));
-                prev = cur;
-            }
-            index = getIndex(input.get(cur)) + 1;
-            keyPurpose = getKeyPurpose(input.get(cur));
-        }
-        result.add(input.subList(prev, input.size()));
-
-        return result;
-    }
-
-    private static int getKeyPurpose(String path) {
-        List<ChildNumber> childNumbers = KeyDerivation.parsePath(path);
-        if(childNumbers.isEmpty()) {
-            return -1;
-        }
-        return childNumbers.get(0).num();
-    }
-
-    private static int getIndex(String path) {
-        List<ChildNumber> childNumbers = KeyDerivation.parsePath(path);
-        if(childNumbers.isEmpty()) {
-            return -1;
-        }
-        return childNumbers.get(childNumbers.size() - 1).num();
-    }
 }

src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java

@@ -25,6 +25,8 @@ import java.io.IOException;
 import java.util.*;
 import java.util.stream.Collectors;
 
+import static com.sparrowwallet.drongo.wallet.WalletNode.nodeRangesToString;
+
 public class WalletForm {
     private static final Logger log = LoggerFactory.getLogger(WalletForm.class);
 
@@ -123,11 +125,14 @@ public class WalletForm {
         refreshHistory(blockHeight, pastWallet, null);
     }
 
-    public void refreshHistory(Integer blockHeight, Wallet pastWallet, WalletNode node) {
+    public void refreshHistory(Integer blockHeight, Wallet pastWallet, Set<WalletNode> nodes) {
         Wallet previousWallet = wallet.copy();
         if(wallet.isValid() && AppServices.isConnected()) {
-            log.debug(node == null ? wallet.getFullName() + " refreshing full wallet history" : wallet.getFullName() + " requesting node wallet history for " + node.getDerivationPath());
+            if(log.isDebugEnabled()) {
-            ElectrumServer.TransactionHistoryService historyService = new ElectrumServer.TransactionHistoryService(wallet, getWalletTransactionNodes(node));
+                log.debug(nodes == null ? wallet.getFullName() + " refreshing full wallet history" : wallet.getFullName() + " requesting node wallet history for " + nodeRangesToString(nodes));
+            }
+
+            ElectrumServer.TransactionHistoryService historyService = new ElectrumServer.TransactionHistoryService(wallet, getWalletTransactionNodes(nodes));
             historyService.setOnSucceeded(workerStateEvent -> {
                 if(historyService.getValue()) {
                     EventManager.get().post(new WalletHistoryFinishedEvent(wallet));
@@ -144,7 +149,7 @@ public class WalletForm {
                 EventManager.get().post(new WalletHistoryFailedEvent(wallet, workerStateEvent.getSource().getException()));
             });
 
-            EventManager.get().post(new WalletHistoryStartedEvent(wallet, node));
+            EventManager.get().post(new WalletHistoryStartedEvent(wallet, nodes));
             historyService.start();
         }
     }
@@ -249,19 +254,21 @@ public class WalletForm {
         walletTransactionNodes.add(transactionNodes);
     }
 
-    private Set<WalletNode> getWalletTransactionNodes(WalletNode walletNode) {
+    private Set<WalletNode> getWalletTransactionNodes(Set<WalletNode> walletNodes) {
-        if(walletNode == null) {
+        if(walletNodes == null) {
             return null;
         }
 
         Set<WalletNode> allNodes = new LinkedHashSet<>();
+        for(WalletNode walletNode : walletNodes) {
             for(Set<WalletNode> nodes : walletTransactionNodes) {
                 if(nodes.contains(walletNode)) {
                     allNodes.addAll(nodes);
                 }
             }
+        }
 
-        return allNodes.isEmpty() ? Set.of(walletNode) : allNodes;
+        return allNodes.isEmpty() ? walletNodes : allNodes;
     }
 
     public NodeEntry getNodeEntry(KeyPurpose keyPurpose) {
@@ -392,7 +399,7 @@ public class WalletForm {
             WalletNode walletNode = event.getWalletNode(wallet);
             if(walletNode != null) {
                 log.debug(wallet.getFullName() + " history event for node " + walletNode + " (" + event.getScriptHash() + ")");
-                refreshHistory(AppServices.getCurrentBlockHeight(), null, walletNode);
+                refreshHistory(AppServices.getCurrentBlockHeight(), null, Set.of(walletNode));
             }
         }
     }
@@ -503,6 +510,26 @@ public class WalletForm {
         }
     }
 
+    @Subscribe
+    public void walletGapLimitChanged(WalletGapLimitChangedEvent event) {
+        if(event.getWallet() == wallet) {
+            Platform.runLater(() -> EventManager.get().post(new WalletDataChangedEvent(wallet)));
+
+            Set<WalletNode> newNodes = new LinkedHashSet<>();
+            for(KeyPurpose keyPurpose : KeyPurpose.DEFAULT_PURPOSES) {
+                Optional<WalletNode> optPurposeNode = wallet.getPurposeNodes().stream().filter(node -> node.getKeyPurpose() == keyPurpose).findFirst();
+                if(optPurposeNode.isPresent()) {
+                    WalletNode purposeNode = optPurposeNode.get();
+                    newNodes.addAll(purposeNode.fillToIndex(wallet.getLookAheadIndex(purposeNode)));
+                }
+            }
+
+            if(!newNodes.isEmpty()) {
+                Platform.runLater(() -> refreshHistory(AppServices.getCurrentBlockHeight(), null, newNodes));
+            }
+        }
+    }
+
     @Subscribe
     public void whirlpoolMixSuccess(WhirlpoolMixSuccessEvent event) {
         if(event.getWallet() == wallet && event.getWalletNode() != null) {

src/main/java/com/sparrowwallet/sparrow/whirlpool/Whirlpool.java

@@ -294,9 +294,15 @@ public class Whirlpool {
 
     public static Wallet getStandardAccountWallet(WhirlpoolAccount whirlpoolAccount, Wallet wallet) {
         StandardAccount standardAccount = getStandardAccount(whirlpoolAccount);
-        if(StandardAccount.WHIRLPOOL_ACCOUNTS.contains(standardAccount)) {
+        if(StandardAccount.WHIRLPOOL_ACCOUNTS.contains(standardAccount) || wallet.getStandardAccountType() != standardAccount) {
-            wallet = wallet.getChildWallet(standardAccount);
+            Wallet standardWallet = wallet.getChildWallet(standardAccount);
+            if(standardWallet == null) {
+                throw new IllegalStateException("Cannot find " + standardAccount + " wallet");
             }
+
+            return standardWallet;
+        }
+
         return wallet;
     }
 

src/main/java/com/sparrowwallet/sparrow/whirlpool/dataSource/SparrowIndexHandler.java

@@ -5,6 +5,7 @@ import com.sparrowwallet.drongo.KeyPurpose;
 import com.sparrowwallet.drongo.wallet.Wallet;
 import com.sparrowwallet.drongo.wallet.WalletNode;
 import com.sparrowwallet.sparrow.EventManager;
+import com.sparrowwallet.sparrow.event.WalletGapLimitChangedEvent;
 import com.sparrowwallet.sparrow.event.WalletMixConfigChangedEvent;
 
 public class SparrowIndexHandler extends AbstractIndexHandler {
@@ -37,6 +38,7 @@ public class SparrowIndexHandler extends AbstractIndexHandler {
     @Override
     public synchronized void set(int value) {
         setStoredIndex(value);
+        ensureSufficientGapLimit(value);
     }
 
     private int getCurrentIndex() {
@@ -67,4 +69,13 @@ public class SparrowIndexHandler extends AbstractIndexHandler {
             EventManager.get().post(new WalletMixConfigChangedEvent(wallet));
         }
     }
+
+    private void ensureSufficientGapLimit(int index) {
+        int highestUsedIndex = getCurrentIndex() - 1;
+        int existingGapLimit = wallet.getGapLimit();
+        if(index > highestUsedIndex + existingGapLimit) {
+            wallet.setGapLimit(Math.max(wallet.getGapLimit(), index - highestUsedIndex));
+            EventManager.get().post(new WalletGapLimitChangedEvent(wallet));
+        }
+    }
 }