improve deep wallet load performance by adding a setting to watch only the last x used addresses

drongo

@@ -1 +1 @@
-Subproject commit 34bd72d87aac7286fd0ca7e94f5a931f00d13cb4
+Subproject commit 8dca2ee3f0ba8dbebf88c3629b2a52c7eecf5b89

src/main/java/com/sparrowwallet/sparrow/event/SettingsChangedEvent.java

@@ -20,6 +20,6 @@ public class SettingsChangedEvent {
     }
 
     public enum Type {
-        POLICY, SCRIPT_TYPE, MUTLISIG_THRESHOLD, MULTISIG_TOTAL, KEYSTORE_LABEL, KEYSTORE_FINGERPRINT, KEYSTORE_DERIVATION, KEYSTORE_XPUB, GAP_LIMIT, BIRTH_DATE;
+        POLICY, SCRIPT_TYPE, MUTLISIG_THRESHOLD, MULTISIG_TOTAL, KEYSTORE_LABEL, KEYSTORE_FINGERPRINT, KEYSTORE_DERIVATION, KEYSTORE_XPUB, GAP_LIMIT, BIRTH_DATE, WATCH_LAST;
     }
 }

src/main/java/com/sparrowwallet/sparrow/event/WalletWatchLastChangedEvent.java

@@ -0,0 +1,16 @@
+package com.sparrowwallet.sparrow.event;
+
+import com.sparrowwallet.drongo.wallet.Wallet;
+
+public class WalletWatchLastChangedEvent extends WalletSettingsChangedEvent {
+    private final Integer watchLast;
+
+    public WalletWatchLastChangedEvent(Wallet wallet, Wallet pastWallet, String walletId, Integer watchLast) {
+        super(wallet, pastWallet, walletId);
+        this.watchLast = watchLast;
+    }
+
+    public Integer getWatchLast() {
+        return watchLast;
+    }
+}

src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java

@@ -274,6 +274,10 @@ public class DbPersistence implements Persistence {
                     walletDao.updateGapLimit(wallet.getId(), dirtyPersistables.gapLimit);
                 }
 
+                if(dirtyPersistables.watchLast != null) {
+                    walletDao.updateWatchLast(wallet.getId(), dirtyPersistables.watchLast);
+                }
+
                 if(!dirtyPersistables.labelEntries.isEmpty()) {
                     BlockTransactionDao blockTransactionDao = handle.attach(BlockTransactionDao.class);
                     WalletNodeDao walletNodeDao = handle.attach(WalletNodeDao.class);
@@ -763,6 +767,13 @@ public class DbPersistence implements Persistence {
         }
     }
 
+    @Subscribe
+    public void walletWatchLastChanged(WalletWatchLastChangedEvent event) {
+        if(persistsFor(event.getWallet())) {
+            updateExecutor.execute(() -> dirtyPersistablesMap.computeIfAbsent(event.getWallet(), key -> new DirtyPersistables()).watchLast = event.getWatchLast());
+        }
+    }
+
     private static class DirtyPersistables {
         public boolean deleteAccount;
         public boolean clearHistory;
@@ -770,6 +781,7 @@ public class DbPersistence implements Persistence {
         public String label;
         public Integer blockHeight = null;
         public Integer gapLimit = null;
+        public Integer watchLast = null;
         public final List<Entry> labelEntries = new ArrayList<>();
         public final List<BlockTransactionHashIndex> utxoStatuses = new ArrayList<>();
         public boolean mixConfig;
@@ -786,6 +798,7 @@ public class DbPersistence implements Persistence {
                     "\nLabel:" + label +
                     "\nBlockHeight:" + blockHeight +
                     "\nGap limit:" + gapLimit +
+                    "\nWatch last:" + watchLast +
                     "\nTx labels:" + labelEntries.stream().filter(entry -> entry instanceof TransactionEntry).map(entry -> ((TransactionEntry)entry).getBlockTransaction().getHash().toString()).collect(Collectors.toList()) +
                     "\nAddress labels:" + labelEntries.stream().filter(entry -> entry instanceof NodeEntry).map(entry -> ((NodeEntry)entry).getNode().toString() + " " + entry.getLabel()).collect(Collectors.toList()) +
                     "\nUTXO labels:" + labelEntries.stream().filter(entry -> entry instanceof HashIndexEntry).map(entry -> ((HashIndexEntry)entry).getHashIndex().toString()).collect(Collectors.toList()) +

src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java

@@ -36,21 +36,21 @@ public interface WalletDao {
     @CreateSqlObject
     UtxoMixDataDao createUtxoMixDataDao();
 
-    @SqlQuery("select wallet.id, wallet.name, wallet.label, wallet.network, wallet.policyType, wallet.scriptType, wallet.storedBlockHeight, wallet.gapLimit, wallet.birthDate, policy.id, policy.name, policy.script from wallet left join policy on wallet.defaultPolicy = policy.id")
+    @SqlQuery("select wallet.id, wallet.name, wallet.label, wallet.network, wallet.policyType, wallet.scriptType, wallet.storedBlockHeight, wallet.gapLimit, wallet.watchLast, wallet.birthDate, policy.id, policy.name, policy.script from wallet left join policy on wallet.defaultPolicy = policy.id")
     @RegisterRowMapper(WalletMapper.class)
     List<Wallet> loadAllWallets();
 
-    @SqlQuery("select wallet.id, wallet.name, wallet.label, wallet.network, wallet.policyType, wallet.scriptType, wallet.storedBlockHeight, wallet.gapLimit, wallet.birthDate, policy.id, policy.name, policy.script from wallet left join policy on wallet.defaultPolicy = policy.id where wallet.id = 1")
+    @SqlQuery("select wallet.id, wallet.name, wallet.label, wallet.network, wallet.policyType, wallet.scriptType, wallet.storedBlockHeight, wallet.gapLimit, wallet.watchLast, wallet.birthDate, policy.id, policy.name, policy.script from wallet left join policy on wallet.defaultPolicy = policy.id where wallet.id = 1")
     @RegisterRowMapper(WalletMapper.class)
     Wallet loadMainWallet();
 
-    @SqlQuery("select wallet.id, wallet.name, wallet.label, wallet.network, wallet.policyType, wallet.scriptType, wallet.storedBlockHeight, wallet.gapLimit, wallet.birthDate, policy.id, policy.name, policy.script from wallet left join policy on wallet.defaultPolicy = policy.id where wallet.id != 1")
+    @SqlQuery("select wallet.id, wallet.name, wallet.label, wallet.network, wallet.policyType, wallet.scriptType, wallet.storedBlockHeight, wallet.gapLimit, wallet.watchLast, wallet.birthDate, policy.id, policy.name, policy.script from wallet left join policy on wallet.defaultPolicy = policy.id where wallet.id != 1")
     @RegisterRowMapper(WalletMapper.class)
     List<Wallet> loadChildWallets();
 
-    @SqlUpdate("insert into wallet (name, label, network, policyType, scriptType, storedBlockHeight, gapLimit, birthDate, defaultPolicy) values (?, ?, ?, ?, ?, ?, ?, ?, ?)")
+    @SqlUpdate("insert into wallet (name, label, network, policyType, scriptType, storedBlockHeight, gapLimit, watchLast, birthDate, defaultPolicy) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
     @GetGeneratedKeys("id")
-    long insert(String name, String label, int network, int policyType, int scriptType, Integer storedBlockHeight, Integer gapLimit, Date birthDate, long defaultPolicy);
+    long insert(String name, String label, int network, int policyType, int scriptType, Integer storedBlockHeight, Integer gapLimit, Integer watchLast, Date birthDate, long defaultPolicy);
 
     @SqlUpdate("update wallet set label = :label where id = :id")
     void updateLabel(@Bind("id") long id, @Bind("label") String label);
@@ -61,6 +61,9 @@ public interface WalletDao {
     @SqlUpdate("update wallet set gapLimit = :gapLimit where id = :id")
     void updateGapLimit(@Bind("id") long id, @Bind("gapLimit") Integer gapLimit);
 
+    @SqlUpdate("update wallet set watchLast = :watchLast where id = :id")
+    void updateWatchLast(@Bind("id") long id, @Bind("watchLast") Integer watchLast);
+
     @SqlUpdate("set schema ?")
     int setSchema(String schema);
 
@@ -111,7 +114,7 @@ public interface WalletDao {
             setSchema(schema);
             createPolicyDao().addPolicy(wallet.getDefaultPolicy());
 
-            long id = insert(truncate(wallet.getName()), truncate(wallet.getLabel()), wallet.getNetwork().ordinal(), wallet.getPolicyType().ordinal(), wallet.getScriptType().ordinal(), wallet.getStoredBlockHeight(), wallet.gapLimit(), wallet.getBirthDate(), wallet.getDefaultPolicy().getId());
+            long id = insert(truncate(wallet.getName()), truncate(wallet.getLabel()), wallet.getNetwork().ordinal(), wallet.getPolicyType().ordinal(), wallet.getScriptType().ordinal(), wallet.getStoredBlockHeight(), wallet.gapLimit(), wallet.getWatchLast(), wallet.getBirthDate(), wallet.getDefaultPolicy().getId());
             wallet.setId(id);
 
             createKeystoreDao().addKeystores(wallet);

src/main/java/com/sparrowwallet/sparrow/io/db/WalletMapper.java

@@ -31,6 +31,8 @@ public class WalletMapper implements RowMapper<Wallet> {
 
         int gapLimit = rs.getInt("wallet.gapLimit");
         wallet.gapLimit(rs.wasNull() ? null : gapLimit);
+        int watchLast = rs.getInt("wallet.watchLast");
+        wallet.setWatchLast(rs.wasNull() ? null : watchLast);
         wallet.setBirthDate(rs.getTimestamp("wallet.birthDate"));
 
         return wallet;

src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java

@@ -298,7 +298,7 @@ public class ElectrumServer {
     public void getHistory(Wallet wallet, KeyPurpose keyPurpose, Map<WalletNode, Set<BlockTransactionHash>> nodeTransactionMap) throws ServerException {
         WalletNode purposeNode = wallet.getNode(keyPurpose);
         //Subscribe to all existing address WalletNodes and add them to nodeTransactionMap as keys to empty sets if they have history that needs to be fetched
-        subscribeWalletNodes(wallet, purposeNode.getChildren(), nodeTransactionMap, 0);
+        subscribeWalletNodes(wallet, getAddressNodes(wallet, purposeNode), nodeTransactionMap, 0);
         //All WalletNode keys in nodeTransactionMap need to have their history fetched (nodes without history will not be keys in the map yet)
         getReferences(wallet, nodeTransactionMap.keySet(), nodeTransactionMap, 0);
         //Fetch all referenced transaction to wallet transactions map. We do this now even though it is done again later to get it done before too many script hashes are subscribed
@@ -309,7 +309,7 @@ public class ElectrumServer {
         log.debug("Fetched history for: " + nodeTransactionMap.keySet());
 
         //Set the remaining WalletNode keys in nodeTransactionMap to empty sets to indicate no history (if no script hash history has already been retrieved in a previous call)
-        purposeNode.getChildren().stream().filter(node -> !nodeTransactionMap.containsKey(node) && retrievedScriptHashes.get(getScriptHash(wallet, node)) == null).forEach(node -> nodeTransactionMap.put(node, Collections.emptySet()));
+        getAddressNodes(wallet, purposeNode).stream().filter(node -> !nodeTransactionMap.containsKey(node) && retrievedScriptHashes.get(getScriptHash(wallet, node)) == null).forEach(node -> nodeTransactionMap.put(node, Collections.emptySet()));
     }
 
     private void getHistoryToGapLimit(Wallet wallet, Map<WalletNode, Set<BlockTransactionHash>> nodeTransactionMap, WalletNode purposeNode) throws ServerException {
@@ -319,7 +319,7 @@ public class ElectrumServer {
         int gapLimitSize = getGapLimitSize(wallet, nodeTransactionMap);
         while(historySize < gapLimitSize) {
             purposeNode.fillToIndex(gapLimitSize - 1);
-            subscribeWalletNodes(wallet, purposeNode.getChildren(), nodeTransactionMap, historySize);
+            subscribeWalletNodes(wallet, getAddressNodes(wallet, purposeNode), nodeTransactionMap, historySize);
             getReferences(wallet, nodeTransactionMap.keySet(), nodeTransactionMap, historySize);
             getReferencedTransactions(wallet, nodeTransactionMap);
             historySize = purposeNode.getChildren().size();
@@ -327,6 +327,17 @@ public class ElectrumServer {
         }
     }
 
+    private Set<WalletNode> getAddressNodes(Wallet wallet, WalletNode purposeNode) {
+        Integer watchLast = wallet.getWatchLast();
+        if(watchLast == null || watchLast < wallet.getGapLimit() || wallet.getStoredBlockHeight() == 0 || wallet.getTransactions().isEmpty()) {
+            return purposeNode.getChildren();
+        }
+
+        int highestUsedIndex = purposeNode.getChildren().stream().filter(WalletNode::isUsed).mapToInt(WalletNode::getIndex).max().orElse(0);
+        int startFromIndex = highestUsedIndex - watchLast;
+        return purposeNode.getChildren().stream().filter(walletNode -> walletNode.getIndex() >= startFromIndex).collect(Collectors.toCollection(TreeSet::new));
+    }
+
     private int getGapLimitSize(Wallet wallet, Map<WalletNode, Set<BlockTransactionHash>> nodeTransactionMap) {
         int highestIndex = nodeTransactionMap.keySet().stream().map(WalletNode::getIndex).max(Comparator.comparing(Integer::valueOf)).orElse(-1);
         return highestIndex + wallet.getGapLimit() + 1;

src/main/java/com/sparrowwallet/sparrow/wallet/AdvancedController.java

@@ -4,24 +4,35 @@ import com.sparrowwallet.drongo.wallet.Wallet;
 import com.sparrowwallet.sparrow.EventManager;
 import com.sparrowwallet.sparrow.control.DateStringConverter;
 import com.sparrowwallet.sparrow.event.SettingsChangedEvent;
+import javafx.collections.FXCollections;
+import javafx.collections.ObservableList;
 import javafx.fxml.FXML;
 import javafx.fxml.Initializable;
+import javafx.scene.control.ComboBox;
 import javafx.scene.control.DatePicker;
 import javafx.scene.control.Spinner;
 import javafx.scene.control.SpinnerValueFactory;
+import javafx.util.StringConverter;
 
 import java.net.URL;
 import java.time.ZoneId;
 import java.util.Date;
+import java.util.List;
 import java.util.ResourceBundle;
+import java.util.stream.Collectors;
 
 public class AdvancedController implements Initializable {
+    private static final List<Integer> DEFAULT_WATCH_LIST_ITEMS = List.of(-1, 100, 500, 1000, 5000, 10000);
+
     @FXML
     private DatePicker birthDate;
 
     @FXML
     private Spinner<Integer> gapLimit;
 
+    @FXML
+    private ComboBox<Integer> watchLast;
+
     @Override
     public void initialize(URL location, ResourceBundle resources) {
 
@@ -39,10 +50,37 @@ public class AdvancedController implements Initializable {
             }
         });
 
-        gapLimit.setValueFactory(new SpinnerValueFactory.IntegerSpinnerValueFactory(Wallet.DEFAULT_LOOKAHEAD, 10000, wallet.getGapLimit()));
+        gapLimit.setValueFactory(new SpinnerValueFactory.IntegerSpinnerValueFactory(Wallet.DEFAULT_LOOKAHEAD, 9999, wallet.getGapLimit()));
         gapLimit.valueProperty().addListener((observable, oldValue, newValue) -> {
             wallet.setGapLimit(newValue);
+            if(!watchLast.getItems().equals(getWatchListItems(wallet))) {
+                Integer value = watchLast.getValue();
+                watchLast.setItems(getWatchListItems(wallet));
+                watchLast.setValue(watchLast.getItems().contains(value) ? value : DEFAULT_WATCH_LIST_ITEMS.stream().filter(val -> val > wallet.getGapLimit()).findFirst().orElse(-1));
+            }
             EventManager.get().post(new SettingsChangedEvent(wallet, SettingsChangedEvent.Type.GAP_LIMIT));
         });
+
+        watchLast.setItems(getWatchListItems(wallet));
+        watchLast.setConverter(new StringConverter<>() {
+            @Override
+            public String toString(Integer value) {
+                return value == null ? "" : (value < 0 ? "All" : "Last " + value + " only");
+            }
+
+            @Override
+            public Integer fromString(String string) {
+                return null;
+            }
+        });
+        watchLast.setValue(wallet.getWatchLast() == null || !watchLast.getItems().contains(wallet.getWatchLast()) ? -1 : wallet.getWatchLast());
+        watchLast.valueProperty().addListener((observable, oldValue, newValue) -> {
+            wallet.setWatchLast(newValue == null || newValue < 0 ? -1 : newValue);
+            EventManager.get().post(new SettingsChangedEvent(wallet, SettingsChangedEvent.Type.WATCH_LAST));
+        });
+    }
+
+    private ObservableList<Integer> getWatchListItems(Wallet wallet) {
+        return FXCollections.observableList(DEFAULT_WATCH_LIST_ITEMS.stream().filter(val -> val < 0 || val > wallet.getGapLimit()).collect(Collectors.toList()));
     }
 }

src/main/java/com/sparrowwallet/sparrow/wallet/SettingsWalletForm.java

@@ -7,10 +7,7 @@ import com.sparrowwallet.drongo.wallet.MasterPrivateExtendedKey;
 import com.sparrowwallet.drongo.wallet.Wallet;
 import com.sparrowwallet.sparrow.AppServices;
 import com.sparrowwallet.sparrow.EventManager;
-import com.sparrowwallet.sparrow.event.KeystoreEncryptionChangedEvent;
+import com.sparrowwallet.sparrow.event.*;
-import com.sparrowwallet.sparrow.event.KeystoreLabelsChangedEvent;
-import com.sparrowwallet.sparrow.event.WalletAddressesChangedEvent;
-import com.sparrowwallet.sparrow.event.WalletPasswordChangedEvent;
 import com.sparrowwallet.sparrow.io.Storage;
 import com.sparrowwallet.sparrow.io.StorageException;
 
@@ -90,6 +87,10 @@ public class SettingsWalletForm extends WalletForm {
                 EventManager.get().post(new KeystoreLabelsChangedEvent(wallet, pastWallet, getWalletId(), labelChangedKeystores));
             }
 
+            if(!Objects.equals(wallet.getWatchLast(), walletCopy.getWatchLast())) {
+                EventManager.get().post(new WalletWatchLastChangedEvent(wallet, pastWallet, getWalletId(), walletCopy.getWatchLast()));
+            }
+
             List<Keystore> encryptionChangedKeystores = getEncryptionChangedKeystores(wallet, walletCopy);
             if(!encryptionChangedKeystores.isEmpty()) {
                 EventManager.get().post(new KeystoreEncryptionChangedEvent(wallet, pastWallet, getWalletId(), encryptionChangedKeystores));

src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java

@@ -391,6 +391,13 @@ public class WalletForm {
         }
     }
 
+    @Subscribe
+    public void walletWatchLastChanged(WalletWatchLastChangedEvent event) {
+        if(event.getWalletId().equals(getWalletId())) {
+            Platform.runLater(() -> EventManager.get().post(new WalletDataChangedEvent(wallet)));
+        }
+    }
+
     @Subscribe
     public void keystoreEncryptionChanged(KeystoreEncryptionChangedEvent event) {
         if(event.getWalletId().equals(getWalletId())) {

src/main/resources/com/sparrowwallet/sparrow/general.css

@@ -11,7 +11,7 @@
 }
 
 .form .relaxedLabelFieldSet.fieldset:horizontal .label-container {
-    -fx-pref-width: 120px;
+    -fx-pref-width: 130px;
     -fx-pref-height: 25px;
 }
 

src/main/resources/com/sparrowwallet/sparrow/sql/V4__Watch.sql

@@ -0,0 +1 @@
+alter table wallet add column watchLast integer after gapLimit;

src/main/resources/com/sparrowwallet/sparrow/wallet/advanced.fxml

@@ -25,7 +25,7 @@
         </rowConstraints>
 
         <Form GridPane.columnIndex="0" GridPane.rowIndex="0">
-            <Fieldset inputGrow="SOMETIMES" text="Advanced Settings">
+            <Fieldset inputGrow="SOMETIMES" text="Advanced Settings" styleClass="relaxedLabelFieldSet">
                 <Field text="Birth date:">
                     <DatePicker editable="false" fx:id="birthDate" prefWidth="140" />
                     <HelpLabel helpText="The date of the earliest transaction (used to avoid scanning the entire blockchain)."/>
@@ -34,6 +34,10 @@
                     <Spinner fx:id="gapLimit" editable="true" prefWidth="90" />
                     <HelpLabel helpText="Change how far ahead to look for additional transactions beyond the highest derivation with previous transaction outputs."/>
                 </Field>
+                <Field text="Watch addresses:">
+                    <ComboBox fx:id="watchLast" />
+                    <HelpLabel helpText="Load deep wallets faster by limiting the number of subscriptions to previously used addresses.\nUsed addresses at lower derivation paths will not be checked for new transactions.\nThis setting will take effect in the next wallet load."/>
+                </Field>
             </Fieldset>
         </Form>
         </GridPane>