From 41d1a1806d5af70978d31084af11b519eb93f5ac Mon Sep 17 00:00:00 2001 From: Craig Raw Date: Wed, 19 Jan 2022 13:50:03 +0200 Subject: [PATCH] improve deep wallet load performance by adding a setting to watch only the last x used addresses --- drongo | 2 +- .../sparrow/event/SettingsChangedEvent.java | 2 +- .../event/WalletWatchLastChangedEvent.java | 16 ++++++++ .../sparrow/io/db/DbPersistence.java | 13 ++++++ .../sparrow/io/db/WalletDao.java | 15 ++++--- .../sparrow/io/db/WalletMapper.java | 2 + .../sparrow/net/ElectrumServer.java | 17 ++++++-- .../sparrow/wallet/AdvancedController.java | 40 ++++++++++++++++++- .../sparrow/wallet/SettingsWalletForm.java | 9 +++-- .../sparrow/wallet/WalletForm.java | 7 ++++ .../com/sparrowwallet/sparrow/general.css | 2 +- .../sparrowwallet/sparrow/sql/V4__Watch.sql | 1 + .../sparrow/wallet/advanced.fxml | 6 ++- 13 files changed, 114 insertions(+), 18 deletions(-) create mode 100644 src/main/java/com/sparrowwallet/sparrow/event/WalletWatchLastChangedEvent.java create mode 100644 src/main/resources/com/sparrowwallet/sparrow/sql/V4__Watch.sql diff --git a/drongo b/drongo index 34bd72d8..8dca2ee3 160000 --- a/drongo +++ b/drongo @@ -1 +1 @@ -Subproject commit 34bd72d87aac7286fd0ca7e94f5a931f00d13cb4 +Subproject commit 8dca2ee3f0ba8dbebf88c3629b2a52c7eecf5b89 diff --git a/src/main/java/com/sparrowwallet/sparrow/event/SettingsChangedEvent.java b/src/main/java/com/sparrowwallet/sparrow/event/SettingsChangedEvent.java index 8ed01c3a..ddfb0997 100644 --- a/src/main/java/com/sparrowwallet/sparrow/event/SettingsChangedEvent.java +++ b/src/main/java/com/sparrowwallet/sparrow/event/SettingsChangedEvent.java @@ -20,6 +20,6 @@ public class SettingsChangedEvent { } public enum Type { - POLICY, SCRIPT_TYPE, MUTLISIG_THRESHOLD, MULTISIG_TOTAL, KEYSTORE_LABEL, KEYSTORE_FINGERPRINT, KEYSTORE_DERIVATION, KEYSTORE_XPUB, GAP_LIMIT, BIRTH_DATE; + POLICY, SCRIPT_TYPE, MUTLISIG_THRESHOLD, MULTISIG_TOTAL, KEYSTORE_LABEL, KEYSTORE_FINGERPRINT, KEYSTORE_DERIVATION, KEYSTORE_XPUB, GAP_LIMIT, BIRTH_DATE, WATCH_LAST; } } diff --git a/src/main/java/com/sparrowwallet/sparrow/event/WalletWatchLastChangedEvent.java b/src/main/java/com/sparrowwallet/sparrow/event/WalletWatchLastChangedEvent.java new file mode 100644 index 00000000..32c22c10 --- /dev/null +++ b/src/main/java/com/sparrowwallet/sparrow/event/WalletWatchLastChangedEvent.java @@ -0,0 +1,16 @@ +package com.sparrowwallet.sparrow.event; + +import com.sparrowwallet.drongo.wallet.Wallet; + +public class WalletWatchLastChangedEvent extends WalletSettingsChangedEvent { + private final Integer watchLast; + + public WalletWatchLastChangedEvent(Wallet wallet, Wallet pastWallet, String walletId, Integer watchLast) { + super(wallet, pastWallet, walletId); + this.watchLast = watchLast; + } + + public Integer getWatchLast() { + return watchLast; + } +} diff --git a/src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java b/src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java index fc6fea3a..04b65660 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java @@ -274,6 +274,10 @@ public class DbPersistence implements Persistence { walletDao.updateGapLimit(wallet.getId(), dirtyPersistables.gapLimit); } + if(dirtyPersistables.watchLast != null) { + walletDao.updateWatchLast(wallet.getId(), dirtyPersistables.watchLast); + } + if(!dirtyPersistables.labelEntries.isEmpty()) { BlockTransactionDao blockTransactionDao = handle.attach(BlockTransactionDao.class); WalletNodeDao walletNodeDao = handle.attach(WalletNodeDao.class); @@ -763,6 +767,13 @@ public class DbPersistence implements Persistence { } } + @Subscribe + public void walletWatchLastChanged(WalletWatchLastChangedEvent event) { + if(persistsFor(event.getWallet())) { + updateExecutor.execute(() -> dirtyPersistablesMap.computeIfAbsent(event.getWallet(), key -> new DirtyPersistables()).watchLast = event.getWatchLast()); + } + } + private static class DirtyPersistables { public boolean deleteAccount; public boolean clearHistory; @@ -770,6 +781,7 @@ public class DbPersistence implements Persistence { public String label; public Integer blockHeight = null; public Integer gapLimit = null; + public Integer watchLast = null; public final List labelEntries = new ArrayList<>(); public final List utxoStatuses = new ArrayList<>(); public boolean mixConfig; @@ -786,6 +798,7 @@ public class DbPersistence implements Persistence { "\nLabel:" + label + "\nBlockHeight:" + blockHeight + "\nGap limit:" + gapLimit + + "\nWatch last:" + watchLast + "\nTx labels:" + labelEntries.stream().filter(entry -> entry instanceof TransactionEntry).map(entry -> ((TransactionEntry)entry).getBlockTransaction().getHash().toString()).collect(Collectors.toList()) + "\nAddress labels:" + labelEntries.stream().filter(entry -> entry instanceof NodeEntry).map(entry -> ((NodeEntry)entry).getNode().toString() + " " + entry.getLabel()).collect(Collectors.toList()) + "\nUTXO labels:" + labelEntries.stream().filter(entry -> entry instanceof HashIndexEntry).map(entry -> ((HashIndexEntry)entry).getHashIndex().toString()).collect(Collectors.toList()) + diff --git a/src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java b/src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java index e9a5d008..938d9e73 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java @@ -36,21 +36,21 @@ public interface WalletDao { @CreateSqlObject UtxoMixDataDao createUtxoMixDataDao(); - @SqlQuery("select wallet.id, wallet.name, wallet.label, wallet.network, wallet.policyType, wallet.scriptType, wallet.storedBlockHeight, wallet.gapLimit, wallet.birthDate, policy.id, policy.name, policy.script from wallet left join policy on wallet.defaultPolicy = policy.id") + @SqlQuery("select wallet.id, wallet.name, wallet.label, wallet.network, wallet.policyType, wallet.scriptType, wallet.storedBlockHeight, wallet.gapLimit, wallet.watchLast, wallet.birthDate, policy.id, policy.name, policy.script from wallet left join policy on wallet.defaultPolicy = policy.id") @RegisterRowMapper(WalletMapper.class) List loadAllWallets(); - @SqlQuery("select wallet.id, wallet.name, wallet.label, wallet.network, wallet.policyType, wallet.scriptType, wallet.storedBlockHeight, wallet.gapLimit, wallet.birthDate, policy.id, policy.name, policy.script from wallet left join policy on wallet.defaultPolicy = policy.id where wallet.id = 1") + @SqlQuery("select wallet.id, wallet.name, wallet.label, wallet.network, wallet.policyType, wallet.scriptType, wallet.storedBlockHeight, wallet.gapLimit, wallet.watchLast, wallet.birthDate, policy.id, policy.name, policy.script from wallet left join policy on wallet.defaultPolicy = policy.id where wallet.id = 1") @RegisterRowMapper(WalletMapper.class) Wallet loadMainWallet(); - @SqlQuery("select wallet.id, wallet.name, wallet.label, wallet.network, wallet.policyType, wallet.scriptType, wallet.storedBlockHeight, wallet.gapLimit, wallet.birthDate, policy.id, policy.name, policy.script from wallet left join policy on wallet.defaultPolicy = policy.id where wallet.id != 1") + @SqlQuery("select wallet.id, wallet.name, wallet.label, wallet.network, wallet.policyType, wallet.scriptType, wallet.storedBlockHeight, wallet.gapLimit, wallet.watchLast, wallet.birthDate, policy.id, policy.name, policy.script from wallet left join policy on wallet.defaultPolicy = policy.id where wallet.id != 1") @RegisterRowMapper(WalletMapper.class) List loadChildWallets(); - @SqlUpdate("insert into wallet (name, label, network, policyType, scriptType, storedBlockHeight, gapLimit, birthDate, defaultPolicy) values (?, ?, ?, ?, ?, ?, ?, ?, ?)") + @SqlUpdate("insert into wallet (name, label, network, policyType, scriptType, storedBlockHeight, gapLimit, watchLast, birthDate, defaultPolicy) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") @GetGeneratedKeys("id") - long insert(String name, String label, int network, int policyType, int scriptType, Integer storedBlockHeight, Integer gapLimit, Date birthDate, long defaultPolicy); + long insert(String name, String label, int network, int policyType, int scriptType, Integer storedBlockHeight, Integer gapLimit, Integer watchLast, Date birthDate, long defaultPolicy); @SqlUpdate("update wallet set label = :label where id = :id") void updateLabel(@Bind("id") long id, @Bind("label") String label); @@ -61,6 +61,9 @@ public interface WalletDao { @SqlUpdate("update wallet set gapLimit = :gapLimit where id = :id") void updateGapLimit(@Bind("id") long id, @Bind("gapLimit") Integer gapLimit); + @SqlUpdate("update wallet set watchLast = :watchLast where id = :id") + void updateWatchLast(@Bind("id") long id, @Bind("watchLast") Integer watchLast); + @SqlUpdate("set schema ?") int setSchema(String schema); @@ -111,7 +114,7 @@ public interface WalletDao { setSchema(schema); createPolicyDao().addPolicy(wallet.getDefaultPolicy()); - long id = insert(truncate(wallet.getName()), truncate(wallet.getLabel()), wallet.getNetwork().ordinal(), wallet.getPolicyType().ordinal(), wallet.getScriptType().ordinal(), wallet.getStoredBlockHeight(), wallet.gapLimit(), wallet.getBirthDate(), wallet.getDefaultPolicy().getId()); + long id = insert(truncate(wallet.getName()), truncate(wallet.getLabel()), wallet.getNetwork().ordinal(), wallet.getPolicyType().ordinal(), wallet.getScriptType().ordinal(), wallet.getStoredBlockHeight(), wallet.gapLimit(), wallet.getWatchLast(), wallet.getBirthDate(), wallet.getDefaultPolicy().getId()); wallet.setId(id); createKeystoreDao().addKeystores(wallet); diff --git a/src/main/java/com/sparrowwallet/sparrow/io/db/WalletMapper.java b/src/main/java/com/sparrowwallet/sparrow/io/db/WalletMapper.java index 3690ff6e..c1886d84 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/db/WalletMapper.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/db/WalletMapper.java @@ -31,6 +31,8 @@ public class WalletMapper implements RowMapper { int gapLimit = rs.getInt("wallet.gapLimit"); wallet.gapLimit(rs.wasNull() ? null : gapLimit); + int watchLast = rs.getInt("wallet.watchLast"); + wallet.setWatchLast(rs.wasNull() ? null : watchLast); wallet.setBirthDate(rs.getTimestamp("wallet.birthDate")); return wallet; diff --git a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java index 14b6601f..b374b10c 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java @@ -298,7 +298,7 @@ public class ElectrumServer { public void getHistory(Wallet wallet, KeyPurpose keyPurpose, Map> nodeTransactionMap) throws ServerException { WalletNode purposeNode = wallet.getNode(keyPurpose); //Subscribe to all existing address WalletNodes and add them to nodeTransactionMap as keys to empty sets if they have history that needs to be fetched - subscribeWalletNodes(wallet, purposeNode.getChildren(), nodeTransactionMap, 0); + subscribeWalletNodes(wallet, getAddressNodes(wallet, purposeNode), nodeTransactionMap, 0); //All WalletNode keys in nodeTransactionMap need to have their history fetched (nodes without history will not be keys in the map yet) getReferences(wallet, nodeTransactionMap.keySet(), nodeTransactionMap, 0); //Fetch all referenced transaction to wallet transactions map. We do this now even though it is done again later to get it done before too many script hashes are subscribed @@ -309,7 +309,7 @@ public class ElectrumServer { log.debug("Fetched history for: " + nodeTransactionMap.keySet()); //Set the remaining WalletNode keys in nodeTransactionMap to empty sets to indicate no history (if no script hash history has already been retrieved in a previous call) - purposeNode.getChildren().stream().filter(node -> !nodeTransactionMap.containsKey(node) && retrievedScriptHashes.get(getScriptHash(wallet, node)) == null).forEach(node -> nodeTransactionMap.put(node, Collections.emptySet())); + getAddressNodes(wallet, purposeNode).stream().filter(node -> !nodeTransactionMap.containsKey(node) && retrievedScriptHashes.get(getScriptHash(wallet, node)) == null).forEach(node -> nodeTransactionMap.put(node, Collections.emptySet())); } private void getHistoryToGapLimit(Wallet wallet, Map> nodeTransactionMap, WalletNode purposeNode) throws ServerException { @@ -319,7 +319,7 @@ public class ElectrumServer { int gapLimitSize = getGapLimitSize(wallet, nodeTransactionMap); while(historySize < gapLimitSize) { purposeNode.fillToIndex(gapLimitSize - 1); - subscribeWalletNodes(wallet, purposeNode.getChildren(), nodeTransactionMap, historySize); + subscribeWalletNodes(wallet, getAddressNodes(wallet, purposeNode), nodeTransactionMap, historySize); getReferences(wallet, nodeTransactionMap.keySet(), nodeTransactionMap, historySize); getReferencedTransactions(wallet, nodeTransactionMap); historySize = purposeNode.getChildren().size(); @@ -327,6 +327,17 @@ public class ElectrumServer { } } + private Set getAddressNodes(Wallet wallet, WalletNode purposeNode) { + Integer watchLast = wallet.getWatchLast(); + if(watchLast == null || watchLast < wallet.getGapLimit() || wallet.getStoredBlockHeight() == 0 || wallet.getTransactions().isEmpty()) { + return purposeNode.getChildren(); + } + + int highestUsedIndex = purposeNode.getChildren().stream().filter(WalletNode::isUsed).mapToInt(WalletNode::getIndex).max().orElse(0); + int startFromIndex = highestUsedIndex - watchLast; + return purposeNode.getChildren().stream().filter(walletNode -> walletNode.getIndex() >= startFromIndex).collect(Collectors.toCollection(TreeSet::new)); + } + private int getGapLimitSize(Wallet wallet, Map> nodeTransactionMap) { int highestIndex = nodeTransactionMap.keySet().stream().map(WalletNode::getIndex).max(Comparator.comparing(Integer::valueOf)).orElse(-1); return highestIndex + wallet.getGapLimit() + 1; diff --git a/src/main/java/com/sparrowwallet/sparrow/wallet/AdvancedController.java b/src/main/java/com/sparrowwallet/sparrow/wallet/AdvancedController.java index 6beb32ab..4c94d754 100644 --- a/src/main/java/com/sparrowwallet/sparrow/wallet/AdvancedController.java +++ b/src/main/java/com/sparrowwallet/sparrow/wallet/AdvancedController.java @@ -4,24 +4,35 @@ import com.sparrowwallet.drongo.wallet.Wallet; import com.sparrowwallet.sparrow.EventManager; import com.sparrowwallet.sparrow.control.DateStringConverter; import com.sparrowwallet.sparrow.event.SettingsChangedEvent; +import javafx.collections.FXCollections; +import javafx.collections.ObservableList; import javafx.fxml.FXML; import javafx.fxml.Initializable; +import javafx.scene.control.ComboBox; import javafx.scene.control.DatePicker; import javafx.scene.control.Spinner; import javafx.scene.control.SpinnerValueFactory; +import javafx.util.StringConverter; import java.net.URL; import java.time.ZoneId; import java.util.Date; +import java.util.List; import java.util.ResourceBundle; +import java.util.stream.Collectors; public class AdvancedController implements Initializable { + private static final List DEFAULT_WATCH_LIST_ITEMS = List.of(-1, 100, 500, 1000, 5000, 10000); + @FXML private DatePicker birthDate; @FXML private Spinner gapLimit; + @FXML + private ComboBox watchLast; + @Override public void initialize(URL location, ResourceBundle resources) { @@ -39,10 +50,37 @@ public class AdvancedController implements Initializable { } }); - gapLimit.setValueFactory(new SpinnerValueFactory.IntegerSpinnerValueFactory(Wallet.DEFAULT_LOOKAHEAD, 10000, wallet.getGapLimit())); + gapLimit.setValueFactory(new SpinnerValueFactory.IntegerSpinnerValueFactory(Wallet.DEFAULT_LOOKAHEAD, 9999, wallet.getGapLimit())); gapLimit.valueProperty().addListener((observable, oldValue, newValue) -> { wallet.setGapLimit(newValue); + if(!watchLast.getItems().equals(getWatchListItems(wallet))) { + Integer value = watchLast.getValue(); + watchLast.setItems(getWatchListItems(wallet)); + watchLast.setValue(watchLast.getItems().contains(value) ? value : DEFAULT_WATCH_LIST_ITEMS.stream().filter(val -> val > wallet.getGapLimit()).findFirst().orElse(-1)); + } EventManager.get().post(new SettingsChangedEvent(wallet, SettingsChangedEvent.Type.GAP_LIMIT)); }); + + watchLast.setItems(getWatchListItems(wallet)); + watchLast.setConverter(new StringConverter<>() { + @Override + public String toString(Integer value) { + return value == null ? "" : (value < 0 ? "All" : "Last " + value + " only"); + } + + @Override + public Integer fromString(String string) { + return null; + } + }); + watchLast.setValue(wallet.getWatchLast() == null || !watchLast.getItems().contains(wallet.getWatchLast()) ? -1 : wallet.getWatchLast()); + watchLast.valueProperty().addListener((observable, oldValue, newValue) -> { + wallet.setWatchLast(newValue == null || newValue < 0 ? -1 : newValue); + EventManager.get().post(new SettingsChangedEvent(wallet, SettingsChangedEvent.Type.WATCH_LAST)); + }); + } + + private ObservableList getWatchListItems(Wallet wallet) { + return FXCollections.observableList(DEFAULT_WATCH_LIST_ITEMS.stream().filter(val -> val < 0 || val > wallet.getGapLimit()).collect(Collectors.toList())); } } diff --git a/src/main/java/com/sparrowwallet/sparrow/wallet/SettingsWalletForm.java b/src/main/java/com/sparrowwallet/sparrow/wallet/SettingsWalletForm.java index 61994685..79d37e81 100644 --- a/src/main/java/com/sparrowwallet/sparrow/wallet/SettingsWalletForm.java +++ b/src/main/java/com/sparrowwallet/sparrow/wallet/SettingsWalletForm.java @@ -7,10 +7,7 @@ import com.sparrowwallet.drongo.wallet.MasterPrivateExtendedKey; import com.sparrowwallet.drongo.wallet.Wallet; import com.sparrowwallet.sparrow.AppServices; import com.sparrowwallet.sparrow.EventManager; -import com.sparrowwallet.sparrow.event.KeystoreEncryptionChangedEvent; -import com.sparrowwallet.sparrow.event.KeystoreLabelsChangedEvent; -import com.sparrowwallet.sparrow.event.WalletAddressesChangedEvent; -import com.sparrowwallet.sparrow.event.WalletPasswordChangedEvent; +import com.sparrowwallet.sparrow.event.*; import com.sparrowwallet.sparrow.io.Storage; import com.sparrowwallet.sparrow.io.StorageException; @@ -90,6 +87,10 @@ public class SettingsWalletForm extends WalletForm { EventManager.get().post(new KeystoreLabelsChangedEvent(wallet, pastWallet, getWalletId(), labelChangedKeystores)); } + if(!Objects.equals(wallet.getWatchLast(), walletCopy.getWatchLast())) { + EventManager.get().post(new WalletWatchLastChangedEvent(wallet, pastWallet, getWalletId(), walletCopy.getWatchLast())); + } + List encryptionChangedKeystores = getEncryptionChangedKeystores(wallet, walletCopy); if(!encryptionChangedKeystores.isEmpty()) { EventManager.get().post(new KeystoreEncryptionChangedEvent(wallet, pastWallet, getWalletId(), encryptionChangedKeystores)); diff --git a/src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java b/src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java index b0c82d95..e2aa10a4 100644 --- a/src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java +++ b/src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java @@ -391,6 +391,13 @@ public class WalletForm { } } + @Subscribe + public void walletWatchLastChanged(WalletWatchLastChangedEvent event) { + if(event.getWalletId().equals(getWalletId())) { + Platform.runLater(() -> EventManager.get().post(new WalletDataChangedEvent(wallet))); + } + } + @Subscribe public void keystoreEncryptionChanged(KeystoreEncryptionChangedEvent event) { if(event.getWalletId().equals(getWalletId())) { diff --git a/src/main/resources/com/sparrowwallet/sparrow/general.css b/src/main/resources/com/sparrowwallet/sparrow/general.css index 218cc7f4..1f8d0789 100644 --- a/src/main/resources/com/sparrowwallet/sparrow/general.css +++ b/src/main/resources/com/sparrowwallet/sparrow/general.css @@ -11,7 +11,7 @@ } .form .relaxedLabelFieldSet.fieldset:horizontal .label-container { - -fx-pref-width: 120px; + -fx-pref-width: 130px; -fx-pref-height: 25px; } diff --git a/src/main/resources/com/sparrowwallet/sparrow/sql/V4__Watch.sql b/src/main/resources/com/sparrowwallet/sparrow/sql/V4__Watch.sql new file mode 100644 index 00000000..0e132df4 --- /dev/null +++ b/src/main/resources/com/sparrowwallet/sparrow/sql/V4__Watch.sql @@ -0,0 +1 @@ +alter table wallet add column watchLast integer after gapLimit; \ No newline at end of file diff --git a/src/main/resources/com/sparrowwallet/sparrow/wallet/advanced.fxml b/src/main/resources/com/sparrowwallet/sparrow/wallet/advanced.fxml index 92a456a6..7852805c 100644 --- a/src/main/resources/com/sparrowwallet/sparrow/wallet/advanced.fxml +++ b/src/main/resources/com/sparrowwallet/sparrow/wallet/advanced.fxml @@ -25,7 +25,7 @@
-
+
@@ -34,6 +34,10 @@ + + + +