From 60aa20ac557479a5fa51443bc16441b1e071adba Mon Sep 17 00:00:00 2001 From: Craig Raw Date: Mon, 18 Jul 2022 16:12:32 +0200 Subject: [PATCH] improve performance on deep wallets by storing addresses --- drongo | 2 +- .../sparrow/control/AddressTreeTable.java | 17 ++++++++------ .../event/WalletHistoryChangedEvent.java | 4 ++-- .../com/sparrowwallet/sparrow/io/IOUtils.java | 2 +- .../sparrow/io/JsonPersistence.java | 22 +++++++++++++++++++ .../sparrow/io/db/DbPersistence.java | 18 ++++++++++----- .../sparrow/io/db/WalletDao.java | 2 +- .../sparrow/io/db/WalletNodeDao.java | 15 ++++++++----- .../sparrow/io/db/WalletNodeMapper.java | 7 +++++- .../sparrow/net/ElectrumServer.java | 7 +++--- .../sparrow/wallet/WalletForm.java | 4 +++- .../sparrow/sql/V7__AddressData.sql | 1 + 12 files changed, 73 insertions(+), 28 deletions(-) create mode 100644 src/main/resources/com/sparrowwallet/sparrow/sql/V7__AddressData.sql diff --git a/drongo b/drongo index 9ae1f68d..5de3abd3 160000 --- a/drongo +++ b/drongo @@ -1 +1 @@ -Subproject commit 9ae1f68dc42529085edcc8c10d9bcfdbf9639448 +Subproject commit 5de3abd36230d545f11bad3b25a21d23ffbbe9cd diff --git a/src/main/java/com/sparrowwallet/sparrow/control/AddressTreeTable.java b/src/main/java/com/sparrowwallet/sparrow/control/AddressTreeTable.java index cf922c79..2f3e620c 100644 --- a/src/main/java/com/sparrowwallet/sparrow/control/AddressTreeTable.java +++ b/src/main/java/com/sparrowwallet/sparrow/control/AddressTreeTable.java @@ -14,9 +14,7 @@ import javafx.collections.ListChangeListener; import javafx.scene.control.*; import javafx.scene.input.MouseButton; -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; +import java.util.*; public class AddressTreeTable extends CoinTreeTable { public void initialize(NodeEntry rootEntry) { @@ -110,10 +108,15 @@ public class AddressTreeTable extends CoinTreeTable { //We only ever add child nodes - never remove in order to keep a full sequence (unless hide empty used addresses is set) NodeEntry rootEntry = (NodeEntry)getRoot().getValue(); + Map childNodes = new HashMap<>(); + for(Entry childEntry : rootEntry.getChildren()) { + NodeEntry nodeEntry = (NodeEntry)childEntry; + childNodes.put(nodeEntry.getNode(), nodeEntry); + } + for(WalletNode updatedNode : updatedNodes) { - Optional optEntry = rootEntry.getChildren().stream().filter(childEntry -> ((NodeEntry)childEntry).getNode().equals(updatedNode)).findFirst(); - if(optEntry.isPresent()) { - NodeEntry existingEntry = (NodeEntry)optEntry.get(); + NodeEntry existingEntry = childNodes.get(updatedNode); + if(existingEntry != null) { existingEntry.refreshChildren(); if(Config.get().isHideEmptyUsedAddresses() && existingEntry.getValue() == 0L) { @@ -125,7 +128,7 @@ public class AddressTreeTable extends CoinTreeTable { if(Config.get().isHideEmptyUsedAddresses()) { int index = 0; for( ; index < rootEntry.getChildren().size(); index++) { - NodeEntry existingEntry = (NodeEntry)rootEntry.getChildren().get(index); + existingEntry = (NodeEntry)rootEntry.getChildren().get(index); if(nodeEntry.compareTo(existingEntry) < 0) { break; } diff --git a/src/main/java/com/sparrowwallet/sparrow/event/WalletHistoryChangedEvent.java b/src/main/java/com/sparrowwallet/sparrow/event/WalletHistoryChangedEvent.java index 73d375f1..0d859378 100644 --- a/src/main/java/com/sparrowwallet/sparrow/event/WalletHistoryChangedEvent.java +++ b/src/main/java/com/sparrowwallet/sparrow/event/WalletHistoryChangedEvent.java @@ -45,10 +45,10 @@ public class WalletHistoryChangedEvent extends WalletChangedEvent { } public List getReceiveNodes() { - return getWallet().getNode(KeyPurpose.RECEIVE).getChildren().stream().filter(historyChangedNodes::contains).collect(Collectors.toList()); + return historyChangedNodes.stream().filter(node -> node.getKeyPurpose() == KeyPurpose.RECEIVE).collect(Collectors.toList()); } public List getChangeNodes() { - return getWallet().getNode(KeyPurpose.CHANGE).getChildren().stream().filter(historyChangedNodes::contains).collect(Collectors.toList()); + return historyChangedNodes.stream().filter(node -> node.getKeyPurpose() == KeyPurpose.CHANGE).collect(Collectors.toList()); } } diff --git a/src/main/java/com/sparrowwallet/sparrow/io/IOUtils.java b/src/main/java/com/sparrowwallet/sparrow/io/IOUtils.java index 39429435..e1221b23 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/IOUtils.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/IOUtils.java @@ -131,7 +131,7 @@ public class IOUtils { if(file.exists()) { long length = file.length(); SecureRandom random = new SecureRandom(); - byte[] data = new byte[64]; + byte[] data = new byte[1024*1024]; random.nextBytes(data); try(RandomAccessFile raf = new RandomAccessFile(file, "rws")) { raf.seek(0); diff --git a/src/main/java/com/sparrowwallet/sparrow/io/JsonPersistence.java b/src/main/java/com/sparrowwallet/sparrow/io/JsonPersistence.java index 16501027..12d89e3f 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/JsonPersistence.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/JsonPersistence.java @@ -3,6 +3,8 @@ package com.sparrowwallet.sparrow.io; import com.google.gson.*; import com.sparrowwallet.drongo.ExtendedKey; import com.sparrowwallet.drongo.Utils; +import com.sparrowwallet.drongo.address.Address; +import com.sparrowwallet.drongo.address.InvalidAddressException; import com.sparrowwallet.drongo.crypto.Argon2KeyDeriver; import com.sparrowwallet.drongo.crypto.AsymmetricKeyDeriver; import com.sparrowwallet.drongo.crypto.ECKey; @@ -331,6 +333,8 @@ public class JsonPersistence implements Persistence { gsonBuilder.registerTypeAdapter(Date.class, new DateDeserializer()); gsonBuilder.registerTypeAdapter(Transaction.class, new TransactionSerializer()); gsonBuilder.registerTypeAdapter(Transaction.class, new TransactionDeserializer()); + gsonBuilder.registerTypeAdapter(Address.class, new AddressSerializer()); + gsonBuilder.registerTypeAdapter(Address.class, new AddressDeserializer()); if(includeWalletSerializers) { gsonBuilder.registerTypeAdapter(Keystore.class, new KeystoreSerializer()); gsonBuilder.registerTypeAdapter(WalletNode.class, new NodeSerializer()); @@ -429,6 +433,24 @@ public class JsonPersistence implements Persistence { } } + private static class AddressSerializer implements JsonSerializer
{ + @Override + public JsonElement serialize(Address src, Type typeOfSrc, JsonSerializationContext context) { + return new JsonPrimitive(src.toString()); + } + } + + private static class AddressDeserializer implements JsonDeserializer
{ + @Override + public Address deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { + try { + return Address.fromString(json.getAsJsonPrimitive().getAsString()); + } catch(InvalidAddressException e) { + throw new IllegalStateException(e); + } + } + } + private static class KeystoreSerializer implements JsonSerializer { @Override public JsonElement serialize(Keystore keystore, Type typeOfSrc, JsonSerializationContext context) { diff --git a/src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java b/src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java index ef61c20d..ff8e9da5 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java @@ -87,6 +87,10 @@ public class DbPersistence implements Persistence { return walletDao.getMainWallet(MASTER_SCHEMA, getWalletName(storage.getWalletFile(), null)); }); + if(masterWallet == null) { + throw new StorageException("The wallet file was corrupted. Check the backups folder for previous copies."); + } + Map childWallets = loadChildWallets(storage, masterWallet, encryptionKey); masterWallet.setChildWallets(childWallets.keySet().stream().map(WalletAndKey::getWallet).collect(Collectors.toList())); @@ -231,12 +235,14 @@ public class DbPersistence implements Persistence { if(addressNode.getId() == null) { WalletNode purposeNode = wallet.getNode(addressNode.getKeyPurpose()); if(purposeNode.getId() == null) { - long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null); + long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null, null); purposeNode.setId(purposeNodeId); } - long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId()); + long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId(), addressNode.getAddressData()); addressNode.setId(nodeId); + } else if(addressNode.getAddress() != null) { + walletNodeDao.updateNodeAddressData(addressNode.getId(), addressNode.getAddressData()); } Set txos = addressNode.getTransactionOutputs().stream().flatMap(txo -> txo.isSpent() ? Stream.of(txo, txo.getSpentBy()) : Stream.of(txo)).collect(Collectors.toSet()); @@ -285,12 +291,14 @@ public class DbPersistence implements Persistence { if(addressNode.getId() == null) { WalletNode purposeNode = wallet.getNode(addressNode.getKeyPurpose()); if(purposeNode.getId() == null) { - long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null); + long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null, null); purposeNode.setId(purposeNodeId); } - long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId()); + long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId(), addressNode.getAddressData()); addressNode.setId(nodeId); + } else if(addressNode.getAddress() != null) { + walletNodeDao.updateNodeAddressData(addressNode.getId(), addressNode.getAddressData()); } walletNodeDao.updateNodeLabel(addressNode.getId(), entry.getLabel()); @@ -666,7 +674,7 @@ public class DbPersistence implements Persistence { } private String getUrl(File walletFile, String password) { - return "jdbc:h2:" + walletFile.getAbsolutePath().replace("." + getType().getExtension(), "") + ";INIT=SET TRACE_LEVEL_FILE=4;TRACE_LEVEL_FILE=4;DATABASE_TO_UPPER=false" + (password == null ? "" : ";CIPHER=AES"); + return "jdbc:h2:" + walletFile.getAbsolutePath().replace("." + getType().getExtension(), "") + ";INIT=SET TRACE_LEVEL_FILE=4;TRACE_LEVEL_FILE=4;DEFRAG_ALWAYS=true;MAX_COMPACT_TIME=5000;DATABASE_TO_UPPER=false" + (password == null ? "" : ";CIPHER=AES"); } private boolean persistsFor(Wallet wallet) { diff --git a/src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java b/src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java index 06abe048..fc53e237 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java @@ -108,7 +108,7 @@ public interface WalletDao { default void loadWallet(Wallet wallet) { wallet.getKeystores().addAll(createKeystoreDao().getForWalletId(wallet.getId())); - List walletNodes = createWalletNodeDao().getForWalletId(wallet.getId()); + List walletNodes = createWalletNodeDao().getForWalletId(wallet.getScriptType().ordinal(), wallet.getId()); wallet.getPurposeNodes().addAll(walletNodes.stream().filter(walletNode -> walletNode.getDerivation().size() == 1).collect(Collectors.toList())); wallet.getPurposeNodes().forEach(walletNode -> walletNode.setWallet(wallet)); diff --git a/src/main/java/com/sparrowwallet/sparrow/io/db/WalletNodeDao.java b/src/main/java/com/sparrowwallet/sparrow/io/db/WalletNodeDao.java index 39cc2c4e..ddcb6dbc 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/db/WalletNodeDao.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/db/WalletNodeDao.java @@ -16,18 +16,18 @@ import java.util.Date; import java.util.List; public interface WalletNodeDao { - @SqlQuery("select walletNode.id, walletNode.derivationPath, walletNode.label, walletNode.parent, " + + @SqlQuery("select walletNode.id, walletNode.derivationPath, walletNode.label, walletNode.parent, walletNode.addressData, ?, " + "blockTransactionHashIndex.id, blockTransactionHashIndex.hash, blockTransactionHashIndex.height, blockTransactionHashIndex.date, blockTransactionHashIndex.fee, blockTransactionHashIndex.label, " + "blockTransactionHashIndex.index, blockTransactionHashIndex.outputValue, blockTransactionHashIndex.status, blockTransactionHashIndex.spentBy, blockTransactionHashIndex.node " + "from walletNode left join blockTransactionHashIndex on walletNode.id = blockTransactionHashIndex.node where walletNode.wallet = ? order by walletNode.parent asc nulls first, blockTransactionHashIndex.spentBy asc nulls first") @RegisterRowMapper(WalletNodeMapper.class) @RegisterRowMapper(BlockTransactionHashIndexMapper.class) @UseRowReducer(WalletNodeReducer.class) - List getForWalletId(Long id); + List getForWalletId(int scriptType, Long id); - @SqlUpdate("insert into walletNode (derivationPath, label, wallet, parent) values (?, ?, ?, ?)") + @SqlUpdate("insert into walletNode (derivationPath, label, wallet, parent, addressData) values (?, ?, ?, ?, ?)") @GetGeneratedKeys("id") - long insertWalletNode(String derivationPath, String label, long wallet, Long parent); + long insertWalletNode(String derivationPath, String label, long wallet, Long parent, byte[] addressData); @SqlUpdate("insert into blockTransactionHashIndex (hash, height, date, fee, label, index, outputValue, status, spentBy, node) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") @GetGeneratedKeys("id") @@ -39,6 +39,9 @@ public interface WalletNodeDao { @SqlUpdate("update walletNode set label = :label where id = :id") void updateNodeLabel(@Bind("id") long id, @Bind("label") String label); + @SqlUpdate("update walletNode set addressData = :addressData where id = :id and addressData is null") + void updateNodeAddressData(@Bind("id") long id, @Bind("addressData") byte[] addressData); + @SqlUpdate("update blockTransactionHashIndex set label = :label where id = :id") void updateTxoLabel(@Bind("id") long id, @Bind("label") String label); @@ -59,12 +62,12 @@ public interface WalletNodeDao { default void addWalletNodes(Wallet wallet) { for(WalletNode purposeNode : wallet.getPurposeNodes()) { - long purposeNodeId = insertWalletNode(purposeNode.getDerivationPath(), truncate(purposeNode.getLabel()), wallet.getId(), null); + long purposeNodeId = insertWalletNode(purposeNode.getDerivationPath(), truncate(purposeNode.getLabel()), wallet.getId(), null, null); purposeNode.setId(purposeNodeId); addTransactionOutputs(purposeNode); List childNodes = new ArrayList<>(purposeNode.getChildren()); for(WalletNode addressNode : childNodes) { - long addressNodeId = insertWalletNode(addressNode.getDerivationPath(), truncate(addressNode.getLabel()), wallet.getId(), purposeNodeId); + long addressNodeId = insertWalletNode(addressNode.getDerivationPath(), truncate(addressNode.getLabel()), wallet.getId(), purposeNodeId, addressNode.getAddressData()); addressNode.setId(addressNodeId); addTransactionOutputs(addressNode); } diff --git a/src/main/java/com/sparrowwallet/sparrow/io/db/WalletNodeMapper.java b/src/main/java/com/sparrowwallet/sparrow/io/db/WalletNodeMapper.java index 2d7e2054..0b1738ed 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/db/WalletNodeMapper.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/db/WalletNodeMapper.java @@ -1,5 +1,6 @@ package com.sparrowwallet.sparrow.io.db; +import com.sparrowwallet.drongo.protocol.ScriptType; import com.sparrowwallet.drongo.wallet.WalletNode; import org.jdbi.v3.core.mapper.RowMapper; import org.jdbi.v3.core.statement.StatementContext; @@ -13,7 +14,11 @@ public class WalletNodeMapper implements RowMapper { WalletNode walletNode = new WalletNode(rs.getString("walletNode.derivationPath")); walletNode.setId(rs.getLong("walletNode.id")); walletNode.setLabel(rs.getString("walletNode.label")); - + byte[] addressData = rs.getBytes("walletNode.addressData"); + if(addressData != null) { + ScriptType scriptType = ScriptType.values()[rs.getInt(6)]; + walletNode.setAddress(scriptType.getAddress(addressData)); + } return walletNode; } } diff --git a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java index 8cc4f1d2..6566378a 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java @@ -432,6 +432,7 @@ public class ElectrumServer { try { Set scriptHashes = new HashSet<>(); Map pathScriptHashes = new LinkedHashMap<>(); + Map pathNodes = new HashMap<>(); for(WalletNode node : nodes) { if(node == null) { log.error("Null node for wallet " + wallet.getFullName() + " subscribing nodes " + nodes + " startIndex " + startIndex, new Throwable()); @@ -448,6 +449,7 @@ public class ElectrumServer { } else if(!subscribedScriptHashes.containsKey(scriptHash) && scriptHashes.add(scriptHash)) { //Unique script hash we are not yet subscribed to pathScriptHashes.put(node.getDerivationPath(), scriptHash); + pathNodes.put(node.getDerivationPath(), node); } } } @@ -463,9 +465,8 @@ public class ElectrumServer { for(String path : result.keySet()) { String status = result.get(path); - Optional optionalNode = nodes.stream().filter(n -> n.getDerivationPath().equals(path)).findFirst(); - if(optionalNode.isPresent()) { - WalletNode node = optionalNode.get(); + WalletNode node = pathNodes.computeIfAbsent(path, p -> nodes.stream().filter(n -> n.getDerivationPath().equals(p)).findFirst().orElse(null)); + if(node != null) { String scriptHash = getScriptHash(wallet, node); //Check if there is history for this script hash, and if the history has changed since last fetched diff --git a/src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java b/src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java index 16ab1bf3..0430a35f 100644 --- a/src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java +++ b/src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java @@ -201,7 +201,9 @@ public class WalletForm { } } } - EventManager.get().post(new ChildWalletsAddedEvent(storage, wallet, addedWallets)); + if(!addedWallets.isEmpty()) { + EventManager.get().post(new ChildWalletsAddedEvent(storage, wallet, addedWallets)); + } }); paymentCodesService.setOnFailed(failedEvent -> { log.error("Could not determine payment codes for wallet " + wallet.getFullName(), failedEvent.getSource().getException()); diff --git a/src/main/resources/com/sparrowwallet/sparrow/sql/V7__AddressData.sql b/src/main/resources/com/sparrowwallet/sparrow/sql/V7__AddressData.sql new file mode 100644 index 00000000..8f056742 --- /dev/null +++ b/src/main/resources/com/sparrowwallet/sparrow/sql/V7__AddressData.sql @@ -0,0 +1 @@ +alter table walletNode add column addressData varbinary(32) after parent; \ No newline at end of file