improve performance on deep wallets by storing addresses

drongo

@@ -1 +1 @@
-Subproject commit 9ae1f68dc42529085edcc8c10d9bcfdbf9639448
+Subproject commit 5de3abd36230d545f11bad3b25a21d23ffbbe9cd

src/main/java/com/sparrowwallet/sparrow/control/AddressTreeTable.java

@@ -14,9 +14,7 @@ import javafx.collections.ListChangeListener;
 import javafx.scene.control.*;
 import javafx.scene.input.MouseButton;
 
-import java.util.List;
+import java.util.*;
-import java.util.Optional;
-import java.util.OptionalInt;
 
 public class AddressTreeTable extends CoinTreeTable {
     public void initialize(NodeEntry rootEntry) {
@@ -110,10 +108,15 @@ public class AddressTreeTable extends CoinTreeTable {
         //We only ever add child nodes - never remove in order to keep a full sequence (unless hide empty used addresses is set)
         NodeEntry rootEntry = (NodeEntry)getRoot().getValue();
 
+        Map<WalletNode, NodeEntry> childNodes = new HashMap<>();
+        for(Entry childEntry : rootEntry.getChildren()) {
+            NodeEntry nodeEntry = (NodeEntry)childEntry;
+            childNodes.put(nodeEntry.getNode(), nodeEntry);
+        }
+
         for(WalletNode updatedNode : updatedNodes) {
-            Optional<Entry> optEntry = rootEntry.getChildren().stream().filter(childEntry -> ((NodeEntry)childEntry).getNode().equals(updatedNode)).findFirst();
+            NodeEntry existingEntry = childNodes.get(updatedNode);
-            if(optEntry.isPresent()) {
+            if(existingEntry != null) {
-                NodeEntry existingEntry = (NodeEntry)optEntry.get();
                 existingEntry.refreshChildren();
 
                 if(Config.get().isHideEmptyUsedAddresses() && existingEntry.getValue() == 0L) {
@@ -125,7 +128,7 @@ public class AddressTreeTable extends CoinTreeTable {
                 if(Config.get().isHideEmptyUsedAddresses()) {
                     int index = 0;
                     for( ; index < rootEntry.getChildren().size(); index++) {
-                        NodeEntry existingEntry = (NodeEntry)rootEntry.getChildren().get(index);
+                        existingEntry = (NodeEntry)rootEntry.getChildren().get(index);
                         if(nodeEntry.compareTo(existingEntry) < 0) {
                              break;
                         }

src/main/java/com/sparrowwallet/sparrow/event/WalletHistoryChangedEvent.java

@@ -45,10 +45,10 @@ public class WalletHistoryChangedEvent extends WalletChangedEvent {
     }
 
     public List<WalletNode> getReceiveNodes() {
-        return getWallet().getNode(KeyPurpose.RECEIVE).getChildren().stream().filter(historyChangedNodes::contains).collect(Collectors.toList());
+        return historyChangedNodes.stream().filter(node -> node.getKeyPurpose() == KeyPurpose.RECEIVE).collect(Collectors.toList());
     }
 
     public List<WalletNode> getChangeNodes() {
-        return getWallet().getNode(KeyPurpose.CHANGE).getChildren().stream().filter(historyChangedNodes::contains).collect(Collectors.toList());
+        return historyChangedNodes.stream().filter(node -> node.getKeyPurpose() == KeyPurpose.CHANGE).collect(Collectors.toList());
     }
 }

src/main/java/com/sparrowwallet/sparrow/io/IOUtils.java

@@ -131,7 +131,7 @@ public class IOUtils {
         if(file.exists()) {
             long length = file.length();
             SecureRandom random = new SecureRandom();
-            byte[] data = new byte[64];
+            byte[] data = new byte[1024*1024];
             random.nextBytes(data);
             try(RandomAccessFile raf = new RandomAccessFile(file, "rws")) {
                 raf.seek(0);

src/main/java/com/sparrowwallet/sparrow/io/JsonPersistence.java

@@ -3,6 +3,8 @@ package com.sparrowwallet.sparrow.io;
 import com.google.gson.*;
 import com.sparrowwallet.drongo.ExtendedKey;
 import com.sparrowwallet.drongo.Utils;
+import com.sparrowwallet.drongo.address.Address;
+import com.sparrowwallet.drongo.address.InvalidAddressException;
 import com.sparrowwallet.drongo.crypto.Argon2KeyDeriver;
 import com.sparrowwallet.drongo.crypto.AsymmetricKeyDeriver;
 import com.sparrowwallet.drongo.crypto.ECKey;
@@ -331,6 +333,8 @@ public class JsonPersistence implements Persistence {
         gsonBuilder.registerTypeAdapter(Date.class, new DateDeserializer());
         gsonBuilder.registerTypeAdapter(Transaction.class, new TransactionSerializer());
         gsonBuilder.registerTypeAdapter(Transaction.class, new TransactionDeserializer());
+        gsonBuilder.registerTypeAdapter(Address.class, new AddressSerializer());
+        gsonBuilder.registerTypeAdapter(Address.class, new AddressDeserializer());
         if(includeWalletSerializers) {
             gsonBuilder.registerTypeAdapter(Keystore.class, new KeystoreSerializer());
             gsonBuilder.registerTypeAdapter(WalletNode.class, new NodeSerializer());
@@ -429,6 +433,24 @@ public class JsonPersistence implements Persistence {
         }
     }
 
+    private static class AddressSerializer implements JsonSerializer<Address> {
+        @Override
+        public JsonElement serialize(Address src, Type typeOfSrc, JsonSerializationContext context) {
+            return new JsonPrimitive(src.toString());
+        }
+    }
+
+    private static class AddressDeserializer implements JsonDeserializer<Address> {
+        @Override
+        public Address deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
+            try {
+                return Address.fromString(json.getAsJsonPrimitive().getAsString());
+            } catch(InvalidAddressException e) {
+                throw new IllegalStateException(e);
+            }
+        }
+    }
+
     private static class KeystoreSerializer implements JsonSerializer<Keystore> {
         @Override
         public JsonElement serialize(Keystore keystore, Type typeOfSrc, JsonSerializationContext context) {

src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java

@@ -87,6 +87,10 @@ public class DbPersistence implements Persistence {
             return walletDao.getMainWallet(MASTER_SCHEMA, getWalletName(storage.getWalletFile(), null));
         });
 
+        if(masterWallet == null) {
+            throw new StorageException("The wallet file was corrupted. Check the backups folder for previous copies.");
+        }
+
         Map<WalletAndKey, Storage> childWallets = loadChildWallets(storage, masterWallet, encryptionKey);
         masterWallet.setChildWallets(childWallets.keySet().stream().map(WalletAndKey::getWallet).collect(Collectors.toList()));
 
@@ -231,12 +235,14 @@ public class DbPersistence implements Persistence {
                         if(addressNode.getId() == null) {
                             WalletNode purposeNode = wallet.getNode(addressNode.getKeyPurpose());
                             if(purposeNode.getId() == null) {
-                                long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null);
+                                long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null, null);
                                 purposeNode.setId(purposeNodeId);
                             }
 
-                            long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId());
+                            long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId(), addressNode.getAddressData());
                             addressNode.setId(nodeId);
+                        } else if(addressNode.getAddress() != null) {
+                            walletNodeDao.updateNodeAddressData(addressNode.getId(), addressNode.getAddressData());
                         }
 
                         Set<BlockTransactionHashIndex> txos = addressNode.getTransactionOutputs().stream().flatMap(txo -> txo.isSpent() ? Stream.of(txo, txo.getSpentBy()) : Stream.of(txo)).collect(Collectors.toSet());
@@ -285,12 +291,14 @@ public class DbPersistence implements Persistence {
                             if(addressNode.getId() == null) {
                                 WalletNode purposeNode = wallet.getNode(addressNode.getKeyPurpose());
                                 if(purposeNode.getId() == null) {
-                                    long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null);
+                                    long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null, null);
                                     purposeNode.setId(purposeNodeId);
                                 }
 
-                                long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId());
+                                long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId(), addressNode.getAddressData());
                                 addressNode.setId(nodeId);
+                            } else if(addressNode.getAddress() != null) {
+                                walletNodeDao.updateNodeAddressData(addressNode.getId(), addressNode.getAddressData());
                             }
 
                             walletNodeDao.updateNodeLabel(addressNode.getId(), entry.getLabel());
@@ -666,7 +674,7 @@ public class DbPersistence implements Persistence {
     }
 
     private String getUrl(File walletFile, String password) {
-        return "jdbc:h2:" + walletFile.getAbsolutePath().replace("." + getType().getExtension(), "") + ";INIT=SET TRACE_LEVEL_FILE=4;TRACE_LEVEL_FILE=4;DATABASE_TO_UPPER=false" + (password == null ? "" : ";CIPHER=AES");
+        return "jdbc:h2:" + walletFile.getAbsolutePath().replace("." + getType().getExtension(), "") + ";INIT=SET TRACE_LEVEL_FILE=4;TRACE_LEVEL_FILE=4;DEFRAG_ALWAYS=true;MAX_COMPACT_TIME=5000;DATABASE_TO_UPPER=false" + (password == null ? "" : ";CIPHER=AES");
     }
 
     private boolean persistsFor(Wallet wallet) {

src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java

@@ -108,7 +108,7 @@ public interface WalletDao {
     default void loadWallet(Wallet wallet) {
         wallet.getKeystores().addAll(createKeystoreDao().getForWalletId(wallet.getId()));
 
-        List<WalletNode> walletNodes = createWalletNodeDao().getForWalletId(wallet.getId());
+        List<WalletNode> walletNodes = createWalletNodeDao().getForWalletId(wallet.getScriptType().ordinal(), wallet.getId());
         wallet.getPurposeNodes().addAll(walletNodes.stream().filter(walletNode -> walletNode.getDerivation().size() == 1).collect(Collectors.toList()));
         wallet.getPurposeNodes().forEach(walletNode -> walletNode.setWallet(wallet));
 

src/main/java/com/sparrowwallet/sparrow/io/db/WalletNodeDao.java

@@ -16,18 +16,18 @@ import java.util.Date;
 import java.util.List;
 
 public interface WalletNodeDao {
-    @SqlQuery("select walletNode.id, walletNode.derivationPath, walletNode.label, walletNode.parent, " +
+    @SqlQuery("select walletNode.id, walletNode.derivationPath, walletNode.label, walletNode.parent, walletNode.addressData, ?, " +
             "blockTransactionHashIndex.id, blockTransactionHashIndex.hash, blockTransactionHashIndex.height, blockTransactionHashIndex.date, blockTransactionHashIndex.fee, blockTransactionHashIndex.label, " +
             "blockTransactionHashIndex.index, blockTransactionHashIndex.outputValue, blockTransactionHashIndex.status, blockTransactionHashIndex.spentBy, blockTransactionHashIndex.node " +
             "from walletNode left join blockTransactionHashIndex on walletNode.id = blockTransactionHashIndex.node where walletNode.wallet = ? order by walletNode.parent asc nulls first, blockTransactionHashIndex.spentBy asc nulls first")
     @RegisterRowMapper(WalletNodeMapper.class)
     @RegisterRowMapper(BlockTransactionHashIndexMapper.class)
     @UseRowReducer(WalletNodeReducer.class)
-    List<WalletNode> getForWalletId(Long id);
+    List<WalletNode> getForWalletId(int scriptType, Long id);
 
-    @SqlUpdate("insert into walletNode (derivationPath, label, wallet, parent) values (?, ?, ?, ?)")
+    @SqlUpdate("insert into walletNode (derivationPath, label, wallet, parent, addressData) values (?, ?, ?, ?, ?)")
     @GetGeneratedKeys("id")
-    long insertWalletNode(String derivationPath, String label, long wallet, Long parent);
+    long insertWalletNode(String derivationPath, String label, long wallet, Long parent, byte[] addressData);
 
     @SqlUpdate("insert into blockTransactionHashIndex (hash, height, date, fee, label, index, outputValue, status, spentBy, node) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
     @GetGeneratedKeys("id")
@@ -39,6 +39,9 @@ public interface WalletNodeDao {
     @SqlUpdate("update walletNode set label = :label where id = :id")
     void updateNodeLabel(@Bind("id") long id, @Bind("label") String label);
 
+    @SqlUpdate("update walletNode set addressData = :addressData where id = :id and addressData is null")
+    void updateNodeAddressData(@Bind("id") long id, @Bind("addressData") byte[] addressData);
+
     @SqlUpdate("update blockTransactionHashIndex set label = :label where id = :id")
     void updateTxoLabel(@Bind("id") long id, @Bind("label") String label);
 
@@ -59,12 +62,12 @@ public interface WalletNodeDao {
 
     default void addWalletNodes(Wallet wallet) {
         for(WalletNode purposeNode : wallet.getPurposeNodes()) {
-            long purposeNodeId = insertWalletNode(purposeNode.getDerivationPath(), truncate(purposeNode.getLabel()), wallet.getId(), null);
+            long purposeNodeId = insertWalletNode(purposeNode.getDerivationPath(), truncate(purposeNode.getLabel()), wallet.getId(), null, null);
             purposeNode.setId(purposeNodeId);
             addTransactionOutputs(purposeNode);
             List<WalletNode> childNodes = new ArrayList<>(purposeNode.getChildren());
             for(WalletNode addressNode : childNodes) {
-                long addressNodeId = insertWalletNode(addressNode.getDerivationPath(), truncate(addressNode.getLabel()), wallet.getId(), purposeNodeId);
+                long addressNodeId = insertWalletNode(addressNode.getDerivationPath(), truncate(addressNode.getLabel()), wallet.getId(), purposeNodeId, addressNode.getAddressData());
                 addressNode.setId(addressNodeId);
                 addTransactionOutputs(addressNode);
             }

src/main/java/com/sparrowwallet/sparrow/io/db/WalletNodeMapper.java

@@ -1,5 +1,6 @@
 package com.sparrowwallet.sparrow.io.db;
 
+import com.sparrowwallet.drongo.protocol.ScriptType;
 import com.sparrowwallet.drongo.wallet.WalletNode;
 import org.jdbi.v3.core.mapper.RowMapper;
 import org.jdbi.v3.core.statement.StatementContext;
@@ -13,7 +14,11 @@ public class WalletNodeMapper implements RowMapper<WalletNode> {
         WalletNode walletNode = new WalletNode(rs.getString("walletNode.derivationPath"));
         walletNode.setId(rs.getLong("walletNode.id"));
         walletNode.setLabel(rs.getString("walletNode.label"));
-
+        byte[] addressData = rs.getBytes("walletNode.addressData");
+        if(addressData != null) {
+            ScriptType scriptType = ScriptType.values()[rs.getInt(6)];
+            walletNode.setAddress(scriptType.getAddress(addressData));
+        }
         return walletNode;
     }
 }

src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java

@@ -432,6 +432,7 @@ public class ElectrumServer {
         try {
             Set<String> scriptHashes = new HashSet<>();
             Map<String, String> pathScriptHashes = new LinkedHashMap<>();
+            Map<String, WalletNode> pathNodes = new HashMap<>();
             for(WalletNode node : nodes) {
                 if(node == null) {
                     log.error("Null node for wallet " + wallet.getFullName() + " subscribing nodes " + nodes + " startIndex " + startIndex, new Throwable());
@@ -448,6 +449,7 @@ public class ElectrumServer {
                     } else if(!subscribedScriptHashes.containsKey(scriptHash) && scriptHashes.add(scriptHash)) {
                         //Unique script hash we are not yet subscribed to
                         pathScriptHashes.put(node.getDerivationPath(), scriptHash);
+                        pathNodes.put(node.getDerivationPath(), node);
                     }
                 }
             }
@@ -463,9 +465,8 @@ public class ElectrumServer {
             for(String path : result.keySet()) {
                 String status = result.get(path);
 
-                Optional<WalletNode> optionalNode = nodes.stream().filter(n -> n.getDerivationPath().equals(path)).findFirst();
+                WalletNode node = pathNodes.computeIfAbsent(path, p -> nodes.stream().filter(n -> n.getDerivationPath().equals(p)).findFirst().orElse(null));
-                if(optionalNode.isPresent()) {
+                if(node != null) {
-                    WalletNode node = optionalNode.get();
                     String scriptHash = getScriptHash(wallet, node);
 
                     //Check if there is history for this script hash, and if the history has changed since last fetched

src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java

@@ -201,7 +201,9 @@ public class WalletForm {
                             }
                         }
                     }
+                    if(!addedWallets.isEmpty()) {
                         EventManager.get().post(new ChildWalletsAddedEvent(storage, wallet, addedWallets));
+                    }
                 });
                 paymentCodesService.setOnFailed(failedEvent -> {
                     log.error("Could not determine payment codes for wallet " + wallet.getFullName(), failedEvent.getSource().getException());

src/main/resources/com/sparrowwallet/sparrow/sql/V7__AddressData.sql

@@ -0,0 +1 @@
+alter table walletNode add column addressData varbinary(32) after parent;