improve wallet loading performance

drongo

@@ -1 +1 @@
-Subproject commit 8dca2ee3f0ba8dbebf88c3629b2a52c7eecf5b89
+Subproject commit fe61c633ae230c3425d52a0c9ac6264ea77e5041

src/main/java/com/sparrowwallet/sparrow/AppController.java

@@ -880,8 +880,21 @@ public class AppController implements Initializable {
         try {
             Storage storage = new Storage(file);
             if(!storage.isEncrypted()) {
-                WalletBackupAndKey walletBackupAndKey = storage.loadUnencryptedWallet();
+                Storage.LoadWalletService loadWalletService = new Storage.LoadWalletService(storage);
+                loadWalletService.setOnSucceeded(workerStateEvent -> {
+                    WalletBackupAndKey walletBackupAndKey = loadWalletService.getValue();
                     openWallet(storage, walletBackupAndKey, this, forceSameWindow);
+                });
+                loadWalletService.setOnFailed(workerStateEvent -> {
+                    Throwable exception = workerStateEvent.getSource().getException();
+                    if(exception instanceof StorageException) {
+                        showErrorDialog("Error Opening Wallet", exception.getMessage());
+                    } else if(!attemptImportWallet(file, null)) {
+                        log.error("Error opening wallet", exception);
+                        showErrorDialog("Error Opening Wallet", exception.getMessage() == null || exception.getMessage().contains("Expected BEGIN_OBJECT") ? "Unsupported wallet file format." : exception.getMessage());
+                    }
+                });
+                loadWalletService.start();
             } else {
                 WalletPasswordDialog dlg = new WalletPasswordDialog(storage.getWalletName(null), WalletPasswordDialog.PasswordRequirement.LOAD);
                 Optional<SecureString> optionalPassword = dlg.showAndWait();
@@ -905,9 +918,11 @@ public class AppController implements Initializable {
                             Platform.runLater(() -> openWalletFile(file, forceSameWindow));
                         }
                     } else {
-                        if(!attemptImportWallet(file, password)) {
+                        if(exception instanceof StorageException) {
+                            showErrorDialog("Error Opening Wallet", exception.getMessage());
+                        } else if(!attemptImportWallet(file, password)) {
                             log.error("Error Opening Wallet", exception);
-                            showErrorDialog("Error Opening Wallet", exception.getMessage() == null ? "Unsupported file format" : exception.getMessage());
+                            showErrorDialog("Error Opening Wallet", exception.getMessage() == null || exception.getMessage().contains("Expected BEGIN_OBJECT") ? "Unsupported wallet file format." : exception.getMessage());
                         }
                         password.clear();
                     }
@@ -915,8 +930,6 @@ public class AppController implements Initializable {
                 EventManager.get().post(new StorageEvent(storage.getWalletId(null), TimedEvent.Action.START, "Decrypting wallet..."));
                 loadWalletService.start();
             }
-        } catch(StorageException e) {
-            showErrorDialog("Error Opening Wallet", e.getMessage());
         } catch(Exception e) {
             if(e instanceof IOException && e.getMessage().startsWith("The process cannot access the file because another process has locked")) {
                 log.error("Error opening wallet", e);

src/main/java/com/sparrowwallet/sparrow/control/BalanceChart.java

@@ -1,20 +1,26 @@
 package com.sparrowwallet.sparrow.control;
 
+import com.google.common.collect.Lists;
 import com.sparrowwallet.drongo.BitcoinUnit;
 import com.sparrowwallet.drongo.wallet.Wallet;
 import com.sparrowwallet.sparrow.io.Config;
+import com.sparrowwallet.sparrow.wallet.Entry;
 import com.sparrowwallet.sparrow.wallet.TransactionEntry;
 import com.sparrowwallet.sparrow.wallet.WalletTransactionsEntry;
 import javafx.beans.NamedArg;
 import javafx.scene.Node;
 import javafx.scene.chart.*;
 
+import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 public class BalanceChart extends LineChart<Number, Number> {
+    private static final int MAX_VALUES = 500;
+
     private XYChart.Series<Number, Number> balanceSeries;
 
     private TransactionEntry selectedEntry;
@@ -37,7 +43,7 @@ public class BalanceChart extends LineChart<Number, Number> {
         setVisible(!walletTransactionsEntry.getChildren().isEmpty());
         balanceSeries.getData().clear();
 
-        List<Data<Number, Number>> balanceDataList = walletTransactionsEntry.getChildren().stream()
+        List<Data<Number, Number>> balanceDataList = getTransactionEntries(walletTransactionsEntry)
                 .map(entry -> (TransactionEntry)entry)
                 .filter(txEntry -> txEntry.getBlockTransaction().getHeight() > 0)
                 .map(txEntry -> new XYChart.Data<>((Number)txEntry.getBlockTransaction().getDate().getTime(), (Number)txEntry.getBalance(), txEntry))
@@ -74,6 +80,24 @@ public class BalanceChart extends LineChart<Number, Number> {
         }
     }
 
+    private Stream<Entry> getTransactionEntries(WalletTransactionsEntry walletTransactionsEntry) {
+        int total = walletTransactionsEntry.getChildren().size();
+        if(walletTransactionsEntry.getChildren().size() <= MAX_VALUES) {
+            return walletTransactionsEntry.getChildren().stream();
+        }
+
+        int bucketSize = total / MAX_VALUES;
+        List<List<Entry>> buckets = Lists.partition(walletTransactionsEntry.getChildren(), bucketSize);
+        List<Entry> reducedEntries = new ArrayList<>(MAX_VALUES);
+        for(List<Entry> bucket : buckets) {
+            long max = bucket.stream().mapToLong(entry -> Math.abs(entry.getValue())).max().orElse(0);
+            Entry bucketEntry = bucket.stream().filter(entry -> entry.getValue() == max || entry.getValue() == -max).findFirst().orElseThrow();
+            reducedEntries.add(bucketEntry);
+        }
+
+        return reducedEntries.stream();
+    }
+
     public void select(TransactionEntry transactionEntry) {
         Set<Node> selectedSymbols = lookupAll(".chart-line-symbol.selected");
         for(Node selectedSymbol : selectedSymbols) {

src/main/java/com/sparrowwallet/sparrow/control/FileImportPane.java

@@ -158,6 +158,7 @@ public abstract class FileImportPane extends TitledDescriptionPane {
                 try {
                     importFile(importer.getName(), null, null);
                 } catch(ImportException e) {
+                    log.error("Error importing QR", e);
                     setError("Import Error", e.getMessage());
                 }
             } else if(result.outputDescriptor != null) {
@@ -165,6 +166,7 @@ public abstract class FileImportPane extends TitledDescriptionPane {
                 try {
                     importFile(importer.getName(), null, null);
                 } catch(ImportException e) {
+                    log.error("Error importing QR", e);
                     setError("Import Error", e.getMessage());
                 }
             } else if(result.payload != null) {

src/main/java/com/sparrowwallet/sparrow/io/CaravanMultisig.java

@@ -78,7 +78,6 @@ public class CaravanMultisig implements WalletImport, WalletExport {
 
             return wallet;
         } catch(Exception e) {
-            log.error("Error importing " + getName() + " wallet", e);
             throw new ImportException("Error importing " + getName() + " wallet", e);
         }
     }

src/main/java/com/sparrowwallet/sparrow/io/CoboVaultSinglesig.java

@@ -57,7 +57,6 @@ public class CoboVaultSinglesig implements KeystoreFileImport, WalletImport {
 
             return keystore;
         } catch (Exception e) {
-            log.error("Error getting " + getName() + " keystore", e);
             throw new ImportException("Error getting " + getName() + " keystore", e);
         }
     }

src/main/java/com/sparrowwallet/sparrow/io/ColdcardMultisig.java

@@ -154,7 +154,6 @@ public class ColdcardMultisig implements WalletImport, KeystoreFileImport, Walle
 
             return wallet;
         } catch(Exception e) {
-            log.error("Error importing " + getName() + " wallet", e);
             throw new ImportException("Error importing " + getName() + " wallet", e);
         }
     }

src/main/java/com/sparrowwallet/sparrow/io/ColdcardSinglesig.java

@@ -86,7 +86,6 @@ public class ColdcardSinglesig implements KeystoreFileImport, WalletImport {
                 }
             }
         } catch (Exception e) {
-            log.error("Error getting " + getName() + " keystore", e);
             throw new ImportException("Error getting " + getName() + " keystore", e);
         }
 

src/main/java/com/sparrowwallet/sparrow/io/Electrum.java

@@ -264,7 +264,6 @@ public class Electrum implements KeystoreFileImport, WalletImport, WalletExport
 
             return wallet;
         } catch (Exception e) {
-            log.error("Error importing Electrum Wallet", e);
             throw new ImportException("Error importing Electrum Wallet", e);
         }
     }

src/main/java/com/sparrowwallet/sparrow/io/KeystoneSinglesig.java

@@ -62,10 +62,8 @@ public class KeystoneSinglesig implements KeystoreFileImport, WalletImport {
 
             return keystore;
         } catch (IllegalArgumentException e) {
-            log.error("Error getting " + getName() + " keystore - not an output descriptor");
+            throw new ImportException("Error getting " + getName() + " keystore - not an output descriptor", e);
-            throw new ImportException("Error getting " + getName() + " keystore", e);
         } catch (Exception e) {
-            log.error("Error getting " + getName() + " keystore", e);
             throw new ImportException("Error getting " + getName() + " keystore", e);
         }
     }

src/main/java/com/sparrowwallet/sparrow/io/Sparrow.java

@@ -121,7 +121,6 @@ public class Sparrow implements WalletImport, WalletExport {
 
             return wallet;
         } catch(IOException | StorageException e) {
-            log.error("Error importing Sparrow wallet", e);
             throw new ImportException("Error importing Sparrow wallet", e);
         } finally {
             if(storage != null) {

src/main/java/com/sparrowwallet/sparrow/io/SpecterDIY.java

@@ -35,7 +35,6 @@ public class SpecterDIY implements KeystoreFileImport, WalletExport {
 
             return keystore;
         } catch(IOException e) {
-            log.error("Error getting " + getName() + " keystore", e);
             throw new ImportException("Error getting " + getName() + " keystore", e);
         }
     }

src/main/java/com/sparrowwallet/sparrow/io/SpecterDesktop.java

@@ -93,7 +93,6 @@ public class SpecterDesktop implements WalletImport, WalletExport {
                 return wallet;
             }
         } catch(Exception e) {
-            log.error("Error importing " + getName() + " wallet", e);
             throw new ImportException("Error importing " + getName() + " wallet", e);
         }
 

src/main/java/com/sparrowwallet/sparrow/io/Storage.java

@@ -534,6 +534,11 @@ public class Storage {
         private final Storage storage;
         private final SecureString password;
 
+        public LoadWalletService(Storage storage) {
+            this.storage = storage;
+            this.password = null;
+        }
+
         public LoadWalletService(Storage storage, SecureString password) {
             this.storage = storage;
             this.password = password;
@@ -543,8 +548,15 @@ public class Storage {
         protected Task<WalletBackupAndKey> createTask() {
             return new Task<>() {
                 protected WalletBackupAndKey call() throws IOException, StorageException {
-                    WalletBackupAndKey walletBackupAndKey = storage.loadEncryptedWallet(password);
+                    WalletBackupAndKey walletBackupAndKey;
+
+                    if(password != null) {
+                        walletBackupAndKey = storage.loadEncryptedWallet(password);
                         password.clear();
+                    } else {
+                        walletBackupAndKey = storage.loadUnencryptedWallet();
+                    }
+
                     return walletBackupAndKey;
                 }
             };

src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java

@@ -578,7 +578,7 @@ public class DbPersistence implements Persistence {
         if(updateExecutor != null) {
             updateExecutor.shutdown();
             try {
-                if(!updateExecutor.awaitTermination(10, TimeUnit.SECONDS)) {
+                if(!updateExecutor.awaitTermination(1, TimeUnit.MINUTES)) {
                     updateExecutor.shutdownNow();
                 }
 

src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java

@@ -166,7 +166,7 @@ public class ElectrumServer {
         }
     }
 
-    public static void addCalculatedScriptHashes(Wallet wallet) {
+    private static void addCalculatedScriptHashes(Wallet wallet) {
         getCalculatedScriptHashes(wallet).forEach(retrievedScriptHashes::putIfAbsent);
     }
 
@@ -1248,8 +1248,12 @@ public class ElectrumServer {
         protected Task<Boolean> createTask() {
             return new Task<>() {
                 protected Boolean call() throws ServerException {
-                    walletSynchronizeLocks.putIfAbsent(wallet, new Object());
+                    boolean initial = (walletSynchronizeLocks.putIfAbsent(wallet, new Object()) == null);
                     synchronized(walletSynchronizeLocks.get(wallet)) {
+                        if(initial) {
+                            addCalculatedScriptHashes(wallet);
+                        }
+
                         if(isConnected()) {
                             ElectrumServer electrumServer = new ElectrumServer();
                             Map<String, String> previousScriptHashes = getCalculatedScriptHashes(wallet);

src/main/java/com/sparrowwallet/sparrow/wallet/TransactionEntry.java

@@ -38,7 +38,18 @@ public class TransactionEntry extends Entry implements Comparable<TransactionEnt
             }
         });
 
-        setConfirmations(calculateConfirmations());
+        confirmations = new IntegerPropertyBase(calculateConfirmations()) {
+            @Override
+            public Object getBean() {
+                return TransactionEntry.this;
+            }
+
+            @Override
+            public String getName() {
+                return "confirmations";
+            }
+        };
+
         if(isFullyConfirming()) {
             EventManager.get().register(this);
         }
@@ -174,33 +185,17 @@ public class TransactionEntry extends Entry implements Comparable<TransactionEnt
     /**
      * Defines the number of confirmations
      */
-    private IntegerProperty confirmations;
+    private final IntegerProperty confirmations;
 
     public final void setConfirmations(int value) {
-        if(confirmations != null || value != 0) {
+        confirmations.set(value);
-            confirmationsProperty().set(value);
-        }
     }
 
     public final int getConfirmations() {
-        return confirmations == null ? 0 : confirmations.get();
+        return confirmations.get();
     }
 
     public final IntegerProperty confirmationsProperty() {
-        if(confirmations == null) {
-            confirmations = new IntegerPropertyBase(0) {
-
-                @Override
-                public Object getBean() {
-                    return TransactionEntry.this;
-                }
-
-                @Override
-                public String getName() {
-                    return "confirmations";
-                }
-            };
-        }
         return confirmations;
     }
 

src/main/java/com/sparrowwallet/sparrow/wallet/TransactionsController.java

@@ -5,7 +5,6 @@ import com.google.common.eventbus.Subscribe;
 import com.sparrowwallet.drongo.BitcoinUnit;
 import com.sparrowwallet.drongo.protocol.Transaction;
 import com.sparrowwallet.sparrow.AppServices;
-import com.sparrowwallet.sparrow.CurrencyRate;
 import com.sparrowwallet.sparrow.EventManager;
 import com.sparrowwallet.sparrow.control.*;
 import com.sparrowwallet.sparrow.event.*;

src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java

@@ -59,7 +59,6 @@ public class WalletForm {
         savedPastWallet = backupWallet;
 
         if(refreshHistory && wallet.isValid()) {
-            ElectrumServer.addCalculatedScriptHashes(wallet);
             refreshHistory(AppServices.getCurrentBlockHeight(), backupWallet);
         }
     }
@@ -252,11 +251,13 @@ public class WalletForm {
     }
 
     private List<WalletNode> getHistoryChangedNodes(Set<WalletNode> previousNodes, Set<WalletNode> currentNodes) {
+        Map<WalletNode, WalletNode> previousNodeMap = new HashMap<>(previousNodes.size());
+        previousNodes.forEach(walletNode -> previousNodeMap.put(walletNode, walletNode));
+
         List<WalletNode> changedNodes = new ArrayList<>();
         for(WalletNode currentNode : currentNodes) {
-            Optional<WalletNode> optPreviousNode = previousNodes.stream().filter(node -> node.equals(currentNode)).findFirst();
+            WalletNode previousNode = previousNodeMap.get(currentNode);
-            if(optPreviousNode.isPresent()) {
+            if(previousNode != null) {
-                WalletNode previousNode = optPreviousNode.get();
                 if(!currentNode.getTransactionOutputs().equals(previousNode.getTransactionOutputs())) {
                     changedNodes.add(currentNode);
                 }

src/main/java/com/sparrowwallet/sparrow/wallet/WalletTransactionsEntry.java

@@ -22,7 +22,7 @@ public class WalletTransactionsEntry extends Entry {
 
     public WalletTransactionsEntry(Wallet wallet) {
         super(wallet, wallet.getName(), getWalletTransactions(wallet).stream().map(WalletTransaction::getTransactionEntry).collect(Collectors.toList()));
-        calculateBalances();
+        calculateBalances(false); //No need to resort
     }
 
     @Override
@@ -30,12 +30,14 @@ public class WalletTransactionsEntry extends Entry {
         return getBalance();
     }
 
-    protected void calculateBalances() {
+    private void calculateBalances(boolean resort) {
         long balance = 0L;
         long mempoolBalance = 0L;
 
+        if(resort) {
             //Note transaction entries must be in ascending order. This sorting is ultimately done according to BlockTransactions' comparator
             getChildren().sort(Comparator.comparing(TransactionEntry.class::cast));
+        }
 
         for(Entry entry : getChildren()) {
             TransactionEntry transactionEntry = (TransactionEntry)entry;
@@ -66,7 +68,7 @@ public class WalletTransactionsEntry extends Entry {
         entriesRemoved.removeAll(current);
         getChildren().removeAll(entriesRemoved);
 
-        calculateBalances();
+        calculateBalances(true);
 
         Map<HashIndex, BlockTransactionHashIndex> walletTxos = getWallet().getWalletTxos().entrySet().stream()
                 .collect(Collectors.toUnmodifiableMap(entry -> new HashIndex(entry.getKey().getHash(), entry.getKey().getIndex()), Map.Entry::getKey));
@@ -86,12 +88,14 @@ public class WalletTransactionsEntry extends Entry {
     }
 
     private static Collection<WalletTransaction> getWalletTransactions(Wallet wallet) {
-        Map<BlockTransaction, WalletTransaction> walletTransactionMap = new TreeMap<>();
+        Map<BlockTransaction, WalletTransaction> walletTransactionMap = new HashMap<>(wallet.getTransactions().size());
 
         getWalletTransactions(wallet, walletTransactionMap, wallet.getNode(KeyPurpose.RECEIVE));
         getWalletTransactions(wallet, walletTransactionMap, wallet.getNode(KeyPurpose.CHANGE));
 
-        return new ArrayList<>(walletTransactionMap.values());
+        List<WalletTransaction> walletTransactions = new ArrayList<>(walletTransactionMap.values());
+        Collections.sort(walletTransactions);
+        return walletTransactions;
     }
 
     private static void getWalletTransactions(Wallet wallet, Map<BlockTransaction, WalletTransaction> walletTransactionMap, WalletNode purposeNode) {
@@ -191,7 +195,7 @@ public class WalletTransactionsEntry extends Entry {
         return mempoolBalance;
     }
 
-    private static class WalletTransaction {
+    private static class WalletTransaction implements Comparable<WalletTransaction> {
         private final Wallet wallet;
         private final BlockTransaction blockTransaction;
         private final Map<BlockTransactionHashIndex, KeyPurpose> incoming = new TreeMap<>();
@@ -205,5 +209,33 @@ public class WalletTransactionsEntry extends Entry {
         public TransactionEntry getTransactionEntry() {
             return new TransactionEntry(wallet, blockTransaction, incoming, outgoing);
         }
+
+        public long getValue() {
+            long value = 0L;
+            for(BlockTransactionHashIndex in : incoming.keySet()) {
+                value += in.getValue();
+            }
+            for(BlockTransactionHashIndex out : outgoing.keySet()) {
+                value -= out.getValue();
+            }
+
+            return value;
+        }
+
+        @Override
+        public int compareTo(WalletTransactionsEntry.WalletTransaction other) {
+            //This comparison must be identical to that of TransactionEntry so we can avoid a resort calculating balances when creating WalletTransactionsEntry
+            int blockOrder = blockTransaction.compareBlockOrder(other.blockTransaction);
+            if(blockOrder != 0) {
+                return blockOrder;
+            }
+
+            int valueOrder = Long.compare(other.getValue(), getValue());
+            if(valueOrder != 0) {
+                return valueOrder;
+            }
+
+            return blockTransaction.compareTo(other.blockTransaction);
+        }
     }
 }