From 9faf036e4d49203a9112dc8c66f92cd2ba401a04 Mon Sep 17 00:00:00 2001 From: Craig Raw Date: Thu, 20 Jan 2022 17:12:38 +0200 Subject: [PATCH] improve wallet loading performance --- drongo | 2 +- .../sparrowwallet/sparrow/AppController.java | 25 +++++++--- .../sparrow/control/BalanceChart.java | 26 +++++++++- .../sparrow/control/FileImportPane.java | 2 + .../sparrow/io/CaravanMultisig.java | 1 - .../sparrow/io/CoboVaultSinglesig.java | 1 - .../sparrow/io/ColdcardMultisig.java | 1 - .../sparrow/io/ColdcardSinglesig.java | 1 - .../sparrowwallet/sparrow/io/Electrum.java | 1 - .../sparrow/io/KeystoneSinglesig.java | 4 +- .../com/sparrowwallet/sparrow/io/Sparrow.java | 1 - .../sparrowwallet/sparrow/io/SpecterDIY.java | 1 - .../sparrow/io/SpecterDesktop.java | 1 - .../com/sparrowwallet/sparrow/io/Storage.java | 16 ++++++- .../sparrow/io/db/DbPersistence.java | 2 +- .../sparrow/net/ElectrumServer.java | 8 +++- .../sparrow/wallet/TransactionEntry.java | 35 ++++++-------- .../wallet/TransactionsController.java | 1 - .../sparrow/wallet/WalletForm.java | 9 ++-- .../wallet/WalletTransactionsEntry.java | 48 +++++++++++++++---- 20 files changed, 129 insertions(+), 57 deletions(-) diff --git a/drongo b/drongo index 8dca2ee3..fe61c633 160000 --- a/drongo +++ b/drongo @@ -1 +1 @@ -Subproject commit 8dca2ee3f0ba8dbebf88c3629b2a52c7eecf5b89 +Subproject commit fe61c633ae230c3425d52a0c9ac6264ea77e5041 diff --git a/src/main/java/com/sparrowwallet/sparrow/AppController.java b/src/main/java/com/sparrowwallet/sparrow/AppController.java index c32df546..ab2b7f25 100644 --- a/src/main/java/com/sparrowwallet/sparrow/AppController.java +++ b/src/main/java/com/sparrowwallet/sparrow/AppController.java @@ -880,8 +880,21 @@ public class AppController implements Initializable { try { Storage storage = new Storage(file); if(!storage.isEncrypted()) { - WalletBackupAndKey walletBackupAndKey = storage.loadUnencryptedWallet(); - openWallet(storage, walletBackupAndKey, this, forceSameWindow); + Storage.LoadWalletService loadWalletService = new Storage.LoadWalletService(storage); + loadWalletService.setOnSucceeded(workerStateEvent -> { + WalletBackupAndKey walletBackupAndKey = loadWalletService.getValue(); + openWallet(storage, walletBackupAndKey, this, forceSameWindow); + }); + loadWalletService.setOnFailed(workerStateEvent -> { + Throwable exception = workerStateEvent.getSource().getException(); + if(exception instanceof StorageException) { + showErrorDialog("Error Opening Wallet", exception.getMessage()); + } else if(!attemptImportWallet(file, null)) { + log.error("Error opening wallet", exception); + showErrorDialog("Error Opening Wallet", exception.getMessage() == null || exception.getMessage().contains("Expected BEGIN_OBJECT") ? "Unsupported wallet file format." : exception.getMessage()); + } + }); + loadWalletService.start(); } else { WalletPasswordDialog dlg = new WalletPasswordDialog(storage.getWalletName(null), WalletPasswordDialog.PasswordRequirement.LOAD); Optional optionalPassword = dlg.showAndWait(); @@ -905,9 +918,11 @@ public class AppController implements Initializable { Platform.runLater(() -> openWalletFile(file, forceSameWindow)); } } else { - if(!attemptImportWallet(file, password)) { + if(exception instanceof StorageException) { + showErrorDialog("Error Opening Wallet", exception.getMessage()); + } else if(!attemptImportWallet(file, password)) { log.error("Error Opening Wallet", exception); - showErrorDialog("Error Opening Wallet", exception.getMessage() == null ? "Unsupported file format" : exception.getMessage()); + showErrorDialog("Error Opening Wallet", exception.getMessage() == null || exception.getMessage().contains("Expected BEGIN_OBJECT") ? "Unsupported wallet file format." : exception.getMessage()); } password.clear(); } @@ -915,8 +930,6 @@ public class AppController implements Initializable { EventManager.get().post(new StorageEvent(storage.getWalletId(null), TimedEvent.Action.START, "Decrypting wallet...")); loadWalletService.start(); } - } catch(StorageException e) { - showErrorDialog("Error Opening Wallet", e.getMessage()); } catch(Exception e) { if(e instanceof IOException && e.getMessage().startsWith("The process cannot access the file because another process has locked")) { log.error("Error opening wallet", e); diff --git a/src/main/java/com/sparrowwallet/sparrow/control/BalanceChart.java b/src/main/java/com/sparrowwallet/sparrow/control/BalanceChart.java index aa9fe0fb..eeaabcf8 100644 --- a/src/main/java/com/sparrowwallet/sparrow/control/BalanceChart.java +++ b/src/main/java/com/sparrowwallet/sparrow/control/BalanceChart.java @@ -1,20 +1,26 @@ package com.sparrowwallet.sparrow.control; +import com.google.common.collect.Lists; import com.sparrowwallet.drongo.BitcoinUnit; import com.sparrowwallet.drongo.wallet.Wallet; import com.sparrowwallet.sparrow.io.Config; +import com.sparrowwallet.sparrow.wallet.Entry; import com.sparrowwallet.sparrow.wallet.TransactionEntry; import com.sparrowwallet.sparrow.wallet.WalletTransactionsEntry; import javafx.beans.NamedArg; import javafx.scene.Node; import javafx.scene.chart.*; +import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; public class BalanceChart extends LineChart { + private static final int MAX_VALUES = 500; + private XYChart.Series balanceSeries; private TransactionEntry selectedEntry; @@ -37,7 +43,7 @@ public class BalanceChart extends LineChart { setVisible(!walletTransactionsEntry.getChildren().isEmpty()); balanceSeries.getData().clear(); - List> balanceDataList = walletTransactionsEntry.getChildren().stream() + List> balanceDataList = getTransactionEntries(walletTransactionsEntry) .map(entry -> (TransactionEntry)entry) .filter(txEntry -> txEntry.getBlockTransaction().getHeight() > 0) .map(txEntry -> new XYChart.Data<>((Number)txEntry.getBlockTransaction().getDate().getTime(), (Number)txEntry.getBalance(), txEntry)) @@ -74,6 +80,24 @@ public class BalanceChart extends LineChart { } } + private Stream getTransactionEntries(WalletTransactionsEntry walletTransactionsEntry) { + int total = walletTransactionsEntry.getChildren().size(); + if(walletTransactionsEntry.getChildren().size() <= MAX_VALUES) { + return walletTransactionsEntry.getChildren().stream(); + } + + int bucketSize = total / MAX_VALUES; + List> buckets = Lists.partition(walletTransactionsEntry.getChildren(), bucketSize); + List reducedEntries = new ArrayList<>(MAX_VALUES); + for(List bucket : buckets) { + long max = bucket.stream().mapToLong(entry -> Math.abs(entry.getValue())).max().orElse(0); + Entry bucketEntry = bucket.stream().filter(entry -> entry.getValue() == max || entry.getValue() == -max).findFirst().orElseThrow(); + reducedEntries.add(bucketEntry); + } + + return reducedEntries.stream(); + } + public void select(TransactionEntry transactionEntry) { Set selectedSymbols = lookupAll(".chart-line-symbol.selected"); for(Node selectedSymbol : selectedSymbols) { diff --git a/src/main/java/com/sparrowwallet/sparrow/control/FileImportPane.java b/src/main/java/com/sparrowwallet/sparrow/control/FileImportPane.java index aad47cf6..73ca49a0 100644 --- a/src/main/java/com/sparrowwallet/sparrow/control/FileImportPane.java +++ b/src/main/java/com/sparrowwallet/sparrow/control/FileImportPane.java @@ -158,6 +158,7 @@ public abstract class FileImportPane extends TitledDescriptionPane { try { importFile(importer.getName(), null, null); } catch(ImportException e) { + log.error("Error importing QR", e); setError("Import Error", e.getMessage()); } } else if(result.outputDescriptor != null) { @@ -165,6 +166,7 @@ public abstract class FileImportPane extends TitledDescriptionPane { try { importFile(importer.getName(), null, null); } catch(ImportException e) { + log.error("Error importing QR", e); setError("Import Error", e.getMessage()); } } else if(result.payload != null) { diff --git a/src/main/java/com/sparrowwallet/sparrow/io/CaravanMultisig.java b/src/main/java/com/sparrowwallet/sparrow/io/CaravanMultisig.java index 729595bd..912d56b7 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/CaravanMultisig.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/CaravanMultisig.java @@ -78,7 +78,6 @@ public class CaravanMultisig implements WalletImport, WalletExport { return wallet; } catch(Exception e) { - log.error("Error importing " + getName() + " wallet", e); throw new ImportException("Error importing " + getName() + " wallet", e); } } diff --git a/src/main/java/com/sparrowwallet/sparrow/io/CoboVaultSinglesig.java b/src/main/java/com/sparrowwallet/sparrow/io/CoboVaultSinglesig.java index 5134d04f..c6948093 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/CoboVaultSinglesig.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/CoboVaultSinglesig.java @@ -57,7 +57,6 @@ public class CoboVaultSinglesig implements KeystoreFileImport, WalletImport { return keystore; } catch (Exception e) { - log.error("Error getting " + getName() + " keystore", e); throw new ImportException("Error getting " + getName() + " keystore", e); } } diff --git a/src/main/java/com/sparrowwallet/sparrow/io/ColdcardMultisig.java b/src/main/java/com/sparrowwallet/sparrow/io/ColdcardMultisig.java index 6ef183e6..fcc21874 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/ColdcardMultisig.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/ColdcardMultisig.java @@ -154,7 +154,6 @@ public class ColdcardMultisig implements WalletImport, KeystoreFileImport, Walle return wallet; } catch(Exception e) { - log.error("Error importing " + getName() + " wallet", e); throw new ImportException("Error importing " + getName() + " wallet", e); } } diff --git a/src/main/java/com/sparrowwallet/sparrow/io/ColdcardSinglesig.java b/src/main/java/com/sparrowwallet/sparrow/io/ColdcardSinglesig.java index 50babc53..7b0c1999 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/ColdcardSinglesig.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/ColdcardSinglesig.java @@ -86,7 +86,6 @@ public class ColdcardSinglesig implements KeystoreFileImport, WalletImport { } } } catch (Exception e) { - log.error("Error getting " + getName() + " keystore", e); throw new ImportException("Error getting " + getName() + " keystore", e); } diff --git a/src/main/java/com/sparrowwallet/sparrow/io/Electrum.java b/src/main/java/com/sparrowwallet/sparrow/io/Electrum.java index 1110abc8..bbf6f4d2 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/Electrum.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/Electrum.java @@ -264,7 +264,6 @@ public class Electrum implements KeystoreFileImport, WalletImport, WalletExport return wallet; } catch (Exception e) { - log.error("Error importing Electrum Wallet", e); throw new ImportException("Error importing Electrum Wallet", e); } } diff --git a/src/main/java/com/sparrowwallet/sparrow/io/KeystoneSinglesig.java b/src/main/java/com/sparrowwallet/sparrow/io/KeystoneSinglesig.java index 88a8eced..65308b8d 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/KeystoneSinglesig.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/KeystoneSinglesig.java @@ -62,10 +62,8 @@ public class KeystoneSinglesig implements KeystoreFileImport, WalletImport { return keystore; } catch (IllegalArgumentException e) { - log.error("Error getting " + getName() + " keystore - not an output descriptor"); - throw new ImportException("Error getting " + getName() + " keystore", e); + throw new ImportException("Error getting " + getName() + " keystore - not an output descriptor", e); } catch (Exception e) { - log.error("Error getting " + getName() + " keystore", e); throw new ImportException("Error getting " + getName() + " keystore", e); } } diff --git a/src/main/java/com/sparrowwallet/sparrow/io/Sparrow.java b/src/main/java/com/sparrowwallet/sparrow/io/Sparrow.java index 582136e1..f7357d54 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/Sparrow.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/Sparrow.java @@ -121,7 +121,6 @@ public class Sparrow implements WalletImport, WalletExport { return wallet; } catch(IOException | StorageException e) { - log.error("Error importing Sparrow wallet", e); throw new ImportException("Error importing Sparrow wallet", e); } finally { if(storage != null) { diff --git a/src/main/java/com/sparrowwallet/sparrow/io/SpecterDIY.java b/src/main/java/com/sparrowwallet/sparrow/io/SpecterDIY.java index ed41467a..ccb78574 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/SpecterDIY.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/SpecterDIY.java @@ -35,7 +35,6 @@ public class SpecterDIY implements KeystoreFileImport, WalletExport { return keystore; } catch(IOException e) { - log.error("Error getting " + getName() + " keystore", e); throw new ImportException("Error getting " + getName() + " keystore", e); } } diff --git a/src/main/java/com/sparrowwallet/sparrow/io/SpecterDesktop.java b/src/main/java/com/sparrowwallet/sparrow/io/SpecterDesktop.java index 149c0a35..6ce62f03 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/SpecterDesktop.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/SpecterDesktop.java @@ -93,7 +93,6 @@ public class SpecterDesktop implements WalletImport, WalletExport { return wallet; } } catch(Exception e) { - log.error("Error importing " + getName() + " wallet", e); throw new ImportException("Error importing " + getName() + " wallet", e); } diff --git a/src/main/java/com/sparrowwallet/sparrow/io/Storage.java b/src/main/java/com/sparrowwallet/sparrow/io/Storage.java index f73a4454..dcd6b940 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/Storage.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/Storage.java @@ -534,6 +534,11 @@ public class Storage { private final Storage storage; private final SecureString password; + public LoadWalletService(Storage storage) { + this.storage = storage; + this.password = null; + } + public LoadWalletService(Storage storage, SecureString password) { this.storage = storage; this.password = password; @@ -543,8 +548,15 @@ public class Storage { protected Task createTask() { return new Task<>() { protected WalletBackupAndKey call() throws IOException, StorageException { - WalletBackupAndKey walletBackupAndKey = storage.loadEncryptedWallet(password); - password.clear(); + WalletBackupAndKey walletBackupAndKey; + + if(password != null) { + walletBackupAndKey = storage.loadEncryptedWallet(password); + password.clear(); + } else { + walletBackupAndKey = storage.loadUnencryptedWallet(); + } + return walletBackupAndKey; } }; diff --git a/src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java b/src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java index 04b65660..b5745e51 100644 --- a/src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java +++ b/src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java @@ -578,7 +578,7 @@ public class DbPersistence implements Persistence { if(updateExecutor != null) { updateExecutor.shutdown(); try { - if(!updateExecutor.awaitTermination(10, TimeUnit.SECONDS)) { + if(!updateExecutor.awaitTermination(1, TimeUnit.MINUTES)) { updateExecutor.shutdownNow(); } diff --git a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java index b374b10c..a6d4939b 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java @@ -166,7 +166,7 @@ public class ElectrumServer { } } - public static void addCalculatedScriptHashes(Wallet wallet) { + private static void addCalculatedScriptHashes(Wallet wallet) { getCalculatedScriptHashes(wallet).forEach(retrievedScriptHashes::putIfAbsent); } @@ -1248,8 +1248,12 @@ public class ElectrumServer { protected Task createTask() { return new Task<>() { protected Boolean call() throws ServerException { - walletSynchronizeLocks.putIfAbsent(wallet, new Object()); + boolean initial = (walletSynchronizeLocks.putIfAbsent(wallet, new Object()) == null); synchronized(walletSynchronizeLocks.get(wallet)) { + if(initial) { + addCalculatedScriptHashes(wallet); + } + if(isConnected()) { ElectrumServer electrumServer = new ElectrumServer(); Map previousScriptHashes = getCalculatedScriptHashes(wallet); diff --git a/src/main/java/com/sparrowwallet/sparrow/wallet/TransactionEntry.java b/src/main/java/com/sparrowwallet/sparrow/wallet/TransactionEntry.java index c25d8ded..f8de49b8 100644 --- a/src/main/java/com/sparrowwallet/sparrow/wallet/TransactionEntry.java +++ b/src/main/java/com/sparrowwallet/sparrow/wallet/TransactionEntry.java @@ -38,7 +38,18 @@ public class TransactionEntry extends Entry implements Comparable getHistoryChangedNodes(Set previousNodes, Set currentNodes) { + Map previousNodeMap = new HashMap<>(previousNodes.size()); + previousNodes.forEach(walletNode -> previousNodeMap.put(walletNode, walletNode)); + List changedNodes = new ArrayList<>(); for(WalletNode currentNode : currentNodes) { - Optional optPreviousNode = previousNodes.stream().filter(node -> node.equals(currentNode)).findFirst(); - if(optPreviousNode.isPresent()) { - WalletNode previousNode = optPreviousNode.get(); + WalletNode previousNode = previousNodeMap.get(currentNode); + if(previousNode != null) { if(!currentNode.getTransactionOutputs().equals(previousNode.getTransactionOutputs())) { changedNodes.add(currentNode); } diff --git a/src/main/java/com/sparrowwallet/sparrow/wallet/WalletTransactionsEntry.java b/src/main/java/com/sparrowwallet/sparrow/wallet/WalletTransactionsEntry.java index dceefce7..51522f85 100644 --- a/src/main/java/com/sparrowwallet/sparrow/wallet/WalletTransactionsEntry.java +++ b/src/main/java/com/sparrowwallet/sparrow/wallet/WalletTransactionsEntry.java @@ -22,7 +22,7 @@ public class WalletTransactionsEntry extends Entry { public WalletTransactionsEntry(Wallet wallet) { super(wallet, wallet.getName(), getWalletTransactions(wallet).stream().map(WalletTransaction::getTransactionEntry).collect(Collectors.toList())); - calculateBalances(); + calculateBalances(false); //No need to resort } @Override @@ -30,12 +30,14 @@ public class WalletTransactionsEntry extends Entry { return getBalance(); } - protected void calculateBalances() { + private void calculateBalances(boolean resort) { long balance = 0L; long mempoolBalance = 0L; - //Note transaction entries must be in ascending order. This sorting is ultimately done according to BlockTransactions' comparator - getChildren().sort(Comparator.comparing(TransactionEntry.class::cast)); + if(resort) { + //Note transaction entries must be in ascending order. This sorting is ultimately done according to BlockTransactions' comparator + getChildren().sort(Comparator.comparing(TransactionEntry.class::cast)); + } for(Entry entry : getChildren()) { TransactionEntry transactionEntry = (TransactionEntry)entry; @@ -66,7 +68,7 @@ public class WalletTransactionsEntry extends Entry { entriesRemoved.removeAll(current); getChildren().removeAll(entriesRemoved); - calculateBalances(); + calculateBalances(true); Map walletTxos = getWallet().getWalletTxos().entrySet().stream() .collect(Collectors.toUnmodifiableMap(entry -> new HashIndex(entry.getKey().getHash(), entry.getKey().getIndex()), Map.Entry::getKey)); @@ -86,12 +88,14 @@ public class WalletTransactionsEntry extends Entry { } private static Collection getWalletTransactions(Wallet wallet) { - Map walletTransactionMap = new TreeMap<>(); + Map walletTransactionMap = new HashMap<>(wallet.getTransactions().size()); getWalletTransactions(wallet, walletTransactionMap, wallet.getNode(KeyPurpose.RECEIVE)); getWalletTransactions(wallet, walletTransactionMap, wallet.getNode(KeyPurpose.CHANGE)); - return new ArrayList<>(walletTransactionMap.values()); + List walletTransactions = new ArrayList<>(walletTransactionMap.values()); + Collections.sort(walletTransactions); + return walletTransactions; } private static void getWalletTransactions(Wallet wallet, Map walletTransactionMap, WalletNode purposeNode) { @@ -191,7 +195,7 @@ public class WalletTransactionsEntry extends Entry { return mempoolBalance; } - private static class WalletTransaction { + private static class WalletTransaction implements Comparable { private final Wallet wallet; private final BlockTransaction blockTransaction; private final Map incoming = new TreeMap<>(); @@ -205,5 +209,33 @@ public class WalletTransactionsEntry extends Entry { public TransactionEntry getTransactionEntry() { return new TransactionEntry(wallet, blockTransaction, incoming, outgoing); } + + public long getValue() { + long value = 0L; + for(BlockTransactionHashIndex in : incoming.keySet()) { + value += in.getValue(); + } + for(BlockTransactionHashIndex out : outgoing.keySet()) { + value -= out.getValue(); + } + + return value; + } + + @Override + public int compareTo(WalletTransactionsEntry.WalletTransaction other) { + //This comparison must be identical to that of TransactionEntry so we can avoid a resort calculating balances when creating WalletTransactionsEntry + int blockOrder = blockTransaction.compareBlockOrder(other.blockTransaction); + if(blockOrder != 0) { + return blockOrder; + } + + int valueOrder = Long.compare(other.getValue(), getValue()); + if(valueOrder != 0) { + return valueOrder; + } + + return blockTransaction.compareTo(other.blockTransaction); + } } }