From d6c7a0b75748e74dcee02c111fb6efd1ec0c4a8a Mon Sep 17 00:00:00 2001 From: Craig Raw Date: Thu, 29 Apr 2021 13:21:52 +0200 Subject: [PATCH] cache fetched script hash status and dont retrieve history again until it changes --- .../sparrowwallet/sparrow/AppController.java | 1 + .../sparrowwallet/sparrow/AppServices.java | 4 ++ .../sparrow/net/ElectrumServer.java | 60 ++++++++++++++++--- .../sparrow/wallet/WalletForm.java | 1 + 4 files changed, 57 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/sparrowwallet/sparrow/AppController.java b/src/main/java/com/sparrowwallet/sparrow/AppController.java index 81bfea33..4a8393bc 100644 --- a/src/main/java/com/sparrowwallet/sparrow/AppController.java +++ b/src/main/java/com/sparrowwallet/sparrow/AppController.java @@ -1010,6 +1010,7 @@ public class AppController implements Initializable { Wallet pastWallet = wallet.copy(); walletTabData.getStorage().backupTempWallet(); wallet.clearHistory(); + AppServices.clearTransactionHistoryCache(wallet); EventManager.get().post(new WalletAddressesChangedEvent(wallet, pastWallet, walletTabData.getStorage().getWalletFile())); } } diff --git a/src/main/java/com/sparrowwallet/sparrow/AppServices.java b/src/main/java/com/sparrowwallet/sparrow/AppServices.java index a1394a2e..a1a030f7 100644 --- a/src/main/java/com/sparrowwallet/sparrow/AppServices.java +++ b/src/main/java/com/sparrowwallet/sparrow/AppServices.java @@ -496,6 +496,10 @@ public class AppServices { payjoinURIs.put(bitcoinURI.getAddress(), bitcoinURI); } + public static void clearTransactionHistoryCache(Wallet wallet) { + ElectrumServer.clearRetrievedScriptHashes(wallet); + } + public static Optional showWarningDialog(String title, String content, ButtonType... buttons) { return showAlertDialog(title, content, Alert.AlertType.WARNING, buttons); } diff --git a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java index 343ed3e1..e3bc5e25 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java @@ -44,6 +44,10 @@ public class ElectrumServer { private static final Map> subscribedScriptHashes = Collections.synchronizedMap(new HashMap<>()); + private static String previousServerAddress; + + private static Map retrievedScriptHashes = Collections.synchronizedMap(new HashMap<>()); + private static ElectrumServerRpc electrumServerRpc = new SimpleElectrumServerRpc(); private static String bwtElectrumServer; @@ -84,6 +88,12 @@ public class ElectrumServer { throw new ServerConfigException("Electrum server URL must start with " + Protocol.TCP.toUrlString() + " or " + Protocol.SSL.toUrlString()); } + //If changing server, don't rely on previous transaction history + if(!electrumServer.equals(previousServerAddress)) { + retrievedScriptHashes.clear(); + } + previousServerAddress = electrumServer; + HostAndPort server = protocol.getServerHostAndPort(electrumServer); if(Config.get().isUseProxy() && proxyServer != null && !proxyServer.isBlank()) { @@ -150,6 +160,11 @@ public class ElectrumServer { } } + public static void clearRetrievedScriptHashes(Wallet wallet) { + wallet.getNode(KeyPurpose.RECEIVE).getChildren().stream().map(node -> getScriptHash(wallet, node)).forEach(scriptHash -> retrievedScriptHashes.remove(scriptHash)); + wallet.getNode(KeyPurpose.CHANGE).getChildren().stream().map(node -> getScriptHash(wallet, node)).forEach(scriptHash -> retrievedScriptHashes.remove(scriptHash)); + } + public Map> getHistory(Wallet wallet) throws ServerException { Map> receiveTransactionMap = new TreeMap<>(); getHistory(wallet, KeyPurpose.RECEIVE, receiveTransactionMap); @@ -177,6 +192,15 @@ public class ElectrumServer { Set newReferences = nodeTransactionMap.values().stream().flatMap(Collection::stream).filter(ref -> !wallet.getTransactions().containsKey(ref.getHash())).collect(Collectors.toSet()); getReferencedTransactions(wallet, nodeTransactionMap); + //Subscribe and retrieve transaction history from child nodes if necessary to maintain gap limit + Set keyPurposes = nodes.stream().map(WalletNode::getKeyPurpose).collect(Collectors.toUnmodifiableSet()); + for(KeyPurpose keyPurpose : keyPurposes) { + WalletNode purposeNode = wallet.getNode(keyPurpose); + getHistoryToGapLimit(wallet, nodeTransactionMap, purposeNode); + } + + log.debug("Fetched nodes history for: " + nodeTransactionMap.keySet()); + if(!newReferences.isEmpty()) { //Look for additional nodes to fetch history for by considering the inputs and outputs of new transactions found log.debug(wallet.getName() + " found new transactions: " + newReferences); @@ -216,13 +240,22 @@ public class ElectrumServer { public void getHistory(Wallet wallet, KeyPurpose keyPurpose, Map> nodeTransactionMap) throws ServerException { WalletNode purposeNode = wallet.getNode(keyPurpose); - //Subscribe to all existing address WalletNodes and add them to nodeTransactionMap as keys to empty sets if they have history + //Subscribe to all existing address WalletNodes and add them to nodeTransactionMap as keys to empty sets if they have history that needs to be fetched subscribeWalletNodes(wallet, purposeNode.getChildren(), nodeTransactionMap, 0); //All WalletNode keys in nodeTransactionMap need to have their history fetched (nodes without history will not be keys in the map yet) getReferences(wallet, nodeTransactionMap.keySet(), nodeTransactionMap, 0); //Fetch all referenced transaction to wallet transactions map. We do this now even though it is done again later to get it done before too many script hashes are subscribed getReferencedTransactions(wallet, nodeTransactionMap); + //Increase child nodes if necessary to maintain gap limit, and ensure they are subscribed and history is fetched + getHistoryToGapLimit(wallet, nodeTransactionMap, purposeNode); + log.debug("Fetched history for: " + nodeTransactionMap.keySet()); + + //Set the remaining WalletNode keys in nodeTransactionMap to empty sets to indicate no history (if no script hash history has already been retrieved in a previous call) + purposeNode.getChildren().stream().filter(node -> !nodeTransactionMap.containsKey(node) && retrievedScriptHashes.get(getScriptHash(wallet, node)) == null).forEach(node -> nodeTransactionMap.put(node, Collections.emptySet())); + } + + private void getHistoryToGapLimit(Wallet wallet, Map> nodeTransactionMap, WalletNode purposeNode) throws ServerException { //Because node children are added sequentially in WalletNode.fillToIndex, we can simply look at the number of children to determine the highest filled index int historySize = purposeNode.getChildren().size(); //The gap limit size takes the highest used index in the retrieved history and adds the gap limit (plus one to be comparable to the number of children since index is zero based) @@ -235,9 +268,6 @@ public class ElectrumServer { historySize = purposeNode.getChildren().size(); gapLimitSize = getGapLimitSize(wallet, nodeTransactionMap); } - - //Set the remaining WalletNode keys in nodeTransactionMap to empty sets to indicate no history - purposeNode.getChildren().stream().filter(node -> !nodeTransactionMap.containsKey(node)).forEach(node -> nodeTransactionMap.put(node, Collections.emptySet())); } private int getGapLimitSize(Wallet wallet, Map> nodeTransactionMap) { @@ -319,9 +349,12 @@ public class ElectrumServer { if(node != null && node.getIndex() >= startIndex) { String scriptHash = getScriptHash(wallet, node); - if(getSubscribedScriptHashStatus(scriptHash) != null) { - //Already subscribed, but still need to fetch history from a used node - nodeTransactionMap.put(node, new TreeSet<>()); + String subscribedStatus = getSubscribedScriptHashStatus(scriptHash); + if(subscribedStatus != null) { + //Already subscribed, but still need to fetch history from a used node if not previously fetched + if(!subscribedStatus.equals(retrievedScriptHashes.get(scriptHash))) { + nodeTransactionMap.put(node, new TreeSet<>()); + } } else if(!subscribedScriptHashes.containsKey(scriptHash) && scriptHashes.add(scriptHash)) { //Unique script hash we are not yet subscribed to pathScriptHashes.put(node.getDerivationPath(), scriptHash); @@ -329,6 +362,8 @@ public class ElectrumServer { } } + log.debug("Subscribe to: " + pathScriptHashes.keySet()); + if(pathScriptHashes.isEmpty()) { return; } @@ -343,8 +378,8 @@ public class ElectrumServer { WalletNode node = optionalNode.get(); String scriptHash = getScriptHash(wallet, node); - //Check if there is history for this script hash - if(status != null) { + //Check if there is history for this script hash, and if the history has changed since last fetched + if(status != null && !status.equals(retrievedScriptHashes.get(scriptHash))) { //Set the value for this node to be an empty set to mark it as requiring a get_history RPC call for this wallet nodeTransactionMap.put(node, new TreeSet<>()); } @@ -1037,6 +1072,13 @@ public class ElectrumServer { Map> nodeTransactionMap = (nodes == null ? electrumServer.getHistory(wallet) : electrumServer.getHistory(wallet, nodes)); electrumServer.getReferencedTransactions(wallet, nodeTransactionMap); electrumServer.calculateNodeHistory(wallet, nodeTransactionMap); + + //Add all of the script hashes we have now fetched the history for so we don't need to fetch again until the script hash status changes + for(WalletNode node : nodeTransactionMap.keySet()) { + String scriptHash = getScriptHash(wallet, node); + retrievedScriptHashes.put(scriptHash, getSubscribedScriptHashStatus(scriptHash)); + } + return true; } diff --git a/src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java b/src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java index 053d169b..ce291f2c 100644 --- a/src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java +++ b/src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java @@ -318,6 +318,7 @@ public class WalletForm { public void walletTabsClosed(WalletTabsClosedEvent event) { for(WalletTabData tabData : event.getClosedWalletTabData()) { if(tabData.getWalletForm() == this) { + AppServices.clearTransactionHistoryCache(wallet); EventManager.get().unregister(this); } }