From e3138f3392dbc05720a7c81c4d21d004308cae50 Mon Sep 17 00:00:00 2001 From: Craig Raw Date: Mon, 28 Apr 2025 14:39:30 +0200 Subject: [PATCH] optimize and reduce electrum server rpc calls --- .../sparrow/net/ElectrumServer.java | 131 +++++++++++------- .../sparrow/net/SubscriptionService.java | 1 + 2 files changed, 82 insertions(+), 50 deletions(-) diff --git a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java index a1cdd655..b03080b3 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java @@ -61,11 +61,13 @@ public class ElectrumServer { private static Server previousServer; - private static Map retrievedScriptHashes = Collections.synchronizedMap(new HashMap<>()); + private static final Map retrievedScriptHashes = Collections.synchronizedMap(new HashMap<>()); - private static Map retrievedTransactions = Collections.synchronizedMap(new HashMap<>()); + private static final Map retrievedTransactions = Collections.synchronizedMap(new HashMap<>()); - private static Set sameHeightTxioScriptHashes = Collections.synchronizedSet(new HashSet<>()); + private static final Map retrievedBlockHeaders = Collections.synchronizedMap(new HashMap<>()); + + private static final Set sameHeightTxioScriptHashes = Collections.synchronizedSet(new HashSet<>()); private static ElectrumServerRpc electrumServerRpc = new SimpleElectrumServerRpc(); @@ -113,6 +115,7 @@ public class ElectrumServer { if(previousServer != null && !electrumServer.equals(previousServer)) { retrievedScriptHashes.clear(); retrievedTransactions.clear(); + retrievedBlockHeaders.clear(); TransactionHistoryService.walletLocks.values().forEach(walletLock -> walletLock.initialized = false); } previousServer = electrumServer; @@ -555,22 +558,29 @@ public class ElectrumServer { } public void getReferencedTransactions(Wallet wallet, Map> nodeTransactionMap) throws ServerException { - Set references = new TreeSet<>(); + Map references = new TreeMap<>(); for(Set nodeReferences : nodeTransactionMap.values()) { - references.addAll(nodeReferences); + for(BlockTransactionHash nodeReference : nodeReferences) { + references.put(nodeReference, null); + } } - for(Iterator iter = references.iterator(); iter.hasNext(); ) { - BlockTransactionHash reference = iter.next(); - BlockTransaction blockTransaction = wallet.getTransactions().get(reference.getHash()); - if(blockTransaction != null && reference.getHeight() == blockTransaction.getHeight()) { - iter.remove(); + for(Iterator> iter = references.entrySet().iterator(); iter.hasNext(); ) { + Map.Entry entry = iter.next(); + BlockTransactionHash reference = entry.getKey(); + BlockTransaction blockTransaction = wallet.getWalletTransaction(reference.getHash()); + if(blockTransaction != null) { + if(reference.getHeight() == blockTransaction.getHeight()) { + iter.remove(); + } else { + entry.setValue(blockTransaction.getTransaction()); + } } } Map transactionMap = new HashMap<>(); if(!references.isEmpty()) { - Map blockHeaderMap = getBlockHeaders(wallet, references); + Map blockHeaderMap = getBlockHeaders(wallet, references.keySet()); transactionMap = getTransactions(wallet, references, blockHeaderMap); } @@ -581,24 +591,29 @@ public class ElectrumServer { public Map getBlockHeaders(Wallet wallet, Set references) throws ServerException { try { + Map blockHeaderMap = new TreeMap<>(); Set blockHeights = new TreeSet<>(); for(BlockTransactionHash reference : references) { if(reference.getHeight() > 0) { - blockHeights.add(reference.getHeight()); + if(retrievedBlockHeaders.get(reference.getHeight()) != null) { + blockHeaderMap.put(reference.getHeight(), retrievedBlockHeaders.get(reference.getHeight())); + } else { + blockHeights.add(reference.getHeight()); + } } } if(blockHeights.isEmpty()) { - return Collections.emptyMap(); + return blockHeaderMap; } Map result = electrumServerRpc.getBlockHeaders(getTransport(), wallet, blockHeights); - Map blockHeaderMap = new TreeMap<>(); for(Integer height : result.keySet()) { byte[] blockHeaderBytes = Utils.hexToBytes(result.get(height)); BlockHeader blockHeader = new BlockHeader(blockHeaderBytes); blockHeaderMap.put(height, blockHeader); + updateRetrievedBlockHeaders(height, blockHeader); blockHeights.remove(height); } @@ -616,51 +631,61 @@ public class ElectrumServer { } } - public Map getTransactions(Wallet wallet, Set references, Map blockHeaderMap) throws ServerException { + public Map getTransactions(Wallet wallet, Map references, Map blockHeaderMap) throws ServerException { try { - Set checkReferences = new TreeSet<>(references); + Map transactionMap = new HashMap<>(); + Set checkReferences = new TreeSet<>(references.keySet()); Set txids = new LinkedHashSet<>(references.size()); - for(BlockTransactionHash reference : references) { - txids.add(reference.getHashAsString()); + for(BlockTransactionHash reference : references.keySet()) { + if(references.get(reference) == null) { + txids.add(reference.getHashAsString()); + } } - Map result = electrumServerRpc.getTransactions(getTransport(), wallet, txids); + if(!txids.isEmpty()) { + Map result = electrumServerRpc.getTransactions(getTransport(), wallet, txids); - String strErrorTx = Sha256Hash.ZERO_HASH.toString(); - Map transactionMap = new HashMap<>(); - for(String txid : result.keySet()) { - Sha256Hash hash = Sha256Hash.wrap(txid); - String strRawTx = result.get(txid); + String strErrorTx = Sha256Hash.ZERO_HASH.toString(); + for(String txid : result.keySet()) { + Sha256Hash hash = Sha256Hash.wrap(txid); + String strRawTx = result.get(txid); - if(strRawTx.equals(strErrorTx)) { - transactionMap.put(hash, UNFETCHABLE_BLOCK_TRANSACTION); - checkReferences.removeIf(ref -> ref.getHash().equals(hash)); - continue; + if(strRawTx.equals(strErrorTx)) { + transactionMap.put(hash, UNFETCHABLE_BLOCK_TRANSACTION); + checkReferences.removeIf(ref -> ref.getHash().equals(hash)); + continue; + } + + byte[] rawtx = Utils.hexToBytes(strRawTx); + Transaction transaction; + + try { + transaction = new Transaction(rawtx); + } catch(ProtocolException e) { + log.error("Could not parse tx: " + strRawTx); + continue; + } + + Optional optionalReference = references.keySet().stream().filter(reference -> reference.getHash().equals(hash)).findFirst(); + if(optionalReference.isEmpty()) { + throw new IllegalStateException("Returned transaction " + hash.toString() + " that was not requested"); + } + BlockTransactionHash reference = optionalReference.get(); + + references.put(reference, transaction); } + } - byte[] rawtx = Utils.hexToBytes(strRawTx); - Transaction transaction; - - try { - transaction = new Transaction(rawtx); - } catch(ProtocolException e) { - log.error("Could not parse tx: " + strRawTx); - continue; - } - - Optional optionalReference = references.stream().filter(reference -> reference.getHash().equals(hash)).findFirst(); - if(optionalReference.isEmpty()) { - throw new IllegalStateException("Returned transaction " + hash.toString() + " that was not requested"); - } - BlockTransactionHash reference = optionalReference.get(); + for(BlockTransactionHash reference : references.keySet()) { + Transaction transaction = references.get(reference); Date blockDate = null; if(reference.getHeight() > 0) { BlockHeader blockHeader = blockHeaderMap.get(reference.getHeight()); if(blockHeader == null) { - transactionMap.put(hash, UNFETCHABLE_BLOCK_TRANSACTION); - checkReferences.removeIf(ref -> ref.getHash().equals(hash)); + transactionMap.put(reference.getHash(), UNFETCHABLE_BLOCK_TRANSACTION); + checkReferences.removeIf(ref -> ref.getHash().equals(reference.getHash())); continue; } blockDate = blockHeader.getTimeAsDate(); @@ -668,7 +693,7 @@ public class ElectrumServer { BlockTransaction blockchainTransaction = new BlockTransaction(reference.getHash(), reference.getHeight(), blockDate, reference.getFee(), transaction); - transactionMap.put(hash, blockchainTransaction); + transactionMap.put(reference.getHash(), blockchainTransaction); checkReferences.remove(reference); } @@ -1024,6 +1049,10 @@ public class ElectrumServer { existingStatuses.add(status); } + public static void updateRetrievedBlockHeaders(Integer blockHeight, BlockHeader blockHeader) { + retrievedBlockHeaders.put(blockHeight, blockHeader); + } + public static ServerCapability getServerCapability(List serverVersion) { if(!serverVersion.isEmpty()) { String server = serverVersion.getFirst().toLowerCase(Locale.ROOT); @@ -1630,15 +1659,17 @@ public class ElectrumServer { ElectrumServer electrumServer = new ElectrumServer(); List> outputTransactionReferences = electrumServer.getOutputTransactionReferences(transaction, indexStart, indexEnd, blockTransactionHashes); - Set setReferences = new HashSet<>(); + Map setReferences = new HashMap<>(); for(Set outputReferences : outputTransactionReferences) { if(outputReferences != null) { - setReferences.addAll(outputReferences); + for(BlockTransactionHash outputReference : outputReferences) { + setReferences.put(outputReference, null); + } } } setReferences.remove(null); setReferences.remove(UNFETCHABLE_BLOCK_TRANSACTION); - setReferences.removeIf(ref -> transactionMap.get(ref.getHash()) != null); + setReferences.keySet().removeIf(ref -> transactionMap.get(ref.getHash()) != null); List blockTransactions = new ArrayList<>(transaction.getOutputs().size()); for(int i = 0; i < transaction.getOutputs().size(); i++) { @@ -1646,7 +1677,7 @@ public class ElectrumServer { } if(!setReferences.isEmpty()) { - Map blockHeaderMap = electrumServer.getBlockHeaders(null, setReferences); + Map blockHeaderMap = electrumServer.getBlockHeaders(null, setReferences.keySet()); transactionMap.putAll(electrumServer.getTransactions(null, setReferences, blockHeaderMap)); } diff --git a/src/main/java/com/sparrowwallet/sparrow/net/SubscriptionService.java b/src/main/java/com/sparrowwallet/sparrow/net/SubscriptionService.java index 20b026b3..44d8de37 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/SubscriptionService.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/SubscriptionService.java @@ -20,6 +20,7 @@ public class SubscriptionService { @JsonRpcMethod("blockchain.headers.subscribe") public void newBlockHeaderTip(@JsonRpcParam("header") final BlockHeaderTip header) { + ElectrumServer.updateRetrievedBlockHeaders(header.height, header.getBlockHeader()); Platform.runLater(() -> EventManager.get().post(new NewBlockEvent(header.height, header.getBlockHeader()))); }