From 3d85491e6b0d447b60221b31e910de5a527be624 Mon Sep 17 00:00:00 2001 From: Craig Raw Date: Mon, 5 May 2025 14:43:42 +0200 Subject: [PATCH 1/8] add block summary service --- .../sparrowwallet/sparrow/AppServices.java | 45 +++++- .../sparrowwallet/sparrow/BlockSummary.java | 65 ++++++++ .../sparrow/event/BlockSummaryEvent.java | 17 ++ .../sparrow/net/BatchedElectrumServerRpc.java | 21 +++ .../sparrow/net/ElectrumServer.java | 147 +++++++++++++++++ .../sparrow/net/ElectrumServerRpc.java | 2 + .../sparrow/net/FeeRatesSource.java | 148 +++++++++++++++++- .../sparrow/net/SimpleElectrumServerRpc.java | 22 ++- .../sparrow/net/SubscriptionService.java | 3 +- 9 files changed, 460 insertions(+), 10 deletions(-) create mode 100644 src/main/java/com/sparrowwallet/sparrow/BlockSummary.java create mode 100644 src/main/java/com/sparrowwallet/sparrow/event/BlockSummaryEvent.java diff --git a/src/main/java/com/sparrowwallet/sparrow/AppServices.java b/src/main/java/com/sparrowwallet/sparrow/AppServices.java index 36222a75..dcfd8c13 100644 --- a/src/main/java/com/sparrowwallet/sparrow/AppServices.java +++ b/src/main/java/com/sparrowwallet/sparrow/AppServices.java @@ -26,6 +26,8 @@ import com.sparrowwallet.sparrow.control.TrayManager; import com.sparrowwallet.sparrow.event.*; import com.sparrowwallet.sparrow.io.*; import com.sparrowwallet.sparrow.net.*; +import io.reactivex.rxjavafx.schedulers.JavaFxScheduler; +import io.reactivex.subjects.PublishSubject; import javafx.application.Application; import javafx.application.Platform; import javafx.beans.property.BooleanProperty; @@ -43,7 +45,6 @@ import javafx.scene.Scene; import javafx.scene.control.*; import javafx.scene.control.Dialog; import javafx.scene.image.Image; -import javafx.scene.image.ImageView; import javafx.scene.input.KeyCode; import javafx.scene.text.Font; import javafx.stage.Screen; @@ -67,6 +68,8 @@ import java.time.ZonedDateTime; import java.time.temporal.ChronoUnit; import java.util.*; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static com.sparrowwallet.sparrow.control.DownloadVerifierDialog.*; @@ -105,6 +108,8 @@ public class AppServices { private TrayManager trayManager; + private final PublishSubject newBlockSubject = PublishSubject.create(); + private static Image windowIcon; private static final BooleanProperty onlineProperty = new SimpleBooleanProperty(false); @@ -127,6 +132,8 @@ public class AppServices { private static BlockHeader latestBlockHeader; + private static final Map blockSummaries = new ConcurrentHashMap<>(); + private static Map targetBlockFeeRates; private static final TreeMap> mempoolHistogram = new TreeMap<>(); @@ -183,6 +190,12 @@ public class AppServices { private AppServices(Application application, InteractionServices interactionServices) { this.application = application; this.interactionServices = interactionServices; + + newBlockSubject.buffer(4, TimeUnit.SECONDS) + .filter(newBlockEvents -> !newBlockEvents.isEmpty()) + .observeOn(JavaFxScheduler.platform()) + .subscribe(this::fetchBlockSummaries, exception -> log.error("Error fetching block summaries", exception)); + EventManager.get().register(this); } @@ -481,6 +494,19 @@ public class AppServices { } } + private void fetchBlockSummaries(List newBlockEvents) { + if(isConnected()) { + ElectrumServer.BlockSummaryService blockSummaryService = new ElectrumServer.BlockSummaryService(newBlockEvents); + blockSummaryService.setOnSucceeded(_ -> { + EventManager.get().post(blockSummaryService.getValue()); + }); + blockSummaryService.setOnFailed(failedState -> { + log.error("Error fetching block summaries", failedState.getSource().getException()); + }); + blockSummaryService.start(); + } + } + public static boolean isTorRunning() { return Tor.getDefault() != null; } @@ -706,6 +732,10 @@ public class AppServices { return latestBlockHeader; } + public static Map getBlockSummaries() { + return blockSummaries; + } + public static Double getDefaultFeeRate() { int defaultTarget = TARGET_BLOCKS_RANGE.get((TARGET_BLOCKS_RANGE.size() / 2) - 1); return getTargetBlockFeeRates() == null ? getFallbackFeeRate() : getTargetBlockFeeRates().get(defaultTarget); @@ -1185,6 +1215,10 @@ public class AppServices { minimumRelayFeeRate = Math.max(event.getMinimumRelayFeeRate(), Transaction.DEFAULT_MIN_RELAY_FEE); latestBlockHeader = event.getBlockHeader(); Config.get().addRecentServer(); + + if(!blockSummaries.containsKey(currentBlockHeight)) { + fetchBlockSummaries(Collections.emptyList()); + } } @Subscribe @@ -1199,6 +1233,15 @@ public class AppServices { latestBlockHeader = event.getBlockHeader(); String status = "Updating to new block height " + event.getHeight(); EventManager.get().post(new StatusEvent(status)); + newBlockSubject.onNext(event); + } + + @Subscribe + public void blockSummary(BlockSummaryEvent event) { + blockSummaries.putAll(event.getBlockSummaryMap()); + if(AppServices.currentBlockHeight != null) { + blockSummaries.keySet().removeIf(height -> AppServices.currentBlockHeight - height > 5); + } } @Subscribe diff --git a/src/main/java/com/sparrowwallet/sparrow/BlockSummary.java b/src/main/java/com/sparrowwallet/sparrow/BlockSummary.java new file mode 100644 index 00000000..1c9c9a41 --- /dev/null +++ b/src/main/java/com/sparrowwallet/sparrow/BlockSummary.java @@ -0,0 +1,65 @@ +package com.sparrowwallet.sparrow; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Date; +import java.util.Optional; + +public class BlockSummary { + private final Integer height; + private final Date timestamp; + private final Double medianFee; + private final Integer transactionCount; + + public BlockSummary(Integer height, Date timestamp) { + this(height, timestamp, null, null); + } + + public BlockSummary(Integer height, Date timestamp, Double medianFee, Integer transactionCount) { + this.height = height; + this.timestamp = timestamp; + this.medianFee = medianFee; + this.transactionCount = transactionCount; + } + + public Integer getHeight() { + return height; + } + + public Date getTimestamp() { + return timestamp; + } + + public Optional getMedianFee() { + return medianFee == null ? Optional.empty() : Optional.of(medianFee); + } + + public Optional getTransactionCount() { + return transactionCount == null ? Optional.empty() : Optional.of(transactionCount); + } + + private static long calculateElapsedSeconds(long timestampUtc) { + Instant timestampInstant = Instant.ofEpochMilli(timestampUtc); + Instant nowInstant = Instant.now(); + return ChronoUnit.SECONDS.between(timestampInstant, nowInstant); + } + + public String getElapsed() { + long elapsed = calculateElapsedSeconds(getTimestamp().getTime()); + if(elapsed < 0) { + return "now"; + } else if(elapsed < 60) { + return elapsed + "s"; + } else if(elapsed < 3600) { + return elapsed / 60 + "m"; + } else if(elapsed < 86400) { + return elapsed / 3600 + "h"; + } else { + return elapsed / 86400 + "d"; + } + } + + public String toString() { + return getElapsed() + ":" + getMedianFee(); + } +} diff --git a/src/main/java/com/sparrowwallet/sparrow/event/BlockSummaryEvent.java b/src/main/java/com/sparrowwallet/sparrow/event/BlockSummaryEvent.java new file mode 100644 index 00000000..4cd74e14 --- /dev/null +++ b/src/main/java/com/sparrowwallet/sparrow/event/BlockSummaryEvent.java @@ -0,0 +1,17 @@ +package com.sparrowwallet.sparrow.event; + +import com.sparrowwallet.sparrow.BlockSummary; + +import java.util.Map; + +public class BlockSummaryEvent { + private final Map blockSummaryMap; + + public BlockSummaryEvent(Map blockSummaryMap) { + this.blockSummaryMap = blockSummaryMap; + } + + public Map getBlockSummaryMap() { + return blockSummaryMap; + } +} diff --git a/src/main/java/com/sparrowwallet/sparrow/net/BatchedElectrumServerRpc.java b/src/main/java/com/sparrowwallet/sparrow/net/BatchedElectrumServerRpc.java index d9b66ad3..588541ed 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/BatchedElectrumServerRpc.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/BatchedElectrumServerRpc.java @@ -151,6 +151,27 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc { } } + @Override + public Map unsubscribeScriptHashes(Transport transport, Set scriptHashes) { + PagedBatchRequestBuilder batchRequest = PagedBatchRequestBuilder.create(transport, idCounter).keysType(String.class).returnType(Boolean.class); + + for(String scriptHash : scriptHashes) { + batchRequest.add(scriptHash, "blockchain.scripthash.unsubscribe", scriptHash); + } + + try { + return batchRequest.execute(); + } catch(JsonRpcBatchException e) { + log.warn("Failed to unsubscribe from script hashes: " + e.getErrors().keySet(), e); + Map unsubscribedScriptHashes = scriptHashes.stream().collect(Collectors.toMap(s -> s, _ -> true)); + unsubscribedScriptHashes.keySet().removeIf(scriptHash -> e.getErrors().containsKey(scriptHash)); + return unsubscribedScriptHashes; + } catch(Exception e) { + log.warn("Failed to unsubscribe from script hashes: " + scriptHashes, e); + return Collections.emptyMap(); + } + } + @Override @SuppressWarnings("unchecked") public Map getBlockHeaders(Transport transport, Wallet wallet, Set blockHeights) { diff --git a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java index 4850d2a9..5b1d5201 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java @@ -12,6 +12,7 @@ import com.sparrowwallet.drongo.bip47.PaymentCode; import com.sparrowwallet.drongo.protocol.*; import com.sparrowwallet.drongo.wallet.*; import com.sparrowwallet.sparrow.AppServices; +import com.sparrowwallet.sparrow.BlockSummary; import com.sparrowwallet.sparrow.EventManager; import com.sparrowwallet.sparrow.event.*; import com.sparrowwallet.sparrow.io.Config; @@ -32,11 +33,13 @@ import org.slf4j.LoggerFactory; import java.io.*; import java.nio.charset.StandardCharsets; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; public class ElectrumServer { @@ -946,6 +949,71 @@ public class ElectrumServer { return Transaction.DEFAULT_MIN_RELAY_FEE; } + public Map getRecentBlockSummaryMap() throws ServerException { + return getBlockSummaryMap(null, null); + } + + public Map getBlockSummaryMap(Integer height, BlockHeader blockHeader) throws ServerException { + FeeRatesSource feeRatesSource = Config.get().getFeeRatesSource(); + feeRatesSource = (feeRatesSource == null ? FeeRatesSource.MEMPOOL_SPACE : feeRatesSource); + + if(feeRatesSource.supportsNetwork(Network.get())) { + try { + if(blockHeader == null) { + return feeRatesSource.getRecentBlockSummaries(); + } else { + Map blockSummaryMap = new HashMap<>(); + BlockSummary blockSummary = feeRatesSource.getBlockSummary(Sha256Hash.twiceOf(blockHeader.bitcoinSerialize())); + if(blockSummary != null && blockSummary.getHeight() != null) { + blockSummaryMap.put(blockSummary.getHeight(), blockSummary); + } + return blockSummaryMap; + } + } catch(Exception e) { + return getServerBlockSummaryMap(height, blockHeader); + } + } else { + return getServerBlockSummaryMap(height, blockHeader); + } + } + + private Map getServerBlockSummaryMap(Integer height, BlockHeader blockHeader) throws ServerException { + if(blockHeader == null || height == null) { + Integer current = AppServices.getCurrentBlockHeight(); + if(current == null) { + return Collections.emptyMap(); + } + Set references = IntStream.range(current - 4, current + 1) + .mapToObj(i -> new BlockTransaction(null, i, null, null, null)).collect(Collectors.toSet()); + Map blockHeaders = getBlockHeaders(null, references); + return blockHeaders.keySet().stream() + .collect(Collectors.toMap(java.util.function.Function.identity(), v -> new BlockSummary(v, blockHeaders.get(v).getTimeAsDate()))); + } else { + Map blockSummaryMap = new HashMap<>(); + blockSummaryMap.put(height, new BlockSummary(height, blockHeader.getTimeAsDate())); + return blockSummaryMap; + } + } + + public List getRecentMempoolTransactions() { + FeeRatesSource feeRatesSource = Config.get().getFeeRatesSource(); + feeRatesSource = (feeRatesSource == null ? FeeRatesSource.MEMPOOL_SPACE : feeRatesSource); + + if(feeRatesSource.supportsNetwork(Network.get())) { + try { + List recentTransactions = feeRatesSource.getRecentMempoolTransactions(); + Map setReferences = new HashMap<>(); + setReferences.put(recentTransactions.getFirst(), null); + Map transactions = getTransactions(null, setReferences, Collections.emptyMap()); + return transactions.values().stream().filter(blxTx -> blxTx.getTransaction() != null).toList(); + } catch(Exception e) { + return Collections.emptyList(); + } + } else { + return Collections.emptyList(); + } + } + public Sha256Hash broadcastTransactionPrivately(Transaction transaction) throws ServerException { //If Tor proxy is configured, try all external broadcast sources in random order before falling back to connected Electrum server if(AppServices.isUsingProxy()) { @@ -1809,6 +1877,85 @@ public class ElectrumServer { } } + public static class BlockSummaryService extends Service { + private final List newBlockEvents; + + public BlockSummaryService(List newBlockEvents) { + this.newBlockEvents = newBlockEvents; + } + + @Override + protected Task createTask() { + return new Task<>() { + protected BlockSummaryEvent call() throws ServerException { + ElectrumServer electrumServer = new ElectrumServer(); + Map blockSummaryMap = new LinkedHashMap<>(); + + int maxHeight = AppServices.getBlockSummaries().keySet().stream().mapToInt(Integer::intValue).max().orElse(0); + int startHeight = newBlockEvents.stream().mapToInt(NewBlockEvent::getHeight).min().orElse(0); + int endHeight = newBlockEvents.stream().mapToInt(NewBlockEvent::getHeight).max().orElse(0); + int totalBlocks = Math.max(0, endHeight - maxHeight); + + if(startHeight == 0 || totalBlocks > 1 || startHeight > maxHeight + 1) { + if(isBlockstorm(totalBlocks)) { + for(int height = maxHeight + 1; height < endHeight; height++) { + blockSummaryMap.put(height, new BlockSummary(height, new Date())); + } + } else { + blockSummaryMap.putAll(electrumServer.getRecentBlockSummaryMap()); + } + } else { + for(NewBlockEvent event : newBlockEvents) { + blockSummaryMap.putAll(electrumServer.getBlockSummaryMap(event.getHeight(), event.getBlockHeader())); + } + } + + Config config = Config.get(); + if(!isBlockstorm(totalBlocks) && !AppServices.isUsingProxy() && config.getServer().getProtocol().equals(Protocol.SSL) + && (config.getServerType() == ServerType.PUBLIC_ELECTRUM_SERVER || config.getServerType() == ServerType.ELECTRUM_SERVER)) { + subscribeRecent(electrumServer); + } + + return new BlockSummaryEvent(blockSummaryMap); + } + }; + } + + private boolean isBlockstorm(int totalBlocks) { + return Network.get() != Network.MAINNET && totalBlocks > 2; + } + + private final static Set subscribedRecent = Collections.newSetFromMap(new ConcurrentHashMap<>()); + + private void subscribeRecent(ElectrumServer electrumServer) { + Set unsubscribeScriptHashes = new HashSet<>(subscribedRecent); + unsubscribeScriptHashes.removeIf(subscribedScriptHashes::containsKey); + electrumServerRpc.unsubscribeScriptHashes(transport, unsubscribeScriptHashes); + subscribedRecent.removeAll(unsubscribeScriptHashes); + + Map subscribeScriptHashes = new HashMap<>(); + List recentTransactions = electrumServer.getRecentMempoolTransactions(); + for(BlockTransaction blkTx : recentTransactions) { + for(int i = 0; i < blkTx.getTransaction().getOutputs().size() && subscribeScriptHashes.size() < 10; i++) { + TransactionOutput txOutput = blkTx.getTransaction().getOutputs().get(i); + String scriptHash = getScriptHash(txOutput); + if(!subscribedScriptHashes.containsKey(scriptHash)) { + subscribeScriptHashes.put("m/" + i, getScriptHash(txOutput)); + } + } + } + + if(!subscribeScriptHashes.isEmpty()) { + try { + electrumServerRpc.subscribeScriptHashes(transport, null, subscribeScriptHashes); + subscribedRecent.addAll(subscribeScriptHashes.values()); + } catch(ElectrumServerRpcException e) { + log.debug("Error subscribing to recent mempool transactions", e); + } + } + } + } + public static class WalletDiscoveryService extends Service> { private final List wallets; diff --git a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServerRpc.java b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServerRpc.java index cc30e196..92e2be52 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServerRpc.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServerRpc.java @@ -22,6 +22,8 @@ public interface ElectrumServerRpc { Map subscribeScriptHashes(Transport transport, Wallet wallet, Map pathScriptHashes); + Map unsubscribeScriptHashes(Transport transport, Set scriptHashes); + Map getBlockHeaders(Transport transport, Wallet wallet, Set blockHeights); Map getTransactions(Transport transport, Wallet wallet, Set txids); diff --git a/src/main/java/com/sparrowwallet/sparrow/net/FeeRatesSource.java b/src/main/java/com/sparrowwallet/sparrow/net/FeeRatesSource.java index 3466f3db..9e7bbd79 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/FeeRatesSource.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/FeeRatesSource.java @@ -1,13 +1,16 @@ package com.sparrowwallet.sparrow.net; import com.sparrowwallet.drongo.Network; +import com.sparrowwallet.drongo.Utils; +import com.sparrowwallet.drongo.protocol.Sha256Hash; +import com.sparrowwallet.drongo.wallet.BlockTransaction; +import com.sparrowwallet.drongo.wallet.BlockTransactionHash; import com.sparrowwallet.sparrow.AppServices; +import com.sparrowwallet.sparrow.BlockSummary; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; +import java.util.*; public enum FeeRatesSource { ELECTRUM_SERVER("Server", false) { @@ -24,11 +27,34 @@ public enum FeeRatesSource { MEMPOOL_SPACE("mempool.space", true) { @Override public Map getBlockTargetFeeRates(Map defaultblockTargetFeeRates) { - String url = AppServices.isUsingProxy() ? "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1/fees/recommended" : "https://mempool.space/api/v1/fees/recommended"; + String url = getApiUrl() + "v1/fees/recommended"; + return getThreeTierFeeRates(this, defaultblockTargetFeeRates, url); + } + + @Override + public BlockSummary getBlockSummary(Sha256Hash blockId) throws Exception { + String url = getApiUrl() + "v1/block/" + Utils.bytesToHex(blockId.getReversedBytes()); + return requestBlockSummary(this, url); + } + + @Override + public Map getRecentBlockSummaries() throws Exception { + String url = getApiUrl() + "v1/blocks"; + return requestBlockSummaries(this, url); + } + + @Override + public List getRecentMempoolTransactions() throws Exception { + String url = getApiUrl() + "mempool/recent"; + return requestRecentMempoolTransactions(this, url); + } + + private String getApiUrl() { + String url = AppServices.isUsingProxy() ? "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/" : "https://mempool.space/api/"; if(Network.get() != Network.MAINNET && supportsNetwork(Network.get())) { url = url.replace("/api/", "/" + Network.get().getName() + "/api/"); } - return getThreeTierFeeRates(this, defaultblockTargetFeeRates, url); + return url; } @Override @@ -101,6 +127,18 @@ public enum FeeRatesSource { public abstract Map getBlockTargetFeeRates(Map defaultblockTargetFeeRates); + public BlockSummary getBlockSummary(Sha256Hash blockId) throws Exception { + throw new UnsupportedOperationException(name + " does not support block summaries"); + } + + public Map getRecentBlockSummaries() throws Exception { + throw new UnsupportedOperationException(name + " does not support block summaries"); + } + + public List getRecentMempoolTransactions() throws Exception { + throw new UnsupportedOperationException(name + " does not support recent mempool transactions"); + } + public abstract boolean supportsNetwork(Network network); public String getName() { @@ -158,6 +196,80 @@ public enum FeeRatesSource { return httpClientService.requestJson(url, ThreeTierRates.class, null); } + protected static BlockSummary requestBlockSummary(FeeRatesSource feeRatesSource, String url) throws Exception { + if(log.isInfoEnabled()) { + log.info("Requesting block summary from " + url); + } + + HttpClientService httpClientService = AppServices.getHttpClientService(); + try { + MempoolBlockSummary mempoolBlockSummary = feeRatesSource.requestBlockSummary(url, httpClientService); + return mempoolBlockSummary.toBlockSummary(); + } catch (Exception e) { + if(log.isDebugEnabled()) { + log.warn("Error retrieving block summary from " + url, e); + } else { + log.warn("Error retrieving block summary from " + url + " (" + e.getMessage() + ")"); + } + + throw e; + } + } + + protected MempoolBlockSummary requestBlockSummary(String url, HttpClientService httpClientService) throws Exception { + return httpClientService.requestJson(url, MempoolBlockSummary.class, null); + } + + protected static Map requestBlockSummaries(FeeRatesSource feeRatesSource, String url) throws Exception { + if(log.isInfoEnabled()) { + log.info("Requesting block summaries from " + url); + } + + Map blockSummaryMap = new LinkedHashMap<>(); + HttpClientService httpClientService = AppServices.getHttpClientService(); + try { + MempoolBlockSummary[] blockSummaries = feeRatesSource.requestBlockSummaries(url, httpClientService); + for(MempoolBlockSummary blockSummary : blockSummaries) { + if(blockSummary.height != null) { + blockSummaryMap.put(blockSummary.height, blockSummary.toBlockSummary()); + } + } + return blockSummaryMap; + } catch (Exception e) { + if(log.isDebugEnabled()) { + log.warn("Error retrieving block summaries from " + url, e); + } else { + log.warn("Error retrieving block summaries from " + url + " (" + e.getMessage() + ")"); + } + + throw e; + } + } + + protected MempoolBlockSummary[] requestBlockSummaries(String url, HttpClientService httpClientService) throws Exception { + return httpClientService.requestJson(url, MempoolBlockSummary[].class, null); + } + + protected List requestRecentMempoolTransactions(FeeRatesSource feeRatesSource, String url) throws Exception { + HttpClientService httpClientService = AppServices.getHttpClientService(); + try { + MempoolRecentTransaction[] recentTransactions = feeRatesSource.requestRecentMempoolTransactions(url, httpClientService); + return Arrays.stream(recentTransactions).sorted().map(tx -> (BlockTransactionHash)new BlockTransaction(tx.txid, 0, null, tx.fee, null)).toList(); + } catch (Exception e) { + if(log.isDebugEnabled()) { + log.warn("Error retrieving recent mempool transactions from " + url, e); + } else { + log.warn("Error retrieving recent mempool from " + url + " (" + e.getMessage() + ")"); + } + + throw e; + } + } + + protected MempoolRecentTransaction[] requestRecentMempoolTransactions(String url, HttpClientService httpClientService) throws Exception { + return httpClientService.requestJson(url, MempoolRecentTransaction[].class, null); + } + @Override public String toString() { return name; @@ -172,4 +284,30 @@ public enum FeeRatesSource { return new ThreeTierRates(recommended_fee_099/1000, recommended_fee_090/1000, recommended_fee_050/1000, null); } } + + protected record MempoolBlockSummary(String id, Integer height, Long timestamp, Integer tx_count, MempoolBlockSummaryExtras extras) { + public Double getMedianFee() { + return extras == null ? null : extras.medianFee(); + } + + public BlockSummary toBlockSummary() { + if(height == null || timestamp == null) { + throw new IllegalStateException("Height = " + height + ", timestamp = " + timestamp + ": both must be specified"); + } + return new BlockSummary(height, new Date(timestamp * 1000), getMedianFee(), tx_count); + } + } + + private record MempoolBlockSummaryExtras(Double medianFee) {} + + protected record MempoolRecentTransaction(Sha256Hash txid, Long fee, Long vsize) implements Comparable { + private Double getFeeRate() { + return fee == null || vsize == null ? 0.0d : (double)fee / vsize; + } + + @Override + public int compareTo(MempoolRecentTransaction o) { + return Double.compare(o.getFeeRate(), getFeeRate()); + } + } } diff --git a/src/main/java/com/sparrowwallet/sparrow/net/SimpleElectrumServerRpc.java b/src/main/java/com/sparrowwallet/sparrow/net/SimpleElectrumServerRpc.java index f407ea5f..d3947a48 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/SimpleElectrumServerRpc.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/SimpleElectrumServerRpc.java @@ -123,9 +123,9 @@ public class SimpleElectrumServerRpc implements ElectrumServerRpc { for(String path : pathScriptHashes.keySet()) { EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Finding transactions for " + path)); try { - String scriptHash = new RetryLogic(MAX_RETRIES, RETRY_DELAY, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(() -> + String scriptHashStatus = new RetryLogic(MAX_RETRIES, RETRY_DELAY, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(() -> client.createRequest().returnAs(String.class).method("blockchain.scripthash.subscribe").id(idCounter.incrementAndGet()).params(pathScriptHashes.get(path)).executeNullable()); - result.put(path, scriptHash); + result.put(path, scriptHashStatus); } catch(Exception e) { //Even if we have some successes, failure to subscribe for all script hashes will result in outdated wallet view. Don't proceed. throw new ElectrumServerRpcException("Failed to subscribe to path: " + path, e); @@ -135,6 +135,24 @@ public class SimpleElectrumServerRpc implements ElectrumServerRpc { return result; } + @Override + public Map unsubscribeScriptHashes(Transport transport, Set scriptHashes) { + JsonRpcClient client = new JsonRpcClient(transport); + + Map result = new LinkedHashMap<>(); + for(String scriptHash : scriptHashes) { + try { + Boolean wasSubscribed = new RetryLogic(MAX_RETRIES, RETRY_DELAY, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(() -> + client.createRequest().returnAs(Boolean.class).method("blockchain.scripthash.unsubscribe").id(idCounter.incrementAndGet()).params(scriptHash).executeNullable()); + result.put(scriptHash, wasSubscribed); + } catch(Exception e) { + log.warn("Failed to unsubscribe from script hash: " + scriptHash, e); + } + } + + return result; + } + @Override public Map getBlockHeaders(Transport transport, Wallet wallet, Set blockHeights) { JsonRpcClient client = new JsonRpcClient(transport); diff --git a/src/main/java/com/sparrowwallet/sparrow/net/SubscriptionService.java b/src/main/java/com/sparrowwallet/sparrow/net/SubscriptionService.java index 44d8de37..1410a0a0 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/SubscriptionService.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/SubscriptionService.java @@ -28,8 +28,7 @@ public class SubscriptionService { public void scriptHashStatusUpdated(@JsonRpcParam("scripthash") final String scriptHash, @JsonRpcOptional @JsonRpcParam("status") final String status) { List existingStatuses = ElectrumServer.getSubscribedScriptHashes().get(scriptHash); if(existingStatuses == null) { - log.debug("Received script hash status update for unsubscribed script hash: " + scriptHash); - ElectrumServer.updateSubscribedScriptHashStatus(scriptHash, status); + log.trace("Received script hash status update for non-wallet script hash: " + scriptHash); } else if(status != null && existingStatuses.contains(status)) { log.debug("Received script hash status update, but status has not changed"); return; From 53c5a8d2df0d8e2e3e7092b19d95f0cb50f52e74 Mon Sep 17 00:00:00 2001 From: Craig Raw Date: Mon, 5 May 2025 14:51:15 +0200 Subject: [PATCH 2/8] update kmp-tor to 2.2.1 and remove runtime module config --- build.gradle | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/build.gradle b/build.gradle index 45d5f124..c83bf27a 100644 --- a/build.gradle +++ b/build.gradle @@ -75,8 +75,8 @@ dependencies { implementation('com.sparrowwallet:hummingbird:1.7.4') implementation('co.nstant.in:cbor:0.9') implementation('org.openpnp:openpnp-capture-java:0.0.28-5') - implementation("io.matthewnelson.kmp-tor:runtime:2.2.0") - implementation("io.matthewnelson.kmp-tor:resource-exec-tor-gpl:408.15.0") + implementation("io.matthewnelson.kmp-tor:runtime:2.2.1") + implementation("io.matthewnelson.kmp-tor:resource-exec-tor-gpl:408.16.0") implementation('org.jetbrains.kotlinx:kotlinx-coroutines-javafx:1.10.1') { exclude group: 'org.jetbrains.kotlin', module: 'kotlin-stdlib-common' } @@ -161,10 +161,6 @@ application { "--add-opens=java.base/java.net=com.sparrowwallet.sparrow", "--add-opens=java.base/java.io=com.google.gson", "--add-opens=java.smartcardio/sun.security.smartcardio=com.sparrowwallet.sparrow", - "--add-opens=io.matthewnelson.kmp.tor.resource.geoip/io.matthewnelson.kmp.tor.resource.geoip=io.matthewnelson.kmp.tor.common.core", - "--add-exports=io.matthewnelson.kmp.tor.runtime.ctrl/io.matthewnelson.kmp.tor.runtime.ctrl.internal=io.matthewnelson.kmp.tor.runtime", - "--add-reads=io.matthewnelson.kmp.tor.runtime=io.matthewnelson.kmp.tor.common.core", - "--add-reads=io.matthewnelson.kmp.tor.runtime.ctrl=io.matthewnelson.kmp.process", "--add-reads=kotlin.stdlib=kotlinx.coroutines.core", "--add-reads=org.flywaydb.core=java.desktop"] @@ -213,10 +209,6 @@ jlink { "--add-opens=java.base/java.net=com.sparrowwallet.sparrow", "--add-opens=java.base/java.io=com.google.gson", "--add-opens=java.smartcardio/sun.security.smartcardio=com.sparrowwallet.sparrow", - "--add-opens=io.matthewnelson.kmp.tor.resource.geoip/io.matthewnelson.kmp.tor.resource.geoip=io.matthewnelson.kmp.tor.common.core", - "--add-exports=io.matthewnelson.kmp.tor.runtime.ctrl/io.matthewnelson.kmp.tor.runtime.ctrl.internal=io.matthewnelson.kmp.tor.runtime", - "--add-reads=io.matthewnelson.kmp.tor.runtime=io.matthewnelson.kmp.tor.common.core", - "--add-reads=io.matthewnelson.kmp.tor.runtime.ctrl=io.matthewnelson.kmp.process", "--add-reads=com.sparrowwallet.merged.module=java.desktop", "--add-reads=com.sparrowwallet.merged.module=java.sql", "--add-reads=com.sparrowwallet.merged.module=com.sparrowwallet.sparrow", From 3698ca8e85bfa35d3d1485fe7ff3fefd36c9a1d6 Mon Sep 17 00:00:00 2001 From: Craig Raw Date: Mon, 5 May 2025 14:53:24 +0200 Subject: [PATCH 3/8] reduce tooltip show delay to 200ms --- src/main/resources/com/sparrowwallet/sparrow/general.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/com/sparrowwallet/sparrow/general.css b/src/main/resources/com/sparrowwallet/sparrow/general.css index 8a15ff49..26d8050f 100644 --- a/src/main/resources/com/sparrowwallet/sparrow/general.css +++ b/src/main/resources/com/sparrowwallet/sparrow/general.css @@ -330,5 +330,5 @@ CellView > .text-input.text-field { } .tooltip { - -fx-show-delay: 400ms; + -fx-show-delay: 200ms; } \ No newline at end of file From c6e42d8fe286029d51cf933ed0a315c8bbdf2e58 Mon Sep 17 00:00:00 2001 From: Craig Raw Date: Mon, 5 May 2025 15:11:04 +0200 Subject: [PATCH 4/8] rename rpm package name from sparrow to sparrowwallet to avoid conflicts --- src/main/deploy/package/linux/sparrow.spec | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/deploy/package/linux/sparrow.spec b/src/main/deploy/package/linux/sparrow.spec index f9e27e06..0d87b60a 100755 --- a/src/main/deploy/package/linux/sparrow.spec +++ b/src/main/deploy/package/linux/sparrow.spec @@ -1,5 +1,5 @@ Summary: Sparrow -Name: sparrow +Name: sparrowwallet Version: 2.1.4 Release: 1 License: ASL 2.0 @@ -13,7 +13,8 @@ URL: Prefix: /opt %endif -Provides: sparrow +Provides: sparrowwallet +Obsoletes: sparrow <= 2.1.3 %if "xutils" != "x" Group: utils @@ -40,7 +41,7 @@ Requires: xdg-utils %define default_filesystem / /opt /usr /usr/bin /usr/lib /usr/local /usr/local/bin /usr/local/lib %description -Sparrow +Sparrow Wallet %global __os_install_post %{nil} From 474f3a4e91ea28ed2a52131bc1909b919b73a8cb Mon Sep 17 00:00:00 2001 From: Craig Raw Date: Wed, 7 May 2025 10:37:56 +0200 Subject: [PATCH 5/8] add custom filterjar plugin to filter out unneeded native binaries per platform --- build.gradle | 9 +++ buildSrc/build.gradle | 4 + .../filterjar/FilterJarExtension.java | 69 ++++++++++++++++ .../filterjar/FilterJarParameters.java | 10 +++ .../filterjar/FilterJarPlugin.java | 33 ++++++++ .../filterjar/FilterJarTransform.java | 79 +++++++++++++++++++ .../filterjar/JarFilterConfig.java | 19 +++++ .../filterjar/JarFilterConfigImpl.java | 64 +++++++++++++++ 8 files changed, 287 insertions(+) create mode 100644 buildSrc/src/main/java/com/sparrowwallet/filterjar/FilterJarExtension.java create mode 100644 buildSrc/src/main/java/com/sparrowwallet/filterjar/FilterJarParameters.java create mode 100644 buildSrc/src/main/java/com/sparrowwallet/filterjar/FilterJarPlugin.java create mode 100644 buildSrc/src/main/java/com/sparrowwallet/filterjar/FilterJarTransform.java create mode 100644 buildSrc/src/main/java/com/sparrowwallet/filterjar/JarFilterConfig.java create mode 100644 buildSrc/src/main/java/com/sparrowwallet/filterjar/JarFilterConfigImpl.java diff --git a/build.gradle b/build.gradle index c83bf27a..567c66ae 100644 --- a/build.gradle +++ b/build.gradle @@ -3,6 +3,7 @@ plugins { id 'org-openjfx-javafxplugin' id 'org.beryx.jlink' version '3.1.1' id 'org.gradlex.extra-java-module-info' version '1.9' + id 'com.sparrowwallet.filterjar' } def sparrowVersion = '2.1.4' @@ -457,4 +458,12 @@ extraJavaModuleInfo { module('com.jcraft:jzlib', 'com.jcraft.jzlib') { exports('com.jcraft.jzlib') } +} + +String torOs = os.macOsX ? "macos" : (os.windows ? "mingw" : "linux-libc") +filterInfo { + filter('io.matthewnelson.kmp-tor', 'resource-lib-tor-gpl-jvm') { + include("io/matthewnelson/kmp/tor/resource/lib/tor/native/${torOs}/${releaseArch}") + exclude('io/matthewnelson/kmp/tor/resource/lib/tor/native/') + } } \ No newline at end of file diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 8cf7a830..f34a90b9 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -20,5 +20,9 @@ gradlePlugin { id = "org-openjfx-javafxplugin" implementationClass = "org.openjfx.gradle.JavaFXPlugin" } + register("com.sparrowwallet.filterjar") { + id = "com.sparrowwallet.filterjar" + implementationClass = "com.sparrowwallet.filterjar.FilterJarPlugin" + } } } diff --git a/buildSrc/src/main/java/com/sparrowwallet/filterjar/FilterJarExtension.java b/buildSrc/src/main/java/com/sparrowwallet/filterjar/FilterJarExtension.java new file mode 100644 index 00000000..a8f756ff --- /dev/null +++ b/buildSrc/src/main/java/com/sparrowwallet/filterjar/FilterJarExtension.java @@ -0,0 +1,69 @@ +package com.sparrowwallet.filterjar; + +import org.gradle.api.Action; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.ConfigurationContainer; +import org.gradle.api.attributes.Attribute; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.provider.MapProperty; +import org.gradle.api.tasks.SourceSet; + +import javax.inject.Inject; + +public abstract class FilterJarExtension { + static Attribute FILTERED_ATTRIBUTE = Attribute.of("filtered", Boolean.class); + + public abstract MapProperty getFilterConfigs(); + + @Inject + protected abstract ObjectFactory getObjects(); + + @Inject + protected abstract ConfigurationContainer getConfigurations(); + + public void filter(String group, String artifact, Action action) { + String name = group + ":" + artifact; + JarFilterConfigImpl config = new JarFilterConfigImpl(name, getObjects()); + config.setGroup(group); + config.setArtifact(artifact); + action.execute(config); + getFilterConfigs().put(name, config); + } + + /** + * Activate the plugin's functionality for dependencies of all scopes of the given source set + * (runtimeClasspath, compileClasspath, annotationProcessor). + * Note that the plugin activates the functionality for all source sets by default. + * Therefore, this method only has an effect for source sets for which a {@link #deactivate(Configuration)} + * has been performed. + * + * @param sourceSet the Source Set to activate (e.g. sourceSets.test) + */ + public void activate(SourceSet sourceSet) { + Configuration runtimeClasspath = getConfigurations().getByName(sourceSet.getRuntimeClasspathConfigurationName()); + Configuration compileClasspath = getConfigurations().getByName(sourceSet.getCompileClasspathConfigurationName()); + Configuration annotationProcessor = getConfigurations().getByName(sourceSet.getAnnotationProcessorConfigurationName()); + + activate(runtimeClasspath); + activate(compileClasspath); + activate(annotationProcessor); + } + + /** + * Activate the plugin's functionality for a single resolvable Configuration. + * + * @param resolvable a resolvable Configuration (e.g. configurations["customClasspath"]) + */ + public void activate(Configuration resolvable) { + resolvable.getAttributes().attribute(FILTERED_ATTRIBUTE, true); + } + + /** + * Deactivate the plugin's functionality for a single resolvable Configuration. + * + * @param resolvable a resolvable Configuration (e.g. configurations.annotationProcessor) + */ + public void deactivate(Configuration resolvable) { + resolvable.getAttributes().attribute(FILTERED_ATTRIBUTE, false); + } +} diff --git a/buildSrc/src/main/java/com/sparrowwallet/filterjar/FilterJarParameters.java b/buildSrc/src/main/java/com/sparrowwallet/filterjar/FilterJarParameters.java new file mode 100644 index 00000000..23565391 --- /dev/null +++ b/buildSrc/src/main/java/com/sparrowwallet/filterjar/FilterJarParameters.java @@ -0,0 +1,10 @@ +package com.sparrowwallet.filterjar; + +import org.gradle.api.artifacts.transform.TransformParameters; +import org.gradle.api.provider.MapProperty; +import org.gradle.api.tasks.Input; + +public interface FilterJarParameters extends TransformParameters { + @Input + MapProperty getFilterConfigs(); +} diff --git a/buildSrc/src/main/java/com/sparrowwallet/filterjar/FilterJarPlugin.java b/buildSrc/src/main/java/com/sparrowwallet/filterjar/FilterJarPlugin.java new file mode 100644 index 00000000..fd284bc0 --- /dev/null +++ b/buildSrc/src/main/java/com/sparrowwallet/filterjar/FilterJarPlugin.java @@ -0,0 +1,33 @@ +package com.sparrowwallet.filterjar; + +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.tasks.SourceSetContainer; + +import static com.sparrowwallet.filterjar.FilterJarExtension.FILTERED_ATTRIBUTE; + +public class FilterJarPlugin implements Plugin { + @Override + public void apply(Project project) { + // Register the extension + FilterJarExtension extension = project.getExtensions().create("filterInfo", FilterJarExtension.class); + + project.getPlugins().withType(JavaPlugin.class).configureEach(_ -> { + // By default, activate plugin for all source sets + project.getExtensions().getByType(SourceSetContainer.class).all(extension::activate); + + // All jars have a filtered=false attribute by default + project.getDependencies().getArtifactTypes().maybeCreate("jar").getAttributes().attribute(FILTERED_ATTRIBUTE, false); + + // Register the transform + project.getDependencies().registerTransform(FilterJarTransform.class, transform -> { + transform.getFrom().attribute(FILTERED_ATTRIBUTE, false); + transform.getTo().attribute(FILTERED_ATTRIBUTE, true); + transform.parameters(params -> { + params.getFilterConfigs().putAll(extension.getFilterConfigs()); + }); + }); + }); + } +} diff --git a/buildSrc/src/main/java/com/sparrowwallet/filterjar/FilterJarTransform.java b/buildSrc/src/main/java/com/sparrowwallet/filterjar/FilterJarTransform.java new file mode 100644 index 00000000..b8f9659d --- /dev/null +++ b/buildSrc/src/main/java/com/sparrowwallet/filterjar/FilterJarTransform.java @@ -0,0 +1,79 @@ +package com.sparrowwallet.filterjar; + + +import org.gradle.api.artifacts.transform.InputArtifact; +import org.gradle.api.artifacts.transform.TransformAction; +import org.gradle.api.artifacts.transform.TransformOutputs; +import org.gradle.api.file.FileSystemLocation; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; + +import java.io.File; +import java.util.Map; +import java.util.Set; +import java.util.HashSet; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; +import java.nio.file.Files; + +public abstract class FilterJarTransform implements TransformAction { + @InputArtifact + @PathSensitive(PathSensitivity.NAME_ONLY) + public abstract Provider getInputArtifact(); + + @Override + public void transform(TransformOutputs outputs) { + File originalJar = getInputArtifact().get().getAsFile(); + String jarName = originalJar.getName(); + + // Get filter configurations from parameters + Map filterConfigs = getParameters().getFilterConfigs().get(); + + //Inclusions are prioritised ahead of exclusions + Set inclusions = new HashSet<>(); + Set exclusions = new HashSet<>(); + + // Check if this JAR matches any configured filters (simplified matching based on artifact name) + filterConfigs.forEach((key, config) -> { + if(jarName.contains(config.getArtifact())) { + inclusions.addAll(config.getInclusions()); + exclusions.addAll(config.getExclusions()); + } + }); + + try { + if(!exclusions.isEmpty()) { + filterJar(originalJar, getFilterJar(outputs, originalJar), inclusions, exclusions); + } else { + outputs.file(originalJar); + } + } catch(Exception e) { + throw new RuntimeException("Failed to transform jar: " + jarName, e); + } + } + + private void filterJar(File inputFile, File outputFile, Set inclusions, Set exclusions) throws Exception { + try(JarFile jarFile = new JarFile(inputFile); JarOutputStream jarOut = new JarOutputStream(Files.newOutputStream(outputFile.toPath()))) { + jarFile.entries().asIterator().forEachRemaining(entry -> { + String entryName = entry.getName(); + boolean shouldInclude = inclusions.stream().anyMatch(entryName::startsWith); + boolean shouldExclude = exclusions.stream().anyMatch(entryName::startsWith); + if(shouldInclude || !shouldExclude) { + try { + jarOut.putNextEntry(new JarEntry(entryName)); + jarFile.getInputStream(entry).transferTo(jarOut); + jarOut.closeEntry(); + } catch(Exception e) { + throw new RuntimeException("Error processing entry: " + entryName, e); + } + } + }); + } + } + + private File getFilterJar(TransformOutputs outputs, File originalJar) { + return outputs.file(originalJar.getName().substring(0, originalJar.getName().lastIndexOf('.')) + "-filtered.jar"); + } +} diff --git a/buildSrc/src/main/java/com/sparrowwallet/filterjar/JarFilterConfig.java b/buildSrc/src/main/java/com/sparrowwallet/filterjar/JarFilterConfig.java new file mode 100644 index 00000000..5bd792c7 --- /dev/null +++ b/buildSrc/src/main/java/com/sparrowwallet/filterjar/JarFilterConfig.java @@ -0,0 +1,19 @@ +package com.sparrowwallet.filterjar; + +import org.gradle.api.tasks.Input; + +import java.util.List; + +public interface JarFilterConfig { + @Input + String getGroup(); + + @Input + String getArtifact(); + + @Input + List getInclusions(); + + @Input + List getExclusions(); +} diff --git a/buildSrc/src/main/java/com/sparrowwallet/filterjar/JarFilterConfigImpl.java b/buildSrc/src/main/java/com/sparrowwallet/filterjar/JarFilterConfigImpl.java new file mode 100644 index 00000000..912ccd4d --- /dev/null +++ b/buildSrc/src/main/java/com/sparrowwallet/filterjar/JarFilterConfigImpl.java @@ -0,0 +1,64 @@ +package com.sparrowwallet.filterjar; + +import org.gradle.api.Named; +import org.gradle.api.model.ObjectFactory; +import javax.inject.Inject; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +public class JarFilterConfigImpl implements Named, JarFilterConfig, Serializable { + private final String name; + private String group; + private String artifact; + private final List inclusions; + private final List exclusions; + + @Inject + public JarFilterConfigImpl(String name, ObjectFactory objectFactory) { + this.name = name; + this.inclusions = new ArrayList<>(); + this.exclusions = new ArrayList<>(); + } + + @Override + public String getName() { + return name; + } + + @Override + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } + + @Override + public String getArtifact() { + return artifact; + } + + public void setArtifact(String artifact) { + this.artifact = artifact; + } + + @Override + public List getInclusions() { + return inclusions; + } + + public void include(String path) { + inclusions.add(path); + } + + @Override + public List getExclusions() { + return exclusions; + } + + public void exclude(String path) { + exclusions.add(path); + } +} From df0c4310ca663b41749de51691b19ee1510b21a5 Mon Sep 17 00:00:00 2001 From: Craig Raw Date: Wed, 7 May 2025 16:03:02 +0200 Subject: [PATCH 6/8] optimize and reduce electrum server rpc calls #3 --- .../sparrow/net/BlockHeaderTip.java | 5 +- .../sparrow/net/ElectrumServer.java | 109 ++++++++++++++---- .../sparrow/paynym/PayNymController.java | 2 +- .../transaction/HeadersController.java | 2 +- .../sparrow/wallet/SendController.java | 2 +- 5 files changed, 94 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/sparrowwallet/sparrow/net/BlockHeaderTip.java b/src/main/java/com/sparrowwallet/sparrow/net/BlockHeaderTip.java index 4b833efd..ed41d9f2 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/BlockHeaderTip.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/BlockHeaderTip.java @@ -2,14 +2,15 @@ package com.sparrowwallet.sparrow.net; import com.sparrowwallet.drongo.Utils; import com.sparrowwallet.drongo.protocol.BlockHeader; +import com.sparrowwallet.drongo.protocol.Sha256Hash; -class BlockHeaderTip { +public class BlockHeaderTip { public int height; public String hex; public BlockHeader getBlockHeader() { if(hex == null) { - return null; + return new BlockHeader(0, Sha256Hash.ZERO_HASH, Sha256Hash.ZERO_HASH, Sha256Hash.ZERO_HASH, 0, 0, 0); } byte[] blockHeaderBytes = Utils.hexToBytes(hex); diff --git a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java index 5b1d5201..8d5dda01 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java @@ -27,6 +27,7 @@ import javafx.beans.property.SimpleIntegerProperty; import javafx.concurrent.ScheduledService; import javafx.concurrent.Service; import javafx.concurrent.Task; +import javafx.util.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,17 +62,19 @@ public class ElectrumServer { private static CloseableTransport transport; - private static final Map> subscribedScriptHashes = Collections.synchronizedMap(new HashMap<>()); + private static final Map> subscribedScriptHashes = new ConcurrentHashMap<>(); private static Server previousServer; private static final Map retrievedScriptHashes = Collections.synchronizedMap(new HashMap<>()); - private static final Map retrievedTransactions = Collections.synchronizedMap(new HashMap<>()); + private static final Map retrievedTransactions = new ConcurrentHashMap<>(); - private static final Map retrievedBlockHeaders = Collections.synchronizedMap(new HashMap<>()); + private static final Map retrievedBlockHeaders = new ConcurrentHashMap<>(); - private static final Set sameHeightTxioScriptHashes = Collections.synchronizedSet(new HashSet<>()); + private static final Map broadcastedTransactions = new ConcurrentHashMap<>(); + + private static final Set sameHeightTxioScriptHashes = ConcurrentHashMap.newKeySet(); private static ElectrumServerRpc electrumServerRpc = new SimpleElectrumServerRpc(); @@ -420,27 +423,47 @@ public class ElectrumServer { return; } - //Optimistic optimization for confirming transactions by matching against the script hash status should all mempool transactions confirm at the current block height + //Optimistic optimizations from guessing the script hash status based on known information for(Map.Entry entry : nodeHashHistory.entrySet()) { WalletNode node = entry.getKey(); String scriptHash = pathScriptHashes.get(node.getDerivationPath()); List statuses = subscribedScriptHashes.get(scriptHash); - if(statuses != null && !statuses.isEmpty() && AppServices.getCurrentBlockHeight() != null && - node.getTransactionOutputs().stream().flatMap(txo -> txo.isSpent() ? Stream.of(txo, txo.getSpentBy()) : Stream.of(txo)) - .anyMatch(txo -> txo.getHeight() <= 0)) { - List scriptHashTxes = getScriptHashes(scriptHash, node); - for(ScriptHashTx scriptHashTx : scriptHashTxes) { - if(scriptHashTx.height <= 0) { - scriptHashTx.height = AppServices.getCurrentBlockHeight(); - scriptHashTx.fee = 0; + if(statuses != null && !statuses.isEmpty()) { + //Optimize for new transactions that have been recently broadcasted + for(Sha256Hash txid : broadcastedTransactions.keySet()) { + BlockTransaction blkTx = broadcastedTransactions.get(txid); + if(blkTx.getTransaction().getOutputs().stream().map(ElectrumServer::getScriptHash).anyMatch(scriptHash::equals) || + blkTx.getTransaction().getInputs().stream().map(txInput -> getPrevOutput(wallet, txInput)) + .filter(Objects::nonNull).map(ElectrumServer::getScriptHash).anyMatch(scriptHash::equals)) { + List scriptHashTxes = new ArrayList<>(getScriptHashes(scriptHash, node)); + scriptHashTxes.add(new ScriptHashTx(0, txid.toString(), blkTx.getFee())); + + String status = getScriptHashStatus(scriptHashTxes); + if(Objects.equals(status, statuses.getLast())) { + entry.setValue(scriptHashTxes.toArray(new ScriptHashTx[0])); + pathScriptHashes.remove(node.getDerivationPath()); + } } } - String status = getScriptHashStatus(scriptHashTxes); - if(Objects.equals(status, statuses.getLast())) { - entry.setValue(scriptHashTxes.toArray(new ScriptHashTx[0])); - pathScriptHashes.remove(node.getDerivationPath()); + //Optimize for new confirmations should all pending transactions confirm at the current block height + if(entry.getValue() == null && AppServices.getCurrentBlockHeight() != null && + node.getTransactionOutputs().stream().flatMap(txo -> txo.isSpent() ? Stream.of(txo, txo.getSpentBy()) : Stream.of(txo)) + .anyMatch(txo -> txo.getHeight() <= 0)) { + List scriptHashTxes = getScriptHashes(scriptHash, node); + for(ScriptHashTx scriptHashTx : scriptHashTxes) { + if(scriptHashTx.height <= 0) { + scriptHashTx.height = AppServices.getCurrentBlockHeight(); + scriptHashTx.fee = 0; + } + } + + String status = getScriptHashStatus(scriptHashTxes); + if(Objects.equals(status, statuses.getLast())) { + entry.setValue(scriptHashTxes.toArray(new ScriptHashTx[0])); + pathScriptHashes.remove(node.getDerivationPath()); + } } } } @@ -623,6 +646,8 @@ public class ElectrumServer { } else { entry.setValue(blockTransaction.getTransaction()); } + } else if(broadcastedTransactions.containsKey(reference.getHash())) { + entry.setValue(broadcastedTransactions.get(reference.getHash()).getTransaction()); } } @@ -634,6 +659,8 @@ public class ElectrumServer { if(!transactionMap.equals(wallet.getTransactions())) { wallet.updateTransactions(transactionMap); + broadcastedTransactions.keySet().removeAll(transactionMap.entrySet().stream().filter(entry -> entry.getValue().getHeight() > 0) + .map(Map.Entry::getKey).collect(Collectors.toSet())); } } @@ -643,7 +670,7 @@ public class ElectrumServer { Set blockHeights = new TreeSet<>(); for(BlockTransactionHash reference : references) { if(reference.getHeight() > 0) { - if(retrievedBlockHeaders.get(reference.getHeight()) != null) { + if(retrievedBlockHeaders.containsKey(reference.getHeight())) { blockHeaderMap.put(reference.getHeight(), retrievedBlockHeaders.get(reference.getHeight())); } else { blockHeights.add(reference.getHeight()); @@ -1014,6 +1041,16 @@ public class ElectrumServer { } } + public Sha256Hash broadcastTransaction(Transaction transaction, Long fee) throws ServerException { + Sha256Hash txid = broadcastTransactionPrivately(transaction); + if(txid != null) { + BlockTransaction blkTx = new BlockTransaction(txid, 0, null, fee, transaction); + broadcastedTransactions.put(txid, blkTx); + } + + return txid; + } + public Sha256Hash broadcastTransactionPrivately(Transaction transaction) throws ServerException { //If Tor proxy is configured, try all external broadcast sources in random order before falling back to connected Electrum server if(AppServices.isUsingProxy()) { @@ -1126,6 +1163,14 @@ public class ElectrumServer { return scriptHashes; } + private static TransactionOutput getPrevOutput(Wallet wallet, TransactionInput txInput) { + try { + return wallet.getWalletTransaction(txInput.getOutpoint().getHash()).getTransaction().getOutputs().get((int)txInput.getOutpoint().getIndex()); + } catch(Exception e) { + return null; + } + } + public static String getScriptHash(WalletNode node) { byte[] hash = Sha256Hash.hash(node.getOutputScript().getProgram()); byte[] reversed = Utils.reverseBytes(hash); @@ -1729,7 +1774,7 @@ public class ElectrumServer { protected Map call() throws ServerException { Map transactionMap = new HashMap<>(); for(Sha256Hash ref : references) { - if(retrievedTransactions.get(ref) != null) { + if(retrievedTransactions.containsKey(ref)) { transactionMap.put(ref, retrievedTransactions.get(ref)); } } @@ -1848,9 +1893,11 @@ public class ElectrumServer { public static class BroadcastTransactionService extends Service { private final Transaction transaction; + private final Long fee; - public BroadcastTransactionService(Transaction transaction) { + public BroadcastTransactionService(Transaction transaction, Long fee) { this.transaction = transaction; + this.fee = fee; } @Override @@ -1858,7 +1905,7 @@ public class ElectrumServer { return new Task<>() { protected Sha256Hash call() throws ServerException { ElectrumServer electrumServer = new ElectrumServer(); - return electrumServer.broadcastTransactionPrivately(transaction); + return electrumServer.broadcastTransaction(transaction, fee); } }; } @@ -1953,6 +2000,26 @@ public class ElectrumServer { log.debug("Error subscribing to recent mempool transactions", e); } } + + ScheduledService broadcastService = new ScheduledService<>() { + @Override + protected Task createTask() { + return new Task<>() { + @Override + protected Void call() throws Exception { + for(BlockTransaction blkTx : recentTransactions) { + electrumServer.broadcastTransaction(blkTx.getTransaction()); + } + return null; + } + }; + } + }; + broadcastService.setDelay(Duration.seconds(Math.random() * 60 * 10)); + broadcastService.setPeriod(Duration.hours(1)); + broadcastService.setOnSucceeded(_ -> broadcastService.cancel()); + broadcastService.setOnFailed(_ -> broadcastService.cancel()); + broadcastService.start(); } } diff --git a/src/main/java/com/sparrowwallet/sparrow/paynym/PayNymController.java b/src/main/java/com/sparrowwallet/sparrow/paynym/PayNymController.java index c8ed7b23..6a2a4b1a 100644 --- a/src/main/java/com/sparrowwallet/sparrow/paynym/PayNymController.java +++ b/src/main/java/com/sparrowwallet/sparrow/paynym/PayNymController.java @@ -550,7 +550,7 @@ public class PayNymController { decryptedWallet.finalise(psbt); Transaction transaction = psbt.extractTransaction(); - ElectrumServer.BroadcastTransactionService broadcastTransactionService = new ElectrumServer.BroadcastTransactionService(transaction); + ElectrumServer.BroadcastTransactionService broadcastTransactionService = new ElectrumServer.BroadcastTransactionService(transaction, psbt.getFee()); broadcastTransactionService.setOnSucceeded(successEvent -> { ElectrumServer.TransactionMempoolService transactionMempoolService = new ElectrumServer.TransactionMempoolService(walletTransaction.getWallet(), transaction.getTxId(), new HashSet<>(walletTransaction.getSelectedUtxos().values())); transactionMempoolService.setDelay(Duration.seconds(2)); diff --git a/src/main/java/com/sparrowwallet/sparrow/transaction/HeadersController.java b/src/main/java/com/sparrowwallet/sparrow/transaction/HeadersController.java index a8db33b0..967d2386 100644 --- a/src/main/java/com/sparrowwallet/sparrow/transaction/HeadersController.java +++ b/src/main/java/com/sparrowwallet/sparrow/transaction/HeadersController.java @@ -1158,7 +1158,7 @@ public class HeadersController extends TransactionFormController implements Init historyService.start(); } - ElectrumServer.BroadcastTransactionService broadcastTransactionService = new ElectrumServer.BroadcastTransactionService(headersForm.getTransaction()); + ElectrumServer.BroadcastTransactionService broadcastTransactionService = new ElectrumServer.BroadcastTransactionService(headersForm.getTransaction(), fee.getValue()); broadcastTransactionService.setOnSucceeded(workerStateEvent -> { //Although we wait for WalletNodeHistoryChangedEvent to indicate tx is in mempool, start a scheduled service to check the script hashes should notifications fail if(headersForm.getSigningWallet() != null) { diff --git a/src/main/java/com/sparrowwallet/sparrow/wallet/SendController.java b/src/main/java/com/sparrowwallet/sparrow/wallet/SendController.java index 73dc6a0a..bbf8e8f7 100644 --- a/src/main/java/com/sparrowwallet/sparrow/wallet/SendController.java +++ b/src/main/java/com/sparrowwallet/sparrow/wallet/SendController.java @@ -1213,7 +1213,7 @@ public class SendController extends WalletFormController implements Initializabl Transaction transaction = psbt.extractTransaction(); ServiceProgressDialog.ProxyWorker proxyWorker = new ServiceProgressDialog.ProxyWorker(); - ElectrumServer.BroadcastTransactionService broadcastTransactionService = new ElectrumServer.BroadcastTransactionService(transaction); + ElectrumServer.BroadcastTransactionService broadcastTransactionService = new ElectrumServer.BroadcastTransactionService(transaction, psbt.getFee()); broadcastTransactionService.setOnSucceeded(successEvent -> { ElectrumServer.TransactionMempoolService transactionMempoolService = new ElectrumServer.TransactionMempoolService(walletTransaction.getWallet(), transaction.getTxId(), new HashSet<>(walletTransaction.getSelectedUtxos().values())); transactionMempoolService.setDelay(Duration.seconds(2)); From 1b0e5e97264b11621e0ae415a5bf9216d6be9a08 Mon Sep 17 00:00:00 2001 From: Craig Raw Date: Wed, 7 May 2025 16:05:17 +0200 Subject: [PATCH 7/8] revert rpm package name change --- src/main/deploy/package/linux/sparrow.spec | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/deploy/package/linux/sparrow.spec b/src/main/deploy/package/linux/sparrow.spec index 0d87b60a..a8aa65e6 100755 --- a/src/main/deploy/package/linux/sparrow.spec +++ b/src/main/deploy/package/linux/sparrow.spec @@ -1,5 +1,5 @@ Summary: Sparrow -Name: sparrowwallet +Name: sparrow Version: 2.1.4 Release: 1 License: ASL 2.0 @@ -13,8 +13,7 @@ URL: Prefix: /opt %endif -Provides: sparrowwallet -Obsoletes: sparrow <= 2.1.3 +Provides: sparrow %if "xutils" != "x" Group: utils From e697313259ccb2f20f94b1960c2c69c12ce3d3a7 Mon Sep 17 00:00:00 2001 From: Craig Raw Date: Thu, 8 May 2025 10:10:50 +0200 Subject: [PATCH 8/8] add accessible text to improve screen reader navigation --- src/main/java/com/sparrowwallet/sparrow/WelcomeDialog.java | 2 +- .../com/sparrowwallet/sparrow/control/MnemonicKeystorePane.java | 1 + .../sparrowwallet/sparrow/control/TitledDescriptionPane.java | 1 + src/main/resources/com/sparrowwallet/sparrow/welcome.fxml | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/sparrowwallet/sparrow/WelcomeDialog.java b/src/main/java/com/sparrowwallet/sparrow/WelcomeDialog.java index 3b2702a0..159f8186 100644 --- a/src/main/java/com/sparrowwallet/sparrow/WelcomeDialog.java +++ b/src/main/java/com/sparrowwallet/sparrow/WelcomeDialog.java @@ -21,7 +21,7 @@ public class WelcomeDialog extends Dialog { welcomeController.initializeView(); dialogPane.setPrefWidth(600); - dialogPane.setPrefHeight(520); + dialogPane.setPrefHeight(540); dialogPane.setMinHeight(dialogPane.getPrefHeight()); AppServices.moveToActiveWindowScreen(this); diff --git a/src/main/java/com/sparrowwallet/sparrow/control/MnemonicKeystorePane.java b/src/main/java/com/sparrowwallet/sparrow/control/MnemonicKeystorePane.java index d3dda2d6..3697dcfd 100644 --- a/src/main/java/com/sparrowwallet/sparrow/control/MnemonicKeystorePane.java +++ b/src/main/java/com/sparrowwallet/sparrow/control/MnemonicKeystorePane.java @@ -321,6 +321,7 @@ public class MnemonicKeystorePane extends TitledDescriptionPane { } }; wordField.setMaxWidth(100); + wordField.setAccessibleText("Word " + (wordNumber + 1)); TextFormatter formatter = new TextFormatter<>((TextFormatter.Change change) -> { String text = change.getText(); // if text was added, fix the text to fit the requirements diff --git a/src/main/java/com/sparrowwallet/sparrow/control/TitledDescriptionPane.java b/src/main/java/com/sparrowwallet/sparrow/control/TitledDescriptionPane.java index 994dd559..84f8c24d 100644 --- a/src/main/java/com/sparrowwallet/sparrow/control/TitledDescriptionPane.java +++ b/src/main/java/com/sparrowwallet/sparrow/control/TitledDescriptionPane.java @@ -25,6 +25,7 @@ public class TitledDescriptionPane extends TitledPane { public TitledDescriptionPane(String title, String description, String content, WalletModel walletModel) { getStylesheets().add(AppServices.class.getResource("general.css").toExternalForm()); getStyleClass().add("titled-description-pane"); + setAccessibleText(title); setPadding(Insets.EMPTY); setGraphic(getTitle(title, description, walletModel)); diff --git a/src/main/resources/com/sparrowwallet/sparrow/welcome.fxml b/src/main/resources/com/sparrowwallet/sparrow/welcome.fxml index 5dc90211..c763e4af 100644 --- a/src/main/resources/com/sparrowwallet/sparrow/welcome.fxml +++ b/src/main/resources/com/sparrowwallet/sparrow/welcome.fxml @@ -20,7 +20,7 @@ - +