From 296223130e216048fa0b07f0e0893a96b7508a64 Mon Sep 17 00:00:00 2001 From: Craig Raw Date: Fri, 30 Jun 2023 09:00:37 +0200 Subject: [PATCH] cormorant: optimize memory used for calculating fee rate histogram --- drongo | 2 +- .../sparrowwallet/sparrow/AppServices.java | 2 +- .../cormorant/bitcoind/BitcoindClient.java | 28 +++++++++-------- .../bitcoind/BitcoindClientService.java | 5 ++-- .../net/cormorant/bitcoind/MempoolEntry.java | 4 +++ .../net/cormorant/bitcoind/VsizeFeerate.java | 28 +++++++++++++++++ .../electrum/ElectrumServerService.java | 30 +++++-------------- 7 files changed, 59 insertions(+), 40 deletions(-) create mode 100644 src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/VsizeFeerate.java diff --git a/drongo b/drongo index d5abf351..4341973a 160000 --- a/drongo +++ b/drongo @@ -1 +1 @@ -Subproject commit d5abf351bedc2e4234f14a1f3883feb9de6803be +Subproject commit 4341973acd9577def1d8fee486718bf5eca9b771 diff --git a/src/main/java/com/sparrowwallet/sparrow/AppServices.java b/src/main/java/com/sparrowwallet/sparrow/AppServices.java index f3b4a53c..3fad7221 100644 --- a/src/main/java/com/sparrowwallet/sparrow/AppServices.java +++ b/src/main/java/com/sparrowwallet/sparrow/AppServices.java @@ -688,7 +688,7 @@ public class AppServices { ZonedDateTime twoHoursAgo = LocalDateTime.now().minusHours(2).atZone(ZoneId.systemDefault()); mempoolHistogram.keySet().removeIf(date -> { ZonedDateTime dateTime = date.toInstant().atZone(ZoneId.systemDefault()); - return dateTime.isBefore(twoHoursAgo) && (dateTime.getMinute() % 10 == 0); + return dateTime.isBefore(twoHoursAgo) && (dateTime.getMinute() % 10 != 0); }); } diff --git a/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindClient.java b/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindClient.java index 977ad127..3302f1cf 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindClient.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindClient.java @@ -81,7 +81,7 @@ public class BitcoindClient { private final List pruneWarnedDescriptors = new ArrayList<>(); - private final Map mempoolEntries = new ConcurrentHashMap<>(); + private final Map mempoolEntries = new ConcurrentHashMap<>(); private MempoolEntriesState mempoolEntriesState = MempoolEntriesState.UNINITIALIZED; private long timerTaskCount; @@ -530,18 +530,20 @@ public class BitcoindClient { mempoolEntriesState = MempoolEntriesState.INITIALIZING; long start = System.currentTimeMillis(); - Set txids = getBitcoindService().getRawMempool(); + Set txids = getBitcoindService().getRawMempool(); long end = System.currentTimeMillis(); if(end - start < 1000) { //Fast system, fetch all mempool data at once - mempoolEntries.putAll(getBitcoindService().getRawMempool(true)); + Map entries = getBitcoindService().getRawMempool(true).entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getVsizeFeerate(), (u, v) -> u, HashMap::new)); + mempoolEntries.putAll(entries); } else { //Slow system, fetch mempool entries one-by-one to avoid risking a node crash - for(String txid : txids) { + for(Sha256Hash txid : txids) { try { - MempoolEntry mempoolEntry = getBitcoindService().getMempoolEntry(txid); - mempoolEntries.put(txid, mempoolEntry); + MempoolEntry mempoolEntry = getBitcoindService().getMempoolEntry(txid.toString()); + mempoolEntries.put(txid, mempoolEntry.getVsizeFeerate()); } catch(JsonRpcException e) { //ignore, probably tx has been removed from mempool } @@ -552,23 +554,23 @@ public class BitcoindClient { } public void updateMempoolEntries() { - Set txids = getBitcoindService().getRawMempool(); + Set txids = getBitcoindService().getRawMempool(); - Set removed = new HashSet<>(Sets.difference(mempoolEntries.keySet(), txids)); + Set removed = new HashSet<>(Sets.difference(mempoolEntries.keySet(), txids)); mempoolEntries.keySet().removeAll(removed); - Set added = Sets.difference(txids, mempoolEntries.keySet()); - for(String txid : added) { + Set added = Sets.difference(txids, mempoolEntries.keySet()); + for(Sha256Hash txid : added) { try { - MempoolEntry mempoolEntry = getBitcoindService().getMempoolEntry(txid); - mempoolEntries.put(txid, mempoolEntry); + MempoolEntry mempoolEntry = getBitcoindService().getMempoolEntry(txid.toString()); + mempoolEntries.put(txid, mempoolEntry.getVsizeFeerate()); } catch(JsonRpcException e) { //ignore, probably tx has been removed from mempool } } } - public Map getMempoolEntries() { + public Map getMempoolEntries() { return mempoolEntries; } diff --git a/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindClientService.java b/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindClientService.java index 1276ce58..a4739103 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindClientService.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindClientService.java @@ -6,6 +6,7 @@ import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcMethod; import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcOptional; import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcParam; import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcService; +import com.sparrowwallet.drongo.protocol.Sha256Hash; import java.util.List; import java.util.Map; @@ -24,10 +25,10 @@ public interface BitcoindClientService { FeeInfo estimateSmartFee(@JsonRpcParam("conf_target") int blocks); @JsonRpcMethod("getrawmempool") - Set getRawMempool(); + Set getRawMempool(); @JsonRpcMethod("getrawmempool") - Map getRawMempool(@JsonRpcParam("verbose") boolean verbose); + Map getRawMempool(@JsonRpcParam("verbose") boolean verbose); @JsonRpcMethod("getmempoolinfo") MempoolInfo getMempoolInfo(); diff --git a/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/MempoolEntry.java b/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/MempoolEntry.java index 378fb1ae..27f42836 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/MempoolEntry.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/MempoolEntry.java @@ -12,4 +12,8 @@ public record MempoolEntry(int vsize, int ancestorsize, boolean bip125_replaceab public TxEntry getTxEntry(String txid) { return new TxEntry(hasUnconfirmedParents() ? -1 : 0, 0, txid); } + + public VsizeFeerate getVsizeFeerate() { + return new VsizeFeerate(vsize, fees().base()); + } } diff --git a/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/VsizeFeerate.java b/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/VsizeFeerate.java new file mode 100644 index 00000000..a0f45fd9 --- /dev/null +++ b/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/VsizeFeerate.java @@ -0,0 +1,28 @@ +package com.sparrowwallet.sparrow.net.cormorant.bitcoind; + +import com.sparrowwallet.drongo.protocol.Transaction; + +public class VsizeFeerate implements Comparable { + private final int vsize; + private final float feerate; + + public VsizeFeerate(int vsize, double fee) { + this.vsize = vsize; + double feeRate = fee / vsize * Transaction.SATOSHIS_PER_BITCOIN; + //Round down to 0.1 sats/vb precision + this.feerate = (float) (Math.floor(10 * feeRate) / 10); + } + + public int getVsize() { + return vsize; + } + + public double getFeerate() { + return feerate; + } + + @Override + public int compareTo(VsizeFeerate o) { + return Float.compare(o.feerate, feerate); + } +} diff --git a/src/main/java/com/sparrowwallet/sparrow/net/cormorant/electrum/ElectrumServerService.java b/src/main/java/com/sparrowwallet/sparrow/net/cormorant/electrum/ElectrumServerService.java index f10a7427..d9ed6e16 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/cormorant/electrum/ElectrumServerService.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/cormorant/electrum/ElectrumServerService.java @@ -5,7 +5,7 @@ import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcMethod; import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcOptional; import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcParam; import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcService; -import com.sparrowwallet.drongo.protocol.Transaction; +import com.sparrowwallet.drongo.protocol.Sha256Hash; import com.sparrowwallet.sparrow.EventManager; import com.sparrowwallet.sparrow.SparrowWallet; import com.sparrowwallet.sparrow.event.MempoolEntriesInitializedEvent; @@ -79,22 +79,22 @@ public class ElectrumServerService { return Collections.emptyList(); } else { - Map mempoolEntries = bitcoindClient.getMempoolEntries(); - - List vsizeFeerates = mempoolEntries.values().stream().map(entry -> new VsizeFeerate(entry.vsize(), entry.fees().base())).sorted().toList(); + Map mempoolEntries = bitcoindClient.getMempoolEntries(); + List vsizeFeerates = new ArrayList<>(mempoolEntries.values()); + Collections.sort(vsizeFeerates); List> histogram = new ArrayList<>(); long binSize = 0; double lastFeerate = 0.0; for(VsizeFeerate vsizeFeerate : vsizeFeerates) { - if(binSize > VSIZE_BIN_WIDTH && Math.abs(lastFeerate - vsizeFeerate.feerate) > 0.0d) { + if(binSize > VSIZE_BIN_WIDTH && Math.abs(lastFeerate - vsizeFeerate.getFeerate()) > 0.0d) { // vsize of transactions paying >= last_feerate histogram.add(List.of(lastFeerate, binSize)); binSize = 0; } - binSize += vsizeFeerate.vsize; - lastFeerate = vsizeFeerate.feerate; + binSize += vsizeFeerate.getVsize(); + lastFeerate = vsizeFeerate.getFeerate(); } if(binSize > 0) { @@ -213,20 +213,4 @@ public class ElectrumServerService { } } - private static class VsizeFeerate implements Comparable { - private final int vsize; - private final double feerate; - - public VsizeFeerate(int vsize, double fee) { - this.vsize = vsize; - double feeRate = fee / vsize * Transaction.SATOSHIS_PER_BITCOIN; - //Round down to 0.1 sats/vb precision - this.feerate = Math.floor(10 * feeRate) / 10; - } - - @Override - public int compareTo(VsizeFeerate o) { - return Double.compare(o.feerate, feerate); - } - } }