cormorant: optimize memory used for calculating fee rate histogram

This commit is contained in:
Craig Raw 2023-06-30 09:00:37 +02:00
parent 87e2da0e01
commit 296223130e
7 changed files with 59 additions and 40 deletions

2
drongo

@ -1 +1 @@
Subproject commit d5abf351bedc2e4234f14a1f3883feb9de6803be Subproject commit 4341973acd9577def1d8fee486718bf5eca9b771

View file

@ -688,7 +688,7 @@ public class AppServices {
ZonedDateTime twoHoursAgo = LocalDateTime.now().minusHours(2).atZone(ZoneId.systemDefault()); ZonedDateTime twoHoursAgo = LocalDateTime.now().minusHours(2).atZone(ZoneId.systemDefault());
mempoolHistogram.keySet().removeIf(date -> { mempoolHistogram.keySet().removeIf(date -> {
ZonedDateTime dateTime = date.toInstant().atZone(ZoneId.systemDefault()); ZonedDateTime dateTime = date.toInstant().atZone(ZoneId.systemDefault());
return dateTime.isBefore(twoHoursAgo) && (dateTime.getMinute() % 10 == 0); return dateTime.isBefore(twoHoursAgo) && (dateTime.getMinute() % 10 != 0);
}); });
} }

View file

@ -81,7 +81,7 @@ public class BitcoindClient {
private final List<String> pruneWarnedDescriptors = new ArrayList<>(); private final List<String> pruneWarnedDescriptors = new ArrayList<>();
private final Map<String, MempoolEntry> mempoolEntries = new ConcurrentHashMap<>(); private final Map<Sha256Hash, VsizeFeerate> mempoolEntries = new ConcurrentHashMap<>();
private MempoolEntriesState mempoolEntriesState = MempoolEntriesState.UNINITIALIZED; private MempoolEntriesState mempoolEntriesState = MempoolEntriesState.UNINITIALIZED;
private long timerTaskCount; private long timerTaskCount;
@ -530,18 +530,20 @@ public class BitcoindClient {
mempoolEntriesState = MempoolEntriesState.INITIALIZING; mempoolEntriesState = MempoolEntriesState.INITIALIZING;
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
Set<String> txids = getBitcoindService().getRawMempool(); Set<Sha256Hash> txids = getBitcoindService().getRawMempool();
long end = System.currentTimeMillis(); long end = System.currentTimeMillis();
if(end - start < 1000) { if(end - start < 1000) {
//Fast system, fetch all mempool data at once //Fast system, fetch all mempool data at once
mempoolEntries.putAll(getBitcoindService().getRawMempool(true)); Map<Sha256Hash, VsizeFeerate> entries = getBitcoindService().getRawMempool(true).entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getVsizeFeerate(), (u, v) -> u, HashMap::new));
mempoolEntries.putAll(entries);
} else { } else {
//Slow system, fetch mempool entries one-by-one to avoid risking a node crash //Slow system, fetch mempool entries one-by-one to avoid risking a node crash
for(String txid : txids) { for(Sha256Hash txid : txids) {
try { try {
MempoolEntry mempoolEntry = getBitcoindService().getMempoolEntry(txid); MempoolEntry mempoolEntry = getBitcoindService().getMempoolEntry(txid.toString());
mempoolEntries.put(txid, mempoolEntry); mempoolEntries.put(txid, mempoolEntry.getVsizeFeerate());
} catch(JsonRpcException e) { } catch(JsonRpcException e) {
//ignore, probably tx has been removed from mempool //ignore, probably tx has been removed from mempool
} }
@ -552,23 +554,23 @@ public class BitcoindClient {
} }
public void updateMempoolEntries() { public void updateMempoolEntries() {
Set<String> txids = getBitcoindService().getRawMempool(); Set<Sha256Hash> txids = getBitcoindService().getRawMempool();
Set<String> removed = new HashSet<>(Sets.difference(mempoolEntries.keySet(), txids)); Set<Sha256Hash> removed = new HashSet<>(Sets.difference(mempoolEntries.keySet(), txids));
mempoolEntries.keySet().removeAll(removed); mempoolEntries.keySet().removeAll(removed);
Set<String> added = Sets.difference(txids, mempoolEntries.keySet()); Set<Sha256Hash> added = Sets.difference(txids, mempoolEntries.keySet());
for(String txid : added) { for(Sha256Hash txid : added) {
try { try {
MempoolEntry mempoolEntry = getBitcoindService().getMempoolEntry(txid); MempoolEntry mempoolEntry = getBitcoindService().getMempoolEntry(txid.toString());
mempoolEntries.put(txid, mempoolEntry); mempoolEntries.put(txid, mempoolEntry.getVsizeFeerate());
} catch(JsonRpcException e) { } catch(JsonRpcException e) {
//ignore, probably tx has been removed from mempool //ignore, probably tx has been removed from mempool
} }
} }
} }
public Map<String, MempoolEntry> getMempoolEntries() { public Map<Sha256Hash, VsizeFeerate> getMempoolEntries() {
return mempoolEntries; return mempoolEntries;
} }

View file

@ -6,6 +6,7 @@ import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcMethod;
import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcOptional; import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcOptional;
import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcParam; import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcParam;
import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcService; import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcService;
import com.sparrowwallet.drongo.protocol.Sha256Hash;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -24,10 +25,10 @@ public interface BitcoindClientService {
FeeInfo estimateSmartFee(@JsonRpcParam("conf_target") int blocks); FeeInfo estimateSmartFee(@JsonRpcParam("conf_target") int blocks);
@JsonRpcMethod("getrawmempool") @JsonRpcMethod("getrawmempool")
Set<String> getRawMempool(); Set<Sha256Hash> getRawMempool();
@JsonRpcMethod("getrawmempool") @JsonRpcMethod("getrawmempool")
Map<String, MempoolEntry> getRawMempool(@JsonRpcParam("verbose") boolean verbose); Map<Sha256Hash, MempoolEntry> getRawMempool(@JsonRpcParam("verbose") boolean verbose);
@JsonRpcMethod("getmempoolinfo") @JsonRpcMethod("getmempoolinfo")
MempoolInfo getMempoolInfo(); MempoolInfo getMempoolInfo();

View file

@ -12,4 +12,8 @@ public record MempoolEntry(int vsize, int ancestorsize, boolean bip125_replaceab
public TxEntry getTxEntry(String txid) { public TxEntry getTxEntry(String txid) {
return new TxEntry(hasUnconfirmedParents() ? -1 : 0, 0, txid); return new TxEntry(hasUnconfirmedParents() ? -1 : 0, 0, txid);
} }
public VsizeFeerate getVsizeFeerate() {
return new VsizeFeerate(vsize, fees().base());
}
} }

View file

@ -0,0 +1,28 @@
package com.sparrowwallet.sparrow.net.cormorant.bitcoind;
import com.sparrowwallet.drongo.protocol.Transaction;
public class VsizeFeerate implements Comparable<VsizeFeerate> {
private final int vsize;
private final float feerate;
public VsizeFeerate(int vsize, double fee) {
this.vsize = vsize;
double feeRate = fee / vsize * Transaction.SATOSHIS_PER_BITCOIN;
//Round down to 0.1 sats/vb precision
this.feerate = (float) (Math.floor(10 * feeRate) / 10);
}
public int getVsize() {
return vsize;
}
public double getFeerate() {
return feerate;
}
@Override
public int compareTo(VsizeFeerate o) {
return Float.compare(o.feerate, feerate);
}
}

View file

@ -5,7 +5,7 @@ import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcMethod;
import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcOptional; import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcOptional;
import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcParam; import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcParam;
import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcService; import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcService;
import com.sparrowwallet.drongo.protocol.Transaction; import com.sparrowwallet.drongo.protocol.Sha256Hash;
import com.sparrowwallet.sparrow.EventManager; import com.sparrowwallet.sparrow.EventManager;
import com.sparrowwallet.sparrow.SparrowWallet; import com.sparrowwallet.sparrow.SparrowWallet;
import com.sparrowwallet.sparrow.event.MempoolEntriesInitializedEvent; import com.sparrowwallet.sparrow.event.MempoolEntriesInitializedEvent;
@ -79,22 +79,22 @@ public class ElectrumServerService {
return Collections.emptyList(); return Collections.emptyList();
} else { } else {
Map<String, MempoolEntry> mempoolEntries = bitcoindClient.getMempoolEntries(); Map<Sha256Hash, VsizeFeerate> mempoolEntries = bitcoindClient.getMempoolEntries();
List<VsizeFeerate> vsizeFeerates = new ArrayList<>(mempoolEntries.values());
List<VsizeFeerate> vsizeFeerates = mempoolEntries.values().stream().map(entry -> new VsizeFeerate(entry.vsize(), entry.fees().base())).sorted().toList(); Collections.sort(vsizeFeerates);
List<List<Number>> histogram = new ArrayList<>(); List<List<Number>> histogram = new ArrayList<>();
long binSize = 0; long binSize = 0;
double lastFeerate = 0.0; double lastFeerate = 0.0;
for(VsizeFeerate vsizeFeerate : vsizeFeerates) { for(VsizeFeerate vsizeFeerate : vsizeFeerates) {
if(binSize > VSIZE_BIN_WIDTH && Math.abs(lastFeerate - vsizeFeerate.feerate) > 0.0d) { if(binSize > VSIZE_BIN_WIDTH && Math.abs(lastFeerate - vsizeFeerate.getFeerate()) > 0.0d) {
// vsize of transactions paying >= last_feerate // vsize of transactions paying >= last_feerate
histogram.add(List.of(lastFeerate, binSize)); histogram.add(List.of(lastFeerate, binSize));
binSize = 0; binSize = 0;
} }
binSize += vsizeFeerate.vsize; binSize += vsizeFeerate.getVsize();
lastFeerate = vsizeFeerate.feerate; lastFeerate = vsizeFeerate.getFeerate();
} }
if(binSize > 0) { if(binSize > 0) {
@ -213,20 +213,4 @@ public class ElectrumServerService {
} }
} }
private static class VsizeFeerate implements Comparable<VsizeFeerate> {
private final int vsize;
private final double feerate;
public VsizeFeerate(int vsize, double fee) {
this.vsize = vsize;
double feeRate = fee / vsize * Transaction.SATOSHIS_PER_BITCOIN;
//Round down to 0.1 sats/vb precision
this.feerate = Math.floor(10 * feeRate) / 10;
}
@Override
public int compareTo(VsizeFeerate o) {
return Double.compare(o.feerate, feerate);
}
}
} }