use unique (per session) integers as ids for all paged server queries

This commit is contained in:
Craig Raw 2021-12-22 11:16:55 +02:00
parent ad091217d6
commit 796f68640c
2 changed files with 58 additions and 29 deletions

View file

@ -73,7 +73,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc {
@Override @Override
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public Map<String, ScriptHashTx[]> getScriptHashHistory(Transport transport, Wallet wallet, Map<String, String> pathScriptHashes, boolean failOnError) { public Map<String, ScriptHashTx[]> getScriptHashHistory(Transport transport, Wallet wallet, Map<String, String> pathScriptHashes, boolean failOnError) {
PagedBatchRequestBuilder<String, ScriptHashTx[]> batchRequest = PagedBatchRequestBuilder.create(transport).keysType(String.class).returnType(ScriptHashTx[].class); PagedBatchRequestBuilder<String, ScriptHashTx[]> batchRequest = PagedBatchRequestBuilder.create(transport, idCounter).keysType(String.class).returnType(ScriptHashTx[].class);
EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Loading transactions for " + nodeRangesToString(pathScriptHashes.keySet()))); EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Loading transactions for " + nodeRangesToString(pathScriptHashes.keySet())));
for(String path : pathScriptHashes.keySet()) { for(String path : pathScriptHashes.keySet()) {
@ -101,7 +101,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc {
@Override @Override
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public Map<String, ScriptHashTx[]> getScriptHashMempool(Transport transport, Wallet wallet, Map<String, String> pathScriptHashes, boolean failOnError) { public Map<String, ScriptHashTx[]> getScriptHashMempool(Transport transport, Wallet wallet, Map<String, String> pathScriptHashes, boolean failOnError) {
PagedBatchRequestBuilder<String, ScriptHashTx[]> batchRequest = PagedBatchRequestBuilder.create(transport).keysType(String.class).returnType(ScriptHashTx[].class); PagedBatchRequestBuilder<String, ScriptHashTx[]> batchRequest = PagedBatchRequestBuilder.create(transport, idCounter).keysType(String.class).returnType(ScriptHashTx[].class);
for(String path : pathScriptHashes.keySet()) { for(String path : pathScriptHashes.keySet()) {
batchRequest.add(path, "blockchain.scripthash.get_mempool", pathScriptHashes.get(path)); batchRequest.add(path, "blockchain.scripthash.get_mempool", pathScriptHashes.get(path));
@ -128,7 +128,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc {
@Override @Override
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public Map<String, String> subscribeScriptHashes(Transport transport, Wallet wallet, Map<String, String> pathScriptHashes) { public Map<String, String> subscribeScriptHashes(Transport transport, Wallet wallet, Map<String, String> pathScriptHashes) {
PagedBatchRequestBuilder<String, String> batchRequest = PagedBatchRequestBuilder.create(transport).keysType(String.class).returnType(String.class); PagedBatchRequestBuilder<String, String> batchRequest = PagedBatchRequestBuilder.create(transport, idCounter).keysType(String.class).returnType(String.class);
EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Finding transactions for " + nodeRangesToString(pathScriptHashes.keySet()))); EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Finding transactions for " + nodeRangesToString(pathScriptHashes.keySet())));
for(String path : pathScriptHashes.keySet()) { for(String path : pathScriptHashes.keySet()) {
@ -148,7 +148,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc {
@Override @Override
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public Map<Integer, String> getBlockHeaders(Transport transport, Wallet wallet, Set<Integer> blockHeights) { public Map<Integer, String> getBlockHeaders(Transport transport, Wallet wallet, Set<Integer> blockHeights) {
PagedBatchRequestBuilder<Integer, String> batchRequest = PagedBatchRequestBuilder.create(transport).keysType(Integer.class).returnType(String.class); PagedBatchRequestBuilder<Integer, String> batchRequest = PagedBatchRequestBuilder.create(transport, idCounter).keysType(Integer.class).returnType(String.class);
EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Retrieving " + blockHeights.size() + " block headers")); EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Retrieving " + blockHeights.size() + " block headers"));
for(Integer height : blockHeights) { for(Integer height : blockHeights) {
@ -167,7 +167,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc {
@Override @Override
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public Map<String, String> getTransactions(Transport transport, Wallet wallet, Set<String> txids) { public Map<String, String> getTransactions(Transport transport, Wallet wallet, Set<String> txids) {
PagedBatchRequestBuilder<String, String> batchRequest = PagedBatchRequestBuilder.create(transport).keysType(String.class).returnType(String.class); PagedBatchRequestBuilder<String, String> batchRequest = PagedBatchRequestBuilder.create(transport, idCounter).keysType(String.class).returnType(String.class);
EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Retrieving " + txids.size() + " transactions")); EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Retrieving " + txids.size() + " transactions"));
for(String txid : txids) { for(String txid : txids) {

View file

@ -11,6 +11,7 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import java.util.*; import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
import static com.sparrowwallet.sparrow.net.BatchedElectrumServerRpc.MAX_RETRIES; import static com.sparrowwallet.sparrow.net.BatchedElectrumServerRpc.MAX_RETRIES;
import static com.sparrowwallet.sparrow.net.BatchedElectrumServerRpc.RETRY_DELAY; import static com.sparrowwallet.sparrow.net.BatchedElectrumServerRpc.RETRY_DELAY;
@ -18,8 +19,10 @@ import static com.sparrowwallet.sparrow.net.BatchedElectrumServerRpc.RETRY_DELAY
public class PagedBatchRequestBuilder<K, V> extends AbstractBuilder { public class PagedBatchRequestBuilder<K, V> extends AbstractBuilder {
public static final int DEFAULT_PAGE_SIZE = 500; public static final int DEFAULT_PAGE_SIZE = 500;
private final AtomicLong counter;
@NotNull @NotNull
private final List<Request> requests; private final List<Request<K>> requests;
/** /**
* Type of request ids * Type of request ids
@ -41,17 +44,19 @@ public class PagedBatchRequestBuilder<K, V> extends AbstractBuilder {
* @param transport transport for request performing * @param transport transport for request performing
* @param mapper mapper for JSON processing * @param mapper mapper for JSON processing
*/ */
public PagedBatchRequestBuilder(@NotNull Transport transport, @NotNull ObjectMapper mapper) { public PagedBatchRequestBuilder(@NotNull Transport transport, @NotNull ObjectMapper mapper, AtomicLong counter) {
this(transport, mapper, new ArrayList<Request>(), null, null); this(transport, mapper, new ArrayList<Request<K>>(), null, null, counter);
} }
public PagedBatchRequestBuilder(@NotNull Transport transport, @NotNull ObjectMapper mapper, public PagedBatchRequestBuilder(@NotNull Transport transport, @NotNull ObjectMapper mapper,
@NotNull List<Request> requests, @NotNull List<Request<K>> requests,
@Nullable Class<K> keysType, @Nullable Class<V> returnType) { @Nullable Class<K> keysType, @Nullable Class<V> returnType,
@Nullable AtomicLong counter) {
super(transport, mapper); super(transport, mapper);
this.requests = requests; this.requests = requests;
this.keysType = keysType; this.keysType = keysType;
this.returnType = returnType; this.returnType = returnType;
this.counter = counter;
} }
/** /**
@ -63,8 +68,8 @@ public class PagedBatchRequestBuilder<K, V> extends AbstractBuilder {
* @return the current builder * @return the current builder
*/ */
@NotNull @NotNull
public PagedBatchRequestBuilder<K, V> add(Object id, @NotNull String method, @NotNull Object param) { public PagedBatchRequestBuilder<K, V> add(K id, @NotNull String method, @NotNull Object param) {
requests.add(new Request(id, method, param)); requests.add(new Request<K>(id, counter == null ? null : counter.incrementAndGet(), method, param));
return this; return this;
} }
@ -77,7 +82,7 @@ public class PagedBatchRequestBuilder<K, V> extends AbstractBuilder {
* @return a new builder * @return a new builder
*/ */
public <NK> PagedBatchRequestBuilder<NK, V> keysType(@NotNull Class<NK> keysClass) { public <NK> PagedBatchRequestBuilder<NK, V> keysType(@NotNull Class<NK> keysClass) {
return new PagedBatchRequestBuilder<NK, V>(transport, mapper, requests, keysClass, returnType); return new PagedBatchRequestBuilder<NK, V>(transport, mapper, new ArrayList<Request<NK>>(), keysClass, returnType, counter);
} }
/** /**
@ -89,7 +94,7 @@ public class PagedBatchRequestBuilder<K, V> extends AbstractBuilder {
* @return a new builder * @return a new builder
*/ */
public <NV> PagedBatchRequestBuilder<K, NV> returnType(@NotNull Class<NV> valuesClass) { public <NV> PagedBatchRequestBuilder<K, NV> returnType(@NotNull Class<NV> valuesClass) {
return new PagedBatchRequestBuilder<K, NV>(transport, mapper, requests, keysType, valuesClass); return new PagedBatchRequestBuilder<K, NV>(transport, mapper, requests, keysType, valuesClass, counter);
} }
/** /**
@ -102,21 +107,35 @@ public class PagedBatchRequestBuilder<K, V> extends AbstractBuilder {
Map<K, V> allResults = new HashMap<>(); Map<K, V> allResults = new HashMap<>();
JsonRpcClient client = new JsonRpcClient(transport); JsonRpcClient client = new JsonRpcClient(transport);
List<List<Request>> pages = Lists.partition(requests, getPageSize()); List<List<Request<K>>> pages = Lists.partition(requests, getPageSize());
for(List<Request> page : pages) { for(List<Request<K>> page : pages) {
BatchRequestBuilder<K, V> batchRequest = client.createBatchRequest().keysType(keysType).returnType(returnType); if(counter != null) {
for(Request request : page) { Map<Long, K> counterIdMap = new HashMap<>();
if(request.id instanceof String strReq) { BatchRequestBuilder<Long, V> batchRequest = client.createBatchRequest().keysType(Long.class).returnType(returnType);
batchRequest.add(strReq, request.method, request.param); for(Request<K> request : page) {
} else if(request.id instanceof Integer intReq) { counterIdMap.put(request.counterId, request.id);
batchRequest.add(intReq, request.method, request.param); batchRequest.add(request.counterId, request.method, request.param);
} else {
throw new IllegalArgumentException("Id of class " + request.id.getClass().getName() + " not supported");
} }
}
Map<K, V> pageResult = new RetryLogic<Map<K, V>>(MAX_RETRIES, RETRY_DELAY, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(batchRequest::execute); Map<Long, V> pageResult = new RetryLogic<Map<Long, V>>(MAX_RETRIES, RETRY_DELAY, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(batchRequest::execute);
allResults.putAll(pageResult); for(Map.Entry<Long, V> pageEntry : pageResult.entrySet()) {
allResults.put(counterIdMap.get(pageEntry.getKey()), pageEntry.getValue());
}
} else {
BatchRequestBuilder<K, V> batchRequest = client.createBatchRequest().keysType(keysType).returnType(returnType);
for(Request<K> request : page) {
if(request.id instanceof String strReq) {
batchRequest.add(strReq, request.method, request.param);
} else if(request.id instanceof Integer intReq) {
batchRequest.add(intReq, request.method, request.param);
} else {
throw new IllegalArgumentException("Id of class " + request.id.getClass().getName() + " not supported");
}
}
Map<K, V> pageResult = new RetryLogic<Map<K, V>>(MAX_RETRIES, RETRY_DELAY, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(batchRequest::execute);
allResults.putAll(pageResult);
}
} }
return allResults; return allResults;
@ -138,8 +157,18 @@ public class PagedBatchRequestBuilder<K, V> extends AbstractBuilder {
*/ */
@NotNull @NotNull
public static PagedBatchRequestBuilder<?, ?> create(Transport transport) { public static PagedBatchRequestBuilder<?, ?> create(Transport transport) {
return new PagedBatchRequestBuilder<Object, Object>(transport, new ObjectMapper()); return new PagedBatchRequestBuilder<Object, Object>(transport, new ObjectMapper(), null);
} }
private static record Request(Object id, String method, Object param) {} /**
* Creates a builder of a JSON-RPC batch request in initial state with a counter for request ids
*
* @return batch request builder
*/
@NotNull
public static PagedBatchRequestBuilder<?, ?> create(Transport transport, AtomicLong counter) {
return new PagedBatchRequestBuilder<Object, Object>(transport, new ObjectMapper(), counter);
}
private static record Request<K>(K id, Long counterId, String method, Object param) {}
} }