sparrowwallet/sparrow
1
0
Fork 0
mirror of https://github.com/sparrowwallet/sparrow.git synced 2024-12-24 12:46:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

cormorant: improve scanning behaviour

Browse source
This commit is contained in:
Craig Raw 2022-12-14 14:55:00 +02:00
parent 61d9ad1875
commit 5ca60699ef
14 changed files with 141 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/main/java/com/sparrowwallet/sparrow/AppController.java
View file

@ -1611,6 +1611,10 @@ public class AppController implements Initializable {
});
subTabs.getSelectionModel().select(subTab);
if(wallet.isValid()) {
Platform.runLater(() -> walletForm.refreshHistory(AppServices.getCurrentBlockHeight()));
}
return walletForm;
} catch(IOException e) {
throw new RuntimeException(e);

5
src/main/java/com/sparrowwallet/sparrow/event/CormorantPruneStatusEvent.java
View file

@ -22,6 +22,11 @@ public class CormorantPruneStatusEvent extends CormorantStatusEvent {
this.legacyWalletExists = legacyWalletExists;
}
@Override
public boolean isFor(Wallet wallet) {
return this.wallet == wallet;
}
public Wallet getWallet() {
return wallet;
}

12
src/main/java/com/sparrowwallet/sparrow/event/CormorantScanStatusEvent.java
View file

@ -1,17 +1,27 @@
package com.sparrowwallet.sparrow.event;
import com.sparrowwallet.drongo.wallet.Wallet;
import java.time.Duration;
import java.util.Set;
public class CormorantScanStatusEvent extends CormorantStatusEvent {
private final Set<Wallet> scanningWallets;
private final int progress;
private final Duration remainingDuration;
public CormorantScanStatusEvent(String status, int progress, Duration remainingDuration) {
public CormorantScanStatusEvent(String status, Set<Wallet> scanningWallets, int progress, Duration remainingDuration) {
super(status);
this.scanningWallets = scanningWallets;
this.progress = progress;
this.remainingDuration = remainingDuration;
}
@Override
public boolean isFor(Wallet wallet) {
return scanningWallets.contains(wallet);
}
public int getProgress() {
return progress;
}

6
src/main/java/com/sparrowwallet/sparrow/event/CormorantStatusEvent.java
View file

@ -1,6 +1,8 @@
package com.sparrowwallet.sparrow.event;
public class CormorantStatusEvent {
import com.sparrowwallet.drongo.wallet.Wallet;
public abstract class CormorantStatusEvent {
private final String status;
public CormorantStatusEvent(String status) {
@ -10,4 +12,6 @@ public class CormorantStatusEvent {
public String getStatus() {
return status;
}
public abstract boolean isFor(Wallet wallet);
}

7
src/main/java/com/sparrowwallet/sparrow/event/CormorantSyncStatusEvent.java
View file

@ -1,5 +1,7 @@
package com.sparrowwallet.sparrow.event;
import com.sparrowwallet.drongo.wallet.Wallet;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
@ -16,6 +18,11 @@ public class CormorantSyncStatusEvent extends CormorantStatusEvent {
this.tip = tip;
}
@Override
public boolean isFor(Wallet wallet) {
return true;
}
public int getProgress() {
return progress;
}

15
src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java
View file

@ -109,6 +109,7 @@ public class ElectrumServer {
if(previousServer != null && !electrumServer.equals(previousServer)) {
retrievedScriptHashes.clear();
retrievedTransactions.clear();
TransactionHistoryService.walletLocks.values().forEach(walletLock -> walletLock.initialized = false);
}
previousServer = electrumServer;
@ -252,6 +253,7 @@ public class ElectrumServer {
public static void clearRetrievedScriptHashes(Wallet wallet) {
wallet.getNode(KeyPurpose.RECEIVE).getChildren().stream().map(ElectrumServer::getScriptHash).forEach(ElectrumServer::clearRetrievedScriptHash);
wallet.getNode(KeyPurpose.CHANGE).getChildren().stream().map(ElectrumServer::getScriptHash).forEach(ElectrumServer::clearRetrievedScriptHash);
TransactionHistoryService.walletLocks.computeIfAbsent(wallet.hashCode(), w -> new WalletLock()).initialized = false;
}
private static void clearRetrievedScriptHash(String scriptHash) {
@ -1339,11 +1341,15 @@ public class ElectrumServer {
}
}
private static class WalletLock {
public boolean initialized;
}
public static class TransactionHistoryService extends Service<Boolean> {
private final Wallet mainWallet;
private final List<Wallet> filterToWallets;
private final Set<WalletNode> filterToNodes;
private final static Map<Wallet, Object> walletSynchronizeLocks = new HashMap<>();
private final static Map<Integer, WalletLock> walletLocks = Collections.synchronizedMap(new HashMap<>());
public TransactionHistoryService(Wallet wallet) {
this.mainWallet = wallet;
@ -1389,10 +1395,11 @@ public class ElectrumServer {
return false;
}
boolean initial = (walletSynchronizeLocks.putIfAbsent(wallet, new Object()) == null);
synchronized(walletSynchronizeLocks.get(wallet)) {
if(initial) {
WalletLock walletLock = walletLocks.computeIfAbsent(wallet.hashCode(), w -> new WalletLock());
synchronized(walletLock) {
if(!walletLock.initialized) {
addCalculatedScriptHashes(wallet);
walletLock.initialized = true;
}
if(isConnected()) {

2
src/main/java/com/sparrowwallet/sparrow/net/cormorant/Cormorant.java
View file

@ -43,6 +43,8 @@ public class Cormorant {
electrumServerThread.setDaemon(true);
electrumServerThread.start();
bitcoindClient.waitUntilInitialImportStarted();
running = true;
return new Server(Protocol.TCP.toUrlString(com.sparrowwallet.sparrow.net.ElectrumServer.CORE_ELECTRUM_HOST, electrumServer.getPort()));
}

101
src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindClient.java
View file

@ -8,6 +8,7 @@ import com.sparrowwallet.drongo.Utils;
import com.sparrowwallet.drongo.wallet.BlockTransactionHash;
import com.sparrowwallet.drongo.wallet.Wallet;
import com.sparrowwallet.drongo.wallet.WalletNode;
import com.sparrowwallet.sparrow.AppServices;
import com.sparrowwallet.sparrow.EventManager;
import com.sparrowwallet.sparrow.event.CormorantPruneStatusEvent;
import com.sparrowwallet.sparrow.event.CormorantScanStatusEvent;
@ -50,6 +51,8 @@ public class BitcoindClient {
private final Map<String, Lock> descriptorLocks = Collections.synchronizedMap(new HashMap<>());
private final Map<String, ScanDate> importedDescriptors = Collections.synchronizedMap(new HashMap<>());
private final Map<String, Date> descriptorBirthDates = new HashMap<>();
private boolean initialized;
private boolean stopped;
@ -62,6 +65,11 @@ public class BitcoindClient {
private boolean syncing;
private final Lock scanningLock = new ReentrantLock();
private final Set<String> scanningDescriptors = Collections.synchronizedSet(new HashSet<>());
private final Lock initialImportLock = new ReentrantLock();
private final Condition initialImportCondition = initialImportLock.newCondition();
private boolean initialImportStarted;
public BitcoindClient() {
BitcoindTransport bitcoindTransport;
@ -127,49 +135,52 @@ public class BitcoindClient {
return getBitcoindService().listSinceBlock(blockHash, 1, true, true, true);
}
public void importWallets(Set<Wallet> wallets) throws ImportFailedException {
public void importWallets(Collection<Wallet> wallets) throws ImportFailedException {
importDescriptors(getWalletDescriptors(wallets));
}
public void importWallet(Wallet wallet) throws ImportFailedException {
importDescriptors(getWalletDescriptors(Set.of(wallet)));
//To avoid unnecessary rescans, get all related wallets
importWallets(wallet.isMasterWallet() ? wallet.getAllWallets() : wallet.getMasterWallet().getAllWallets());
}
private Map<String, ScanDate> getWalletDescriptors(Set<Wallet> wallets) throws ImportFailedException {
private Map<String, ScanDate> getWalletDescriptors(Collection<Wallet> wallets) throws ImportFailedException {
List<Wallet> validWallets = wallets.stream().filter(Wallet::isValid).collect(Collectors.toList());
Date earliestBirthDate = validWallets.stream().map(Wallet::getBirthDate).filter(Objects::nonNull).sorted().findFirst().orElse(null);
Map<String, ScanDate> outputDescriptors = new LinkedHashMap<>();
for(Wallet wallet : validWallets) {
if(pruned) {
Optional<Date> optPrunedDate = getPrunedDate();
if(optPrunedDate.isPresent() && wallet.getBirthDate() != null) {
if(optPrunedDate.isPresent() && earliestBirthDate != null) {
Date prunedDate = optPrunedDate.get();
Date earliestScanDate = wallet.getBirthDate();
if(earliestScanDate.before(prunedDate)) {
if(earliestBirthDate.before(prunedDate)) {
if(!prunedWarningShown) {
prunedWarningShown = true;
Platform.runLater(() -> EventManager.get().post(new CormorantPruneStatusEvent("Error: Wallet birthday earlier than Bitcoin Core prune date", wallet, earliestScanDate, prunedDate, legacyWalletExists)));
Platform.runLater(() -> EventManager.get().post(new CormorantPruneStatusEvent("Error: Wallet birthday earlier than Bitcoin Core prune date", wallet, earliestBirthDate, prunedDate, legacyWalletExists)));
}
throw new ImportFailedException("Wallet birthday earlier than prune date");
}
}
}
OutputDescriptor receiveOutputDescriptor = OutputDescriptor.getOutputDescriptor(wallet, KeyPurpose.RECEIVE);
outputDescriptors.put(OutputDescriptor.normalize(receiveOutputDescriptor.toString(false, false)), getScanDate(wallet, KeyPurpose.RECEIVE));
OutputDescriptor changeOutputDescriptor = OutputDescriptor.getOutputDescriptor(wallet, KeyPurpose.CHANGE);
outputDescriptors.put(OutputDescriptor.normalize(changeOutputDescriptor.toString(false, false)), getScanDate(wallet, KeyPurpose.CHANGE));
String receiveOutputDescriptor = OutputDescriptor.getOutputDescriptor(wallet, KeyPurpose.RECEIVE).toString(false, false);
addOutputDescriptor(outputDescriptors, receiveOutputDescriptor, wallet, KeyPurpose.RECEIVE, earliestBirthDate);
String changeOutputDescriptor = OutputDescriptor.getOutputDescriptor(wallet, KeyPurpose.CHANGE).toString(false, false);
addOutputDescriptor(outputDescriptors, changeOutputDescriptor, wallet, KeyPurpose.CHANGE, earliestBirthDate);
if(wallet.isMasterWallet() && wallet.hasPaymentCode()) {
Wallet notificationWallet = wallet.getNotificationWallet();
WalletNode notificationNode = notificationWallet.getNode(KeyPurpose.NOTIFICATION);
outputDescriptors.put(OutputDescriptor.normalize(OutputDescriptor.toDescriptorString(notificationNode.getAddress())), getScanDate(wallet, null));
String notificationOutputDescriptor = OutputDescriptor.toDescriptorString(notificationNode.getAddress());
addOutputDescriptor(outputDescriptors, notificationOutputDescriptor, wallet, null, earliestBirthDate);
for(Wallet childWallet : wallet.getChildWallets()) {
if(childWallet.isNested()) {
for(KeyPurpose keyPurpose : KeyPurpose.DEFAULT_PURPOSES) {
for(WalletNode addressNode : childWallet.getNode(keyPurpose).getChildren()) {
outputDescriptors.put(OutputDescriptor.normalize(OutputDescriptor.toDescriptorString(addressNode.getAddress())), getScanDate(childWallet, null));
String addressOutputDescriptor = OutputDescriptor.toDescriptorString(addressNode.getAddress());
addOutputDescriptor(outputDescriptors, addressOutputDescriptor, childWallet, null, earliestBirthDate);
}
}
}
@ -180,6 +191,12 @@ public class BitcoindClient {
return outputDescriptors;
}
private void addOutputDescriptor(Map<String, ScanDate> outputDescriptors, String outputDescriptor, Wallet wallet, KeyPurpose keyPurpose, Date earliestBirthDate) {
String normalizedDescriptor = OutputDescriptor.normalize(outputDescriptor);
ScanDate scanDate = getScanDate(normalizedDescriptor, wallet, keyPurpose, earliestBirthDate);
outputDescriptors.put(normalizedDescriptor, scanDate);
}
private Optional<Date> getPrunedDate() {
BlockchainInfo blockchainInfo = getBitcoindService().getBlockchainInfo();
if(blockchainInfo.pruned()) {
@ -191,24 +208,30 @@ public class BitcoindClient {
return Optional.empty();
}
private ScanDate getScanDate(Wallet wallet, KeyPurpose keyPurpose) {
private ScanDate getScanDate(String normalizedDescriptor, Wallet wallet, KeyPurpose keyPurpose, Date earliestBirthDate) {
Integer range = (keyPurpose == null ? null : wallet.getFreshNode(keyPurpose).getIndex() + GAP_LIMIT);
//Force a rescan if loading a wallet with a birthday later than existing transactions, or if the wallet birthdate has been set or changed to an earlier date from the last check
boolean forceRescan = false;
Date txBirthDate = wallet.getTransactions().values().stream().map(BlockTransactionHash::getDate).filter(Objects::nonNull).min(Date::compareTo).orElse(null);
if((wallet.getBirthDate() != null && txBirthDate != null && wallet.getBirthDate().before(txBirthDate)) || (txBirthDate == null && wallet.getStoredBlockHeight() != null && wallet.getStoredBlockHeight() == 0)) {
Date lastBirthDate = descriptorBirthDates.get(normalizedDescriptor);
if((wallet.getBirthDate() != null && txBirthDate != null && wallet.getBirthDate().before(txBirthDate))
|| (descriptorBirthDates.containsKey(normalizedDescriptor) && earliestBirthDate != null && (lastBirthDate == null || earliestBirthDate.before(lastBirthDate)))) {
forceRescan = true;
}
return new ScanDate(wallet.getBirthDate() == null && !wallet.isMasterWallet() ? wallet.getMasterWallet().getBirthDate() : wallet.getBirthDate(), range, forceRescan);
return new ScanDate(earliestBirthDate, range, forceRescan);
}
private void importDescriptors(Map<String, ScanDate> descriptors) {
for(String descriptor : descriptors.keySet()) {
Lock lock = descriptorLocks.computeIfAbsent(descriptor, desc -> new ReentrantLock());
lock.lock();
descriptorBirthDates.put(descriptor, descriptors.get(descriptor).rescanSince);
}
signalInitialImportStarted();
try {
Set<String> addedDescriptors = addDescriptors(descriptors);
if(!addedDescriptors.isEmpty()) {
@ -268,10 +291,13 @@ public class BitcoindClient {
List<ImportDescriptorResult> results;
scanningLock.lock();
try {
scanningDescriptors.addAll(importingDescriptors.keySet());
results = getBitcoindService().importDescriptors(importDescriptors);
} finally {
scanningLock.unlock();
Platform.runLater(() -> EventManager.get().post(new CormorantScanStatusEvent("Scanning completed", 100, Duration.ZERO)));
Set<Wallet> scanningWallets = getScanningWallets();
scanningDescriptors.clear();
Platform.runLater(() -> EventManager.get().post(new CormorantScanStatusEvent("Scanning completed", scanningWallets, 100, Duration.ZERO)));
}
for(int i = 0; i < importDescriptors.size(); i++) {
@ -416,6 +442,31 @@ public class BitcoindClient {
}
}
public void waitUntilInitialImportStarted() {
initialImportLock.lock();
try {
if(!initialImportStarted) {
initialImportCondition.await();
}
} catch(InterruptedException e) {
//ignore
} finally {
initialImportLock.unlock();
}
}
private void signalInitialImportStarted() {
if(!initialImportStarted) {
initialImportLock.lock();
try {
initialImportStarted = true;
initialImportCondition.signal();
} finally {
initialImportLock.unlock();
}
}
}
public Store getStore() {
return store;
}
@ -481,9 +532,10 @@ public class BitcoindClient {
} else {
WalletInfo walletInfo = getBitcoindService().getWalletInfo();
if(walletInfo.scanning().isScanning()) {
Set<Wallet> scanningWallets = getScanningWallets();
int percent = walletInfo.scanning().getPercent();
Duration remainingDuration = walletInfo.scanning().getRemaining();
Platform.runLater(() -> EventManager.get().post(new CormorantScanStatusEvent("Scanning" + (percent < 100 ? " (" + percent + "%)" : ""), percent, remainingDuration)));
Platform.runLater(() -> EventManager.get().post(new CormorantScanStatusEvent("Scanning" + (percent < 100 ? " (" + percent + "%)" : ""), scanningWallets, percent, remainingDuration)));
}
}
} catch(Exception e) {
@ -492,6 +544,19 @@ public class BitcoindClient {
}
}
private Set<Wallet> getScanningWallets() {
Set<Wallet> scanningWallets = new HashSet<>();
Set<Wallet> openWallets = AppServices.get().getOpenWallets().keySet();
for(Wallet openWallet : openWallets) {
String normalizedDescriptor = OutputDescriptor.normalize(OutputDescriptor.getOutputDescriptor(openWallet, KeyPurpose.RECEIVE).toString(false, false));
if(scanningDescriptors.contains(normalizedDescriptor)) {
scanningWallets.add(openWallet);
}
}
return scanningWallets;
}
private record ScanDate(Date rescanSince, Integer range, boolean forceRescan) {
public Object getTimestamp() {
return rescanSince == null ? "now" : rescanSince.getTime() / 1000;

4
src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindTransport.java
View file

@ -67,7 +67,7 @@ public class BitcoindTransport implements Transport {
connection.setDoOutput(true);
log.debug("> " + request);
log.trace("> " + request);
try(OutputStream os = connection.getOutputStream()) {
byte[] jsonBytes = request.getBytes(StandardCharsets.UTF_8);
@ -93,7 +93,7 @@ public class BitcoindTransport implements Transport {
}
String response = res.toString();
log.debug("< " + response);
log.trace("< " + response);
return response;
}

4
src/main/java/com/sparrowwallet/sparrow/terminal/SparrowTerminal.java
View file

@ -142,6 +142,10 @@ public class SparrowTerminal extends Application {
.map(data -> new WalletTabData(TabData.TabType.WALLET, data.getWalletForm())).collect(Collectors.toList());
EventManager.get().post(new OpenWalletsEvent(DEFAULT_WINDOW, walletTabDataList));
if(wallet.isValid()) {
Platform.runLater(() -> walletForm.refreshHistory(AppServices.getCurrentBlockHeight()));
}
Set<File> walletFiles = new LinkedHashSet<>();
walletFiles.add(storage.getWalletFile());
if(Config.get().getRecentWalletFiles() != null) {

2
src/main/java/com/sparrowwallet/sparrow/wallet/SettingsWalletForm.java
View file

@ -24,7 +24,7 @@ public class SettingsWalletForm extends WalletForm {
private Wallet walletCopy;
public SettingsWalletForm(Storage storage, Wallet currentWallet) {
super(storage, currentWallet, false);
super(storage, currentWallet);
this.walletCopy = currentWallet.copy();
this.walletCopy.setMasterWallet(walletCopy.isMasterWallet() ? null : walletCopy.getMasterWallet().copy());
}

4
src/main/java/com/sparrowwallet/sparrow/wallet/TransactionsController.java
View file

@ -244,7 +244,9 @@ public class TransactionsController extends WalletFormController implements Init
@Subscribe
public void cormorantStatus(CormorantStatusEvent event) {
walletHistoryStatus(new WalletHistoryStatusEvent(walletForm.getWallet(), true, event.getStatus()));
if(event.isFor(walletForm.getWallet())) {
walletHistoryStatus(new WalletHistoryStatusEvent(walletForm.getWallet(), true, event.getStatus()));
}
}
@Subscribe

4
src/main/java/com/sparrowwallet/sparrow/wallet/UtxosController.java
View file

@ -571,7 +571,9 @@ public class UtxosController extends WalletFormController implements Initializab
@Subscribe
public void cormorantStatus(CormorantStatusEvent event) {
walletHistoryStatus(new WalletHistoryStatusEvent(walletForm.getWallet(), true, event.getStatus()));
if(event.isFor(walletForm.getWallet())) {
walletHistoryStatus(new WalletHistoryStatusEvent(walletForm.getWallet(), true, event.getStatus()));
}
}
@Subscribe

8
src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java
View file

@ -54,10 +54,6 @@ public class WalletForm {
private final BooleanProperty lockedProperty = new SimpleBooleanProperty(false);
public WalletForm(Storage storage, Wallet currentWallet) {
this(storage, currentWallet, true);
}
public WalletForm(Storage storage, Wallet currentWallet, boolean refreshHistory) {
this.storage = storage;
this.wallet = currentWallet;
@ -70,10 +66,6 @@ public class WalletForm {
}, exception -> {
log.error("Error refreshing nodes", exception);
});
if(refreshHistory && wallet.isValid()) {
refreshHistory(AppServices.getCurrentBlockHeight());
}
}
public Wallet getWallet() {