cormorant: improve scanning behaviour

src/main/java/com/sparrowwallet/sparrow/AppController.java

@@ -1611,6 +1611,10 @@ public class AppController implements Initializable {
             });
             subTabs.getSelectionModel().select(subTab);
 
+            if(wallet.isValid()) {
+                Platform.runLater(() -> walletForm.refreshHistory(AppServices.getCurrentBlockHeight()));
+            }
+
             return walletForm;
         } catch(IOException e) {
             throw new RuntimeException(e);

src/main/java/com/sparrowwallet/sparrow/event/CormorantPruneStatusEvent.java

@@ -22,6 +22,11 @@ public class CormorantPruneStatusEvent extends CormorantStatusEvent {
         this.legacyWalletExists = legacyWalletExists;
     }
 
+    @Override
+    public boolean isFor(Wallet wallet) {
+        return this.wallet == wallet;
+    }
+
     public Wallet getWallet() {
         return wallet;
     }

src/main/java/com/sparrowwallet/sparrow/event/CormorantScanStatusEvent.java

@@ -1,17 +1,27 @@
 package com.sparrowwallet.sparrow.event;
 
+import com.sparrowwallet.drongo.wallet.Wallet;
+
 import java.time.Duration;
+import java.util.Set;
 
 public class CormorantScanStatusEvent extends CormorantStatusEvent {
+    private final Set<Wallet> scanningWallets;
     private final int progress;
     private final Duration remainingDuration;
 
-    public CormorantScanStatusEvent(String status, int progress, Duration remainingDuration) {
+    public CormorantScanStatusEvent(String status, Set<Wallet> scanningWallets, int progress, Duration remainingDuration) {
         super(status);
+        this.scanningWallets = scanningWallets;
         this.progress = progress;
         this.remainingDuration = remainingDuration;
     }
 
+    @Override
+    public boolean isFor(Wallet wallet) {
+        return scanningWallets.contains(wallet);
+    }
+
     public int getProgress() {
         return progress;
     }

src/main/java/com/sparrowwallet/sparrow/event/CormorantStatusEvent.java

@@ -1,6 +1,8 @@
 package com.sparrowwallet.sparrow.event;
 
-public class CormorantStatusEvent {
+import com.sparrowwallet.drongo.wallet.Wallet;
+
+public abstract class CormorantStatusEvent {
     private final String status;
 
     public CormorantStatusEvent(String status) {
@@ -10,4 +12,6 @@ public class CormorantStatusEvent {
     public String getStatus() {
         return status;
     }
+
+    public abstract boolean isFor(Wallet wallet);
 }

src/main/java/com/sparrowwallet/sparrow/event/CormorantSyncStatusEvent.java

@@ -1,5 +1,7 @@
 package com.sparrowwallet.sparrow.event;
 
+import com.sparrowwallet.drongo.wallet.Wallet;
+
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.Date;
@@ -16,6 +18,11 @@ public class CormorantSyncStatusEvent extends CormorantStatusEvent {
         this.tip = tip;
     }
 
+    @Override
+    public boolean isFor(Wallet wallet) {
+        return true;
+    }
+
     public int getProgress() {
         return progress;
     }

src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java

@@ -109,6 +109,7 @@ public class ElectrumServer {
                 if(previousServer != null && !electrumServer.equals(previousServer)) {
                     retrievedScriptHashes.clear();
                     retrievedTransactions.clear();
+                    TransactionHistoryService.walletLocks.values().forEach(walletLock -> walletLock.initialized = false);
                 }
                 previousServer = electrumServer;
 
@@ -252,6 +253,7 @@ public class ElectrumServer {
     public static void clearRetrievedScriptHashes(Wallet wallet) {
         wallet.getNode(KeyPurpose.RECEIVE).getChildren().stream().map(ElectrumServer::getScriptHash).forEach(ElectrumServer::clearRetrievedScriptHash);
         wallet.getNode(KeyPurpose.CHANGE).getChildren().stream().map(ElectrumServer::getScriptHash).forEach(ElectrumServer::clearRetrievedScriptHash);
+        TransactionHistoryService.walletLocks.computeIfAbsent(wallet.hashCode(), w -> new WalletLock()).initialized = false;
     }
 
     private static void clearRetrievedScriptHash(String scriptHash) {
@@ -1339,11 +1341,15 @@ public class ElectrumServer {
         }
     }
 
+    private static class WalletLock {
+        public boolean initialized;
+    }
+
     public static class TransactionHistoryService extends Service<Boolean> {
         private final Wallet mainWallet;
         private final List<Wallet> filterToWallets;
         private final Set<WalletNode> filterToNodes;
-        private final static Map<Wallet, Object> walletSynchronizeLocks = new HashMap<>();
+        private final static Map<Integer, WalletLock> walletLocks = Collections.synchronizedMap(new HashMap<>());
 
         public TransactionHistoryService(Wallet wallet) {
             this.mainWallet = wallet;
@@ -1389,10 +1395,11 @@ public class ElectrumServer {
                 return false;
             }
 
-            boolean initial = (walletSynchronizeLocks.putIfAbsent(wallet, new Object()) == null);
+            WalletLock walletLock = walletLocks.computeIfAbsent(wallet.hashCode(), w -> new WalletLock());
-            synchronized(walletSynchronizeLocks.get(wallet)) {
+            synchronized(walletLock) {
-                if(initial) {
+                if(!walletLock.initialized) {
                     addCalculatedScriptHashes(wallet);
+                    walletLock.initialized = true;
                 }
 
                 if(isConnected()) {

src/main/java/com/sparrowwallet/sparrow/net/cormorant/Cormorant.java

@@ -43,6 +43,8 @@ public class Cormorant {
         electrumServerThread.setDaemon(true);
         electrumServerThread.start();
 
+        bitcoindClient.waitUntilInitialImportStarted();
+
         running = true;
         return new Server(Protocol.TCP.toUrlString(com.sparrowwallet.sparrow.net.ElectrumServer.CORE_ELECTRUM_HOST, electrumServer.getPort()));
     }

src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindClient.java

@@ -8,6 +8,7 @@ import com.sparrowwallet.drongo.Utils;
 import com.sparrowwallet.drongo.wallet.BlockTransactionHash;
 import com.sparrowwallet.drongo.wallet.Wallet;
 import com.sparrowwallet.drongo.wallet.WalletNode;
+import com.sparrowwallet.sparrow.AppServices;
 import com.sparrowwallet.sparrow.EventManager;
 import com.sparrowwallet.sparrow.event.CormorantPruneStatusEvent;
 import com.sparrowwallet.sparrow.event.CormorantScanStatusEvent;
@@ -50,6 +51,8 @@ public class BitcoindClient {
     private final Map<String, Lock> descriptorLocks = Collections.synchronizedMap(new HashMap<>());
     private final Map<String, ScanDate> importedDescriptors = Collections.synchronizedMap(new HashMap<>());
 
+    private final Map<String, Date> descriptorBirthDates = new HashMap<>();
+
     private boolean initialized;
     private boolean stopped;
 
@@ -62,6 +65,11 @@ public class BitcoindClient {
     private boolean syncing;
 
     private final Lock scanningLock = new ReentrantLock();
+    private final Set<String> scanningDescriptors = Collections.synchronizedSet(new HashSet<>());
+
+    private final Lock initialImportLock = new ReentrantLock();
+    private final Condition initialImportCondition = initialImportLock.newCondition();
+    private boolean initialImportStarted;
 
     public BitcoindClient() {
         BitcoindTransport bitcoindTransport;
@@ -127,49 +135,52 @@ public class BitcoindClient {
         return getBitcoindService().listSinceBlock(blockHash, 1, true, true, true);
     }
 
-    public void importWallets(Set<Wallet> wallets) throws ImportFailedException {
+    public void importWallets(Collection<Wallet> wallets) throws ImportFailedException {
         importDescriptors(getWalletDescriptors(wallets));
     }
 
     public void importWallet(Wallet wallet) throws ImportFailedException {
-        importDescriptors(getWalletDescriptors(Set.of(wallet)));
+        //To avoid unnecessary rescans, get all related wallets
+        importWallets(wallet.isMasterWallet() ? wallet.getAllWallets() : wallet.getMasterWallet().getAllWallets());
     }
 
-    private Map<String, ScanDate> getWalletDescriptors(Set<Wallet> wallets) throws ImportFailedException {
+    private Map<String, ScanDate> getWalletDescriptors(Collection<Wallet> wallets) throws ImportFailedException {
         List<Wallet> validWallets = wallets.stream().filter(Wallet::isValid).collect(Collectors.toList());
 
+        Date earliestBirthDate = validWallets.stream().map(Wallet::getBirthDate).filter(Objects::nonNull).sorted().findFirst().orElse(null);
         Map<String, ScanDate> outputDescriptors = new LinkedHashMap<>();
         for(Wallet wallet : validWallets) {
             if(pruned) {
                 Optional<Date> optPrunedDate = getPrunedDate();
-                if(optPrunedDate.isPresent() && wallet.getBirthDate() != null) {
+                if(optPrunedDate.isPresent() && earliestBirthDate != null) {
                     Date prunedDate = optPrunedDate.get();
-                    Date earliestScanDate = wallet.getBirthDate();
+                    if(earliestBirthDate.before(prunedDate)) {
-                    if(earliestScanDate.before(prunedDate)) {
                         if(!prunedWarningShown) {
                             prunedWarningShown = true;
-                            Platform.runLater(() -> EventManager.get().post(new CormorantPruneStatusEvent("Error: Wallet birthday earlier than Bitcoin Core prune date", wallet, earliestScanDate, prunedDate, legacyWalletExists)));
+                            Platform.runLater(() -> EventManager.get().post(new CormorantPruneStatusEvent("Error: Wallet birthday earlier than Bitcoin Core prune date", wallet, earliestBirthDate, prunedDate, legacyWalletExists)));
                         }
                         throw new ImportFailedException("Wallet birthday earlier than prune date");
                     }
                 }
             }
 
-            OutputDescriptor receiveOutputDescriptor = OutputDescriptor.getOutputDescriptor(wallet, KeyPurpose.RECEIVE);
+            String receiveOutputDescriptor = OutputDescriptor.getOutputDescriptor(wallet, KeyPurpose.RECEIVE).toString(false, false);
-            outputDescriptors.put(OutputDescriptor.normalize(receiveOutputDescriptor.toString(false, false)), getScanDate(wallet, KeyPurpose.RECEIVE));
+            addOutputDescriptor(outputDescriptors, receiveOutputDescriptor, wallet, KeyPurpose.RECEIVE, earliestBirthDate);
-            OutputDescriptor changeOutputDescriptor = OutputDescriptor.getOutputDescriptor(wallet, KeyPurpose.CHANGE);
+            String changeOutputDescriptor = OutputDescriptor.getOutputDescriptor(wallet, KeyPurpose.CHANGE).toString(false, false);
-            outputDescriptors.put(OutputDescriptor.normalize(changeOutputDescriptor.toString(false, false)), getScanDate(wallet, KeyPurpose.CHANGE));
+            addOutputDescriptor(outputDescriptors, changeOutputDescriptor, wallet, KeyPurpose.CHANGE, earliestBirthDate);
 
             if(wallet.isMasterWallet() && wallet.hasPaymentCode()) {
                 Wallet notificationWallet = wallet.getNotificationWallet();
                 WalletNode notificationNode = notificationWallet.getNode(KeyPurpose.NOTIFICATION);
-                outputDescriptors.put(OutputDescriptor.normalize(OutputDescriptor.toDescriptorString(notificationNode.getAddress())), getScanDate(wallet, null));
+                String notificationOutputDescriptor = OutputDescriptor.toDescriptorString(notificationNode.getAddress());
+                addOutputDescriptor(outputDescriptors, notificationOutputDescriptor, wallet, null, earliestBirthDate);
 
                 for(Wallet childWallet : wallet.getChildWallets()) {
                     if(childWallet.isNested()) {
                         for(KeyPurpose keyPurpose : KeyPurpose.DEFAULT_PURPOSES) {
                             for(WalletNode addressNode : childWallet.getNode(keyPurpose).getChildren()) {
-                                outputDescriptors.put(OutputDescriptor.normalize(OutputDescriptor.toDescriptorString(addressNode.getAddress())), getScanDate(childWallet, null));
+                                String addressOutputDescriptor = OutputDescriptor.toDescriptorString(addressNode.getAddress());
+                                addOutputDescriptor(outputDescriptors, addressOutputDescriptor, childWallet, null, earliestBirthDate);
                             }
                         }
                     }
@@ -180,6 +191,12 @@ public class BitcoindClient {
         return outputDescriptors;
     }
 
+    private void addOutputDescriptor(Map<String, ScanDate> outputDescriptors, String outputDescriptor, Wallet wallet, KeyPurpose keyPurpose, Date earliestBirthDate) {
+        String normalizedDescriptor = OutputDescriptor.normalize(outputDescriptor);
+        ScanDate scanDate = getScanDate(normalizedDescriptor, wallet, keyPurpose, earliestBirthDate);
+        outputDescriptors.put(normalizedDescriptor, scanDate);
+    }
+
     private Optional<Date> getPrunedDate() {
         BlockchainInfo blockchainInfo = getBitcoindService().getBlockchainInfo();
         if(blockchainInfo.pruned()) {
@@ -191,24 +208,30 @@ public class BitcoindClient {
         return Optional.empty();
     }
 
-    private ScanDate getScanDate(Wallet wallet, KeyPurpose keyPurpose) {
+    private ScanDate getScanDate(String normalizedDescriptor, Wallet wallet, KeyPurpose keyPurpose, Date earliestBirthDate) {
         Integer range = (keyPurpose == null ? null : wallet.getFreshNode(keyPurpose).getIndex() + GAP_LIMIT);
 
+        //Force a rescan if loading a wallet with a birthday later than existing transactions, or if the wallet birthdate has been set or changed to an earlier date from the last check
         boolean forceRescan = false;
         Date txBirthDate = wallet.getTransactions().values().stream().map(BlockTransactionHash::getDate).filter(Objects::nonNull).min(Date::compareTo).orElse(null);
-        if((wallet.getBirthDate() != null && txBirthDate != null && wallet.getBirthDate().before(txBirthDate)) || (txBirthDate == null && wallet.getStoredBlockHeight() != null && wallet.getStoredBlockHeight() == 0)) {
+        Date lastBirthDate = descriptorBirthDates.get(normalizedDescriptor);
+        if((wallet.getBirthDate() != null && txBirthDate != null && wallet.getBirthDate().before(txBirthDate))
+                || (descriptorBirthDates.containsKey(normalizedDescriptor) && earliestBirthDate != null && (lastBirthDate == null || earliestBirthDate.before(lastBirthDate)))) {
             forceRescan = true;
         }
 
-        return new ScanDate(wallet.getBirthDate() == null && !wallet.isMasterWallet() ? wallet.getMasterWallet().getBirthDate() : wallet.getBirthDate(), range, forceRescan);
+        return new ScanDate(earliestBirthDate, range, forceRescan);
     }
 
     private void importDescriptors(Map<String, ScanDate> descriptors) {
         for(String descriptor : descriptors.keySet()) {
             Lock lock = descriptorLocks.computeIfAbsent(descriptor, desc -> new ReentrantLock());
             lock.lock();
+            descriptorBirthDates.put(descriptor, descriptors.get(descriptor).rescanSince);
         }
 
+        signalInitialImportStarted();
+
         try {
             Set<String> addedDescriptors = addDescriptors(descriptors);
             if(!addedDescriptors.isEmpty()) {
@@ -268,10 +291,13 @@ public class BitcoindClient {
             List<ImportDescriptorResult> results;
             scanningLock.lock();
             try {
+                scanningDescriptors.addAll(importingDescriptors.keySet());
                 results = getBitcoindService().importDescriptors(importDescriptors);
             } finally {
                 scanningLock.unlock();
-                Platform.runLater(() -> EventManager.get().post(new CormorantScanStatusEvent("Scanning completed", 100, Duration.ZERO)));
+                Set<Wallet> scanningWallets = getScanningWallets();
+                scanningDescriptors.clear();
+                Platform.runLater(() -> EventManager.get().post(new CormorantScanStatusEvent("Scanning completed", scanningWallets, 100, Duration.ZERO)));
             }
 
             for(int i = 0; i < importDescriptors.size(); i++) {
@@ -416,6 +442,31 @@ public class BitcoindClient {
         }
     }
 
+    public void waitUntilInitialImportStarted() {
+        initialImportLock.lock();
+        try {
+            if(!initialImportStarted) {
+                initialImportCondition.await();
+            }
+        } catch(InterruptedException e) {
+            //ignore
+        } finally {
+            initialImportLock.unlock();
+        }
+    }
+
+    private void signalInitialImportStarted() {
+        if(!initialImportStarted) {
+            initialImportLock.lock();
+            try {
+                initialImportStarted = true;
+                initialImportCondition.signal();
+            } finally {
+                initialImportLock.unlock();
+            }
+        }
+    }
+
     public Store getStore() {
         return store;
     }
@@ -481,9 +532,10 @@ public class BitcoindClient {
                 } else {
                     WalletInfo walletInfo = getBitcoindService().getWalletInfo();
                     if(walletInfo.scanning().isScanning()) {
+                        Set<Wallet> scanningWallets = getScanningWallets();
                         int percent = walletInfo.scanning().getPercent();
                         Duration remainingDuration = walletInfo.scanning().getRemaining();
-                        Platform.runLater(() -> EventManager.get().post(new CormorantScanStatusEvent("Scanning" + (percent < 100 ? " (" + percent + "%)" : ""), percent, remainingDuration)));
+                        Platform.runLater(() -> EventManager.get().post(new CormorantScanStatusEvent("Scanning" + (percent < 100 ? " (" + percent + "%)" : ""), scanningWallets, percent, remainingDuration)));
                     }
                 }
             } catch(Exception e) {
@@ -492,6 +544,19 @@ public class BitcoindClient {
         }
     }
 
+    private Set<Wallet> getScanningWallets() {
+        Set<Wallet> scanningWallets = new HashSet<>();
+        Set<Wallet> openWallets = AppServices.get().getOpenWallets().keySet();
+        for(Wallet openWallet : openWallets) {
+            String normalizedDescriptor = OutputDescriptor.normalize(OutputDescriptor.getOutputDescriptor(openWallet, KeyPurpose.RECEIVE).toString(false, false));
+            if(scanningDescriptors.contains(normalizedDescriptor)) {
+                scanningWallets.add(openWallet);
+            }
+        }
+
+        return scanningWallets;
+    }
+
     private record ScanDate(Date rescanSince, Integer range, boolean forceRescan) {
         public Object getTimestamp() {
             return rescanSince == null ? "now" : rescanSince.getTime() / 1000;

src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindTransport.java

@@ -67,7 +67,7 @@ public class BitcoindTransport implements Transport {
 
         connection.setDoOutput(true);
 
-        log.debug("> " + request);
+        log.trace("> " + request);
 
         try(OutputStream os = connection.getOutputStream()) {
             byte[] jsonBytes = request.getBytes(StandardCharsets.UTF_8);
@@ -93,7 +93,7 @@ public class BitcoindTransport implements Transport {
         }
 
         String response = res.toString();
-        log.debug("< " + response);
+        log.trace("< " + response);
 
         return response;
     }

src/main/java/com/sparrowwallet/sparrow/terminal/SparrowTerminal.java

@@ -142,6 +142,10 @@ public class SparrowTerminal extends Application {
                     .map(data -> new WalletTabData(TabData.TabType.WALLET, data.getWalletForm())).collect(Collectors.toList());
             EventManager.get().post(new OpenWalletsEvent(DEFAULT_WINDOW, walletTabDataList));
 
+            if(wallet.isValid()) {
+                Platform.runLater(() -> walletForm.refreshHistory(AppServices.getCurrentBlockHeight()));
+            }
+
             Set<File> walletFiles = new LinkedHashSet<>();
             walletFiles.add(storage.getWalletFile());
             if(Config.get().getRecentWalletFiles() != null) {

src/main/java/com/sparrowwallet/sparrow/wallet/SettingsWalletForm.java

@@ -24,7 +24,7 @@ public class SettingsWalletForm extends WalletForm {
     private Wallet walletCopy;
 
     public SettingsWalletForm(Storage storage, Wallet currentWallet) {
-        super(storage, currentWallet, false);
+        super(storage, currentWallet);
         this.walletCopy = currentWallet.copy();
         this.walletCopy.setMasterWallet(walletCopy.isMasterWallet() ? null : walletCopy.getMasterWallet().copy());
     }

src/main/java/com/sparrowwallet/sparrow/wallet/TransactionsController.java

@@ -244,8 +244,10 @@ public class TransactionsController extends WalletFormController implements Init
 
     @Subscribe
     public void cormorantStatus(CormorantStatusEvent event) {
+        if(event.isFor(walletForm.getWallet())) {
             walletHistoryStatus(new WalletHistoryStatusEvent(walletForm.getWallet(), true, event.getStatus()));
         }
+    }
 
     @Subscribe
     public void bwtSyncStatus(BwtSyncStatusEvent event) {

src/main/java/com/sparrowwallet/sparrow/wallet/UtxosController.java

@@ -571,8 +571,10 @@ public class UtxosController extends WalletFormController implements Initializab
 
     @Subscribe
     public void cormorantStatus(CormorantStatusEvent event) {
+        if(event.isFor(walletForm.getWallet())) {
             walletHistoryStatus(new WalletHistoryStatusEvent(walletForm.getWallet(), true, event.getStatus()));
         }
+    }
 
     @Subscribe
     public void bwtSyncStatus(BwtSyncStatusEvent event) {

src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java

@@ -54,10 +54,6 @@ public class WalletForm {
     private final BooleanProperty lockedProperty = new SimpleBooleanProperty(false);
 
     public WalletForm(Storage storage, Wallet currentWallet) {
-        this(storage, currentWallet, true);
-    }
-
-    public WalletForm(Storage storage, Wallet currentWallet, boolean refreshHistory) {
         this.storage = storage;
         this.wallet = currentWallet;
 
@@ -70,10 +66,6 @@ public class WalletForm {
                 }, exception -> {
                     log.error("Error refreshing nodes", exception);
                 });
-
-        if(refreshHistory && wallet.isValid()) {
-            refreshHistory(AppServices.getCurrentBlockHeight());
-        }
     }
 
     public Wallet getWallet() {