improvements to fetching and settings

drongo

@@ -1 +1 @@
-Subproject commit 601c11bd50405cc781bc35f8c680ba4c5e48ae91
+Subproject commit 81378b28b25d02dca8cdfc21a6b4fae0421d82b1

src/main/java/com/sparrowwallet/sparrow/AppController.java

@@ -515,6 +515,7 @@ public class AppController implements Initializable {
             tab.setContent(walletLoader.load());
             WalletController controller = walletLoader.getController();
             WalletForm walletForm = new WalletForm(storage, wallet);
+            EventManager.get().register(walletForm);
             controller.setWalletForm(walletForm);
 
             if(!storage.getWalletFile().exists() || wallet.containsSource(KeystoreSource.HW_USB)) {

src/main/java/com/sparrowwallet/sparrow/event/WalletHistoryChangedEvent.java

@@ -0,0 +1,19 @@
+package com.sparrowwallet.sparrow.event;
+
+import com.sparrowwallet.drongo.wallet.Wallet;
+import com.sparrowwallet.drongo.wallet.WalletNode;
+
+import java.util.List;
+
+public class WalletHistoryChangedEvent extends WalletChangedEvent {
+    private final List<WalletNode> historyChangedNodes;
+
+    public WalletHistoryChangedEvent(Wallet wallet, List<WalletNode> historyChangedNodes) {
+        super(wallet);
+        this.historyChangedNodes = historyChangedNodes;
+    }
+
+    public List<WalletNode> getHistoryChangedNodes() {
+        return historyChangedNodes;
+    }
+}

src/main/java/com/sparrowwallet/sparrow/io/ElectrumServer.java

@@ -127,33 +127,47 @@ public class ElectrumServer {
     }
 
     public Map<WalletNode, Set<BlockTransactionHash>> getHistory(Wallet wallet) throws ServerException {
-        Map<WalletNode, Set<BlockTransactionHash>> nodeTransactionMap = new HashMap<>();
+        Map<WalletNode, Set<BlockTransactionHash>> receiveTransactionMap = new TreeMap<>();
-        getHistory(wallet, KeyPurpose.RECEIVE, nodeTransactionMap);
+        getHistory(wallet, KeyPurpose.RECEIVE, receiveTransactionMap);
-        getHistory(wallet, KeyPurpose.CHANGE, nodeTransactionMap);
 
-        return nodeTransactionMap;
+        Map<WalletNode, Set<BlockTransactionHash>> changeTransactionMap = new TreeMap<>();
+        getHistory(wallet, KeyPurpose.CHANGE, changeTransactionMap);
+
+        receiveTransactionMap.putAll(changeTransactionMap);
+        return receiveTransactionMap;
     }
 
     public void getHistory(Wallet wallet, KeyPurpose keyPurpose, Map<WalletNode, Set<BlockTransactionHash>> nodeTransactionMap) throws ServerException {
-        getHistory(wallet, wallet.getNode(keyPurpose).getChildren(), nodeTransactionMap);
+        WalletNode purposeNode = wallet.getNode(keyPurpose);
-        //Not necessary, mempool transactions included in history
+        getHistory(wallet, purposeNode.getChildren(), nodeTransactionMap, 0);
-        //getMempool(wallet, wallet.getNode(keyPurpose).getChildren(), nodeTransactionMap);
+
+        int historySize = purposeNode.getChildren().size();
+        int gapLimitSize = nodeTransactionMap.size() + Wallet.DEFAULT_LOOKAHEAD;
+        while(historySize < gapLimitSize) {
+            purposeNode.fillToIndex(gapLimitSize - 1);
+            getHistory(wallet, purposeNode.getChildren(), nodeTransactionMap, historySize);
+            historySize = purposeNode.getChildren().size();
+            gapLimitSize = nodeTransactionMap.size() + Wallet.DEFAULT_LOOKAHEAD;
+        }
     }
 
-    public void getHistory(Wallet wallet, Collection<WalletNode> nodes, Map<WalletNode, Set<BlockTransactionHash>> nodeTransactionMap) throws ServerException {
+    public void getHistory(Wallet wallet, Collection<WalletNode> nodes, Map<WalletNode, Set<BlockTransactionHash>> nodeTransactionMap, int startIndex) throws ServerException {
-        getReferences(wallet, "blockchain.scripthash.get_history", nodes, nodeTransactionMap);
+        getReferences(wallet, "blockchain.scripthash.get_history", nodes, nodeTransactionMap, startIndex);
     }
 
-    public void getMempool(Wallet wallet, Collection<WalletNode> nodes, Map<WalletNode, Set<BlockTransactionHash>> nodeTransactionMap) throws ServerException {
+    public void getMempool(Wallet wallet, Collection<WalletNode> nodes, Map<WalletNode, Set<BlockTransactionHash>> nodeTransactionMap, int startIndex) throws ServerException {
-        getReferences(wallet, "blockchain.scripthash.get_mempool", nodes, nodeTransactionMap);
+        getReferences(wallet, "blockchain.scripthash.get_mempool", nodes, nodeTransactionMap, startIndex);
     }
 
-    public void getReferences(Wallet wallet, String method, Collection<WalletNode> nodes, Map<WalletNode, Set<BlockTransactionHash>> nodeTransactionMap) throws ServerException {
+    public void getReferences(Wallet wallet, String method, Collection<WalletNode> nodes, Map<WalletNode, Set<BlockTransactionHash>> nodeTransactionMap, int startIndex) throws ServerException {
         try {
             JsonRpcClient client = new JsonRpcClient(getTransport());
             BatchRequestBuilder<String, ScriptHashTx[]> batchRequest = client.createBatchRequest().keysType(String.class).returnType(ScriptHashTx[].class);
+
             for(WalletNode node : nodes) {
-                batchRequest.add(node.getDerivationPath(), method, getScriptHash(wallet, node));
+                if(node.getIndex() >= startIndex) {
+                    batchRequest.add(node.getDerivationPath(), method, getScriptHash(wallet, node));
+                }
             }
 
             Map<String, ScriptHashTx[]> result;
@@ -392,16 +406,17 @@ public class ElectrumServer {
     public void calculateNodeHistory(Wallet wallet, Map<WalletNode, Set<BlockTransactionHash>> nodeTransactionMap, WalletNode node) {
         Set<BlockTransactionHashIndex> transactionOutputs = new TreeSet<>();
 
+        //First check all provided txes that pay to this node
         Script nodeScript = wallet.getOutputScript(node);
         Set<BlockTransactionHash> history = nodeTransactionMap.get(node);
         for(BlockTransactionHash reference : history) {
             BlockTransaction blockTransaction = wallet.getTransactions().get(reference.getHash());
-            if (blockTransaction == null || blockTransaction.equals(UNFETCHABLE_BLOCK_TRANSACTION)) {
+            if(blockTransaction == null || blockTransaction.equals(UNFETCHABLE_BLOCK_TRANSACTION)) {
                 throw new IllegalStateException("Could not retrieve transaction for hash " + reference.getHashAsString());
             }
             Transaction transaction = blockTransaction.getTransaction();
 
-            for (int outputIndex = 0; outputIndex < transaction.getOutputs().size(); outputIndex++) {
+            for(int outputIndex = 0; outputIndex < transaction.getOutputs().size(); outputIndex++) {
                 TransactionOutput output = transaction.getOutputs().get(outputIndex);
                 if (output.getScript().equals(nodeScript)) {
                     BlockTransactionHashIndex receivingTXO = new BlockTransactionHashIndex(reference.getHash(), reference.getHeight(), blockTransaction.getDate(), reference.getFee(), output.getIndex(), output.getValue());
@@ -410,9 +425,10 @@ public class ElectrumServer {
             }
         }
 
+        //Then check all provided txes that pay from this node
         for(BlockTransactionHash reference : history) {
             BlockTransaction blockTransaction = wallet.getTransactions().get(reference.getHash());
-            if (blockTransaction == null || blockTransaction.equals(UNFETCHABLE_BLOCK_TRANSACTION)) {
+            if(blockTransaction == null || blockTransaction.equals(UNFETCHABLE_BLOCK_TRANSACTION)) {
                 throw new IllegalStateException("Could not retrieve transaction for hash " + reference.getHashAsString());
             }
             Transaction transaction = blockTransaction.getTransaction();
@@ -443,7 +459,7 @@ public class ElectrumServer {
                     BlockTransactionHashIndex spendingTXI = new BlockTransactionHashIndex(reference.getHash(), reference.getHeight(), blockTransaction.getDate(), reference.getFee(), inputIndex, spentOutput.getValue());
                     BlockTransactionHashIndex spentTXO = new BlockTransactionHashIndex(spentTxHash.getHash(), spentTxHash.getHeight(), previousTransaction.getDate(), spentTxHash.getFee(), spentOutput.getIndex(), spentOutput.getValue(), spendingTXI);
 
-                    Optional<BlockTransactionHashIndex> optionalReference = transactionOutputs.stream().filter(receivedTXO -> receivedTXO.equals(spentTXO)).findFirst();
+                    Optional<BlockTransactionHashIndex> optionalReference = transactionOutputs.stream().filter(receivedTXO -> receivedTXO.getHash().equals(spentTXO.getHash()) && receivedTXO.getIndex() == spentTXO.getIndex()).findFirst();
                     if(optionalReference.isEmpty()) {
                         throw new IllegalStateException("Found spent transaction output " + spentTXO + " but no record of receiving it");
                     }

src/main/java/com/sparrowwallet/sparrow/wallet/SettingsController.java

@@ -32,6 +32,8 @@ import tornadofx.control.Fieldset;
 
 import java.io.IOException;
 import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Optional;
 import java.util.ResourceBundle;
 import java.util.stream.Collectors;
@@ -72,6 +74,8 @@ public class SettingsController extends WalletFormController implements Initiali
 
     private final SimpleIntegerProperty totalKeystores = new SimpleIntegerProperty(0);
 
+    private boolean initialising = true;
+
     @Override
     public void initialize(URL location, ResourceBundle resources) {
         EventManager.get().register(this);
@@ -90,9 +94,10 @@ public class SettingsController extends WalletFormController implements Initiali
                 scriptType.getSelectionModel().select(policyType.getDefaultScriptType());
             }
 
-            if(oldValue != null) {
+            if(!initialising) {
                 clearKeystoreTabs();
             }
+            initialising = false;
 
             multisigFieldset.setVisible(policyType.equals(PolicyType.MULTI));
             if(policyType.equals(PolicyType.MULTI)) {
@@ -133,7 +138,9 @@ public class SettingsController extends WalletFormController implements Initiali
                 keystore.setWalletModel(WalletModel.SPARROW);
                 walletForm.getWallet().getKeystores().add(keystore);
             }
-            walletForm.getWallet().setKeystores(walletForm.getWallet().getKeystores().subList(0, numCosigners.intValue()));
+            List<Keystore> newKeystoreList = new ArrayList<>(walletForm.getWallet().getKeystores().subList(0, numCosigners.intValue()));
+            walletForm.getWallet().getKeystores().clear();
+            walletForm.getWallet().getKeystores().addAll(newKeystoreList);
 
             for(int i = 0; i < walletForm.getWallet().getKeystores().size(); i++) {
                 Keystore keystore = walletForm.getWallet().getKeystores().get(i);
@@ -155,7 +162,8 @@ public class SettingsController extends WalletFormController implements Initiali
             keystoreTabs.getTabs().removeAll(keystoreTabs.getTabs());
             totalKeystores.unbind();
             totalKeystores.setValue(0);
-            walletForm.revertAndRefresh();
+            walletForm.revert();
+            initialising = true;
             setFieldsFromWallet(walletForm.getWallet());
         });
 

src/main/java/com/sparrowwallet/sparrow/wallet/SettingsWalletForm.java

@@ -0,0 +1,34 @@
+package com.sparrowwallet.sparrow.wallet;
+
+import com.sparrowwallet.drongo.wallet.Wallet;
+import com.sparrowwallet.sparrow.io.Storage;
+
+import java.io.IOException;
+
+public class SettingsWalletForm extends WalletForm {
+    private Wallet walletCopy;
+
+    public SettingsWalletForm(Storage storage, Wallet currentWallet) {
+        super(storage, currentWallet);
+        this.walletCopy = currentWallet.copy();
+    }
+
+    @Override
+    public Wallet getWallet() {
+        return walletCopy;
+    }
+
+    @Override
+    public void revert() {
+        this.walletCopy = super.getWallet().copy();
+    }
+
+    @Override
+    public void saveAndRefresh() throws IOException {
+        //TODO: Detect trivial changes and don't clear history
+        walletCopy.clearHistory();
+        wallet = walletCopy.copy();
+        save();
+        refreshHistory(wallet.getStoredBlockHeight());
+    }
+}

src/main/java/com/sparrowwallet/sparrow/wallet/WalletController.java

@@ -60,7 +60,13 @@ public class WalletController extends WalletFormController implements Initializa
                     Node walletFunction = functionLoader.load();
                     walletFunction.setUserData(function);
                     WalletFormController controller = functionLoader.getController();
-                    controller.setWalletForm(getWalletForm());
+
+                    WalletForm walletForm = getWalletForm();
+                    if(function.equals(Function.SETTINGS)) {
+                        walletForm = new SettingsWalletForm(getWalletForm().getStorage(), getWalletForm().getWallet());
+                    }
+
+                    controller.setWalletForm(walletForm);
                     walletFunction.setViewOrder(1);
                     walletPane.getChildren().add(walletFunction);
                 }

src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java

@@ -4,32 +4,30 @@ import com.google.common.eventbus.Subscribe;
 import com.sparrowwallet.drongo.KeyPurpose;
 import com.sparrowwallet.drongo.wallet.Wallet;
 import com.sparrowwallet.drongo.wallet.WalletNode;
+import com.sparrowwallet.sparrow.AppController;
 import com.sparrowwallet.sparrow.EventManager;
+import com.sparrowwallet.sparrow.event.ConnectionEvent;
 import com.sparrowwallet.sparrow.event.NewBlockEvent;
 import com.sparrowwallet.sparrow.event.WalletChangedEvent;
+import com.sparrowwallet.sparrow.event.WalletHistoryChangedEvent;
 import com.sparrowwallet.sparrow.io.ElectrumServer;
 import com.sparrowwallet.sparrow.io.Storage;
+import javafx.application.Platform;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.List;
-import java.util.Optional;
 
 public class WalletForm {
     private final Storage storage;
-    private Wallet oldWallet;
+    protected Wallet wallet;
-    private Wallet wallet;
 
     private final List<NodeEntry> accountEntries = new ArrayList<>();
 
     public WalletForm(Storage storage, Wallet currentWallet) {
         this.storage = storage;
-        this.oldWallet = currentWallet.copy();
         this.wallet = currentWallet;
-        refreshHistory();
+        refreshHistory(wallet.getStoredBlockHeight());
-
-        EventManager.get().register(this);
     }
 
     public Wallet getWallet() {
@@ -44,41 +42,60 @@ public class WalletForm {
         return storage.getWalletFile();
     }
 
-    public void revertAndRefresh() {
+    public void revert() {
-        this.wallet = oldWallet.copy();
+        throw new UnsupportedOperationException("Only SettingsWalletForm supports revert");
-        refreshHistory();
     }
 
     public void save() throws IOException {
         storage.storeWallet(wallet);
-        oldWallet = wallet.copy();
     }
 
     public void saveAndRefresh() throws IOException {
-        //TODO: Detect trivial changes and don't clear history
         wallet.clearHistory();
         save();
-        refreshHistory();
+        refreshHistory(wallet.getStoredBlockHeight());
     }
 
-    public void refreshHistory() {
+    public void refreshHistory(Integer blockHeight) {
-        if(wallet.isValid()) {
+        Wallet previousWallet = wallet.copy();
+        if(wallet.isValid() && AppController.isOnline()) {
             ElectrumServer.TransactionHistoryService historyService = new ElectrumServer.TransactionHistoryService(wallet);
             historyService.setOnSucceeded(workerStateEvent -> {
-                //TODO: Show connected
+                wallet.setStoredBlockHeight(blockHeight);
-                try {
+                notifyIfHistoryChanged(previousWallet);
-                    storage.storeWallet(wallet);
-                } catch (IOException e) {
-                    e.printStackTrace();
-                }
             });
             historyService.setOnFailed(workerStateEvent -> {
-                //TODO: Show not connected, log exception
+                workerStateEvent.getSource().getException().printStackTrace();
             });
             historyService.start();
         }
     }
 
+    private void notifyIfHistoryChanged(Wallet previousWallet) {
+        List<WalletNode> historyChangedNodes = new ArrayList<>();
+        historyChangedNodes.addAll(getHistoryChangedNodes(previousWallet.getNode(KeyPurpose.RECEIVE).getChildren(), wallet.getNode(KeyPurpose.RECEIVE).getChildren()));
+        historyChangedNodes.addAll(getHistoryChangedNodes(previousWallet.getNode(KeyPurpose.CHANGE).getChildren(), wallet.getNode(KeyPurpose.CHANGE).getChildren()));
+
+        if(!historyChangedNodes.isEmpty()) {
+            Platform.runLater(() -> EventManager.get().post(new WalletHistoryChangedEvent(wallet, historyChangedNodes)));
+        }
+    }
+
+    private List<WalletNode> getHistoryChangedNodes(Set<WalletNode> previousNodes, Set<WalletNode> currentNodes) {
+        List<WalletNode> changedNodes = new ArrayList<>();
+        for(WalletNode currentNode : currentNodes) {
+            Optional<WalletNode> optPreviousNode = previousNodes.stream().filter(node -> node.equals(currentNode)).findFirst();
+            if(optPreviousNode.isPresent()) {
+                WalletNode previousNode = optPreviousNode.get();
+                if(!currentNode.getTransactionOutputs().equals(previousNode.getTransactionOutputs())) {
+                    changedNodes.add(currentNode);
+                }
+            }
+        }
+
+        return changedNodes;
+    }
+
     public NodeEntry getNodeEntry(KeyPurpose keyPurpose) {
         NodeEntry purposeEntry;
         Optional<NodeEntry> optionalPurposeEntry = accountEntries.stream().filter(entry -> entry.getNode().getKeyPurpose().equals(keyPurpose)).findFirst();
@@ -123,7 +140,11 @@ public class WalletForm {
 
     @Subscribe
     public void newBlock(NewBlockEvent event) {
-        refreshHistory();
+        refreshHistory(event.getHeight());
-        wallet.setStoredBlockHeight(event.getHeight());
+    }
+
+    @Subscribe
+    public void connected(ConnectionEvent event) {
+        refreshHistory(event.getBlockHeight());
     }
 }