testnet related changes and network improvements and logging

README.md

@@ -8,7 +8,9 @@ More information (and release binaries) can be found at https://sparrowwallet.co
 
 ## Building
 
-To clone this project, use `git clone --recursive git@github.com:sparrowwallet/sparrow.git`
+To clone this project, use 
+
+`git clone --recursive git@github.com:sparrowwallet/sparrow.git`
 
 In order to build, Sparrow requires Java 14 to be installed. The release packages can be built using
 
@@ -18,13 +20,35 @@ In order to build, Sparrow requires Java 14 to be installed. The release package
 
 If you prefer to run Sparrow directly from source, it can be launched with
 
-`./gradlew run`
+`./sparrow`
 
 Java 14 must be installed.
 
 ## Configuration
 
-Sparrow stores it's configuration, log file and wallets in a location appropriate to the operating system:
+Sparrow has a number of command line options, for example to change it's home folder or use testnet:
+
+```
+./sparrow -h
+
+Usage: sparrow [options]
+  Options:
+    --dir, -d
+      Path to Sparrow home folder
+    --help, -h
+      Show usage
+    --network, -n
+      Network to use
+      Possible Values: [mainnet, testnet, regtest]
+```
+
+As a fallback, the network (mainnet, testnet or regtest) can also be set using an environment variable `SPARROW_NETWORK`. For example:
+
+`export SPARROW_NETWORK=testnet`
+
+Note that if you are connecting to an Electrum server when using testnet, that server will need to running on testnet configuration as well.
+
+When not explicitly configured using the command line argument above, Sparrow stores it's mainnet config file, log file and wallets in a home folder location appropriate to the operating system:
 
 Platform | Location
 -------- | --------
@@ -32,6 +56,8 @@ OSX      | ~/.sparrow
 Linux    | ~/.sparrow
 Windows  | %APPDATA%/Sparrow
 
+Testnet and regtest configurations (along with their wallets) are stored in subfolders to allow easy switching between networks.
+
 ## Reporting Issues
 
 Please use the [Issues](https://github.com/sparrowwallet/sparrow/issues) tab above to report an issue. If possible, look in the sparrow.log file in the configuration directory for information helpful in debugging. 

drongo

@@ -1 +1 @@
-Subproject commit b877e94cd09adb3fbc17ecba95897ae5c428ffe9
+Subproject commit 9c6d3ec94b3c4aa961ceb1197348fd8ca3854b4e

src/main/java/com/sparrowwallet/sparrow/Args.java

@@ -7,7 +7,7 @@ public class Args {
     @Parameter(names = { "--dir", "-d" }, description = "Path to Sparrow home folder")
     public String dir;
 
-    @Parameter(names = { "--network", "-n" }, description = "Network to use (mainnet, testnet or regtest)")
+    @Parameter(names = { "--network", "-n" }, description = "Network to use")
     public Network network;
 
     @Parameter(names = { "--help", "-h" }, description = "Show usage", help = true)

src/main/java/com/sparrowwallet/sparrow/net/BatchedElectrumServerRpc.java

@@ -14,15 +14,18 @@ import org.slf4j.LoggerFactory;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
 
 public class BatchedElectrumServerRpc implements ElectrumServerRpc {
     private static final Logger log = LoggerFactory.getLogger(BatchedElectrumServerRpc.class);
 
+    private final AtomicLong idCounter = new AtomicLong();
+
     @Override
     public void ping(Transport transport) {
         try {
             JsonRpcClient client = new JsonRpcClient(transport);
-            client.createRequest().method("server.ping").id(1).executeNullable();
+            client.createRequest().method("server.ping").id(idCounter.incrementAndGet()).executeNullable();
         } catch(JsonRpcException e) {
             throw new ElectrumServerRpcException("Error pinging server", e);
         }
@@ -32,7 +35,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc {
     public List<String> getServerVersion(Transport transport, String clientName, String[] supportedVersions) {
         try {
             JsonRpcClient client = new JsonRpcClient(transport);
-            return client.createRequest().returnAsList(String.class).method("server.version").id(1).param("client_name", clientName).param("protocol_version", supportedVersions).execute();
+            return client.createRequest().returnAsList(String.class).method("server.version").id(idCounter.incrementAndGet()).param("client_name", clientName).param("protocol_version", supportedVersions).execute();
         } catch(JsonRpcException e) {
             throw new ElectrumServerRpcException("Error getting server version", e);
         }
@@ -42,7 +45,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc {
     public String getServerBanner(Transport transport) {
         try {
             JsonRpcClient client = new JsonRpcClient(transport);
-            return client.createRequest().returnAs(String.class).method("server.banner").id(1).execute();
+            return client.createRequest().returnAs(String.class).method("server.banner").id(idCounter.incrementAndGet()).execute();
         } catch(JsonRpcException e) {
             throw new ElectrumServerRpcException("Error getting server banner", e);
         }
@@ -52,7 +55,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc {
     public BlockHeaderTip subscribeBlockHeaders(Transport transport) {
         try {
             JsonRpcClient client = new JsonRpcClient(transport);
-            return client.createRequest().returnAs(BlockHeaderTip.class).method("blockchain.headers.subscribe").id(1).execute();
+            return client.createRequest().returnAs(BlockHeaderTip.class).method("blockchain.headers.subscribe").id(idCounter.incrementAndGet()).execute();
         } catch(JsonRpcException e) {
             throw new ElectrumServerRpcException("Error subscribing to block headers", e);
         }
@@ -209,7 +212,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc {
     public Double getMinimumRelayFee(Transport transport) {
         try {
             JsonRpcClient client = new JsonRpcClient(transport);
-            return client.createRequest().returnAs(Double.class).method("blockchain.relayfee").id(1).execute();
+            return client.createRequest().returnAs(Double.class).method("blockchain.relayfee").id(idCounter.incrementAndGet()).execute();
         } catch(JsonRpcException e) {
             throw new ElectrumServerRpcException("Error getting minimum relay fee", e);
         }
@@ -219,7 +222,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc {
     public String broadcastTransaction(Transport transport, String txHex) {
         try {
             JsonRpcClient client = new JsonRpcClient(transport);
-            return client.createRequest().returnAs(String.class).method("blockchain.transaction.broadcast").id(1).param("raw_tx", txHex).execute();
+            return client.createRequest().returnAs(String.class).method("blockchain.transaction.broadcast").id(idCounter.incrementAndGet()).param("raw_tx", txHex).execute();
         } catch(JsonRpcException e) {
             throw new ElectrumServerRpcException(e.getErrorMessage().getMessage(), e);
         }

src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java

@@ -528,7 +528,8 @@ public class ElectrumServer {
 
             Map<Integer, Double> targetBlocksFeeRatesSats = new TreeMap<>();
             for(Integer target : targetBlocksFeeRatesBtcKb.keySet()) {
-                targetBlocksFeeRatesSats.put(target, targetBlocksFeeRatesBtcKb.get(target) * Transaction.SATOSHIS_PER_BITCOIN / 1000);
+                long minFeeRateSatsKb = (long)(targetBlocksFeeRatesBtcKb.get(target) * Transaction.SATOSHIS_PER_BITCOIN);
+                targetBlocksFeeRatesSats.put(target, minFeeRateSatsKb / 1000d);
             }
 
             return targetBlocksFeeRatesSats;

src/main/java/com/sparrowwallet/sparrow/net/SimpleElectrumServerRpc.java

@@ -13,16 +13,19 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.*;
+import java.util.concurrent.atomic.AtomicLong;
 
 public class SimpleElectrumServerRpc implements ElectrumServerRpc {
     private static final Logger log = LoggerFactory.getLogger(SimpleElectrumServerRpc.class);
     private static final int MAX_TARGET_BLOCKS = 25;
 
+    private final AtomicLong idCounter = new AtomicLong();
+
     @Override
     public void ping(Transport transport) {
         try {
             JsonRpcClient client = new JsonRpcClient(transport);
-            client.createRequest().method("server.ping").id(1).executeNullable();
+            client.createRequest().method("server.ping").id(idCounter.incrementAndGet()).executeNullable();
         } catch(JsonRpcException | IllegalStateException | IllegalArgumentException e) {
             throw new ElectrumServerRpcException("Error pinging server", e);
         }
@@ -33,7 +36,7 @@ public class SimpleElectrumServerRpc implements ElectrumServerRpc {
         try {
             JsonRpcClient client = new JsonRpcClient(transport);
             //Using 1.4 as the version number as EPS tries to parse this number to a float
-            return client.createRequest().returnAsList(String.class).method("server.version").id(1).params(clientName, "1.4").execute();
+            return client.createRequest().returnAsList(String.class).method("server.version").id(idCounter.incrementAndGet()).params(clientName, "1.4").execute();
         } catch(JsonRpcException | IllegalStateException | IllegalArgumentException e) {
             throw new ElectrumServerRpcException("Error getting server version", e);
         }
@@ -43,7 +46,7 @@ public class SimpleElectrumServerRpc implements ElectrumServerRpc {
     public String getServerBanner(Transport transport) {
         try {
             JsonRpcClient client = new JsonRpcClient(transport);
-            return client.createRequest().returnAs(String.class).method("server.banner").id(1).execute();
+            return client.createRequest().returnAs(String.class).method("server.banner").id(idCounter.incrementAndGet()).execute();
         } catch(JsonRpcException | IllegalStateException | IllegalArgumentException e) {
             throw new ElectrumServerRpcException("Error getting server banner", e);
         }
@@ -53,7 +56,7 @@ public class SimpleElectrumServerRpc implements ElectrumServerRpc {
     public BlockHeaderTip subscribeBlockHeaders(Transport transport) {
         try {
             JsonRpcClient client = new JsonRpcClient(transport);
-            return client.createRequest().returnAs(BlockHeaderTip.class).method("blockchain.headers.subscribe").id(1).execute();
+            return client.createRequest().returnAs(BlockHeaderTip.class).method("blockchain.headers.subscribe").id(idCounter.incrementAndGet()).execute();
         } catch(JsonRpcException | IllegalStateException | IllegalArgumentException e) {
             throw new ElectrumServerRpcException("Error subscribing to block headers", e);
         }
@@ -67,7 +70,7 @@ public class SimpleElectrumServerRpc implements ElectrumServerRpc {
         for(String path : pathScriptHashes.keySet()) {
             EventManager.get().post(new WalletHistoryStatusEvent(false, "Loading transactions for " + path));
             try {
-                ScriptHashTx[] scriptHashTxes = client.createRequest().returnAs(ScriptHashTx[].class).method("blockchain.scripthash.get_history").id(path).params(pathScriptHashes.get(path)).execute();
+                ScriptHashTx[] scriptHashTxes = client.createRequest().returnAs(ScriptHashTx[].class).method("blockchain.scripthash.get_history").id(path + "-" + idCounter.incrementAndGet()).params(pathScriptHashes.get(path)).execute();
                 result.put(path, scriptHashTxes);
             } catch(JsonRpcException | IllegalStateException | IllegalArgumentException e) {
                 if(failOnError) {
@@ -88,7 +91,7 @@ public class SimpleElectrumServerRpc implements ElectrumServerRpc {
         Map<String, ScriptHashTx[]> result = new LinkedHashMap<>();
         for(String path : pathScriptHashes.keySet()) {
             try {
-                ScriptHashTx[] scriptHashTxes = client.createRequest().returnAs(ScriptHashTx[].class).method("blockchain.scripthash.get_mempool").id(path).params(pathScriptHashes.get(path)).execute();
+                ScriptHashTx[] scriptHashTxes = client.createRequest().returnAs(ScriptHashTx[].class).method("blockchain.scripthash.get_mempool").id(path + "-" + idCounter.incrementAndGet()).params(pathScriptHashes.get(path)).execute();
                 result.put(path, scriptHashTxes);
             } catch(JsonRpcException | IllegalStateException | IllegalArgumentException e) {
                 if(failOnError) {
@@ -110,7 +113,7 @@ public class SimpleElectrumServerRpc implements ElectrumServerRpc {
         for(String path : pathScriptHashes.keySet()) {
             EventManager.get().post(new WalletHistoryStatusEvent(false, "Finding transactions for " + path));
             try {
-                String scriptHash = client.createRequest().returnAs(String.class).method("blockchain.scripthash.subscribe").id(path).params(pathScriptHashes.get(path)).executeNullable();
+                String scriptHash = client.createRequest().returnAs(String.class).method("blockchain.scripthash.subscribe").id(path + "-" + idCounter.incrementAndGet()).params(pathScriptHashes.get(path)).executeNullable();
                 result.put(path, scriptHash);
             } catch(JsonRpcException | IllegalStateException | IllegalArgumentException e) {
                 //Even if we have some successes, failure to subscribe for all script hashes will result in outdated wallet view. Don't proceed.
@@ -129,7 +132,7 @@ public class SimpleElectrumServerRpc implements ElectrumServerRpc {
         for(Integer blockHeight : blockHeights) {
             EventManager.get().post(new WalletHistoryStatusEvent(false, "Retrieving block at height " + blockHeight));
             try {
-                String blockHeader = client.createRequest().returnAs(String.class).method("blockchain.block.header").id(blockHeight).params(blockHeight).execute();
+                String blockHeader = client.createRequest().returnAs(String.class).method("blockchain.block.header").id(idCounter.incrementAndGet()).params(blockHeight).execute();
                 result.put(blockHeight, blockHeader);
             } catch(IllegalStateException | IllegalArgumentException e) {
                 log.warn("Failed to retrieve block header for block height: " + blockHeight + " (" + e.getMessage() + ")");
@@ -149,7 +152,7 @@ public class SimpleElectrumServerRpc implements ElectrumServerRpc {
         for(String txid : txids) {
             EventManager.get().post(new WalletHistoryStatusEvent(false, "Retrieving transaction [" + txid.substring(0, 6) + "]"));
             try {
-                String rawTxHex = client.createRequest().returnAs(String.class).method("blockchain.transaction.get").id(txid).params(txid).execute();
+                String rawTxHex = client.createRequest().returnAs(String.class).method("blockchain.transaction.get").id(idCounter.incrementAndGet()).params(txid).execute();
                 result.put(txid, rawTxHex);
             } catch(JsonRpcException | IllegalStateException | IllegalArgumentException e) {
                 result.put(txid, Sha256Hash.ZERO_HASH.toString());
@@ -166,7 +169,7 @@ public class SimpleElectrumServerRpc implements ElectrumServerRpc {
         Map<String, VerboseTransaction> result = new LinkedHashMap<>();
         for(String txid : txids) {
             try {
-                VerboseTransaction verboseTransaction = client.createRequest().returnAs(VerboseTransaction.class).method("blockchain.transaction.get").id(txid).params(txid, true).execute();
+                VerboseTransaction verboseTransaction = client.createRequest().returnAs(VerboseTransaction.class).method("blockchain.transaction.get").id(idCounter.incrementAndGet()).params(txid, true).execute();
                 result.put(txid, verboseTransaction);
             } catch(Exception e) {
                 //electrs does not currently support the verbose parameter, so try to fetch an incomplete VerboseTransaction without it
@@ -175,13 +178,13 @@ public class SimpleElectrumServerRpc implements ElectrumServerRpc {
                 log.debug("Error retrieving transaction: " + txid + " (" + (e.getCause() != null ? e.getCause().getMessage() : e.getMessage()) + ")");
 
                 try {
-                    String rawTxHex = client.createRequest().returnAs(String.class).method("blockchain.transaction.get").id(txid).params(txid).execute();
+                    String rawTxHex = client.createRequest().returnAs(String.class).method("blockchain.transaction.get").id(idCounter.incrementAndGet()).params(txid).execute();
                     Transaction tx = new Transaction(Utils.hexToBytes(rawTxHex));
                     String id = tx.getTxId().toString();
                     int height = 0;
 
                     if(scriptHash != null) {
-                        ScriptHashTx[] scriptHashTxes = client.createRequest().returnAs(ScriptHashTx[].class).method("blockchain.scripthash.get_history").id(id).params(scriptHash).execute();
+                        ScriptHashTx[] scriptHashTxes = client.createRequest().returnAs(ScriptHashTx[].class).method("blockchain.scripthash.get_history").id(idCounter.incrementAndGet()).params(scriptHash).execute();
                         for(ScriptHashTx scriptHashTx : scriptHashTxes) {
                             if(scriptHashTx.tx_hash.equals(id)) {
                                 height = scriptHashTx.height;
@@ -213,7 +216,7 @@ public class SimpleElectrumServerRpc implements ElectrumServerRpc {
         for(Integer targetBlock : targetBlocks) {
             if(targetBlock <= MAX_TARGET_BLOCKS) {
                 try {
-                    Double targetBlocksFeeRateBtcKb = client.createRequest().returnAs(Double.class).method("blockchain.estimatefee").id(targetBlock).params(targetBlock).execute();
+                    Double targetBlocksFeeRateBtcKb = client.createRequest().returnAs(Double.class).method("blockchain.estimatefee").id(idCounter.incrementAndGet()).params(targetBlock).execute();
                     result.put(targetBlock, targetBlocksFeeRateBtcKb);
                 } catch(IllegalStateException | IllegalArgumentException e) {
                     log.warn("Failed to retrieve fee rate for target blocks: " + targetBlock + " (" + e.getMessage() + ")");
@@ -233,7 +236,7 @@ public class SimpleElectrumServerRpc implements ElectrumServerRpc {
     public Double getMinimumRelayFee(Transport transport) {
         try {
             JsonRpcClient client = new JsonRpcClient(transport);
-            return client.createRequest().returnAs(Double.class).method("blockchain.relayfee").id(1).execute();
+            return client.createRequest().returnAs(Double.class).method("blockchain.relayfee").id(idCounter.incrementAndGet()).execute();
         } catch(JsonRpcException e) {
             throw new ElectrumServerRpcException("Error getting minimum relay fee", e);
         }
@@ -243,7 +246,7 @@ public class SimpleElectrumServerRpc implements ElectrumServerRpc {
     public String broadcastTransaction(Transport transport, String txHex) {
         try {
             JsonRpcClient client = new JsonRpcClient(transport);
-            return client.createRequest().returnAs(String.class).method("blockchain.transaction.broadcast").id(1).params(txHex).execute();
+            return client.createRequest().returnAs(String.class).method("blockchain.transaction.broadcast").id(idCounter.incrementAndGet()).params(txHex).execute();
         } catch(IllegalStateException | IllegalArgumentException e) {
             throw new ElectrumServerRpcException(e.getMessage(), e);
         } catch(JsonRpcException e) {

src/main/java/com/sparrowwallet/sparrow/wallet/SendController.java

@@ -16,6 +16,7 @@ import com.sparrowwallet.sparrow.control.*;
 import com.sparrowwallet.sparrow.event.*;
 import com.sparrowwallet.sparrow.io.Config;
 import com.sparrowwallet.sparrow.io.ExchangeSource;
+import com.sparrowwallet.sparrow.net.ElectrumServer;
 import javafx.beans.property.BooleanProperty;
 import javafx.beans.property.ObjectProperty;
 import javafx.beans.property.SimpleBooleanProperty;
@@ -32,14 +33,19 @@ import org.controlsfx.validation.ValidationResult;
 import org.controlsfx.validation.ValidationSupport;
 import org.controlsfx.validation.Validator;
 import org.controlsfx.validation.decoration.StyleClassValidationDecoration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.net.URL;
 import java.text.DecimalFormat;
 import java.text.DecimalFormatSymbols;
 import java.util.*;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 public class SendController extends WalletFormController implements Initializable {
+    private static final Logger log = LoggerFactory.getLogger(SendController.class);
+
     public static final List<Integer> TARGET_BLOCKS_RANGE = List.of(1, 2, 3, 4, 5, 10, 25, 50, 100, 500);
 
     public static final double FALLBACK_FEE_RATE = 20000d / 1000;
@@ -586,6 +592,15 @@ public class SendController extends WalletFormController implements Initializabl
     }
 
     public void createTransaction(ActionEvent event) {
+        if(log.isDebugEnabled()) {
+            Map<WalletNode, String> nodeHashes = walletTransactionProperty.get().getSelectedUtxos().values().stream().collect(Collectors.toMap(Function.identity(), node -> ElectrumServer.getScriptHash(walletForm.getWallet(), node)));
+            Map<WalletNode, String> changeHash = Collections.emptyMap();
+            if(walletTransactionProperty.get().getChangeNode() != null) {
+                changeHash = Map.of(walletTransactionProperty.get().getChangeNode(), ElectrumServer.getScriptHash(walletForm.getWallet(), walletTransactionProperty.get().getChangeNode()));
+            }
+            log.debug("Creating tx " + walletTransactionProperty.get().getTransaction().getTxId() + ", expecting notifications for \ninputs \n" + nodeHashes + " and \nchange \n" + changeHash);
+        }
+
         createdWalletTransactionProperty.set(walletTransactionProperty.get());
         PSBT psbt = walletTransactionProperty.get().createPSBT();
         EventManager.get().post(new ViewPSBTEvent(label.getText(), psbt));

src/main/java/com/sparrowwallet/sparrow/wallet/TransactionEntry.java

@@ -116,13 +116,13 @@ public class TransactionEntry extends Entry implements Comparable<TransactionEnt
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         TransactionEntry that = (TransactionEntry) o;
-        return wallet.equals(that.wallet) &&
+        return wallet.equals(that.wallet) && blockTransaction.equals(that.blockTransaction) &&
-                blockTransaction.equals(that.blockTransaction);
+                getChildren().equals(that.getChildren());
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(wallet, blockTransaction);
+        return Objects.hash(wallet, blockTransaction, getChildren());
     }
 
     @Override

src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java

@@ -228,6 +228,7 @@ public class WalletForm {
         if(wallet.isValid()) {
             WalletNode walletNode = event.getWalletNode(wallet);
             if(walletNode != null) {
+                log.debug(wallet.getName() + " history event for node " + walletNode);
                 refreshHistory(AppController.getCurrentBlockHeight(), walletNode);
             }
         }