sparrowwallet/sparrow
1
0
Fork 0
mirror of https://github.com/sparrowwallet/sparrow.git synced 2024-12-24 12:46:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

improve performance on deep wallets by storing addresses

Browse source
This commit is contained in:
Craig Raw 2022-07-18 16:12:32 +02:00
parent 11cda40a40
commit 60aa20ac55
12 changed files with 73 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
drongo

@ -1 +1 @@
Subproject commit 9ae1f68dc42529085edcc8c10d9bcfdbf9639448
Subproject commit 5de3abd36230d545f11bad3b25a21d23ffbbe9cd

17
src/main/java/com/sparrowwallet/sparrow/control/AddressTreeTable.java
View file

@ -14,9 +14,7 @@ import javafx.collections.ListChangeListener;
import javafx.scene.control.*;
import javafx.scene.input.MouseButton;
import java.util.List;
import java.util.Optional;
import java.util.OptionalInt;
import java.util.*;
public class AddressTreeTable extends CoinTreeTable {
public void initialize(NodeEntry rootEntry) {
@ -110,10 +108,15 @@ public class AddressTreeTable extends CoinTreeTable {
//We only ever add child nodes - never remove in order to keep a full sequence (unless hide empty used addresses is set)
NodeEntry rootEntry = (NodeEntry)getRoot().getValue();
Map<WalletNode, NodeEntry> childNodes = new HashMap<>();
for(Entry childEntry : rootEntry.getChildren()) {
NodeEntry nodeEntry = (NodeEntry)childEntry;
childNodes.put(nodeEntry.getNode(), nodeEntry);
}
for(WalletNode updatedNode : updatedNodes) {
Optional<Entry> optEntry = rootEntry.getChildren().stream().filter(childEntry -> ((NodeEntry)childEntry).getNode().equals(updatedNode)).findFirst();
if(optEntry.isPresent()) {
NodeEntry existingEntry = (NodeEntry)optEntry.get();
NodeEntry existingEntry = childNodes.get(updatedNode);
if(existingEntry != null) {
existingEntry.refreshChildren();
if(Config.get().isHideEmptyUsedAddresses() && existingEntry.getValue() == 0L) {
@ -125,7 +128,7 @@ public class AddressTreeTable extends CoinTreeTable {
if(Config.get().isHideEmptyUsedAddresses()) {
int index = 0;
for( ; index < rootEntry.getChildren().size(); index++) {
NodeEntry existingEntry = (NodeEntry)rootEntry.getChildren().get(index);
existingEntry = (NodeEntry)rootEntry.getChildren().get(index);
if(nodeEntry.compareTo(existingEntry) < 0) {
break;
}

4
src/main/java/com/sparrowwallet/sparrow/event/WalletHistoryChangedEvent.java
View file

@ -45,10 +45,10 @@ public class WalletHistoryChangedEvent extends WalletChangedEvent {
}
public List<WalletNode> getReceiveNodes() {
return getWallet().getNode(KeyPurpose.RECEIVE).getChildren().stream().filter(historyChangedNodes::contains).collect(Collectors.toList());
return historyChangedNodes.stream().filter(node -> node.getKeyPurpose() == KeyPurpose.RECEIVE).collect(Collectors.toList());
}
public List<WalletNode> getChangeNodes() {
return getWallet().getNode(KeyPurpose.CHANGE).getChildren().stream().filter(historyChangedNodes::contains).collect(Collectors.toList());
return historyChangedNodes.stream().filter(node -> node.getKeyPurpose() == KeyPurpose.CHANGE).collect(Collectors.toList());
}
}

2
src/main/java/com/sparrowwallet/sparrow/io/IOUtils.java
View file

@ -131,7 +131,7 @@ public class IOUtils {
if(file.exists()) {
long length = file.length();
SecureRandom random = new SecureRandom();
byte[] data = new byte[64];
byte[] data = new byte[1024*1024];
random.nextBytes(data);
try(RandomAccessFile raf = new RandomAccessFile(file, "rws")) {
raf.seek(0);

22
src/main/java/com/sparrowwallet/sparrow/io/JsonPersistence.java
View file

@ -3,6 +3,8 @@ package com.sparrowwallet.sparrow.io;
import com.google.gson.*;
import com.sparrowwallet.drongo.ExtendedKey;
import com.sparrowwallet.drongo.Utils;
import com.sparrowwallet.drongo.address.Address;
import com.sparrowwallet.drongo.address.InvalidAddressException;
import com.sparrowwallet.drongo.crypto.Argon2KeyDeriver;
import com.sparrowwallet.drongo.crypto.AsymmetricKeyDeriver;
import com.sparrowwallet.drongo.crypto.ECKey;
@ -331,6 +333,8 @@ public class JsonPersistence implements Persistence {
gsonBuilder.registerTypeAdapter(Date.class, new DateDeserializer());
gsonBuilder.registerTypeAdapter(Transaction.class, new TransactionSerializer());
gsonBuilder.registerTypeAdapter(Transaction.class, new TransactionDeserializer());
gsonBuilder.registerTypeAdapter(Address.class, new AddressSerializer());
gsonBuilder.registerTypeAdapter(Address.class, new AddressDeserializer());
if(includeWalletSerializers) {
gsonBuilder.registerTypeAdapter(Keystore.class, new KeystoreSerializer());
gsonBuilder.registerTypeAdapter(WalletNode.class, new NodeSerializer());
@ -429,6 +433,24 @@ public class JsonPersistence implements Persistence {
}
}
private static class AddressSerializer implements JsonSerializer<Address> {
@Override
public JsonElement serialize(Address src, Type typeOfSrc, JsonSerializationContext context) {
return new JsonPrimitive(src.toString());
}
}
private static class AddressDeserializer implements JsonDeserializer<Address> {
@Override
public Address deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
try {
return Address.fromString(json.getAsJsonPrimitive().getAsString());
} catch(InvalidAddressException e) {
throw new IllegalStateException(e);
}
}
}
private static class KeystoreSerializer implements JsonSerializer<Keystore> {
@Override
public JsonElement serialize(Keystore keystore, Type typeOfSrc, JsonSerializationContext context) {

18
src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java
View file

@ -87,6 +87,10 @@ public class DbPersistence implements Persistence {
return walletDao.getMainWallet(MASTER_SCHEMA, getWalletName(storage.getWalletFile(), null));
});
if(masterWallet == null) {
throw new StorageException("The wallet file was corrupted. Check the backups folder for previous copies.");
}
Map<WalletAndKey, Storage> childWallets = loadChildWallets(storage, masterWallet, encryptionKey);
masterWallet.setChildWallets(childWallets.keySet().stream().map(WalletAndKey::getWallet).collect(Collectors.toList()));
@ -231,12 +235,14 @@ public class DbPersistence implements Persistence {
if(addressNode.getId() == null) {
WalletNode purposeNode = wallet.getNode(addressNode.getKeyPurpose());
if(purposeNode.getId() == null) {
long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null);
long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null, null);
purposeNode.setId(purposeNodeId);
}
long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId());
long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId(), addressNode.getAddressData());
addressNode.setId(nodeId);
} else if(addressNode.getAddress() != null) {
walletNodeDao.updateNodeAddressData(addressNode.getId(), addressNode.getAddressData());
}
Set<BlockTransactionHashIndex> txos = addressNode.getTransactionOutputs().stream().flatMap(txo -> txo.isSpent() ? Stream.of(txo, txo.getSpentBy()) : Stream.of(txo)).collect(Collectors.toSet());
@ -285,12 +291,14 @@ public class DbPersistence implements Persistence {
if(addressNode.getId() == null) {
WalletNode purposeNode = wallet.getNode(addressNode.getKeyPurpose());
if(purposeNode.getId() == null) {
long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null);
long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null, null);
purposeNode.setId(purposeNodeId);
}
long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId());
long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId(), addressNode.getAddressData());
addressNode.setId(nodeId);
} else if(addressNode.getAddress() != null) {
walletNodeDao.updateNodeAddressData(addressNode.getId(), addressNode.getAddressData());
}
walletNodeDao.updateNodeLabel(addressNode.getId(), entry.getLabel());
@ -666,7 +674,7 @@ public class DbPersistence implements Persistence {
}
private String getUrl(File walletFile, String password) {
return "jdbc:h2:" + walletFile.getAbsolutePath().replace("." + getType().getExtension(), "") + ";INIT=SET TRACE_LEVEL_FILE=4;TRACE_LEVEL_FILE=4;DATABASE_TO_UPPER=false" + (password == null ? "" : ";CIPHER=AES");
return "jdbc:h2:" + walletFile.getAbsolutePath().replace("." + getType().getExtension(), "") + ";INIT=SET TRACE_LEVEL_FILE=4;TRACE_LEVEL_FILE=4;DEFRAG_ALWAYS=true;MAX_COMPACT_TIME=5000;DATABASE_TO_UPPER=false" + (password == null ? "" : ";CIPHER=AES");
}
private boolean persistsFor(Wallet wallet) {

2
src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java
View file

@ -108,7 +108,7 @@ public interface WalletDao {
default void loadWallet(Wallet wallet) {
wallet.getKeystores().addAll(createKeystoreDao().getForWalletId(wallet.getId()));
List<WalletNode> walletNodes = createWalletNodeDao().getForWalletId(wallet.getId());
List<WalletNode> walletNodes = createWalletNodeDao().getForWalletId(wallet.getScriptType().ordinal(), wallet.getId());
wallet.getPurposeNodes().addAll(walletNodes.stream().filter(walletNode -> walletNode.getDerivation().size() == 1).collect(Collectors.toList()));
wallet.getPurposeNodes().forEach(walletNode -> walletNode.setWallet(wallet));

15
src/main/java/com/sparrowwallet/sparrow/io/db/WalletNodeDao.java
View file

@ -16,18 +16,18 @@ import java.util.Date;
import java.util.List;
public interface WalletNodeDao {
@SqlQuery("select walletNode.id, walletNode.derivationPath, walletNode.label, walletNode.parent, " +
@SqlQuery("select walletNode.id, walletNode.derivationPath, walletNode.label, walletNode.parent, walletNode.addressData, ?, " +
"blockTransactionHashIndex.id, blockTransactionHashIndex.hash, blockTransactionHashIndex.height, blockTransactionHashIndex.date, blockTransactionHashIndex.fee, blockTransactionHashIndex.label, " +
"blockTransactionHashIndex.index, blockTransactionHashIndex.outputValue, blockTransactionHashIndex.status, blockTransactionHashIndex.spentBy, blockTransactionHashIndex.node " +
"from walletNode left join blockTransactionHashIndex on walletNode.id = blockTransactionHashIndex.node where walletNode.wallet = ? order by walletNode.parent asc nulls first, blockTransactionHashIndex.spentBy asc nulls first")
@RegisterRowMapper(WalletNodeMapper.class)
@RegisterRowMapper(BlockTransactionHashIndexMapper.class)
@UseRowReducer(WalletNodeReducer.class)
List<WalletNode> getForWalletId(Long id);
List<WalletNode> getForWalletId(int scriptType, Long id);
@SqlUpdate("insert into walletNode (derivationPath, label, wallet, parent) values (?, ?, ?, ?)")
@SqlUpdate("insert into walletNode (derivationPath, label, wallet, parent, addressData) values (?, ?, ?, ?, ?)")
@GetGeneratedKeys("id")
long insertWalletNode(String derivationPath, String label, long wallet, Long parent);
long insertWalletNode(String derivationPath, String label, long wallet, Long parent, byte[] addressData);
@SqlUpdate("insert into blockTransactionHashIndex (hash, height, date, fee, label, index, outputValue, status, spentBy, node) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
@GetGeneratedKeys("id")
@ -39,6 +39,9 @@ public interface WalletNodeDao {
@SqlUpdate("update walletNode set label = :label where id = :id")
void updateNodeLabel(@Bind("id") long id, @Bind("label") String label);
@SqlUpdate("update walletNode set addressData = :addressData where id = :id and addressData is null")
void updateNodeAddressData(@Bind("id") long id, @Bind("addressData") byte[] addressData);
@SqlUpdate("update blockTransactionHashIndex set label = :label where id = :id")
void updateTxoLabel(@Bind("id") long id, @Bind("label") String label);
@ -59,12 +62,12 @@ public interface WalletNodeDao {
default void addWalletNodes(Wallet wallet) {
for(WalletNode purposeNode : wallet.getPurposeNodes()) {
long purposeNodeId = insertWalletNode(purposeNode.getDerivationPath(), truncate(purposeNode.getLabel()), wallet.getId(), null);
long purposeNodeId = insertWalletNode(purposeNode.getDerivationPath(), truncate(purposeNode.getLabel()), wallet.getId(), null, null);
purposeNode.setId(purposeNodeId);
addTransactionOutputs(purposeNode);
List<WalletNode> childNodes = new ArrayList<>(purposeNode.getChildren());
for(WalletNode addressNode : childNodes) {
long addressNodeId = insertWalletNode(addressNode.getDerivationPath(), truncate(addressNode.getLabel()), wallet.getId(), purposeNodeId);
long addressNodeId = insertWalletNode(addressNode.getDerivationPath(), truncate(addressNode.getLabel()), wallet.getId(), purposeNodeId, addressNode.getAddressData());
addressNode.setId(addressNodeId);
addTransactionOutputs(addressNode);
}

7
src/main/java/com/sparrowwallet/sparrow/io/db/WalletNodeMapper.java
View file

@ -1,5 +1,6 @@
package com.sparrowwallet.sparrow.io.db;
import com.sparrowwallet.drongo.protocol.ScriptType;
import com.sparrowwallet.drongo.wallet.WalletNode;
import org.jdbi.v3.core.mapper.RowMapper;
import org.jdbi.v3.core.statement.StatementContext;
@ -13,7 +14,11 @@ public class WalletNodeMapper implements RowMapper<WalletNode> {
WalletNode walletNode = new WalletNode(rs.getString("walletNode.derivationPath"));
walletNode.setId(rs.getLong("walletNode.id"));
walletNode.setLabel(rs.getString("walletNode.label"));
byte[] addressData = rs.getBytes("walletNode.addressData");
if(addressData != null) {
ScriptType scriptType = ScriptType.values()[rs.getInt(6)];
walletNode.setAddress(scriptType.getAddress(addressData));
}
return walletNode;
}
}

7
src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java
View file

@ -432,6 +432,7 @@ public class ElectrumServer {
try {
Set<String> scriptHashes = new HashSet<>();
Map<String, String> pathScriptHashes = new LinkedHashMap<>();
Map<String, WalletNode> pathNodes = new HashMap<>();
for(WalletNode node : nodes) {
if(node == null) {
log.error("Null node for wallet " + wallet.getFullName() + " subscribing nodes " + nodes + " startIndex " + startIndex, new Throwable());
@ -448,6 +449,7 @@ public class ElectrumServer {
} else if(!subscribedScriptHashes.containsKey(scriptHash) && scriptHashes.add(scriptHash)) {
//Unique script hash we are not yet subscribed to
pathScriptHashes.put(node.getDerivationPath(), scriptHash);
pathNodes.put(node.getDerivationPath(), node);
}
}
}
@ -463,9 +465,8 @@ public class ElectrumServer {
for(String path : result.keySet()) {
String status = result.get(path);
Optional<WalletNode> optionalNode = nodes.stream().filter(n -> n.getDerivationPath().equals(path)).findFirst();
if(optionalNode.isPresent()) {
WalletNode node = optionalNode.get();
WalletNode node = pathNodes.computeIfAbsent(path, p -> nodes.stream().filter(n -> n.getDerivationPath().equals(p)).findFirst().orElse(null));
if(node != null) {
String scriptHash = getScriptHash(wallet, node);
//Check if there is history for this script hash, and if the history has changed since last fetched

4
src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java
View file

@ -201,7 +201,9 @@ public class WalletForm {
}
}
}
EventManager.get().post(new ChildWalletsAddedEvent(storage, wallet, addedWallets));
if(!addedWallets.isEmpty()) {
EventManager.get().post(new ChildWalletsAddedEvent(storage, wallet, addedWallets));
}
});
paymentCodesService.setOnFailed(failedEvent -> {
log.error("Could not determine payment codes for wallet " + wallet.getFullName(), failedEvent.getSource().getException());

1
src/main/resources/com/sparrowwallet/sparrow/sql/V7__AddressData.sql Normal file
View file

@ -0,0 +1 @@
alter table walletNode add column addressData varbinary(32) after parent;