improve performance on deep wallets by storing addresses

This commit is contained in:
Craig Raw 2022-07-18 16:12:32 +02:00
parent 11cda40a40
commit 60aa20ac55
12 changed files with 73 additions and 28 deletions

2
drongo

@ -1 +1 @@
Subproject commit 9ae1f68dc42529085edcc8c10d9bcfdbf9639448 Subproject commit 5de3abd36230d545f11bad3b25a21d23ffbbe9cd

View file

@ -14,9 +14,7 @@ import javafx.collections.ListChangeListener;
import javafx.scene.control.*; import javafx.scene.control.*;
import javafx.scene.input.MouseButton; import javafx.scene.input.MouseButton;
import java.util.List; import java.util.*;
import java.util.Optional;
import java.util.OptionalInt;
public class AddressTreeTable extends CoinTreeTable { public class AddressTreeTable extends CoinTreeTable {
public void initialize(NodeEntry rootEntry) { public void initialize(NodeEntry rootEntry) {
@ -110,10 +108,15 @@ public class AddressTreeTable extends CoinTreeTable {
//We only ever add child nodes - never remove in order to keep a full sequence (unless hide empty used addresses is set) //We only ever add child nodes - never remove in order to keep a full sequence (unless hide empty used addresses is set)
NodeEntry rootEntry = (NodeEntry)getRoot().getValue(); NodeEntry rootEntry = (NodeEntry)getRoot().getValue();
Map<WalletNode, NodeEntry> childNodes = new HashMap<>();
for(Entry childEntry : rootEntry.getChildren()) {
NodeEntry nodeEntry = (NodeEntry)childEntry;
childNodes.put(nodeEntry.getNode(), nodeEntry);
}
for(WalletNode updatedNode : updatedNodes) { for(WalletNode updatedNode : updatedNodes) {
Optional<Entry> optEntry = rootEntry.getChildren().stream().filter(childEntry -> ((NodeEntry)childEntry).getNode().equals(updatedNode)).findFirst(); NodeEntry existingEntry = childNodes.get(updatedNode);
if(optEntry.isPresent()) { if(existingEntry != null) {
NodeEntry existingEntry = (NodeEntry)optEntry.get();
existingEntry.refreshChildren(); existingEntry.refreshChildren();
if(Config.get().isHideEmptyUsedAddresses() && existingEntry.getValue() == 0L) { if(Config.get().isHideEmptyUsedAddresses() && existingEntry.getValue() == 0L) {
@ -125,7 +128,7 @@ public class AddressTreeTable extends CoinTreeTable {
if(Config.get().isHideEmptyUsedAddresses()) { if(Config.get().isHideEmptyUsedAddresses()) {
int index = 0; int index = 0;
for( ; index < rootEntry.getChildren().size(); index++) { for( ; index < rootEntry.getChildren().size(); index++) {
NodeEntry existingEntry = (NodeEntry)rootEntry.getChildren().get(index); existingEntry = (NodeEntry)rootEntry.getChildren().get(index);
if(nodeEntry.compareTo(existingEntry) < 0) { if(nodeEntry.compareTo(existingEntry) < 0) {
break; break;
} }

View file

@ -45,10 +45,10 @@ public class WalletHistoryChangedEvent extends WalletChangedEvent {
} }
public List<WalletNode> getReceiveNodes() { public List<WalletNode> getReceiveNodes() {
return getWallet().getNode(KeyPurpose.RECEIVE).getChildren().stream().filter(historyChangedNodes::contains).collect(Collectors.toList()); return historyChangedNodes.stream().filter(node -> node.getKeyPurpose() == KeyPurpose.RECEIVE).collect(Collectors.toList());
} }
public List<WalletNode> getChangeNodes() { public List<WalletNode> getChangeNodes() {
return getWallet().getNode(KeyPurpose.CHANGE).getChildren().stream().filter(historyChangedNodes::contains).collect(Collectors.toList()); return historyChangedNodes.stream().filter(node -> node.getKeyPurpose() == KeyPurpose.CHANGE).collect(Collectors.toList());
} }
} }

View file

@ -131,7 +131,7 @@ public class IOUtils {
if(file.exists()) { if(file.exists()) {
long length = file.length(); long length = file.length();
SecureRandom random = new SecureRandom(); SecureRandom random = new SecureRandom();
byte[] data = new byte[64]; byte[] data = new byte[1024*1024];
random.nextBytes(data); random.nextBytes(data);
try(RandomAccessFile raf = new RandomAccessFile(file, "rws")) { try(RandomAccessFile raf = new RandomAccessFile(file, "rws")) {
raf.seek(0); raf.seek(0);

View file

@ -3,6 +3,8 @@ package com.sparrowwallet.sparrow.io;
import com.google.gson.*; import com.google.gson.*;
import com.sparrowwallet.drongo.ExtendedKey; import com.sparrowwallet.drongo.ExtendedKey;
import com.sparrowwallet.drongo.Utils; import com.sparrowwallet.drongo.Utils;
import com.sparrowwallet.drongo.address.Address;
import com.sparrowwallet.drongo.address.InvalidAddressException;
import com.sparrowwallet.drongo.crypto.Argon2KeyDeriver; import com.sparrowwallet.drongo.crypto.Argon2KeyDeriver;
import com.sparrowwallet.drongo.crypto.AsymmetricKeyDeriver; import com.sparrowwallet.drongo.crypto.AsymmetricKeyDeriver;
import com.sparrowwallet.drongo.crypto.ECKey; import com.sparrowwallet.drongo.crypto.ECKey;
@ -331,6 +333,8 @@ public class JsonPersistence implements Persistence {
gsonBuilder.registerTypeAdapter(Date.class, new DateDeserializer()); gsonBuilder.registerTypeAdapter(Date.class, new DateDeserializer());
gsonBuilder.registerTypeAdapter(Transaction.class, new TransactionSerializer()); gsonBuilder.registerTypeAdapter(Transaction.class, new TransactionSerializer());
gsonBuilder.registerTypeAdapter(Transaction.class, new TransactionDeserializer()); gsonBuilder.registerTypeAdapter(Transaction.class, new TransactionDeserializer());
gsonBuilder.registerTypeAdapter(Address.class, new AddressSerializer());
gsonBuilder.registerTypeAdapter(Address.class, new AddressDeserializer());
if(includeWalletSerializers) { if(includeWalletSerializers) {
gsonBuilder.registerTypeAdapter(Keystore.class, new KeystoreSerializer()); gsonBuilder.registerTypeAdapter(Keystore.class, new KeystoreSerializer());
gsonBuilder.registerTypeAdapter(WalletNode.class, new NodeSerializer()); gsonBuilder.registerTypeAdapter(WalletNode.class, new NodeSerializer());
@ -429,6 +433,24 @@ public class JsonPersistence implements Persistence {
} }
} }
private static class AddressSerializer implements JsonSerializer<Address> {
@Override
public JsonElement serialize(Address src, Type typeOfSrc, JsonSerializationContext context) {
return new JsonPrimitive(src.toString());
}
}
private static class AddressDeserializer implements JsonDeserializer<Address> {
@Override
public Address deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
try {
return Address.fromString(json.getAsJsonPrimitive().getAsString());
} catch(InvalidAddressException e) {
throw new IllegalStateException(e);
}
}
}
private static class KeystoreSerializer implements JsonSerializer<Keystore> { private static class KeystoreSerializer implements JsonSerializer<Keystore> {
@Override @Override
public JsonElement serialize(Keystore keystore, Type typeOfSrc, JsonSerializationContext context) { public JsonElement serialize(Keystore keystore, Type typeOfSrc, JsonSerializationContext context) {

View file

@ -87,6 +87,10 @@ public class DbPersistence implements Persistence {
return walletDao.getMainWallet(MASTER_SCHEMA, getWalletName(storage.getWalletFile(), null)); return walletDao.getMainWallet(MASTER_SCHEMA, getWalletName(storage.getWalletFile(), null));
}); });
if(masterWallet == null) {
throw new StorageException("The wallet file was corrupted. Check the backups folder for previous copies.");
}
Map<WalletAndKey, Storage> childWallets = loadChildWallets(storage, masterWallet, encryptionKey); Map<WalletAndKey, Storage> childWallets = loadChildWallets(storage, masterWallet, encryptionKey);
masterWallet.setChildWallets(childWallets.keySet().stream().map(WalletAndKey::getWallet).collect(Collectors.toList())); masterWallet.setChildWallets(childWallets.keySet().stream().map(WalletAndKey::getWallet).collect(Collectors.toList()));
@ -231,12 +235,14 @@ public class DbPersistence implements Persistence {
if(addressNode.getId() == null) { if(addressNode.getId() == null) {
WalletNode purposeNode = wallet.getNode(addressNode.getKeyPurpose()); WalletNode purposeNode = wallet.getNode(addressNode.getKeyPurpose());
if(purposeNode.getId() == null) { if(purposeNode.getId() == null) {
long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null); long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null, null);
purposeNode.setId(purposeNodeId); purposeNode.setId(purposeNodeId);
} }
long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId()); long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId(), addressNode.getAddressData());
addressNode.setId(nodeId); addressNode.setId(nodeId);
} else if(addressNode.getAddress() != null) {
walletNodeDao.updateNodeAddressData(addressNode.getId(), addressNode.getAddressData());
} }
Set<BlockTransactionHashIndex> txos = addressNode.getTransactionOutputs().stream().flatMap(txo -> txo.isSpent() ? Stream.of(txo, txo.getSpentBy()) : Stream.of(txo)).collect(Collectors.toSet()); Set<BlockTransactionHashIndex> txos = addressNode.getTransactionOutputs().stream().flatMap(txo -> txo.isSpent() ? Stream.of(txo, txo.getSpentBy()) : Stream.of(txo)).collect(Collectors.toSet());
@ -285,12 +291,14 @@ public class DbPersistence implements Persistence {
if(addressNode.getId() == null) { if(addressNode.getId() == null) {
WalletNode purposeNode = wallet.getNode(addressNode.getKeyPurpose()); WalletNode purposeNode = wallet.getNode(addressNode.getKeyPurpose());
if(purposeNode.getId() == null) { if(purposeNode.getId() == null) {
long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null); long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null, null);
purposeNode.setId(purposeNodeId); purposeNode.setId(purposeNodeId);
} }
long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId()); long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId(), addressNode.getAddressData());
addressNode.setId(nodeId); addressNode.setId(nodeId);
} else if(addressNode.getAddress() != null) {
walletNodeDao.updateNodeAddressData(addressNode.getId(), addressNode.getAddressData());
} }
walletNodeDao.updateNodeLabel(addressNode.getId(), entry.getLabel()); walletNodeDao.updateNodeLabel(addressNode.getId(), entry.getLabel());
@ -666,7 +674,7 @@ public class DbPersistence implements Persistence {
} }
private String getUrl(File walletFile, String password) { private String getUrl(File walletFile, String password) {
return "jdbc:h2:" + walletFile.getAbsolutePath().replace("." + getType().getExtension(), "") + ";INIT=SET TRACE_LEVEL_FILE=4;TRACE_LEVEL_FILE=4;DATABASE_TO_UPPER=false" + (password == null ? "" : ";CIPHER=AES"); return "jdbc:h2:" + walletFile.getAbsolutePath().replace("." + getType().getExtension(), "") + ";INIT=SET TRACE_LEVEL_FILE=4;TRACE_LEVEL_FILE=4;DEFRAG_ALWAYS=true;MAX_COMPACT_TIME=5000;DATABASE_TO_UPPER=false" + (password == null ? "" : ";CIPHER=AES");
} }
private boolean persistsFor(Wallet wallet) { private boolean persistsFor(Wallet wallet) {

View file

@ -108,7 +108,7 @@ public interface WalletDao {
default void loadWallet(Wallet wallet) { default void loadWallet(Wallet wallet) {
wallet.getKeystores().addAll(createKeystoreDao().getForWalletId(wallet.getId())); wallet.getKeystores().addAll(createKeystoreDao().getForWalletId(wallet.getId()));
List<WalletNode> walletNodes = createWalletNodeDao().getForWalletId(wallet.getId()); List<WalletNode> walletNodes = createWalletNodeDao().getForWalletId(wallet.getScriptType().ordinal(), wallet.getId());
wallet.getPurposeNodes().addAll(walletNodes.stream().filter(walletNode -> walletNode.getDerivation().size() == 1).collect(Collectors.toList())); wallet.getPurposeNodes().addAll(walletNodes.stream().filter(walletNode -> walletNode.getDerivation().size() == 1).collect(Collectors.toList()));
wallet.getPurposeNodes().forEach(walletNode -> walletNode.setWallet(wallet)); wallet.getPurposeNodes().forEach(walletNode -> walletNode.setWallet(wallet));

View file

@ -16,18 +16,18 @@ import java.util.Date;
import java.util.List; import java.util.List;
public interface WalletNodeDao { public interface WalletNodeDao {
@SqlQuery("select walletNode.id, walletNode.derivationPath, walletNode.label, walletNode.parent, " + @SqlQuery("select walletNode.id, walletNode.derivationPath, walletNode.label, walletNode.parent, walletNode.addressData, ?, " +
"blockTransactionHashIndex.id, blockTransactionHashIndex.hash, blockTransactionHashIndex.height, blockTransactionHashIndex.date, blockTransactionHashIndex.fee, blockTransactionHashIndex.label, " + "blockTransactionHashIndex.id, blockTransactionHashIndex.hash, blockTransactionHashIndex.height, blockTransactionHashIndex.date, blockTransactionHashIndex.fee, blockTransactionHashIndex.label, " +
"blockTransactionHashIndex.index, blockTransactionHashIndex.outputValue, blockTransactionHashIndex.status, blockTransactionHashIndex.spentBy, blockTransactionHashIndex.node " + "blockTransactionHashIndex.index, blockTransactionHashIndex.outputValue, blockTransactionHashIndex.status, blockTransactionHashIndex.spentBy, blockTransactionHashIndex.node " +
"from walletNode left join blockTransactionHashIndex on walletNode.id = blockTransactionHashIndex.node where walletNode.wallet = ? order by walletNode.parent asc nulls first, blockTransactionHashIndex.spentBy asc nulls first") "from walletNode left join blockTransactionHashIndex on walletNode.id = blockTransactionHashIndex.node where walletNode.wallet = ? order by walletNode.parent asc nulls first, blockTransactionHashIndex.spentBy asc nulls first")
@RegisterRowMapper(WalletNodeMapper.class) @RegisterRowMapper(WalletNodeMapper.class)
@RegisterRowMapper(BlockTransactionHashIndexMapper.class) @RegisterRowMapper(BlockTransactionHashIndexMapper.class)
@UseRowReducer(WalletNodeReducer.class) @UseRowReducer(WalletNodeReducer.class)
List<WalletNode> getForWalletId(Long id); List<WalletNode> getForWalletId(int scriptType, Long id);
@SqlUpdate("insert into walletNode (derivationPath, label, wallet, parent) values (?, ?, ?, ?)") @SqlUpdate("insert into walletNode (derivationPath, label, wallet, parent, addressData) values (?, ?, ?, ?, ?)")
@GetGeneratedKeys("id") @GetGeneratedKeys("id")
long insertWalletNode(String derivationPath, String label, long wallet, Long parent); long insertWalletNode(String derivationPath, String label, long wallet, Long parent, byte[] addressData);
@SqlUpdate("insert into blockTransactionHashIndex (hash, height, date, fee, label, index, outputValue, status, spentBy, node) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") @SqlUpdate("insert into blockTransactionHashIndex (hash, height, date, fee, label, index, outputValue, status, spentBy, node) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
@GetGeneratedKeys("id") @GetGeneratedKeys("id")
@ -39,6 +39,9 @@ public interface WalletNodeDao {
@SqlUpdate("update walletNode set label = :label where id = :id") @SqlUpdate("update walletNode set label = :label where id = :id")
void updateNodeLabel(@Bind("id") long id, @Bind("label") String label); void updateNodeLabel(@Bind("id") long id, @Bind("label") String label);
@SqlUpdate("update walletNode set addressData = :addressData where id = :id and addressData is null")
void updateNodeAddressData(@Bind("id") long id, @Bind("addressData") byte[] addressData);
@SqlUpdate("update blockTransactionHashIndex set label = :label where id = :id") @SqlUpdate("update blockTransactionHashIndex set label = :label where id = :id")
void updateTxoLabel(@Bind("id") long id, @Bind("label") String label); void updateTxoLabel(@Bind("id") long id, @Bind("label") String label);
@ -59,12 +62,12 @@ public interface WalletNodeDao {
default void addWalletNodes(Wallet wallet) { default void addWalletNodes(Wallet wallet) {
for(WalletNode purposeNode : wallet.getPurposeNodes()) { for(WalletNode purposeNode : wallet.getPurposeNodes()) {
long purposeNodeId = insertWalletNode(purposeNode.getDerivationPath(), truncate(purposeNode.getLabel()), wallet.getId(), null); long purposeNodeId = insertWalletNode(purposeNode.getDerivationPath(), truncate(purposeNode.getLabel()), wallet.getId(), null, null);
purposeNode.setId(purposeNodeId); purposeNode.setId(purposeNodeId);
addTransactionOutputs(purposeNode); addTransactionOutputs(purposeNode);
List<WalletNode> childNodes = new ArrayList<>(purposeNode.getChildren()); List<WalletNode> childNodes = new ArrayList<>(purposeNode.getChildren());
for(WalletNode addressNode : childNodes) { for(WalletNode addressNode : childNodes) {
long addressNodeId = insertWalletNode(addressNode.getDerivationPath(), truncate(addressNode.getLabel()), wallet.getId(), purposeNodeId); long addressNodeId = insertWalletNode(addressNode.getDerivationPath(), truncate(addressNode.getLabel()), wallet.getId(), purposeNodeId, addressNode.getAddressData());
addressNode.setId(addressNodeId); addressNode.setId(addressNodeId);
addTransactionOutputs(addressNode); addTransactionOutputs(addressNode);
} }

View file

@ -1,5 +1,6 @@
package com.sparrowwallet.sparrow.io.db; package com.sparrowwallet.sparrow.io.db;
import com.sparrowwallet.drongo.protocol.ScriptType;
import com.sparrowwallet.drongo.wallet.WalletNode; import com.sparrowwallet.drongo.wallet.WalletNode;
import org.jdbi.v3.core.mapper.RowMapper; import org.jdbi.v3.core.mapper.RowMapper;
import org.jdbi.v3.core.statement.StatementContext; import org.jdbi.v3.core.statement.StatementContext;
@ -13,7 +14,11 @@ public class WalletNodeMapper implements RowMapper<WalletNode> {
WalletNode walletNode = new WalletNode(rs.getString("walletNode.derivationPath")); WalletNode walletNode = new WalletNode(rs.getString("walletNode.derivationPath"));
walletNode.setId(rs.getLong("walletNode.id")); walletNode.setId(rs.getLong("walletNode.id"));
walletNode.setLabel(rs.getString("walletNode.label")); walletNode.setLabel(rs.getString("walletNode.label"));
byte[] addressData = rs.getBytes("walletNode.addressData");
if(addressData != null) {
ScriptType scriptType = ScriptType.values()[rs.getInt(6)];
walletNode.setAddress(scriptType.getAddress(addressData));
}
return walletNode; return walletNode;
} }
} }

View file

@ -432,6 +432,7 @@ public class ElectrumServer {
try { try {
Set<String> scriptHashes = new HashSet<>(); Set<String> scriptHashes = new HashSet<>();
Map<String, String> pathScriptHashes = new LinkedHashMap<>(); Map<String, String> pathScriptHashes = new LinkedHashMap<>();
Map<String, WalletNode> pathNodes = new HashMap<>();
for(WalletNode node : nodes) { for(WalletNode node : nodes) {
if(node == null) { if(node == null) {
log.error("Null node for wallet " + wallet.getFullName() + " subscribing nodes " + nodes + " startIndex " + startIndex, new Throwable()); log.error("Null node for wallet " + wallet.getFullName() + " subscribing nodes " + nodes + " startIndex " + startIndex, new Throwable());
@ -448,6 +449,7 @@ public class ElectrumServer {
} else if(!subscribedScriptHashes.containsKey(scriptHash) && scriptHashes.add(scriptHash)) { } else if(!subscribedScriptHashes.containsKey(scriptHash) && scriptHashes.add(scriptHash)) {
//Unique script hash we are not yet subscribed to //Unique script hash we are not yet subscribed to
pathScriptHashes.put(node.getDerivationPath(), scriptHash); pathScriptHashes.put(node.getDerivationPath(), scriptHash);
pathNodes.put(node.getDerivationPath(), node);
} }
} }
} }
@ -463,9 +465,8 @@ public class ElectrumServer {
for(String path : result.keySet()) { for(String path : result.keySet()) {
String status = result.get(path); String status = result.get(path);
Optional<WalletNode> optionalNode = nodes.stream().filter(n -> n.getDerivationPath().equals(path)).findFirst(); WalletNode node = pathNodes.computeIfAbsent(path, p -> nodes.stream().filter(n -> n.getDerivationPath().equals(p)).findFirst().orElse(null));
if(optionalNode.isPresent()) { if(node != null) {
WalletNode node = optionalNode.get();
String scriptHash = getScriptHash(wallet, node); String scriptHash = getScriptHash(wallet, node);
//Check if there is history for this script hash, and if the history has changed since last fetched //Check if there is history for this script hash, and if the history has changed since last fetched

View file

@ -201,7 +201,9 @@ public class WalletForm {
} }
} }
} }
EventManager.get().post(new ChildWalletsAddedEvent(storage, wallet, addedWallets)); if(!addedWallets.isEmpty()) {
EventManager.get().post(new ChildWalletsAddedEvent(storage, wallet, addedWallets));
}
}); });
paymentCodesService.setOnFailed(failedEvent -> { paymentCodesService.setOnFailed(failedEvent -> {
log.error("Could not determine payment codes for wallet " + wallet.getFullName(), failedEvent.getSource().getException()); log.error("Could not determine payment codes for wallet " + wallet.getFullName(), failedEvent.getSource().getException());

View file

@ -0,0 +1 @@
alter table walletNode add column addressData varbinary(32) after parent;