Merge pull request #5790 from chimp1984/extract-persistence-of-blocks

Extract persistence of BSQ blocks out of DaoStateStore [5]

.gitattributes

@@ -13,3 +13,4 @@
 *.jpeg binary
 *.png binary
 p2p/src/main/resources/*BTC_MAINNET filter=lfs diff=lfs merge=lfs -text
+p2p/src/main/resources/BsqBlocks_BTC_MAINNET/* filter=lfs diff=lfs merge=lfs -text

build.gradle

@@ -238,7 +238,7 @@ configure(project(':common')) {
 
  javafx {
  version = "$javafxVersion"
- modules =  ['javafx.graphics']
+ modules = ['javafx.graphics']
  }
 
  dependencies {
@@ -284,7 +284,7 @@ configure(project(':p2p')) {
 
  javafx {
  version = "$javafxVersion"
- modules =  ['javafx.base']
+ modules = ['javafx.base']
  }
 
  dependencies {
@@ -324,7 +324,7 @@ configure(project(':core')) {
 
  javafx {
  version = "$javafxVersion"
- modules =  ['javafx.base']
+ modules = ['javafx.base']
  }
 
  dependencies {
@@ -411,7 +411,7 @@ configure(project(':desktop')) {
 
  javafx {
  version = "$javafxVersion"
- modules =  ['javafx.controls', 'javafx.fxml']
+ modules = ['javafx.controls', 'javafx.fxml']
  }
 
  version = '1.7.5-SNAPSHOT'
@@ -459,7 +459,7 @@ configure(project(':monitor')) {
 
  javafx {
  version = "$javafxVersion"
- modules =  ['javafx.base']
+ modules = ['javafx.base']
  }
 
  mainClassName = 'bisq.monitor.Monitor'

common/src/main/java/bisq/common/util/GcUtil.java

@@ -18,7 +18,6 @@
 package bisq.common.util;
 
 import bisq.common.UserThread;
-import bisq.common.app.DevEnv;
 
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
@@ -27,8 +26,10 @@
 public class GcUtil {
  @Setter
  private static boolean DISABLE_GC_CALLS = false;
- private static int TRIGGER_MEM = 1000;
- private static int TRIGGER_MAX_MEM = 3000;
+ private static final int TRIGGER_MEM = 1000;
+ private static final int TRIGGER_MAX_MEM = 3000;
+ private static int totalInvocations;
+ private static long totalGCTime;
 
  public static void autoReleaseMemory() {
  if (DISABLE_GC_CALLS)
@@ -59,20 +60,25 @@ private static void maybeReleaseMemory(long trigger) {
  long preGcMemory = Runtime.getRuntime().totalMemory();
  if (preGcMemory > trigger * 1024 * 1024) {
  System.gc();
+ totalInvocations++;
  long postGcMemory = Runtime.getRuntime().totalMemory();
- log.info("GC reduced memory by {}. Total memory before/after: {}/{}. Took {} ms.",
+ long duration = System.currentTimeMillis() - ts;
+ totalGCTime += duration;
+ log.info("GC reduced memory by {}. Total memory before/after: {}/{}. Took {} ms. Total GC invocations: {} / Total GC time {} sec",
  Utilities.readableFileSize(preGcMemory - postGcMemory),
  Utilities.readableFileSize(preGcMemory),
  Utilities.readableFileSize(postGcMemory),
- System.currentTimeMillis() - ts);
- if (DevEnv.isDevMode()) {
+ duration,
+ totalInvocations,
+ totalGCTime / 1000d);
+ /* if (DevEnv.isDevMode()) {
  try {
  // To see from where we got called
  throw new RuntimeException("Dummy Exception for print stacktrace at maybeReleaseMemory");
  } catch (Throwable t) {
  t.printStackTrace();
  }
- }
+ }*/
  }
  }
 }

core/src/main/java/bisq/core/dao/DaoFacade.java

@@ -35,6 +35,7 @@
 import bisq.core.dao.governance.myvote.MyVote;
 import bisq.core.dao.governance.myvote.MyVoteListService;
 import bisq.core.dao.governance.param.Param;
+import bisq.core.dao.governance.period.CycleService;
 import bisq.core.dao.governance.period.PeriodService;
 import bisq.core.dao.governance.proposal.MyProposalListService;
 import bisq.core.dao.governance.proposal.ProposalConsensus;
@@ -99,14 +100,13 @@
 
 import java.util.ArrayList;
 import java.util.Date;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
+import java.util.stream.Collectors;
 
 import lombok.extern.slf4j.Slf4j;
 
@@ -128,6 +128,7 @@ public class DaoFacade implements DaoSetupService {
  private final DaoStateService daoStateService;
  private final DaoStateMonitoringService daoStateMonitoringService;
  private final PeriodService periodService;
+ private final CycleService cycleService;
  private final MyBlindVoteListService myBlindVoteListService;
  private final MyVoteListService myVoteListService;
  private final CompensationProposalFactory compensationProposalFactory;
@@ -155,6 +156,7 @@ public DaoFacade(MyProposalListService myProposalListService,
  DaoStateService daoStateService,
  DaoStateMonitoringService daoStateMonitoringService,
  PeriodService periodService,
+ CycleService cycleService,
  MyBlindVoteListService myBlindVoteListService,
  MyVoteListService myVoteListService,
  CompensationProposalFactory compensationProposalFactory,
@@ -178,6 +180,7 @@ public DaoFacade(MyProposalListService myProposalListService,
  this.daoStateService = daoStateService;
  this.daoStateMonitoringService = daoStateMonitoringService;
  this.periodService = periodService;
+ this.cycleService = cycleService;
  this.myBlindVoteListService = myBlindVoteListService;
  this.myVoteListService = myVoteListService;
  this.compensationProposalFactory = compensationProposalFactory;
@@ -438,12 +441,10 @@ public int getFirstBlockOfPhaseForDisplay(int height, DaoPhase.Phase phase) {
  }
 
  public Map<Integer, Date> getBlockStartDateByCycleIndex() {
- AtomicInteger index = new AtomicInteger();
- Map<Integer, Date> map = new HashMap<>();
- periodService.getCycles()
- .forEach(cycle -> daoStateService.getBlockAtHeight(cycle.getHeightOfFirstBlock())
- .ifPresent(block -> map.put(index.getAndIncrement(), new Date(block.getTime()))));
- return map;
+ return periodService.getCycles().stream().collect(Collectors.toMap(
+ cycleService::getCycleIndex,
+ cycle -> new Date(daoStateService.getBlockTimeAtBlockHeight(cycle.getHeightOfFirstBlock()))
+ ));
  }
 
  // Because last block in request and voting phases must not be used for making a tx as it will get confirmed in the

core/src/main/java/bisq/core/dao/node/lite/LiteNode.java

@@ -35,6 +35,7 @@
 
 import bisq.common.Timer;
 import bisq.common.UserThread;
+import bisq.common.util.MathUtils;
 
 import com.google.inject.Inject;
 
@@ -222,8 +223,11 @@ private void onRequestedBlocksReceived(List<RawBlock> blockList, Runnable onPars
 
  runDelayedBatchProcessing(new ArrayList<>(blockList),
  () -> {
- log.info("Parsing {} blocks took {} seconds.", blockList.size(),
- (System.currentTimeMillis() - ts) / 1000d);
+ double duration = System.currentTimeMillis() - ts;
+ log.info("Parsing {} blocks took {} seconds ({} min.) / {} ms in average / block", blockList.size(),
+ MathUtils.roundDouble(duration / 1000d, 2),
+ MathUtils.roundDouble(duration / 1000d / 60, 2),
+ MathUtils.roundDouble(duration / blockList.size(), 2));
  // We only request again if wallet is synced, otherwise we would get repeated calls we want to avoid.
  // We deal with that case at the setupWalletBestBlockListener method above.
  if (walletsSetup.isDownloadComplete() &&

core/src/main/java/bisq/core/dao/node/parser/BlockParser.java

@@ -135,6 +135,8 @@ private void validateIfBlockIsConnecting(RawBlock rawBlock) throws BlockHashNotC
  }
 
  private boolean isBlockAlreadyAdded(RawBlock rawBlock) {
- return daoStateService.isBlockHashKnown(rawBlock.getHash());
+ return daoStateService.getBlockAtHeight(rawBlock.getHeight())
+ .map(block -> block.getHash().equals(rawBlock.getHash()))
+ .orElse(false);
  }
 }

core/src/main/java/bisq/core/dao/state/DaoStateService.java

@@ -149,6 +149,10 @@ public DaoState getClone() {
  return DaoState.getClone(daoState);
  }
 
+ public protobuf.DaoState getBsqStateCloneExcludingBlocks() {
+ return DaoState.getBsqStateCloneExcludingBlocks(daoState);
+ }
+
  public byte[] getSerializedStateForHashChain() {
  return daoState.getSerializedStateForHashChain();
  }
@@ -286,17 +290,6 @@ public List<Block> getBlocks() {
  return daoState.getBlocks();
  }
 
- /**
- * Whether specified block hash belongs to a block we already know about.
- *
- * @param blockHash The hash of a {@link Block}.
- * @return True if the hash belongs to a {@link Block} we know about, otherwise
- * {@code false}.
- */
- public boolean isBlockHashKnown(String blockHash) {
- return daoState.getBlockHashes().contains(blockHash);
- }
-
  public Optional<Block> getLastBlock() {
  if (!getBlocks().isEmpty())
  return Optional.of(daoState.getLastBlock());
@@ -316,6 +309,10 @@ public Optional<Block> getBlockAtHeight(int height) {
  return Optional.ofNullable(daoState.getBlocksByHeight().get(height));
  }
 
+ public long getBlockTimeAtBlockHeight(int height) {
+ return getBlockAtHeight(height).map(Block::getTime).orElse(0L);
+ }
+
  public boolean containsBlock(Block block) {
  return getBlocks().contains(block);
  }
@@ -324,12 +321,15 @@ public long getBlockTime(int height) {
  return getBlockAtHeight(height).map(Block::getTime).orElse(0L);
  }
 
+ public List<Block> getBlocksFromBlockHeight(int fromBlockHeight) {
+ return getBlocksFromBlockHeight(fromBlockHeight, Integer.MAX_VALUE);
+ }
+
  public List<Block> getBlocksFromBlockHeight(int fromBlockHeight, int numMaxBlocks) {
  // We limit requests to numMaxBlocks blocks, to avoid performance issues and too
  // large network data in case a node requests too far back in history.
  return getBlocks().stream()
  .filter(block -> block.getHeight() >= fromBlockHeight)
- .sorted(Comparator.comparing(Block::getHeight))
  .limit(numMaxBlocks)
  .collect(Collectors.toList());
  }