Delete reuse space

engine/src/main/java/com/arcadedb/database/TransactionContext.java

@@ -20,6 +20,7 @@
 
 import com.arcadedb.GlobalConfiguration;
 import com.arcadedb.engine.BasePage;
+import com.arcadedb.engine.Bucket;
 import com.arcadedb.engine.ComponentFile;
 import com.arcadedb.engine.ImmutablePage;
 import com.arcadedb.engine.LocalBucket;
@@ -37,6 +38,8 @@
 import com.arcadedb.index.IndexInternal;
 import com.arcadedb.index.lsm.LSMTreeIndexAbstract;
 import com.arcadedb.log.LogManager;
+import com.arcadedb.schema.LocalSchema;
+import com.arcadedb.schema.Schema;
 
 import java.io.*;
 import java.util.*;
@@ -572,6 +575,20 @@ public TransactionPhase1 commit1stPhase(final boolean isLeader) {
  final List<MutablePage> pages = new ArrayList<>();
  final PageManager pageManager = database.getPageManager();
 
+ // COMPRESS PAGES BEFORE SAVING THEM
+ final LocalSchema localSchema = database.getSchema().getEmbedded();
+
+ for (MutablePage page : modifiedPages.values()) {
+ final LocalBucket bucket = localSchema.getBucketById(page.getPageId().getFileId(), false);
+ if (bucket != null)
+ bucket.compressPage(page);
+ }
+ for (MutablePage page : newPages.values()) {
+ final LocalBucket bucket = localSchema.getBucketById(page.getPageId().getFileId(), false);
+ if (bucket != null)
+ bucket.compressPage(page);
+ }
+
  for (final Iterator<MutablePage> it = modifiedPages.values().iterator(); it.hasNext(); ) {
  final MutablePage p = it.next();
 

engine/src/main/java/com/arcadedb/schema/LocalSchema.java

@@ -284,12 +284,23 @@ public Bucket getBucketByName(final String name) {
 
  @Override
  public LocalBucket getBucketById(final int id) {
+ return getBucketById(id, true);
+ }
+
+ public LocalBucket getBucketById(final int id, final boolean throwExceptionIfNotFound) {
  if (id < 0 || id >= files.size())
- throw new SchemaException("Bucket with id '" + id + "' was not found");
+ if (throwExceptionIfNotFound)
+ throw new SchemaException("Bucket with id '" + id + "' was not found");
+ else
+ return null;
 
  final Component p = files.get(id);
- if (!(p instanceof LocalBucket))
- throw new SchemaException("Bucket with id '" + id + "' was not found");
+ if (!(p instanceof LocalBucket)) {
+ if (throwExceptionIfNotFound)
+ throw new SchemaException("Bucket with id '" + id + "' was not found");
+ else
+ return null;
+ }
  return (LocalBucket) p;
  }
 

engine/src/test/java/com/arcadedb/AsyncTest.java

@@ -71,7 +71,8 @@ public void testSyncScanAndAsyncUpdate() {
 
  database.scanType(TYPE_NAME, true, record -> {
  callbackInvoked.incrementAndGet();
- database.async().updateRecord(record.modify().set("updated", true), newRecord -> updatedRecords.incrementAndGet());
+ record.modify().set("updated", true).save();
+ updatedRecords.incrementAndGet();
  return true;
  });
 
@@ -84,7 +85,10 @@ public void testSyncScanAndAsyncUpdate() {
  assertThat(callbackInvoked.get()).isEqualTo(TOT);
  assertThat(updatedRecords.get()).isEqualTo(TOT);
 
- final ResultSet resultSet = database.query("sql", "select count(*) as count from " + TYPE_NAME + " where updated = true");
+ ResultSet resultSet = database.query("sql", "select from " + TYPE_NAME + " where updated <> true");
+ Assertions.assertFalse(resultSet.hasNext());
+
+ resultSet = database.query("sql", "select count(*) as count from " + TYPE_NAME + " where updated = true");
 
  assertThat(resultSet.hasNext()).isTrue();
  assertThat(((Number) resultSet.next().getProperty("count")).intValue()).isEqualTo(TOT);

engine/src/test/java/com/arcadedb/RandomTestMultiThreadsTest.java

@@ -300,6 +300,8 @@ private int updateRecords(final Database database, final int threadId) {
  }
 
  private int deleteRecords(final Database database, final int threadId) {
+// if (true)
+// return 0;
  if (totalTransactionRecords.get() == 0)
  return 0;
 

engine/src/test/java/com/arcadedb/database/CheckDatabaseTest.java

@@ -173,7 +173,7 @@ public void checkBrokenDeletedEdges() {
  assertThat(row.<String>getProperty("operation")).isEqualTo("check database");
  assertThat((Long) row.getProperty("autoFix")).isEqualTo(0);
  assertThat((Long) row.getProperty("totalActiveVertices")).isEqualTo(TOTAL);
- assertThat((Long) row.getProperty("totalAllocatedEdges")).isEqualTo(TOTAL - 1);
+ assertThat((Long) row.getProperty("totalAllocatedEdges")).isEqualTo(TOTAL - 2);
  assertThat((Long) row.getProperty("totalActiveEdges")).isEqualTo(TOTAL - 2);
  assertThat((Long) row.getProperty("totalDeletedRecords")).isEqualTo(1);
  assertThat(((Collection) row.getProperty("corruptedRecords")).size()).isEqualTo(0);
@@ -214,16 +214,16 @@ public void checkBrokenDeletedVertex() {
  assertThat(result.hasNext()).isTrue();
  Result row = result.next();
 
- assertThat(row.<String>getProperty("operation")).isEqualTo("check database");
- assertThat((Long) row.getProperty("autoFix")).isEqualTo((TOTAL - 1) * 2);
- assertThat((Long) row.getProperty("totalActiveVertices")).isEqualTo(TOTAL - 1);
- assertThat((Long) row.getProperty("totalAllocatedEdges")).isEqualTo(TOTAL - 1);
- assertThat((Long) row.getProperty("totalActiveEdges")).isEqualTo(0);
- assertThat((Long) row.getProperty("totalDeletedRecords")).isEqualTo(TOTAL);
- assertThat(((Collection) row.getProperty("corruptedRecords")).size()).isEqualTo(TOTAL - 1);
- assertThat((Long) row.getProperty("missingReferenceBack")).isEqualTo(0);
- assertThat((Long) row.getProperty("invalidLinks")).isEqualTo((TOTAL - 1) * 2);
- assertThat(((Collection) row.getProperty("warnings")).size()).isEqualTo((TOTAL - 1) * 2);
+ Assertions.assertEquals("check database", row.getProperty("operation"));
+ Assertions.assertEquals((TOTAL - 1L) * 2L, (Long) row.getProperty("autoFix"));
+ Assertions.assertEquals(TOTAL - 1L, (Long) row.getProperty("totalActiveVertices"));
+ Assertions.assertEquals(0L, (Long) row.getProperty("totalAllocatedEdges"));
+ Assertions.assertEquals(0L, (Long) row.getProperty("totalActiveEdges"));
+ Assertions.assertEquals(1, (Long) row.getProperty("totalDeletedRecords"));
+ Assertions.assertEquals(TOTAL - 1L, ((Collection) row.getProperty("corruptedRecords")).size());
+ Assertions.assertEquals(0L, (Long) row.getProperty("missingReferenceBack"));
+ Assertions.assertEquals((TOTAL - 1L) * 2L, (Long) row.getProperty("invalidLinks"));
+ Assertions.assertEquals((TOTAL - 1L) * 2L, ((Collection) row.getProperty("warnings")).size());
 
  result = database.command("sql", "check database");
  assertThat(result.hasNext()).isTrue();

engine/src/test/java/com/arcadedb/engine/RandomDeleteTest.java

@@ -0,0 +1,87 @@
+package com.arcadedb.engine;
+
+import com.arcadedb.database.Database;
+import com.arcadedb.database.DatabaseFactory;
+import com.arcadedb.database.RID;
+import com.arcadedb.database.Record;
+import com.arcadedb.graph.MutableVertex;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.*;
+
+public class RandomDeleteTest {
+ private final static int TOT_RECORDS = 100_000;
+ private final static String TYPE = "Product";
+ private static final int CYCLES = 20;
+
+ @Test
+ public void testSmallRecords() {
+ try (DatabaseFactory databaseFactory = new DatabaseFactory("databases/randomDeleteTest")) {
+ if (databaseFactory.exists())
+ databaseFactory.open().drop();
+
+ try (Database db = databaseFactory.create()) {
+ db.getSchema().createVertexType(TYPE, 1);
+
+ db.transaction(() -> {
+ List<RID> rids = insert(db);
+ Assertions.assertEquals(TOT_RECORDS, db.countType(TYPE, true));
+
+ // DELETE FROM 1 TO N
+ for (int i = 0; i < TOT_RECORDS; i++)
+ db.deleteRecord(rids.get(i).asVertex());
+
+ Assertions.assertEquals(0, db.countType(TYPE, true));
+
+ // DELETE RANDOMLY X TIMES
+ for (int cycle = 0; cycle < CYCLES; cycle++) {
+ rids = insert(db);
+ checkRecords(db, rids);
+
+ for (int deleted = 0; deleted < TOT_RECORDS; ) {
+ final int i = new Random().nextInt(TOT_RECORDS);
+ final RID rid = rids.get(i);
+ if (rid != null) {
+ db.deleteRecord(rid.asVertex());
+ rids.set(i, null);
+ ++deleted;
+ }
+ }
+ }
+
+ Assertions.assertEquals(0, db.countType(TYPE, true));
+
+ });
+ } finally {
+ if (databaseFactory.exists())
+ databaseFactory.open().drop();
+ }
+ }
+ }
+
+ private void checkRecords(Database db, List<RID> rids) {
+ for (int i = 0; i < rids.size(); i++)
+ Assertions.assertNotNull(rids.get(i).asVertex());
+
+ final List<RID> found = new ArrayList<>();
+ for (Iterator<Record> it = db.iterateType(TYPE, true); it.hasNext(); )
+ found.add(it.next().asVertex().getIdentity());
+
+ Assertions.assertEquals(rids, found);
+
+ Assertions.assertEquals(rids.size(), db.countType(TYPE, true));
+ }
+
+ private static List<RID> insert(final Database db) {
+ final List<RID> rids = new ArrayList<>(TOT_RECORDS);
+ for (int i = 0; i < TOT_RECORDS; i++) {
+ final MutableVertex v = db.newVertex(TYPE)//
+ .set("id", i)//
+ .save();
+
+ rids.add(v.getIdentity());
+ }
+ return rids;
+ }
+}