From 907c803ac777faafd9ad728b14ee966f6f075346 Mon Sep 17 00:00:00 2001 From: lvca Date: Tue, 19 Dec 2023 01:25:53 -0500 Subject: [PATCH 01/11] first version of compress pages after a delete The compression must work with previously deleted records with v < 23.12.1 --- .../java/com/arcadedb/engine/LocalBucket.java | 158 +++++++++++++++--- .../com/arcadedb/engine/RandomDeleteTest.java | 87 ++++++++++ 2 files changed, 218 insertions(+), 27 deletions(-) create mode 100644 engine/src/test/java/com/arcadedb/engine/RandomDeleteTest.java diff --git a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java index 896dfc569..b3409418c 100644 --- a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java +++ b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java @@ -157,7 +157,7 @@ public boolean existsRecord(final RID rid) { final int recordPositionInPage = (int) page.readUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); if (recordPositionInPage == 0) - // CLEANED CORRUPTED RECORD + // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) return false; final long[] recordSize = page.readNumberAndSize(recordPositionInPage); @@ -194,7 +194,7 @@ public void scan(final RawRecordCallback callback, final ErrorRecordCallback err final int recordPositionInPage = (int) page.readUnsignedInt( PAGE_RECORD_TABLE_OFFSET + recordIdInPage * INT_SERIALIZED_SIZE); if (recordPositionInPage == 0) - // CLEANED CORRUPTED RECORD + // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) continue; final long[] recordSize = page.readNumberAndSize(recordPositionInPage); @@ -309,7 +309,7 @@ public long count() { final int recordPositionInPage = (int) page.readUnsignedInt( PAGE_RECORD_TABLE_OFFSET + recordIdInPage * INT_SERIALIZED_SIZE); if (recordPositionInPage == 0) - // CLEANED CORRUPTED RECORD + // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) continue; final long[] recordSize = page.readNumberAndSize(recordPositionInPage); @@ -373,7 +373,7 @@ public Map check(final int verboseLevel, final boolean fix) { PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); if (recordPositionInPage == 0) { - // CLEANED CORRUPTED RECORD + // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) pageDeletedRecords++; totalDeletedRecords++; @@ -533,7 +533,7 @@ public Binary getRecordInternal(final RID rid, final boolean readPlaceHolderCont final int recordPositionInPage = (int) page.readUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); if (recordPositionInPage == 0) - // CLEANED CORRUPTED RECORD + // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) return null; final long[] recordSize = page.readNumberAndSize(recordPositionInPage); @@ -674,7 +674,7 @@ private boolean updateRecordInternal(final Record record, final RID rid, final b final int recordPositionInPage = (int) page.readUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); if (recordPositionInPage == 0) - // CLEANED CORRUPTED RECORD + // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) throw new RecordNotFoundException("Record " + rid + " not found", rid); final long[] recordSize = page.readNumberAndSize(recordPositionInPage); @@ -818,6 +818,8 @@ private boolean updateRecordInternal(final Record record, final RID rid, final b if (!discardRecordAfter) ((RecordInternal) record).setBuffer(buffer.getNotReusable()); + //compressPage(page); + return true; } catch (final IOException e) { @@ -845,7 +847,7 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder final int recordPositionInPage = (int) page.readUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); if (recordPositionInPage < 1) - // CLEANED CORRUPTED RECORD + // DELETED RECORD (from 23.12.1) throw new RecordNotFoundException("Record " + rid + " not found", rid); // AVOID DELETION OF CONTENT IN CORRUPTED RECORD @@ -897,32 +899,32 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder throw new RecordNotFoundException("Record " + rid + " not found", rid); } - // CONTENT SIZE = 0 MEANS DELETED - page.writeNumber(recordPositionInPage, 0L); + // POINTER = 0 MEANS DELETED + page.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE, 0); + + if (positionInPage == recordCountInPage - 1) { + // LAST POSITION IN THE PAGE, UPDATE THE TOTAL RECORDS IN THE PAGE GOING BACK FROM THE CURRENT RECORD + int newRecordCount = -1; + for (int i = positionInPage - 1; i > -1; i--) { + final int pos = (int) page.readUnsignedInt(PAGE_RECORD_TABLE_OFFSET + i * INT_SERIALIZED_SIZE); + if (pos == 0 || page.readNumberAndSize(pos)[0] == 0) + // DELETE RECORD, + newRecordCount = i; + else + break; + } + + if (newRecordCount > -1) + // UPDATE TOTAL RECORDS IN THE PAGE + page.writeShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET, (short) newRecordCount); + } } else { // CORRUPTED RECORD: WRITE ZERO AS POINTER TO RECORD page.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE, 0L); } - // TODO: EVALUATE COMPACTING THE PAGE FOR REUSING THE SPACE -// recordPositionInPage++; -// -// AVOID COMPACTION DURING DELETE -// // COMPACT PAGE BY SHIFTING THE RECORDS TO THE LEFT -// for (int pos = positionInPage + 1; pos < recordCountInPage; ++pos) { -// final int nextRecordPosInPage = (int) page.readUnsignedInt(PAGE_RECORD_TABLE_OFFSET + pos * INT_SERIALIZED_SIZE); -// final byte[] record = page.readBytes(nextRecordPosInPage); -// -// final int bytesWritten = page.writeBytes(recordPositionInPage, record); -// -// // OVERWRITE POS TABLE WITH NEW POSITION -// page.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + pos * INT_SERIALIZED_SIZE, recordPositionInPage); -// -// recordPositionInPage += bytesWritten; -// } -// -// page.writeShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET, (short) (recordCountInPage - 1)); + compressPage(page); LogManager.instance() .log(this, Level.FINE, "Deleted record %s (%s threadId=%d)", null, rid, page, Thread.currentThread().getId()); @@ -932,6 +934,108 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder } } + private void compressPage(final MutablePage page) { + final short recordCountInPage = page.readShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET); + + final List orderedRecordContentInPage = new ArrayList<>(DEF_MAX_RECORDS_IN_PAGE); + + for (int positionInPage = 0; positionInPage < recordCountInPage; positionInPage++) { + final int recordPositionInPage = (int) page.readUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); + if (recordPositionInPage == 0 || recordPositionInPage >= page.getContentSize()) + // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) + continue; + + final long[] recordSize = page.readNumberAndSize(recordPositionInPage); + + if (recordSize[0] == RECORD_PLACEHOLDER_POINTER) + orderedRecordContentInPage.add(new int[] { recordPositionInPage, LONG_SERIALIZED_SIZE + (int) recordSize[1] }); + else if (recordSize[0] == FIRST_CHUNK || recordSize[0] == NEXT_CHUNK) { + final int chunkSize = page.readInt((recordPositionInPage + (int) recordSize[1])); + orderedRecordContentInPage.add( + new int[] { recordPositionInPage, chunkSize + (int) recordSize[1] + INT_SERIALIZED_SIZE + LONG_SERIALIZED_SIZE }); + } else if (recordSize[0] != 0) + orderedRecordContentInPage.add(new int[] { recordPositionInPage, (int) recordSize[0] + (int) recordSize[1] }); + } + + if (orderedRecordContentInPage.isEmpty()) + return; + + orderedRecordContentInPage.sort(Comparator.comparingLong(a -> a[0])); + + final List holes = new ArrayList<>(128); + + // USE INT BECAUSE OF THE LIMITED SIZE OF THE PAGE + int[] lastPointer = new int[] { PAGE_RECORD_TABLE_OFFSET + maxRecordsInPage * INT_SERIALIZED_SIZE, 0 }; + for (int i = 0; i < orderedRecordContentInPage.size(); i++) { + final int[] pointer = orderedRecordContentInPage.get(i); + + final int lastPointerEnd = lastPointer[0] + lastPointer[1]; + if (pointer[0] != lastPointerEnd) { + final int[] lastHole = holes.isEmpty() ? null : holes.get(holes.size() - 1); + if (lastHole != null && lastHole[0] + lastHole[1] == pointer[0]) + // UPDATE PREVIOUS HOLE + lastHole[1] += pointer[1]; + else + // CREATE A NEW HOLE + holes.add(new int[] { lastPointerEnd, pointer[0] - lastPointerEnd }); + } + lastPointer = pointer; + } + + if (holes.size() > 1) + System.out.println("FOUND " + holes.size() + " HOLES"); + + for (int i = 0; i < holes.size(); i++) { + final int[] hole = holes.get(i); + System.out.printf("Hole %d/%d %d-(%d)->%d %n", i + 1, holes.size(), hole[0], hole[1], hole[0] + hole[1]); + } + + int gap = 0; + for (int i = 0; i < holes.size(); i++) { + final int[] hole = holes.get(i); + final int marginEnd = i < holes.size() - 1 ? holes.get(i + 1)[0] : page.getContentSize(); + + final byte[] buffer = page.content.getContent(); + + final int from = hole[0] + hole[1]; + final int length = marginEnd - from; + if (length < 1) + System.out.println("ERROR"); + + if (BasePage.PAGE_HEADER_SIZE + hole[0] + length > page.getPhysicalSize()) { + System.out.printf("hole %d/%d %d <-> %d = %db%n", i + 1, holes.size(), hole[0], hole[0] + hole[1], hole[1]); + + System.out.printf("Moving %d <- %d %d (marginEnd=%d)%n", BasePage.PAGE_HEADER_SIZE + hole[0] + hole[1],// + BasePage.PAGE_HEADER_SIZE + hole[0], marginEnd - hole[0] - BasePage.PAGE_HEADER_SIZE,// + (BasePage.PAGE_HEADER_SIZE + hole[0]) + (marginEnd - hole[0] - BasePage.PAGE_HEADER_SIZE)); + } + + System.out.printf("Removing hole %d/%d %d-(%d)->%d MOVING %d<-%d (gap %d)%n", i + 1, holes.size(), hole[0], hole[1], + hole[0] + hole[1],// + hole[0] - gap, from, gap); + + System.arraycopy(buffer, BasePage.PAGE_HEADER_SIZE + from,// + buffer, BasePage.PAGE_HEADER_SIZE + hole[0] - gap, length); + + page.updateModifiedRange(hole[0] - gap, from + length); + + // SHIFT ALL THE POINTERS FROM THE HOLE TO THE LAST + for (int positionInPage = 0; positionInPage < recordCountInPage; positionInPage++) { + final int recordPositionInPage = (int) page.readUnsignedInt( + PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); + if (recordPositionInPage < 1 || recordPositionInPage >= page.getContentSize()) + // AVOID TOUCHING CORRUPTED RECORD + continue; + + if (recordPositionInPage >= from && recordPositionInPage <= from + length) + page.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE, + recordPositionInPage - hole[1] - gap); + } + + gap += hole[1]; + } + } + private Binary loadMultiPageRecord(final RID originalRID, BasePage page, int recordPositionInPage, long[] recordSize) throws IOException { // READ ALL THE CHUNKS TILL THE END diff --git a/engine/src/test/java/com/arcadedb/engine/RandomDeleteTest.java b/engine/src/test/java/com/arcadedb/engine/RandomDeleteTest.java new file mode 100644 index 000000000..a69e403ff --- /dev/null +++ b/engine/src/test/java/com/arcadedb/engine/RandomDeleteTest.java @@ -0,0 +1,87 @@ +package com.arcadedb.engine; + +import com.arcadedb.database.Database; +import com.arcadedb.database.DatabaseFactory; +import com.arcadedb.database.RID; +import com.arcadedb.database.Record; +import com.arcadedb.graph.MutableVertex; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.*; + +public class RandomDeleteTest { + private final static int TOT_RECORDS = 100_000; + private final static String TYPE = "Product"; + private static final int CYCLES = 20; + + @Test + public void testSmallRecords() { + try (DatabaseFactory databaseFactory = new DatabaseFactory("databases/randomDeleteTest")) { + if (databaseFactory.exists()) + databaseFactory.open().drop(); + + try (Database db = databaseFactory.create()) { + db.getSchema().createVertexType(TYPE, 1); + + db.transaction(() -> { + List rids = insert(db); + Assertions.assertEquals(TOT_RECORDS, db.countType(TYPE, true)); + + // DELETE FROM 1 TO N + for (int i = 0; i < TOT_RECORDS; i++) + db.deleteRecord(rids.get(i).asVertex()); + + Assertions.assertEquals(0, db.countType(TYPE, true)); + + // DELETE RANDOMLY X TIMES + for (int cycle = 0; cycle < CYCLES; cycle++) { + rids = insert(db); + checkRecords(db, rids); + + for (int deleted = 0; deleted < TOT_RECORDS; ) { + final int i = new Random().nextInt(TOT_RECORDS); + final RID rid = rids.get(i); + if (rid != null) { + db.deleteRecord(rid.asVertex()); + rids.set(i, null); + ++deleted; + } + } + } + + Assertions.assertEquals(0, db.countType(TYPE, true)); + + }); + } finally { + if (databaseFactory.exists()) + databaseFactory.open().drop(); + } + } + } + + private void checkRecords(Database db, List rids) { + for (int i = 0; i < rids.size(); i++) + Assertions.assertNotNull(rids.get(i).asVertex()); + + final List found = new ArrayList<>(); + for (Iterator it = db.iterateType(TYPE, true); it.hasNext(); ) + found.add(it.next().asVertex().getIdentity()); + + Assertions.assertEquals(rids, found); + + Assertions.assertEquals(rids.size(), db.countType(TYPE, true)); + } + + private static List insert(final Database db) { + final List rids = new ArrayList<>(TOT_RECORDS); + for (int i = 0; i < TOT_RECORDS; i++) { + final MutableVertex v = db.newVertex(TYPE)// + .set("id", i)// + .save(); + + rids.add(v.getIdentity()); + } + return rids; + } +} From 4d20a470596722f45ab1e6533c4a96ced7c61cd7 Mon Sep 17 00:00:00 2001 From: lvca Date: Tue, 19 Dec 2023 01:38:57 -0500 Subject: [PATCH 02/11] chore: additional check on page pointer --- .../java/com/arcadedb/engine/LocalBucket.java | 54 +++++++++---------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java index b3409418c..122bfe5fb 100644 --- a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java +++ b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java @@ -155,7 +155,7 @@ public boolean existsRecord(final RID rid) { if (positionInPage >= recordCountInPage) return false; - final int recordPositionInPage = (int) page.readUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); + final int recordPositionInPage = getRecordPositionInPage(page, positionInPage); if (recordPositionInPage == 0) // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) return false; @@ -191,8 +191,7 @@ public void scan(final RawRecordCallback callback, final ErrorRecordCallback err final RID rid = new RID(database, fileId, ((long) pageId) * maxRecordsInPage + recordIdInPage); try { - final int recordPositionInPage = (int) page.readUnsignedInt( - PAGE_RECORD_TABLE_OFFSET + recordIdInPage * INT_SERIALIZED_SIZE); + final int recordPositionInPage = getRecordPositionInPage(page, recordIdInPage); if (recordPositionInPage == 0) // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) continue; @@ -306,8 +305,7 @@ public long count() { if (recordCountInPage > 0) { for (int recordIdInPage = 0; recordIdInPage < recordCountInPage; ++recordIdInPage) { - final int recordPositionInPage = (int) page.readUnsignedInt( - PAGE_RECORD_TABLE_OFFSET + recordIdInPage * INT_SERIALIZED_SIZE); + final int recordPositionInPage = getRecordPositionInPage(page, recordIdInPage); if (recordPositionInPage == 0) // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) continue; @@ -369,9 +367,7 @@ public Map check(final int verboseLevel, final boolean fix) { for (int positionInPage = 0; positionInPage < recordCountInPage; ++positionInPage) { final RID rid = new RID(database, file.getFileId(), positionInPage); - final int recordPositionInPage = (int) page.readUnsignedInt( - PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); - + final int recordPositionInPage = getRecordPositionInPage(page, positionInPage); if (recordPositionInPage == 0) { // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) pageDeletedRecords++; @@ -531,7 +527,7 @@ public Binary getRecordInternal(final RID rid, final boolean readPlaceHolderCont if (positionInPage >= recordCountInPage) throw new RecordNotFoundException("Record " + rid + " not found", rid); - final int recordPositionInPage = (int) page.readUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); + final int recordPositionInPage = getRecordPositionInPage(page, positionInPage); if (recordPositionInPage == 0) // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) return null; @@ -672,7 +668,7 @@ private boolean updateRecordInternal(final Record record, final RID rid, final b if (positionInPage >= recordCountInPage) throw new RecordNotFoundException("Record " + rid + " not found", rid); - final int recordPositionInPage = (int) page.readUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); + final int recordPositionInPage = getRecordPositionInPage(page, positionInPage); if (recordPositionInPage == 0) // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) throw new RecordNotFoundException("Record " + rid + " not found", rid); @@ -725,8 +721,7 @@ private boolean updateRecordInternal(final Record record, final RID rid, final b if (bufferSize > recordSize[0]) { // UPDATED RECORD IS LARGER THAN THE PREVIOUS VERSION: MAKE ROOM IN THE PAGE IF POSSIBLE - final int lastRecordPositionInPage = (int) page.readUnsignedInt( - PAGE_RECORD_TABLE_OFFSET + (recordCountInPage - 1) * INT_SERIALIZED_SIZE); + final int lastRecordPositionInPage = getRecordPositionInPage(page, recordCountInPage - 1); final long[] lastRecordSize = page.readNumberAndSize(lastRecordPositionInPage); if (lastRecordSize[0] == RECORD_PLACEHOLDER_POINTER) { @@ -749,17 +744,14 @@ private boolean updateRecordInternal(final Record record, final RID rid, final b // THERE IS SPACE LEFT IN THE PAGE, SHIFT ON THE RIGHT THE EXISTENT RECORDS if (positionInPage < recordCountInPage - 1) { // NOT LAST RECORD IN PAGE, SHIFT NEXT RECORDS - final int nextRecordPositionInPage = (int) page.readUnsignedInt( - PAGE_RECORD_TABLE_OFFSET + (positionInPage + 1) * INT_SERIALIZED_SIZE); - + final int nextRecordPositionInPage = getRecordPositionInPage(page, positionInPage + 1); final int newPos = nextRecordPositionInPage + additionalSpaceNeeded; page.move(nextRecordPositionInPage, newPos, pageOccupiedInBytes - nextRecordPositionInPage); // TODO: CALCULATE THE REAL SIZE TO COMPACT DELETED RECORDS/PLACEHOLDERS for (int pos = positionInPage + 1; pos < recordCountInPage; ++pos) { - final int nextRecordPosInPage = (int) page.readUnsignedInt(PAGE_RECORD_TABLE_OFFSET + pos * INT_SERIALIZED_SIZE); - + final int nextRecordPosInPage = getRecordPositionInPage(page, pos); if (nextRecordPosInPage == 0) page.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + pos * INT_SERIALIZED_SIZE, 0); else @@ -845,7 +837,7 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder if (positionInPage >= recordCountInPage) throw new RecordNotFoundException("Record " + rid + " not found", rid); - final int recordPositionInPage = (int) page.readUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); + final int recordPositionInPage = getRecordPositionInPage(page, positionInPage); if (recordPositionInPage < 1) // DELETED RECORD (from 23.12.1) throw new RecordNotFoundException("Record " + rid + " not found", rid); @@ -878,8 +870,7 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder final int chunkPositionInPage = (int) (nextChunkPointer % maxRecordsInPage); chunkPage = database.getTransaction().getPageToModify(new PageId(file.getFileId(), chunkPageId), pageSize, false); - chunkRecordPositionInPage = (int) chunkPage.readUnsignedInt( - PAGE_RECORD_TABLE_OFFSET + chunkPositionInPage * INT_SERIALIZED_SIZE); + chunkRecordPositionInPage = getRecordPositionInPage(chunkPage, chunkPositionInPage); recordSize = chunkPage.readNumberAndSize(chunkRecordPositionInPage); if (recordSize[0] != NEXT_CHUNK) @@ -906,7 +897,7 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder // LAST POSITION IN THE PAGE, UPDATE THE TOTAL RECORDS IN THE PAGE GOING BACK FROM THE CURRENT RECORD int newRecordCount = -1; for (int i = positionInPage - 1; i > -1; i--) { - final int pos = (int) page.readUnsignedInt(PAGE_RECORD_TABLE_OFFSET + i * INT_SERIALIZED_SIZE); + final int pos = getRecordPositionInPage(page, i); if (pos == 0 || page.readNumberAndSize(pos)[0] == 0) // DELETE RECORD, newRecordCount = i; @@ -940,7 +931,7 @@ private void compressPage(final MutablePage page) { final List orderedRecordContentInPage = new ArrayList<>(DEF_MAX_RECORDS_IN_PAGE); for (int positionInPage = 0; positionInPage < recordCountInPage; positionInPage++) { - final int recordPositionInPage = (int) page.readUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); + final int recordPositionInPage = getRecordPositionInPage(page, positionInPage); if (recordPositionInPage == 0 || recordPositionInPage >= page.getContentSize()) // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) continue; @@ -1021,8 +1012,7 @@ else if (recordSize[0] == FIRST_CHUNK || recordSize[0] == NEXT_CHUNK) { // SHIFT ALL THE POINTERS FROM THE HOLE TO THE LAST for (int positionInPage = 0; positionInPage < recordCountInPage; positionInPage++) { - final int recordPositionInPage = (int) page.readUnsignedInt( - PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); + final int recordPositionInPage = getRecordPositionInPage(page, positionInPage); if (recordPositionInPage < 1 || recordPositionInPage >= page.getContentSize()) // AVOID TOUCHING CORRUPTED RECORD continue; @@ -1056,7 +1046,7 @@ private Binary loadMultiPageRecord(final RID originalRID, BasePage page, int rec final int chunkPositionInPage = (int) (nextChunkPointer % maxRecordsInPage); page = database.getTransaction().getPage(new PageId(file.getFileId(), chunkPageId), pageSize); - recordPositionInPage = (int) page.readUnsignedInt(PAGE_RECORD_TABLE_OFFSET + chunkPositionInPage * INT_SERIALIZED_SIZE); + recordPositionInPage = getRecordPositionInPage(page, chunkPositionInPage); recordSize = page.readNumberAndSize(recordPositionInPage); if (recordSize[0] != NEXT_CHUNK) @@ -1067,6 +1057,14 @@ private Binary loadMultiPageRecord(final RID originalRID, BasePage page, int rec return record; } + private int getRecordPositionInPage(final BasePage page, final int positionInPage) { + final int recordPositionInPage = (int) page.readUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); + if (recordPositionInPage < PAGE_RECORD_TABLE_OFFSET + maxRecordsInPage * INT_SERIALIZED_SIZE) + throw new DatabaseOperationException( + "Invalid record #" + fileId + ":" + (page.pageId.getPageNumber() * maxRecordsInPage + positionInPage)); + return recordPositionInPage; + } + private void writeMultiPageRecord(final Binary buffer, MutablePage currentPage, int newPosition, final int availableSpaceForFirstChunk) throws IOException { int bufferSize = buffer.size(); @@ -1196,8 +1194,7 @@ private void updateMultiPageRecord(final RID originalRID, final Binary buffer, M final int chunkPositionInPage = (int) (nextChunkPointer % maxRecordsInPage); nextPage = database.getTransaction().getPageToModify(new PageId(file.getFileId(), chunkPageId), pageSize, false); - final int recordPositionInPage = (int) nextPage.readUnsignedInt( - PAGE_RECORD_TABLE_OFFSET + chunkPositionInPage * INT_SERIALIZED_SIZE); + final int recordPositionInPage = getRecordPositionInPage(nextPage, chunkPositionInPage); final long[] recordSize = nextPage.readNumberAndSize(recordPositionInPage); if (recordSize[0] != NEXT_CHUNK) @@ -1280,8 +1277,7 @@ private AvailableSpace checkForSpaceInPage(final int pageNumber, final boolean i result.createNewPage = true; else if (result.recordCountInPage > 0) { // GET FIRST EMPTY POSITION - final int lastRecordPositionInPage = (int) result.page.readUnsignedInt( - PAGE_RECORD_TABLE_OFFSET + (result.recordCountInPage - 1) * INT_SERIALIZED_SIZE); + final int lastRecordPositionInPage = getRecordPositionInPage(result.page, result.recordCountInPage - 1); if (lastRecordPositionInPage == 0) // CLEANED CORRUPTED RECORD result.createNewPage = true; From 9e844dc30c5981d574ec2e1269e62b056741398f Mon Sep 17 00:00:00 2001 From: lvca Date: Thu, 21 Dec 2023 13:04:21 -0500 Subject: [PATCH 03/11] Fixed issue with compacting holes The issue was invalid pointers while deleting multi chunk records. Some pages was defrag while the old pointers pointed to the old place --- .../arcadedb/database/TransactionContext.java | 17 ++ .../java/com/arcadedb/engine/LocalBucket.java | 150 +++++++++--------- .../java/com/arcadedb/schema/LocalSchema.java | 17 +- 3 files changed, 107 insertions(+), 77 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/database/TransactionContext.java b/engine/src/main/java/com/arcadedb/database/TransactionContext.java index 9ba83b424..6f35b740e 100644 --- a/engine/src/main/java/com/arcadedb/database/TransactionContext.java +++ b/engine/src/main/java/com/arcadedb/database/TransactionContext.java @@ -20,6 +20,7 @@ import com.arcadedb.GlobalConfiguration; import com.arcadedb.engine.BasePage; +import com.arcadedb.engine.Bucket; import com.arcadedb.engine.ComponentFile; import com.arcadedb.engine.ImmutablePage; import com.arcadedb.engine.LocalBucket; @@ -37,6 +38,8 @@ import com.arcadedb.index.IndexInternal; import com.arcadedb.index.lsm.LSMTreeIndexAbstract; import com.arcadedb.log.LogManager; +import com.arcadedb.schema.LocalSchema; +import com.arcadedb.schema.Schema; import java.io.*; import java.util.*; @@ -568,6 +571,20 @@ public TransactionPhase1 commit1stPhase(final boolean isLeader) { final List pages = new ArrayList<>(); final PageManager pageManager = database.getPageManager(); + // COMPRESS PAGES BEFORE SAVING THEM + final LocalSchema localSchema = database.getSchema().getEmbedded(); + + for (MutablePage page : modifiedPages.values()) { + final LocalBucket bucket = localSchema.getBucketById(page.getPageId().getFileId(), false); + if (bucket != null) + bucket.compressPage(page); + } + for (MutablePage page : newPages.values()) { + final LocalBucket bucket = localSchema.getBucketById(page.getPageId().getFileId(), false); + if (bucket != null) + bucket.compressPage(page); + } + for (final Iterator it = modifiedPages.values().iterator(); it.hasNext(); ) { final MutablePage p = it.next(); diff --git a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java index 3df550cc8..3016bb095 100644 --- a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java +++ b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java @@ -46,13 +46,18 @@ /** * PAGE CONTENT = [version(long:8),recordCountInPage(short:2),recordOffsetsInPage(2048*uint=8192)] *

- * Record size is the length of the record: + * Record size is the length of the record, managing also the following special cases: *
    + *
  • > 0 = active record
    • *
  • 0 = deleted record
    • *
  • -1 = placeholder pointer that points to another record in another page
    • - *
  • <-1 = placeholder content pointed from another record in another page
    • + *
  • -2 = first chunk of a multi page record
    • + *
  • -3 = after first chunk of a multi page record
    • + *
  • <-5 = placeholder content pointed from another record in another page
    • *
- * The record size is stored as a varint (variable integer size). The minimum size of a record stored in a page is 5 bytes. If the record is smaller than 5 bytes, + * The record size is stored as a varint (variable integer size) + the length of the size itself. + * So if the record size is 10, it would take 1 byte (to store the number 10 in 7 bits) + 1 byte (to store how many bytes is 10)) + * The minimum size of a record stored in a page is 5 bytes. If the record is smaller than 5 bytes, * it is filled with blanks. * * @author Luca Garulli (l.garulli@arcadedata.com) @@ -157,7 +162,7 @@ public boolean existsRecord(final RID rid) { final int recordPositionInPage = getRecordPositionInPage(page, positionInPage); if (recordPositionInPage == 0) - // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) + // DELETED RECORD (>= 24.1.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) return false; final long[] recordSize = page.readNumberAndSize(recordPositionInPage); @@ -193,7 +198,7 @@ public void scan(final RawRecordCallback callback, final ErrorRecordCallback err try { final int recordPositionInPage = getRecordPositionInPage(page, recordIdInPage); if (recordPositionInPage == 0) - // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) + // DELETED RECORD (>= 24.1.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) continue; final long[] recordSize = page.readNumberAndSize(recordPositionInPage); @@ -307,7 +312,7 @@ public long count() { for (int recordIdInPage = 0; recordIdInPage < recordCountInPage; ++recordIdInPage) { final int recordPositionInPage = getRecordPositionInPage(page, recordIdInPage); if (recordPositionInPage == 0) - // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) + // DELETED RECORD (>= 24.1.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) continue; final long[] recordSize = page.readNumberAndSize(recordPositionInPage); @@ -370,7 +375,7 @@ public Map check(final int verboseLevel, final boolean fix) { final int recordPositionInPage = (int) page.readUnsignedInt( PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); if (recordPositionInPage == 0) { - // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) + // DELETED RECORD (>= 24.1.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) pageDeletedRecords++; totalDeletedRecords++; @@ -530,7 +535,7 @@ public Binary getRecordInternal(final RID rid, final boolean readPlaceHolderCont final int recordPositionInPage = getRecordPositionInPage(page, positionInPage); if (recordPositionInPage == 0) - // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) + // DELETED RECORD (>= 24.1.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) return null; final long[] recordSize = page.readNumberAndSize(recordPositionInPage); @@ -623,7 +628,7 @@ private RID createRecordInternal(final Record record, final boolean isPlaceHolde if (spaceNeeded > spaceAvailableInCurrentPage) { // MULTI-PAGE RECORD - writeMultiPageRecord(buffer, selectedPage, newPosition, spaceAvailableInCurrentPage); + writeMultiPageRecord(rid, buffer, selectedPage, newPosition, spaceAvailableInCurrentPage); } else { final int byteWritten = selectedPage.writeNumber(newPosition, isPlaceHolder ? (-1L * bufferSize) : bufferSize); @@ -671,7 +676,7 @@ private boolean updateRecordInternal(final Record record, final RID rid, final b final int recordPositionInPage = getRecordPositionInPage(page, positionInPage); if (recordPositionInPage == 0) - // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) + // DELETED RECORD (>= 24.1.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) throw new RecordNotFoundException("Record " + rid + " not found", rid); final long[] recordSize = page.readNumberAndSize(recordPositionInPage); @@ -754,6 +759,7 @@ private boolean updateRecordInternal(final Record record, final RID rid, final b for (int pos = positionInPage + 1; pos < recordCountInPage; ++pos) { final int nextRecordPosInPage = getRecordPositionInPage(page, pos); if (nextRecordPosInPage == 0) + // TODO: WHY DO WE NEED TO WRITE BACK 0? page.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + pos * INT_SERIALIZED_SIZE, 0); else page.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + pos * INT_SERIALIZED_SIZE, @@ -790,7 +796,7 @@ private boolean updateRecordInternal(final Record record, final RID rid, final b page, Thread.currentThread().getId()); } else { // SPLIT THE RECORD IN CHUNKS AS LINKED LIST AND STORE THE FIRST PART ON CURRENT PAGE ISSUE https://github.com/ArcadeData/arcadedb/issues/332 - writeMultiPageRecord(buffer, page, recordPositionInPage, availableSpaceForChunk); + writeMultiPageRecord(rid, buffer, page, recordPositionInPage, availableSpaceForChunk); LogManager.instance().log(this, Level.FINE, "Updated record %s by splitting it in multiple chunks to be saved in multiple pages (%s threadId=%d)", null, rid, @@ -799,6 +805,7 @@ private boolean updateRecordInternal(final Record record, final RID rid, final b } } else { // UPDATED RECORD CONTENT IS NOT LARGER THAN PREVIOUS VERSION: OVERWRITE THE CONTENT + // CREATE A HOLE (REMOVED LATER BY COMPRESS-PAGE) recordSize[1] = page.writeNumber(recordPositionInPage, isPlaceHolder ? -1L * bufferSize : bufferSize); final int recordContentPositionInPage = (int) (recordPositionInPage + recordSize[1]); page.writeByteArray(recordContentPositionInPage, buffer.getContent(), buffer.getContentBeginOffset(), bufferSize); @@ -811,8 +818,6 @@ private boolean updateRecordInternal(final Record record, final RID rid, final b if (!discardRecordAfter) ((RecordInternal) record).setBuffer(buffer.getNotReusable()); - //compressPage(page); - return true; } catch (final IOException e) { @@ -840,7 +845,7 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder final int recordPositionInPage = getRecordPositionInPage(page, positionInPage); if (recordPositionInPage < 1) - // DELETED RECORD (from 23.12.1) + // CLEANED CORRUPTED RECORD throw new RecordNotFoundException("Record " + rid + " not found", rid); // AVOID DELETION OF CONTENT IN CORRUPTED RECORD @@ -862,6 +867,7 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder for (int chunkId = 0; ; ++chunkId) { final long nextChunkPointer = chunkPage.readLong( (int) (chunkRecordPositionInPage + recordSize[1] + INT_SERIALIZED_SIZE)); + if (nextChunkPointer == 0) // LAST CHUNK break; @@ -892,32 +898,33 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder } // POINTER = 0 MEANS DELETED + //page.writeNumber(recordPositionInPage, 0L); page.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE, 0); - if (positionInPage == recordCountInPage - 1) { - // LAST POSITION IN THE PAGE, UPDATE THE TOTAL RECORDS IN THE PAGE GOING BACK FROM THE CURRENT RECORD - int newRecordCount = -1; - for (int i = positionInPage - 1; i > -1; i--) { - final int pos = getRecordPositionInPage(page, i); - if (pos == 0 || page.readNumberAndSize(pos)[0] == 0) - // DELETE RECORD, - newRecordCount = i; - else - break; - } - - if (newRecordCount > -1) - // UPDATE TOTAL RECORDS IN THE PAGE - page.writeShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET, (short) newRecordCount); - } +// TODO: UPDATE TOTAL RECORDS IN PAGE +// +// if (positionInPage == recordCountInPage - 1) { +// // LAST POSITION IN THE PAGE, UPDATE THE TOTAL RECORDS IN THE PAGE GOING BACK FROM THE CURRENT RECORD +// int newRecordCount = -1; +// for (int i = positionInPage - 1; i > -1; i--) { +// final int pos = getRecordPositionInPage(page, i); +// if (pos == 0 || page.readNumberAndSize(pos)[0] == 0) +// // DELETE RECORD, +// newRecordCount = i; +// else +// break; +// } +// +// if (newRecordCount > -1) +// // UPDATE TOTAL RECORDS IN THE PAGE +// page.writeShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET, (short) newRecordCount); +// } } else { // CORRUPTED RECORD: WRITE ZERO AS POINTER TO RECORD page.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE, 0L); } - compressPage(page); - LogManager.instance() .log(this, Level.FINE, "Deleted record %s (%s threadId=%d)", null, rid, page, Thread.currentThread().getId()); @@ -926,18 +933,21 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder } } - private void compressPage(final MutablePage page) { + public void compressPage(final MutablePage page) { final short recordCountInPage = page.readShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET); final List orderedRecordContentInPage = new ArrayList<>(DEF_MAX_RECORDS_IN_PAGE); for (int positionInPage = 0; positionInPage < recordCountInPage; positionInPage++) { - final int recordPositionInPage = getRecordPositionInPage(page, positionInPage); - if (recordPositionInPage == 0 || recordPositionInPage >= page.getContentSize()) - // DELETED RECORD (from 23.12.1, IT WAS CLEANED CORRUPTED RECORD BEFORE) + final int recordPositionInPage = (int) page.readUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); + if (recordPositionInPage < 1 || recordPositionInPage >= page.getContentSize()) + // SKIP DELETED RECORD (>=24.1.1), OR CORRUPTED continue; final long[] recordSize = page.readNumberAndSize(recordPositionInPage); + if (recordSize[0] == 0) + // DELETED <24.1.1 + continue; if (recordSize[0] == RECORD_PLACEHOLDER_POINTER) orderedRecordContentInPage.add(new int[] { recordPositionInPage, LONG_SERIALIZED_SIZE + (int) recordSize[1] }); @@ -945,7 +955,10 @@ else if (recordSize[0] == FIRST_CHUNK || recordSize[0] == NEXT_CHUNK) { final int chunkSize = page.readInt((recordPositionInPage + (int) recordSize[1])); orderedRecordContentInPage.add( new int[] { recordPositionInPage, chunkSize + (int) recordSize[1] + INT_SERIALIZED_SIZE + LONG_SERIALIZED_SIZE }); - } else if (recordSize[0] != 0) + } else if (recordSize[0] < RECORD_PLACEHOLDER_CONTENT) + // PLACEHOLDER CONTENT, CONSIDER THE RECORD SIZE (CONVERTED FROM NEGATIVE NUMBER) + VARINT SIZE + orderedRecordContentInPage.add(new int[] { recordPositionInPage, (int) (-1 * recordSize[0]) + (int) recordSize[1] }); + else orderedRecordContentInPage.add(new int[] { recordPositionInPage, (int) recordSize[0] + (int) recordSize[1] }); } @@ -964,63 +977,40 @@ else if (recordSize[0] == FIRST_CHUNK || recordSize[0] == NEXT_CHUNK) { final int lastPointerEnd = lastPointer[0] + lastPointer[1]; if (pointer[0] != lastPointerEnd) { final int[] lastHole = holes.isEmpty() ? null : holes.get(holes.size() - 1); - if (lastHole != null && lastHole[0] + lastHole[1] == pointer[0]) + if (lastHole != null && lastHole[0] + lastHole[1] == pointer[0]) { // UPDATE PREVIOUS HOLE lastHole[1] += pointer[1]; - else + } else // CREATE A NEW HOLE holes.add(new int[] { lastPointerEnd, pointer[0] - lastPointerEnd }); } lastPointer = pointer; } - if (holes.size() > 1) - System.out.println("FOUND " + holes.size() + " HOLES"); - - for (int i = 0; i < holes.size(); i++) { - final int[] hole = holes.get(i); - System.out.printf("Hole %d/%d %d-(%d)->%d %n", i + 1, holes.size(), hole[0], hole[1], hole[0] + hole[1]); - } - int gap = 0; for (int i = 0; i < holes.size(); i++) { final int[] hole = holes.get(i); final int marginEnd = i < holes.size() - 1 ? holes.get(i + 1)[0] : page.getContentSize(); - final byte[] buffer = page.content.getContent(); - final int from = hole[0] + hole[1]; final int length = marginEnd - from; if (length < 1) - System.out.println("ERROR"); - - if (BasePage.PAGE_HEADER_SIZE + hole[0] + length > page.getPhysicalSize()) { - System.out.printf("hole %d/%d %d <-> %d = %db%n", i + 1, holes.size(), hole[0], hole[0] + hole[1], hole[1]); - - System.out.printf("Moving %d <- %d %d (marginEnd=%d)%n", BasePage.PAGE_HEADER_SIZE + hole[0] + hole[1],// - BasePage.PAGE_HEADER_SIZE + hole[0], marginEnd - hole[0] - BasePage.PAGE_HEADER_SIZE,// - (BasePage.PAGE_HEADER_SIZE + hole[0]) + (marginEnd - hole[0] - BasePage.PAGE_HEADER_SIZE)); - } - - System.out.printf("Removing hole %d/%d %d-(%d)->%d MOVING %d<-%d (gap %d)%n", i + 1, holes.size(), hole[0], hole[1], - hole[0] + hole[1],// - hole[0] - gap, from, gap); + LogManager.instance().log(this, Level.SEVERE, "Error on reusing hole, invalid length " + length); - System.arraycopy(buffer, BasePage.PAGE_HEADER_SIZE + from,// - buffer, BasePage.PAGE_HEADER_SIZE + hole[0] - gap, length); - - page.updateModifiedRange(hole[0] - gap, from + length); + page.move(from, hole[0], length); // SHIFT ALL THE POINTERS FROM THE HOLE TO THE LAST for (int positionInPage = 0; positionInPage < recordCountInPage; positionInPage++) { - final int recordPositionInPage = getRecordPositionInPage(page, positionInPage); - if (recordPositionInPage < 1 || recordPositionInPage >= page.getContentSize()) - // AVOID TOUCHING CORRUPTED RECORD + final int recordPositionInPage = (int) page.readUnsignedInt( + PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); + if (recordPositionInPage == 0 || recordPositionInPage >= page.getContentSize()) + // AVOID TOUCHING DELETED OR CORRUPTED RECORD continue; - if (recordPositionInPage >= from && recordPositionInPage <= from + length) + if (recordPositionInPage >= from && recordPositionInPage <= from + length) { page.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE, recordPositionInPage - hole[1] - gap); + } } gap += hole[1]; @@ -1038,7 +1028,6 @@ private Binary loadMultiPageRecord(final RID originalRID, BasePage page, int rec final Binary chunk = page.getImmutableView( (int) (recordPositionInPage + recordSize[1] + INT_SERIALIZED_SIZE + LONG_SERIALIZED_SIZE), chunkSize); record.append(chunk); - if (nextChunkPointer == 0) // LAST CHUNK break; @@ -1046,8 +1035,17 @@ private Binary loadMultiPageRecord(final RID originalRID, BasePage page, int rec final int chunkPageId = (int) (nextChunkPointer / maxRecordsInPage); final int chunkPositionInPage = (int) (nextChunkPointer % maxRecordsInPage); + if (chunkPageId >= getTotalPages()) { + Integer pageCounter = database.getTransaction().getPageCounter(chunkPageId); + if (pageCounter == null || chunkPageId >= pageCounter) + throw new DatabaseOperationException("Invalid pointer to a chunk for record " + originalRID); + } + page = database.getTransaction().getPage(new PageId(file.getFileId(), chunkPageId), pageSize); recordPositionInPage = getRecordPositionInPage(page, chunkPositionInPage); + if (recordPositionInPage == 0) + throw new DatabaseOperationException("Chunk of record " + originalRID + " was deleted"); + recordSize = page.readNumberAndSize(recordPositionInPage); if (recordSize[0] != NEXT_CHUNK) @@ -1060,12 +1058,12 @@ private Binary loadMultiPageRecord(final RID originalRID, BasePage page, int rec private int getRecordPositionInPage(final BasePage page, final int positionInPage) throws IOException { final int recordPositionInPage = (int) page.readUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE); - if (recordPositionInPage != 0 && recordPositionInPage < PAGE_RECORD_TABLE_OFFSET + maxRecordsInPage * INT_SERIALIZED_SIZE) + if (recordPositionInPage != 0 && recordPositionInPage < contentHeaderSize) throw new IOException("Invalid record #" + fileId + ":" + (page.pageId.getPageNumber() * maxRecordsInPage + positionInPage)); return recordPositionInPage; } - private void writeMultiPageRecord(final Binary buffer, MutablePage currentPage, int newPosition, + private void writeMultiPageRecord(final RID originalRID, final Binary buffer, MutablePage currentPage, int newPosition, final int availableSpaceForFirstChunk) throws IOException { int bufferSize = buffer.size(); @@ -1277,8 +1275,12 @@ private AvailableSpace checkForSpaceInPage(final int pageNumber, final boolean i result.createNewPage = true; else if (result.recordCountInPage > 0) { // GET FIRST EMPTY POSITION + + // TODO: LOOK FOR ACTUAL EMPTY SPACE, NOT JUST FROM THE LAST RECORD + final int lastRecordPositionInPage = getRecordPositionInPage(result.page, result.recordCountInPage - 1); if (lastRecordPositionInPage == 0) + // TODO: NOW THIS MEANS DELETED! // CLEANED CORRUPTED RECORD result.createNewPage = true; else { @@ -1289,7 +1291,7 @@ else if (result.recordCountInPage > 0) { result.newPosition = lastRecordPositionInPage + (int) lastRecordSize[0] + (int) lastRecordSize[1]; else if (lastRecordSize[0] == RECORD_PLACEHOLDER_POINTER) // PLACEHOLDER, CONSIDER NEXT 9 BYTES - result.newPosition = lastRecordPositionInPage + LONG_SERIALIZED_SIZE + 1; + result.newPosition = lastRecordPositionInPage + LONG_SERIALIZED_SIZE + (int) lastRecordSize[1]; else if (lastRecordSize[0] == FIRST_CHUNK || lastRecordSize[0] == NEXT_CHUNK) { // CHUNK final int chunkSize = result.page.readInt((int) (lastRecordPositionInPage + lastRecordSize[1])); diff --git a/engine/src/main/java/com/arcadedb/schema/LocalSchema.java b/engine/src/main/java/com/arcadedb/schema/LocalSchema.java index 5a1d7463e..96ed13e98 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalSchema.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalSchema.java @@ -285,12 +285,23 @@ public Bucket getBucketByName(final String name) { @Override public LocalBucket getBucketById(final int id) { + return getBucketById(id, true); + } + + public LocalBucket getBucketById(final int id, final boolean throwExceptionIfNotFound) { if (id < 0 || id >= files.size()) - throw new SchemaException("Bucket with id '" + id + "' was not found"); + if (throwExceptionIfNotFound) + throw new SchemaException("Bucket with id '" + id + "' was not found"); + else + return null; final Component p = files.get(id); - if (!(p instanceof LocalBucket)) - throw new SchemaException("Bucket with id '" + id + "' was not found"); + if (!(p instanceof LocalBucket)) { + if (throwExceptionIfNotFound) + throw new SchemaException("Bucket with id '" + id + "' was not found"); + else + return null; + } return (LocalBucket) p; } From 1f52172db16e4afcf59b4ce52539f296a56c485f Mon Sep 17 00:00:00 2001 From: lvca Date: Fri, 22 Dec 2023 01:32:46 -0500 Subject: [PATCH 04/11] Fixed move of records in update --- .../java/com/arcadedb/engine/LocalBucket.java | 232 +++++++++++------- 1 file changed, 145 insertions(+), 87 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java index ac3410457..eb3971bf6 100644 --- a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java +++ b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java @@ -80,10 +80,10 @@ public class LocalBucket extends PaginatedComponent implements Bucket { private final AtomicLong cachedRecordCount = new AtomicLong(-1); private static class AvailableSpace { - public BasePage page = null; - public int newPosition = -1; - public int recordCountInPage = -1; - public boolean createNewPage = false; + public BasePage page = null; + public int newRecordPositionInPage = -1; + public int availablePositionIndex = -1; + public boolean createNewPage = false; } public static class PaginatedComponentFactoryHandler implements ComponentFactory.PaginatedComponentFactoryHandler { @@ -591,8 +591,8 @@ private RID createRecordInternal(final Record record, final boolean isPlaceHolde final int bufferSize = buffer.size(); try { - int newPosition = -1; - int recordCountInPage = -1; + int newRecordPositionInPage = -1; + int availablePositionIndex = -1; final boolean createNewPage; BasePage lastPage = null; @@ -600,11 +600,11 @@ private RID createRecordInternal(final Record record, final boolean isPlaceHolde if (txPageCounter > 0) { // CHECK IF THERE IS SPACE IN THE LAST PAGE - final AvailableSpace availableSpace = checkForSpaceInPage(txPageCounter - 1, isPlaceHolder, bufferSize); + final AvailableSpace availableSpace = getAvailableSpaceInPage(txPageCounter - 1, isPlaceHolder, bufferSize); lastPage = availableSpace.page; - newPosition = availableSpace.newPosition; - recordCountInPage = availableSpace.recordCountInPage; + newRecordPositionInPage = availableSpace.newRecordPositionInPage; + availablePositionIndex = availableSpace.availablePositionIndex; createNewPage = availableSpace.createNewPage; } else @@ -613,33 +613,36 @@ private RID createRecordInternal(final Record record, final boolean isPlaceHolde final MutablePage selectedPage; if (createNewPage) { selectedPage = database.getTransaction().addPage(new PageId(file.getFileId(), txPageCounter), pageSize); - newPosition = contentHeaderSize; - recordCountInPage = 0; + newRecordPositionInPage = contentHeaderSize; + availablePositionIndex = 0; } else selectedPage = database.getTransaction().getPageToModify(lastPage); - LogManager.instance().log(this, Level.FINE, "Creating record (%s records=%d threadId=%d)", selectedPage, recordCountInPage, - Thread.currentThread().getId()); + LogManager.instance() + .log(this, Level.FINE, "Creating record (%s records=%d threadId=%d)", selectedPage, availablePositionIndex, + Thread.currentThread().getId()); final RID rid = new RID(database, file.getFileId(), - ((long) selectedPage.getPageId().getPageNumber()) * maxRecordsInPage + recordCountInPage); + ((long) selectedPage.getPageId().getPageNumber()) * maxRecordsInPage + availablePositionIndex); - final int spaceAvailableInCurrentPage = selectedPage.getMaxContentSize() - newPosition; + final int spaceAvailableInCurrentPage = selectedPage.getMaxContentSize() - newRecordPositionInPage; final int spaceNeeded = Binary.getNumberSpace(isPlaceHolder ? (-1L * bufferSize) : bufferSize) + bufferSize; if (spaceNeeded > spaceAvailableInCurrentPage) { // MULTI-PAGE RECORD - writeMultiPageRecord(rid, buffer, selectedPage, newPosition, spaceAvailableInCurrentPage); + writeMultiPageRecord(buffer, selectedPage, newRecordPositionInPage, spaceAvailableInCurrentPage); } else { - final int byteWritten = selectedPage.writeNumber(newPosition, isPlaceHolder ? (-1L * bufferSize) : bufferSize); - selectedPage.writeByteArray(newPosition + byteWritten, buffer.getContent(), buffer.getContentBeginOffset(), bufferSize); + final int byteWritten = selectedPage.writeNumber(newRecordPositionInPage, isPlaceHolder ? (-1L * bufferSize) : bufferSize); + selectedPage.writeByteArray(newRecordPositionInPage + byteWritten, buffer.getContent(), buffer.getContentBeginOffset(), + bufferSize); } - selectedPage.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + recordCountInPage * INT_SERIALIZED_SIZE, newPosition); - selectedPage.writeShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET, (short) ++recordCountInPage); + selectedPage.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + availablePositionIndex * INT_SERIALIZED_SIZE, + newRecordPositionInPage); + selectedPage.writeShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET, (short) ++availablePositionIndex); LogManager.instance() - .log(this, Level.FINE, "Created record %s (%s records=%d threadId=%d)", rid, selectedPage, recordCountInPage, + .log(this, Level.FINE, "Created record %s (%s records=%d threadId=%d)", rid, selectedPage, availablePositionIndex, Thread.currentThread().getId()); if (!discardRecordAfter) @@ -726,42 +729,50 @@ private boolean updateRecordInternal(final Record record, final RID rid, final b final int bufferSize = buffer.size(); if (bufferSize > recordSize[0]) { // UPDATED RECORD IS LARGER THAN THE PREVIOUS VERSION: MAKE ROOM IN THE PAGE IF POSSIBLE + final int lastRecordPositionInPage = getLastRecordPositionInPage(page, recordCountInPage); + + final long[] lastRecordSize; + if (lastRecordPositionInPage != recordPositionInPage) { + // CURRENT RECORD IS NOT THE LATEST RIGHT IN THE PAGE + + if (lastRecordPositionInPage < contentHeaderSize) + // IT SHOULD NEVER OCCUR BECAUSE THE CURRENT RECORD IS PRESENT IN THIS PAGE + throw new DatabaseOperationException("Invalid position on expanding existing record " + rid); + + lastRecordSize = page.readNumberAndSize(lastRecordPositionInPage); + + if (lastRecordSize[0] == RECORD_PLACEHOLDER_POINTER) { + lastRecordSize[0] = LONG_SERIALIZED_SIZE; + lastRecordSize[1] = 1L; + } else if (lastRecordSize[0] == FIRST_CHUNK || lastRecordSize[0] == NEXT_CHUNK) { + // CONSIDER THE CHUNK SIZE + lastRecordSize[0] = page.readInt((int) (lastRecordPositionInPage + lastRecordSize[1])); + lastRecordSize[1] = INT_SERIALIZED_SIZE + LONG_SERIALIZED_SIZE; + } else if (lastRecordSize[0] < RECORD_PLACEHOLDER_CONTENT) { + lastRecordSize[0] *= -1L; + } + } else + lastRecordSize = recordSize; - final int lastRecordPositionInPage = getRecordPositionInPage(page, recordCountInPage - 1); - final long[] lastRecordSize = page.readNumberAndSize(lastRecordPositionInPage); - - if (lastRecordSize[0] == RECORD_PLACEHOLDER_POINTER) { - lastRecordSize[0] = LONG_SERIALIZED_SIZE; - lastRecordSize[1] = 1L; - } else if (lastRecordSize[0] == FIRST_CHUNK || lastRecordSize[0] == NEXT_CHUNK) { - // CONSIDER THE CHUNK SIZE - lastRecordSize[0] = page.readInt((int) (lastRecordPositionInPage + lastRecordSize[1])); - lastRecordSize[1] = INT_SERIALIZED_SIZE + LONG_SERIALIZED_SIZE; - } else if (lastRecordSize[0] < RECORD_PLACEHOLDER_CONTENT) { - lastRecordSize[0] *= -1L; - } - - final int pageOccupiedInBytes = (int) (lastRecordPositionInPage + lastRecordSize[0] + lastRecordSize[1]) + 1; - final int spaceAvailableInCurrentPage = page.getMaxContentSize() - pageOccupiedInBytes - 1; + final int pageOccupiedInBytes = (int) (lastRecordPositionInPage + lastRecordSize[0] + lastRecordSize[1]); + final int spaceAvailableInCurrentPage = page.getMaxContentSize() - pageOccupiedInBytes; final int bufferSizeLength = Binary.getNumberSpace(isPlaceHolder ? -1L * bufferSize : bufferSize); final int additionalSpaceNeeded = (int) (bufferSize + bufferSizeLength - recordSize[0] - recordSize[1]); - if (additionalSpaceNeeded <= spaceAvailableInCurrentPage) { + if (additionalSpaceNeeded < spaceAvailableInCurrentPage) { // THERE IS SPACE LEFT IN THE PAGE, SHIFT ON THE RIGHT THE EXISTENT RECORDS - if (positionInPage < recordCountInPage - 1) { + if (lastRecordPositionInPage != recordPositionInPage) { // NOT LAST RECORD IN PAGE, SHIFT NEXT RECORDS - final int nextRecordPositionInPage = getRecordPositionInPage(page, positionInPage + 1); - final int newPos = nextRecordPositionInPage + additionalSpaceNeeded; + final int from = (int) (recordPositionInPage + recordSize[0] + recordSize[1]); - page.move(nextRecordPositionInPage, newPos, pageOccupiedInBytes - nextRecordPositionInPage); + page.move(from, from + additionalSpaceNeeded, pageOccupiedInBytes - from); // TODO: CALCULATE THE REAL SIZE TO COMPACT DELETED RECORDS/PLACEHOLDERS - for (int pos = positionInPage + 1; pos < recordCountInPage; ++pos) { + for (int pos = 0; pos < recordCountInPage; ++pos) { final int nextRecordPosInPage = getRecordPositionInPage(page, pos); - if (nextRecordPosInPage == 0) - // TODO: WHY DO WE NEED TO WRITE BACK 0? - page.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + pos * INT_SERIALIZED_SIZE, 0); - else + if (nextRecordPosInPage != 0 &&// + nextRecordPosInPage >= from &&// + nextRecordPosInPage <= pageOccupiedInBytes) page.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + pos * INT_SERIALIZED_SIZE, nextRecordPosInPage + additionalSpaceNeeded); @@ -796,7 +807,7 @@ private boolean updateRecordInternal(final Record record, final RID rid, final b page, Thread.currentThread().getId()); } else { // SPLIT THE RECORD IN CHUNKS AS LINKED LIST AND STORE THE FIRST PART ON CURRENT PAGE ISSUE https://github.com/ArcadeData/arcadedb/issues/332 - writeMultiPageRecord(rid, buffer, page, recordPositionInPage, availableSpaceForChunk); + writeMultiPageRecord(buffer, page, recordPositionInPage, availableSpaceForChunk); LogManager.instance().log(this, Level.FINE, "Updated record %s by splitting it in multiple chunks to be saved in multiple pages (%s threadId=%d)", null, rid, @@ -878,6 +889,9 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder chunkPage = database.getTransaction().getPageToModify(new PageId(file.getFileId(), chunkPageId), pageSize, false); chunkRecordPositionInPage = getRecordPositionInPage(chunkPage, chunkPositionInPage); + if (chunkRecordPositionInPage == 0) + throw new DatabaseOperationException("Error on fetching multi page record " + rid + " chunk " + chunkId); + recordSize = chunkPage.readNumberAndSize(chunkRecordPositionInPage); if (recordSize[0] != NEXT_CHUNK) @@ -898,8 +912,8 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder } // POINTER = 0 MEANS DELETED - //page.writeNumber(recordPositionInPage, 0L); - page.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE, 0); + page.writeNumber(recordPositionInPage, 0L); +// page.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE, 0); // TODO: UPDATE TOTAL RECORDS IN PAGE // @@ -934,6 +948,8 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder } public void compressPage(final MutablePage page) { +// if (true) +// return; final short recordCountInPage = page.readShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET); final List orderedRecordContentInPage = new ArrayList<>(DEF_MAX_RECORDS_IN_PAGE); @@ -949,21 +965,36 @@ public void compressPage(final MutablePage page) { // DELETED <24.1.1 continue; + final int size; if (recordSize[0] == RECORD_PLACEHOLDER_POINTER) - orderedRecordContentInPage.add(new int[] { recordPositionInPage, LONG_SERIALIZED_SIZE + (int) recordSize[1] }); + size = LONG_SERIALIZED_SIZE + (int) recordSize[1]; else if (recordSize[0] == FIRST_CHUNK || recordSize[0] == NEXT_CHUNK) { final int chunkSize = page.readInt((recordPositionInPage + (int) recordSize[1])); - orderedRecordContentInPage.add( - new int[] { recordPositionInPage, chunkSize + (int) recordSize[1] + INT_SERIALIZED_SIZE + LONG_SERIALIZED_SIZE }); + size = chunkSize + (int) recordSize[1] + INT_SERIALIZED_SIZE + LONG_SERIALIZED_SIZE; } else if (recordSize[0] < RECORD_PLACEHOLDER_CONTENT) // PLACEHOLDER CONTENT, CONSIDER THE RECORD SIZE (CONVERTED FROM NEGATIVE NUMBER) + VARINT SIZE - orderedRecordContentInPage.add(new int[] { recordPositionInPage, (int) (-1 * recordSize[0]) + (int) recordSize[1] }); + size = (int) (-1 * recordSize[0]) + (int) recordSize[1]; else - orderedRecordContentInPage.add(new int[] { recordPositionInPage, (int) recordSize[0] + (int) recordSize[1] }); + size = (int) recordSize[0] + (int) recordSize[1]; + + if (size < 0 || size > getPageSize() - contentHeaderSize) + // INVALID SIZE + throw new DatabaseOperationException( + "Invalid record size " + size + " for record #" + fileId + ":" + (page.pageId.getPageNumber() * maxRecordsInPage) + + positionInPage); + + orderedRecordContentInPage.add(new int[] { recordPositionInPage, size }); } - if (orderedRecordContentInPage.isEmpty()) + if (orderedRecordContentInPage.isEmpty()) { + if (recordCountInPage > 0) { + // RESET RECORD COUNTER TO 0 + page.writeShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET, (short) 0); + LogManager.instance() + .log(this, Level.FINE, "Compressed the whole page %s with %d deleted records", page.pageId, recordCountInPage); + } return; + } orderedRecordContentInPage.sort(Comparator.comparingLong(a -> a[0])); @@ -973,7 +1004,6 @@ else if (recordSize[0] == FIRST_CHUNK || recordSize[0] == NEXT_CHUNK) { int[] lastPointer = new int[] { PAGE_RECORD_TABLE_OFFSET + maxRecordsInPage * INT_SERIALIZED_SIZE, 0 }; for (int i = 0; i < orderedRecordContentInPage.size(); i++) { final int[] pointer = orderedRecordContentInPage.get(i); - final int lastPointerEnd = lastPointer[0] + lastPointer[1]; if (pointer[0] != lastPointerEnd) { final int[] lastHole = holes.isEmpty() ? null : holes.get(holes.size() - 1); @@ -1015,6 +1045,9 @@ else if (recordSize[0] == FIRST_CHUNK || recordSize[0] == NEXT_CHUNK) { gap += hole[1]; } + + if (!holes.isEmpty()) + LogManager.instance().log(this, Level.FINE, "Compressed page %s removed %d holes", page.pageId, holes.size()); } private Binary loadMultiPageRecord(final RID originalRID, BasePage page, int recordPositionInPage, long[] recordSize) @@ -1063,7 +1096,7 @@ private int getRecordPositionInPage(final BasePage page, final int positionInPag return recordPositionInPage; } - private void writeMultiPageRecord(final RID originalRID, final Binary buffer, MutablePage currentPage, int newPosition, + private void writeMultiPageRecord(final Binary buffer, MutablePage currentPage, int newPosition, final int availableSpaceForFirstChunk) throws IOException { int bufferSize = buffer.size(); @@ -1098,13 +1131,14 @@ private void writeMultiPageRecord(final RID originalRID, final Binary buffer, Mu int recordIdInPage = 0; if (currentPage.getPageId().getPageNumber() < txPageCounter - 1) { // LOOK IN THE LAST PAGE FOR SPACE - final AvailableSpace availableSpace = checkForSpaceInPage(txPageCounter - 1, false, bufferSize); + final AvailableSpace availableSpace = getAvailableSpaceInPage(txPageCounter - 1, false, bufferSize); if (!availableSpace.createNewPage) { nextPage = database.getTransaction().getPageToModify(availableSpace.page.pageId, pageSize, false); - newPosition = availableSpace.newPosition; - recordIdInPage = availableSpace.recordCountInPage; + newPosition = availableSpace.newRecordPositionInPage; + recordIdInPage = availableSpace.availablePositionIndex; - nextPage.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + availableSpace.recordCountInPage * INT_SERIALIZED_SIZE, newPosition); + nextPage.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + availableSpace.availablePositionIndex * INT_SERIALIZED_SIZE, + newPosition); nextPage.writeShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET, (short) (recordIdInPage + 1)); } } @@ -1193,6 +1227,9 @@ private void updateMultiPageRecord(final RID originalRID, final Binary buffer, M nextPage = database.getTransaction().getPageToModify(new PageId(file.getFileId(), chunkPageId), pageSize, false); final int recordPositionInPage = getRecordPositionInPage(nextPage, chunkPositionInPage); + if (recordPositionInPage == 0) + throw new DatabaseOperationException("Error on fetching multi page record " + originalRID); + final long[] recordSize = nextPage.readNumberAndSize(recordPositionInPage); if (recordSize[0] != NEXT_CHUNK) @@ -1207,13 +1244,13 @@ private void updateMultiPageRecord(final RID originalRID, final Binary buffer, M int recordIdInPage = 0; if (currentPage.getPageId().getPageNumber() < txPageCounter - 1) { // LOOK IN THE LAST PAGE FOR SPACE - final AvailableSpace availableSpace = checkForSpaceInPage(txPageCounter - 1, false, bufferSize); + final AvailableSpace availableSpace = getAvailableSpaceInPage(txPageCounter - 1, false, bufferSize); if (!availableSpace.createNewPage) { nextPage = database.getTransaction().getPageToModify(availableSpace.page.pageId, pageSize, false); - newPosition = availableSpace.newPosition; - recordIdInPage = availableSpace.recordCountInPage; + newPosition = availableSpace.newRecordPositionInPage; + recordIdInPage = availableSpace.availablePositionIndex; - nextPage.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + availableSpace.recordCountInPage * INT_SERIALIZED_SIZE, + nextPage.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + availableSpace.availablePositionIndex * INT_SERIALIZED_SIZE, newPosition); nextPage.writeShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET, (short) (recordIdInPage + 1)); } @@ -1263,57 +1300,78 @@ private void updateMultiPageRecord(final RID originalRID, final Binary buffer, M } } - private AvailableSpace checkForSpaceInPage(final int pageNumber, final boolean isPlaceHolder, final int bufferSize) + private int getLastRecordPositionInPage(final MutablePage page, final int totalRecords) throws IOException { + int lastRecordPositionInPage = -1; + for (int i = 0; i < totalRecords; i++) { + final int recordPositionInPage = getRecordPositionInPage(page, i); + if (recordPositionInPage != 0 && recordPositionInPage > lastRecordPositionInPage) + lastRecordPositionInPage = recordPositionInPage; + } + return lastRecordPositionInPage; + } + + private AvailableSpace getAvailableSpaceInPage(final int pageNumber, final boolean isPlaceHolder, final int bufferSize) throws IOException { final AvailableSpace result = new AvailableSpace(); result.page = database.getTransaction().getPage(new PageId(file.getFileId(), pageNumber), pageSize); - result.recordCountInPage = result.page.readShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET); - if (result.recordCountInPage >= maxRecordsInPage) + final short totalRecords = result.page.readShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET); + if (totalRecords >= maxRecordsInPage) // MAX NUMBER OF RECORDS IN A PAGE REACHED, USE A NEW PAGE result.createNewPage = true; - else if (result.recordCountInPage > 0) { - // GET FIRST EMPTY POSITION + else if (totalRecords > 0) { + // GET FIRST EMPTY POSITION IF ANY + result.availablePositionIndex = -1; + int lastRecordPositionInPage = -1; + for (int i = 0; i < totalRecords; i++) { + final int recordPositionInPage = getRecordPositionInPage(result.page, i); + if (recordPositionInPage == 0) { + // REUSE THE FIRST AVAILABLE POSITION FROM DELETED RECORD + if (result.availablePositionIndex == -1) + result.availablePositionIndex = i; + } else if (recordPositionInPage > lastRecordPositionInPage) + lastRecordPositionInPage = recordPositionInPage; + } - // TODO: LOOK FOR ACTUAL EMPTY SPACE, NOT JUST FROM THE LAST RECORD + if (result.availablePositionIndex == -1) + // USE NEW POSITION + result.availablePositionIndex = totalRecords; - final int lastRecordPositionInPage = getRecordPositionInPage(result.page, result.recordCountInPage - 1); - if (lastRecordPositionInPage == 0) - // TODO: NOW THIS MEANS DELETED! - // CLEANED CORRUPTED RECORD - result.createNewPage = true; + if (lastRecordPositionInPage == -1) + // TOTALLY EMPTY PAGE AFTER DELETION, GET THE FIRST POSITION + result.newRecordPositionInPage = contentHeaderSize; else { final long[] lastRecordSize = result.page.readNumberAndSize(lastRecordPositionInPage); - if (lastRecordSize[0] == 0) // DELETED (V<24.1.1) - result.newPosition = lastRecordPositionInPage + (int) lastRecordSize[1]; + result.newRecordPositionInPage = lastRecordPositionInPage + (int) lastRecordSize[1]; else if (lastRecordSize[0] > 0) // RECORD PRESENT, CONSIDER THE RECORD SIZE + VARINT SIZE - result.newPosition = lastRecordPositionInPage + (int) lastRecordSize[0] + (int) lastRecordSize[1]; + result.newRecordPositionInPage = lastRecordPositionInPage + (int) lastRecordSize[0] + (int) lastRecordSize[1]; else if (lastRecordSize[0] == RECORD_PLACEHOLDER_POINTER) // PLACEHOLDER, CONSIDER NEXT 9 BYTES - result.newPosition = lastRecordPositionInPage + LONG_SERIALIZED_SIZE + (int) lastRecordSize[1]; + result.newRecordPositionInPage = lastRecordPositionInPage + LONG_SERIALIZED_SIZE + (int) lastRecordSize[1]; else if (lastRecordSize[0] == FIRST_CHUNK || lastRecordSize[0] == NEXT_CHUNK) { // CHUNK final int chunkSize = result.page.readInt((int) (lastRecordPositionInPage + lastRecordSize[1])); - result.newPosition = + result.newRecordPositionInPage = lastRecordPositionInPage + (int) lastRecordSize[1] + INT_SERIALIZED_SIZE + LONG_SERIALIZED_SIZE + chunkSize; } else // PLACEHOLDER CONTENT, CONSIDER THE RECORD SIZE (CONVERTED FROM NEGATIVE NUMBER) + VARINT SIZE - result.newPosition = lastRecordPositionInPage + (int) (-1 * lastRecordSize[0]) + (int) lastRecordSize[1]; + result.newRecordPositionInPage = lastRecordPositionInPage + (int) (-1 * lastRecordSize[0]) + (int) lastRecordSize[1]; - final long spaceAvailableInCurrentPage = result.page.getMaxContentSize() - result.newPosition; + final long spaceAvailableInCurrentPage = result.page.getMaxContentSize() - result.newRecordPositionInPage; final int spaceNeeded = Binary.getNumberSpace(isPlaceHolder ? (-1L * bufferSize) : bufferSize) + bufferSize; if (spaceNeeded > spaceAvailableInCurrentPage && spaceAvailableInCurrentPage < MINIMUM_SPACE_LEFT_IN_PAGE) // RECORD TOO BIG FOR THIS PAGE, USE A NEW PAGE result.createNewPage = true; } - } else + } else { // FIRST RECORD, START RIGHT AFTER THE HEADER - result.newPosition = contentHeaderSize; - + result.availablePositionIndex = 0; + result.newRecordPositionInPage = contentHeaderSize; + } return result; } } From 2c3f38e8713f5671ee07f4dd00384eb2466a0c21 Mon Sep 17 00:00:00 2001 From: lvca Date: Sun, 24 Dec 2023 01:30:36 -0500 Subject: [PATCH 05/11] Fixed compression of page Issue #1391 --- .../java/com/arcadedb/engine/LocalBucket.java | 57 ++++++++++--------- .../arcadedb/RandomTestMultiThreadsTest.java | 11 ++-- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java index eb3971bf6..a02dcb38f 100644 --- a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java +++ b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java @@ -912,27 +912,7 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder } // POINTER = 0 MEANS DELETED - page.writeNumber(recordPositionInPage, 0L); -// page.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE, 0); - -// TODO: UPDATE TOTAL RECORDS IN PAGE -// -// if (positionInPage == recordCountInPage - 1) { -// // LAST POSITION IN THE PAGE, UPDATE THE TOTAL RECORDS IN THE PAGE GOING BACK FROM THE CURRENT RECORD -// int newRecordCount = -1; -// for (int i = positionInPage - 1; i > -1; i--) { -// final int pos = getRecordPositionInPage(page, i); -// if (pos == 0 || page.readNumberAndSize(pos)[0] == 0) -// // DELETE RECORD, -// newRecordCount = i; -// else -// break; -// } -// -// if (newRecordCount > -1) -// // UPDATE TOTAL RECORDS IN THE PAGE -// page.writeShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET, (short) newRecordCount); -// } + page.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE, 0); } else { // CORRUPTED RECORD: WRITE ZERO AS POINTER TO RECORD @@ -947,9 +927,7 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder } } - public void compressPage(final MutablePage page) { -// if (true) -// return; + public void compressPage(final MutablePage page) throws IOException { final short recordCountInPage = page.readShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET); final List orderedRecordContentInPage = new ArrayList<>(DEF_MAX_RECORDS_IN_PAGE); @@ -990,8 +968,7 @@ else if (recordSize[0] == FIRST_CHUNK || recordSize[0] == NEXT_CHUNK) { if (recordCountInPage > 0) { // RESET RECORD COUNTER TO 0 page.writeShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET, (short) 0); - LogManager.instance() - .log(this, Level.FINE, "Compressed the whole page %s with %d deleted records", page.pageId, recordCountInPage); + LogManager.instance().log(this, Level.FINE, "Update record count from %d to 0 in page %s", recordCountInPage, page.pageId); } return; } @@ -1023,11 +1000,13 @@ else if (recordSize[0] == FIRST_CHUNK || recordSize[0] == NEXT_CHUNK) { final int marginEnd = i < holes.size() - 1 ? holes.get(i + 1)[0] : page.getContentSize(); final int from = hole[0] + hole[1]; + final int to = hole[0] - gap; final int length = marginEnd - from; if (length < 1) LogManager.instance().log(this, Level.SEVERE, "Error on reusing hole, invalid length " + length); - page.move(from, hole[0], length); + LogManager.instance().log(this, Level.FINE, "Moving segment page %s %d-(%d)->%d...", page.pageId, from, length, to); + page.move(from, to, length); // SHIFT ALL THE POINTERS FROM THE HOLE TO THE LAST for (int positionInPage = 0; positionInPage < recordCountInPage; positionInPage++) { @@ -1040,14 +1019,36 @@ else if (recordSize[0] == FIRST_CHUNK || recordSize[0] == NEXT_CHUNK) { if (recordPositionInPage >= from && recordPositionInPage <= from + length) { page.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + positionInPage * INT_SERIALIZED_SIZE, recordPositionInPage - hole[1] - gap); + LogManager.instance().log(this, Level.FINE, "- record %d %d->%d", positionInPage, recordPositionInPage, + recordPositionInPage - hole[1] - gap); } } gap += hole[1]; } - if (!holes.isEmpty()) + if (!holes.isEmpty()) { LogManager.instance().log(this, Level.FINE, "Compressed page %s removed %d holes", page.pageId, holes.size()); + + // UPDATE THE RECORD COUNT + // LAST POSITION IN THE PAGE, UPDATE THE TOTAL RECORDS IN THE PAGE GOING BACK FROM THE CURRENT RECORD + int newRecordCount = -1; + for (int i = recordCountInPage - 1; i > -1; i--) { + final int pos = getRecordPositionInPage(page, i); + if (pos == 0 || page.readNumberAndSize(pos)[0] == 0) + // DELETE RECORD, + newRecordCount = i; + else + break; + } + + if (newRecordCount > -1) { + // UPDATE TOTAL RECORDS IN THE PAGE + LogManager.instance() + .log(this, Level.FINE, "Update record count from %d to %d in page %s", recordCountInPage, newRecordCount, page.pageId); + page.writeShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET, (short) newRecordCount); + } + } } private Binary loadMultiPageRecord(final RID originalRID, BasePage page, int recordPositionInPage, long[] recordSize) diff --git a/engine/src/test/java/com/arcadedb/RandomTestMultiThreadsTest.java b/engine/src/test/java/com/arcadedb/RandomTestMultiThreadsTest.java index 4067e3317..6a9e3f927 100644 --- a/engine/src/test/java/com/arcadedb/RandomTestMultiThreadsTest.java +++ b/engine/src/test/java/com/arcadedb/RandomTestMultiThreadsTest.java @@ -81,8 +81,9 @@ public void run() { try { final int op = getRandom(100); if (i % 5000 == 0) - LogManager.instance().log(this, Level.FINE, "Operations %d/%d totalTransactionInCurrentTx=%d totalTransactions=%d (thread=%d)", i, CYCLES, - totalTransactionInCurrentTx, totalTransactionRecords.get(), threadId); + LogManager.instance() + .log(this, Level.FINE, "Operations %d/%d totalTransactionInCurrentTx=%d totalTransactions=%d (thread=%d)", i, + CYCLES, totalTransactionInCurrentTx, totalTransactionRecords.get(), threadId); LogManager.instance().log(this, Level.FINE, "Operation %d %d/%d (thread=%d)", op, i, CYCLES, threadId); @@ -165,8 +166,8 @@ public void run() { if (getRandom(50) == 0) LogManager.instance() - .log(this, Level.FINE, "Found %d Transaction records, ram counter=%d (thread=%d)...", newCounter, totalTransactionRecords.get(), - threadId); + .log(this, Level.FINE, "Found %d Transaction records, ram counter=%d (thread=%d)...", newCounter, + totalTransactionRecords.get(), threadId); totalTransactionInCurrentTx -= deleteRecords(database, threadId); @@ -290,6 +291,8 @@ private int updateRecords(final Database database, final int threadId) { } private int deleteRecords(final Database database, final int threadId) { +// if (true) +// return 0; if (totalTransactionRecords.get() == 0) return 0; From 9a10ece63cafbfafdb9a5b5abe94a31c52c60a89 Mon Sep 17 00:00:00 2001 From: lvca Date: Tue, 26 Dec 2023 19:20:34 -0500 Subject: [PATCH 06/11] Fixed last issue with update in multi page record --- .../java/com/arcadedb/engine/LocalBucket.java | 34 +++++++++++-------- .../src/test/java/com/arcadedb/AsyncTest.java | 8 +++-- .../arcadedb/database/CheckDatabaseTest.java | 34 +++++++++---------- 3 files changed, 43 insertions(+), 33 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java index a02dcb38f..f9462728b 100644 --- a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java +++ b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java @@ -20,6 +20,7 @@ import com.arcadedb.database.Binary; import com.arcadedb.database.DatabaseInternal; +import com.arcadedb.database.ImmutableDocument; import com.arcadedb.database.RID; import com.arcadedb.database.Record; import com.arcadedb.database.RecordEventsRegistry; @@ -84,6 +85,7 @@ private static class AvailableSpace { public int newRecordPositionInPage = -1; public int availablePositionIndex = -1; public boolean createNewPage = false; + public int totalRecordsInPage = -1; } public static class PaginatedComponentFactoryHandler implements ComponentFactory.PaginatedComponentFactoryHandler { @@ -345,7 +347,7 @@ public Map check(final int verboseLevel, final boolean fix) { long totalActiveRecords = 0L; long totalPlaceholderRecords = 0L; long totalSurrogateRecords = 0L; - long totalMultiPageeRecords = 0L; + long totalMultiPageRecords = 0L; long totalDeletedRecords = 0L; long totalMaxOffset = 0L; long totalChunks = 0L; @@ -404,7 +406,7 @@ public Map check(final int verboseLevel, final boolean fix) { } else if (recordSize[0] == FIRST_CHUNK) { pageActiveRecords++; pageMultiPageRecords++; - totalMultiPageeRecords++; + totalMultiPageRecords++; recordSize[0] = page.readInt((int) (recordPositionInPage + recordSize[1])); } else if (recordSize[0] == NEXT_CHUNK) { totalChunks++; @@ -484,6 +486,8 @@ public Map check(final int verboseLevel, final boolean fix) { stats.put("totalSurrogateRecords", totalSurrogateRecords); stats.put("totalDeletedRecords", totalDeletedRecords); stats.put("totalMaxOffset", totalMaxOffset); + stats.put("totalMultiPageRecords", totalMultiPageRecords); + stats.put("totalChunks", totalChunks); final DocumentType type = database.getSchema().getTypeByBucketId(fileId); if (type instanceof LocalVertexType) { @@ -629,7 +633,7 @@ private RID createRecordInternal(final Record record, final boolean isPlaceHolde if (spaceNeeded > spaceAvailableInCurrentPage) { // MULTI-PAGE RECORD - writeMultiPageRecord(buffer, selectedPage, newRecordPositionInPage, spaceAvailableInCurrentPage); + writeMultiPageRecord(rid, buffer, selectedPage, newRecordPositionInPage, spaceAvailableInCurrentPage); } else { final int byteWritten = selectedPage.writeNumber(newRecordPositionInPage, isPlaceHolder ? (-1L * bufferSize) : bufferSize); @@ -747,7 +751,7 @@ private boolean updateRecordInternal(final Record record, final RID rid, final b } else if (lastRecordSize[0] == FIRST_CHUNK || lastRecordSize[0] == NEXT_CHUNK) { // CONSIDER THE CHUNK SIZE lastRecordSize[0] = page.readInt((int) (lastRecordPositionInPage + lastRecordSize[1])); - lastRecordSize[1] = INT_SERIALIZED_SIZE + LONG_SERIALIZED_SIZE; + lastRecordSize[1] = 1L + INT_SERIALIZED_SIZE + LONG_SERIALIZED_SIZE; } else if (lastRecordSize[0] < RECORD_PLACEHOLDER_CONTENT) { lastRecordSize[0] *= -1L; } @@ -807,7 +811,7 @@ private boolean updateRecordInternal(final Record record, final RID rid, final b page, Thread.currentThread().getId()); } else { // SPLIT THE RECORD IN CHUNKS AS LINKED LIST AND STORE THE FIRST PART ON CURRENT PAGE ISSUE https://github.com/ArcadeData/arcadedb/issues/332 - writeMultiPageRecord(buffer, page, recordPositionInPage, availableSpaceForChunk); + writeMultiPageRecord(rid, buffer, page, recordPositionInPage, availableSpaceForChunk); LogManager.instance().log(this, Level.FINE, "Updated record %s by splitting it in multiple chunks to be saved in multiple pages (%s threadId=%d)", null, rid, @@ -1062,6 +1066,7 @@ private Binary loadMultiPageRecord(final RID originalRID, BasePage page, int rec final Binary chunk = page.getImmutableView( (int) (recordPositionInPage + recordSize[1] + INT_SERIALIZED_SIZE + LONG_SERIALIZED_SIZE), chunkSize); record.append(chunk); + if (nextChunkPointer == 0) // LAST CHUNK break; @@ -1097,7 +1102,7 @@ private int getRecordPositionInPage(final BasePage page, final int positionInPag return recordPositionInPage; } - private void writeMultiPageRecord(final Binary buffer, MutablePage currentPage, int newPosition, + private void writeMultiPageRecord(final RID rid, final Binary buffer, MutablePage currentPage, int newPosition, final int availableSpaceForFirstChunk) throws IOException { int bufferSize = buffer.size(); @@ -1138,9 +1143,10 @@ private void writeMultiPageRecord(final Binary buffer, MutablePage currentPage, newPosition = availableSpace.newRecordPositionInPage; recordIdInPage = availableSpace.availablePositionIndex; - nextPage.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + availableSpace.availablePositionIndex * INT_SERIALIZED_SIZE, - newPosition); - nextPage.writeShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET, (short) (recordIdInPage + 1)); + nextPage.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + recordIdInPage * INT_SERIALIZED_SIZE, newPosition); + + if (recordIdInPage >= availableSpace.totalRecordsInPage) + nextPage.writeShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET, (short) (recordIdInPage + 1)); } } @@ -1317,15 +1323,15 @@ private AvailableSpace getAvailableSpaceInPage(final int pageNumber, final boole result.page = database.getTransaction().getPage(new PageId(file.getFileId(), pageNumber), pageSize); - final short totalRecords = result.page.readShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET); - if (totalRecords >= maxRecordsInPage) + result.totalRecordsInPage = result.page.readShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET); + if (result.totalRecordsInPage >= maxRecordsInPage) // MAX NUMBER OF RECORDS IN A PAGE REACHED, USE A NEW PAGE result.createNewPage = true; - else if (totalRecords > 0) { + else if (result.totalRecordsInPage > 0) { // GET FIRST EMPTY POSITION IF ANY result.availablePositionIndex = -1; int lastRecordPositionInPage = -1; - for (int i = 0; i < totalRecords; i++) { + for (int i = 0; i < result.totalRecordsInPage; i++) { final int recordPositionInPage = getRecordPositionInPage(result.page, i); if (recordPositionInPage == 0) { // REUSE THE FIRST AVAILABLE POSITION FROM DELETED RECORD @@ -1337,7 +1343,7 @@ else if (totalRecords > 0) { if (result.availablePositionIndex == -1) // USE NEW POSITION - result.availablePositionIndex = totalRecords; + result.availablePositionIndex = result.totalRecordsInPage; if (lastRecordPositionInPage == -1) // TOTALLY EMPTY PAGE AFTER DELETION, GET THE FIRST POSITION diff --git a/engine/src/test/java/com/arcadedb/AsyncTest.java b/engine/src/test/java/com/arcadedb/AsyncTest.java index 0267255ef..1d7fbf86e 100644 --- a/engine/src/test/java/com/arcadedb/AsyncTest.java +++ b/engine/src/test/java/com/arcadedb/AsyncTest.java @@ -68,7 +68,8 @@ public void testSyncScanAndAsyncUpdate() { database.scanType(TYPE_NAME, true, record -> { callbackInvoked.incrementAndGet(); - database.async().updateRecord(record.modify().set("updated", true), newRecord -> updatedRecords.incrementAndGet()); + record.modify().set("updated", true).save(); + updatedRecords.incrementAndGet(); return true; }); @@ -81,7 +82,10 @@ public void testSyncScanAndAsyncUpdate() { Assertions.assertEquals(TOT, callbackInvoked.get()); Assertions.assertEquals(TOT, updatedRecords.get()); - final ResultSet resultSet = database.query("sql", "select count(*) as count from " + TYPE_NAME + " where updated = true"); + ResultSet resultSet = database.query("sql", "select from " + TYPE_NAME + " where updated <> true"); + Assertions.assertFalse(resultSet.hasNext()); + + resultSet = database.query("sql", "select count(*) as count from " + TYPE_NAME + " where updated = true"); Assertions.assertTrue(resultSet.hasNext()); Assertions.assertEquals(TOT, ((Number) resultSet.next().getProperty("count")).intValue()); diff --git a/engine/src/test/java/com/arcadedb/database/CheckDatabaseTest.java b/engine/src/test/java/com/arcadedb/database/CheckDatabaseTest.java index b5c2a444b..7427060f4 100644 --- a/engine/src/test/java/com/arcadedb/database/CheckDatabaseTest.java +++ b/engine/src/test/java/com/arcadedb/database/CheckDatabaseTest.java @@ -95,9 +95,9 @@ public void checkRegularDeleteEdges() { Assertions.assertEquals("check database", row.getProperty("operation")); Assertions.assertEquals(0L, (Long) row.getProperty("autoFix")); Assertions.assertEquals(TOTAL, (Long) row.getProperty("totalActiveVertices")); - Assertions.assertEquals(TOTAL - 1, (Long) row.getProperty("totalAllocatedEdges")); + Assertions.assertEquals(0, (Long) row.getProperty("totalAllocatedEdges")); Assertions.assertEquals(0L, (Long) row.getProperty("totalActiveEdges")); - Assertions.assertEquals(TOTAL - 1, (Long) row.getProperty("totalDeletedRecords")); + Assertions.assertEquals(0, (Long) row.getProperty("totalDeletedRecords")); Assertions.assertEquals(0, ((Collection) row.getProperty("corruptedRecords")).size()); Assertions.assertEquals(0L, (Long) row.getProperty("missingReferenceBack")); Assertions.assertEquals(0L, (Long) row.getProperty("invalidLinks")); @@ -130,7 +130,7 @@ public void checkBrokenDeletedEdges() { Assertions.assertEquals("check database", row.getProperty("operation")); Assertions.assertEquals(0, (Long) row.getProperty("autoFix")); Assertions.assertEquals(TOTAL, (Long) row.getProperty("totalActiveVertices")); - Assertions.assertEquals(TOTAL - 1, (Long) row.getProperty("totalAllocatedEdges")); + Assertions.assertEquals(TOTAL - 2, (Long) row.getProperty("totalAllocatedEdges")); Assertions.assertEquals(TOTAL - 2, (Long) row.getProperty("totalActiveEdges")); Assertions.assertEquals(1, (Long) row.getProperty("totalDeletedRecords")); Assertions.assertEquals(1, ((Collection) row.getProperty("corruptedRecords")).size()); @@ -150,7 +150,7 @@ public void checkBrokenDeletedEdges() { Assertions.assertEquals("check database", row.getProperty("operation")); Assertions.assertEquals(1, (Long) row.getProperty("autoFix")); Assertions.assertEquals(TOTAL, (Long) row.getProperty("totalActiveVertices")); - Assertions.assertEquals(TOTAL - 1, (Long) row.getProperty("totalAllocatedEdges")); + Assertions.assertEquals(TOTAL - 2, (Long) row.getProperty("totalAllocatedEdges")); Assertions.assertEquals(TOTAL - 2, (Long) row.getProperty("totalActiveEdges")); Assertions.assertEquals(1, (Long) row.getProperty("totalDeletedRecords")); Assertions.assertEquals(1, ((Collection) row.getProperty("corruptedRecords")).size()); @@ -170,7 +170,7 @@ public void checkBrokenDeletedEdges() { Assertions.assertEquals("check database", row.getProperty("operation")); Assertions.assertEquals(0, (Long) row.getProperty("autoFix")); Assertions.assertEquals(TOTAL, (Long) row.getProperty("totalActiveVertices")); - Assertions.assertEquals(TOTAL - 1, (Long) row.getProperty("totalAllocatedEdges")); + Assertions.assertEquals(TOTAL - 2, (Long) row.getProperty("totalAllocatedEdges")); Assertions.assertEquals(TOTAL - 2, (Long) row.getProperty("totalActiveEdges")); Assertions.assertEquals(1, (Long) row.getProperty("totalDeletedRecords")); Assertions.assertEquals(0, ((Collection) row.getProperty("corruptedRecords")).size()); @@ -212,15 +212,15 @@ public void checkBrokenDeletedVertex() { Result row = result.next(); Assertions.assertEquals("check database", row.getProperty("operation")); - Assertions.assertEquals((TOTAL - 1) * 2, (Long) row.getProperty("autoFix")); - Assertions.assertEquals(TOTAL - 1, (Long) row.getProperty("totalActiveVertices")); - Assertions.assertEquals(TOTAL - 1, (Long) row.getProperty("totalAllocatedEdges")); - Assertions.assertEquals(0, (Long) row.getProperty("totalActiveEdges")); - Assertions.assertEquals(TOTAL, (Long) row.getProperty("totalDeletedRecords")); - Assertions.assertEquals(TOTAL - 1, ((Collection) row.getProperty("corruptedRecords")).size()); - Assertions.assertEquals(0, (Long) row.getProperty("missingReferenceBack")); - Assertions.assertEquals((TOTAL - 1) * 2, (Long) row.getProperty("invalidLinks")); - Assertions.assertEquals((TOTAL - 1) * 2, ((Collection) row.getProperty("warnings")).size()); + Assertions.assertEquals((TOTAL - 1L) * 2L, (Long) row.getProperty("autoFix")); + Assertions.assertEquals(TOTAL - 1L, (Long) row.getProperty("totalActiveVertices")); + Assertions.assertEquals(0L, (Long) row.getProperty("totalAllocatedEdges")); + Assertions.assertEquals(0L, (Long) row.getProperty("totalActiveEdges")); + Assertions.assertEquals(1, (Long) row.getProperty("totalDeletedRecords")); + Assertions.assertEquals(TOTAL - 1L, ((Collection) row.getProperty("corruptedRecords")).size()); + Assertions.assertEquals(0L, (Long) row.getProperty("missingReferenceBack")); + Assertions.assertEquals((TOTAL - 1L) * 2L, (Long) row.getProperty("invalidLinks")); + Assertions.assertEquals((TOTAL - 1L) * 2L, ((Collection) row.getProperty("warnings")).size()); result = database.command("sql", "check database"); Assertions.assertTrue(result.hasNext()); @@ -229,10 +229,10 @@ public void checkBrokenDeletedVertex() { Assertions.assertEquals("check database", row.getProperty("operation")); Assertions.assertEquals(0, (Long) row.getProperty("autoFix")); - Assertions.assertEquals(TOTAL - 1, (Long) row.getProperty("totalActiveVertices")); - Assertions.assertEquals(TOTAL - 1, (Long) row.getProperty("totalAllocatedEdges")); + Assertions.assertEquals(TOTAL - 1L, (Long) row.getProperty("totalActiveVertices")); + Assertions.assertEquals(0, (Long) row.getProperty("totalAllocatedEdges")); Assertions.assertEquals(0, (Long) row.getProperty("totalActiveEdges")); - Assertions.assertEquals(TOTAL, (Long) row.getProperty("totalDeletedRecords")); + Assertions.assertEquals(1, (Long) row.getProperty("totalDeletedRecords")); Assertions.assertEquals(0, ((Collection) row.getProperty("corruptedRecords")).size()); Assertions.assertEquals(0, (Long) row.getProperty("missingReferenceBack")); Assertions.assertEquals(0, (Long) row.getProperty("invalidLinks")); From 577e787430f9c1597d8c207171a5f1637aad908b Mon Sep 17 00:00:00 2001 From: lvca Date: Wed, 27 Dec 2023 15:37:03 -0500 Subject: [PATCH 07/11] feat: deleted extra space from multi-page records when on update the record shrunk Issue https://github.com/ArcadeData/arcadedb/issues/1391 --- .../java/com/arcadedb/engine/LocalBucket.java | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java index e1bbab8d1..40d4fcc48 100644 --- a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java +++ b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java @@ -1220,6 +1220,7 @@ private void updateMultiPageRecord(final RID originalRID, final Binary buffer, M // WRITE ALL THE REMAINING CHUNKS IN NEW PAGES int txPageCounter = getTotalPages(); + long chunkToDeletePointer = 0L; while (bufferSize > 0) { MutablePage nextPage = null; @@ -1280,9 +1281,11 @@ private void updateMultiPageRecord(final RID originalRID, final Binary buffer, M // WRITE CHUNK SIZE final boolean lastChunk = bufferSize <= chunkSize; - if (bufferSize < chunkSize) + if (bufferSize < chunkSize) { // LAST CHUNK chunkSize = bufferSize; + chunkToDeletePointer = nextChunkPointer; + } nextPage.writeInt(newPosition, chunkSize); @@ -1301,6 +1304,36 @@ private void updateMultiPageRecord(final RID originalRID, final Binary buffer, M contentOffset += chunkSize; currentPage = nextPage; } + + // CHECK TO DELETE REMAINING CHUNKS IF THE RECORD SHRUNK + while (chunkToDeletePointer > 0) { + final int chunkPageId = (int) (chunkToDeletePointer / maxRecordsInPage); + final int chunkPositionInPage = (int) (chunkToDeletePointer % maxRecordsInPage); + + final MutablePage nextPage = database.getTransaction() + .getPageToModify(new PageId(file.getFileId(), chunkPageId), pageSize, false); + final int recordPositionInPage = getRecordPositionInPage(nextPage, chunkPositionInPage); + + // DELETE THE CHUNK AS RECORD IN THE PAGE + nextPage.writeUnsignedInt(PAGE_RECORD_TABLE_OFFSET + chunkPositionInPage * INT_SERIALIZED_SIZE, 0L); + + if (recordPositionInPage == 0) { + LogManager.instance() + .log(this, Level.WARNING, "Error on deleting extra part of multi-page record %s after it shrunk", originalRID); + break; + } + + final long[] recordSize = nextPage.readNumberAndSize(recordPositionInPage); + + if (recordSize[0] != NEXT_CHUNK) { + LogManager.instance() + .log(this, Level.WARNING, "Error on deleting extra part of multi-page record %s after it shrunk", originalRID); + break; + } + + newPosition = (int) (recordPositionInPage + recordSize[1]); + chunkToDeletePointer = nextPage.readLong(newPosition + INT_SERIALIZED_SIZE); + } } private int getLastRecordPositionInPage(final MutablePage page, final int totalRecords) throws IOException { From c7fac0a8e3594697fb6a62dbaf4d7805f6fbc689 Mon Sep 17 00:00:00 2001 From: lvca Date: Thu, 4 Apr 2024 22:49:09 -0400 Subject: [PATCH 08/11] test: added new test case that creates and delete multiple times a graph --- .../com/arcadedb/engine/DeleteAllTest.java | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 engine/src/test/java/com/arcadedb/engine/DeleteAllTest.java diff --git a/engine/src/test/java/com/arcadedb/engine/DeleteAllTest.java b/engine/src/test/java/com/arcadedb/engine/DeleteAllTest.java new file mode 100644 index 000000000..e8f589899 --- /dev/null +++ b/engine/src/test/java/com/arcadedb/engine/DeleteAllTest.java @@ -0,0 +1,66 @@ +package com.arcadedb.engine; + +import com.arcadedb.database.Database; +import com.arcadedb.database.DatabaseFactory; +import com.arcadedb.graph.MutableVertex; +import com.arcadedb.utility.FileUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.*; +import java.util.*; + +public class DeleteAllTest { + private final static int TOT_RECORDS = 100_000; + private final static String VERTEX_TYPE = "Product"; + private final static String EDGE_TYPE = "LinkedTo"; + private static final int CYCLES = 10; + + @Test + public void testCreateAndDeleteGraph() { + try (DatabaseFactory databaseFactory = new DatabaseFactory("databases/DeleteAllTest")) { + if (databaseFactory.exists()) + databaseFactory.open().drop(); + + try (Database db = databaseFactory.create()) { + db.getSchema().createVertexType(VERTEX_TYPE, 1); + db.getSchema().createEdgeType(EDGE_TYPE, 1); + + for (int i = 0; i < CYCLES; i++) { + System.out.println("Cycle " + i); + List.of(new File(databaseFactory.getDatabasePath()).listFiles()) + .forEach(f -> System.out.println("- " + f.getName() + ": " + FileUtils.getSizeAsString( + f.length()))); + + db.transaction(() -> { + final MutableVertex root = db.newVertex(VERTEX_TYPE)// + .set("id", 0)// + .save(); + + for (int k = 1; k < TOT_RECORDS; k++) { + final MutableVertex v = db.newVertex(VERTEX_TYPE)// + .set("id", k)// + .save(); + + root.newEdge(EDGE_TYPE, v, true, "something", k); + } + }); + + db.transaction(() -> { + Assertions.assertEquals(TOT_RECORDS, db.countType(VERTEX_TYPE, true)); + Assertions.assertEquals(TOT_RECORDS - 1, db.countType(EDGE_TYPE, true)); + + db.command("sql", "delete from " + VERTEX_TYPE); + + Assertions.assertEquals(0, db.countType(VERTEX_TYPE, true)); + Assertions.assertEquals(0, db.countType(EDGE_TYPE, true)); + }); + } + } finally { + System.out.println(databaseFactory.getDatabasePath()); + if (databaseFactory.exists()) + databaseFactory.open().drop(); + } + } + } +} From dc49c4b967d2b8307d2c6ac3344ccb611150738d Mon Sep 17 00:00:00 2001 From: lvca Date: Thu, 4 Apr 2024 23:02:34 -0400 Subject: [PATCH 09/11] test: added check database after test --- .../src/test/java/com/arcadedb/engine/DeleteAllTest.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/engine/src/test/java/com/arcadedb/engine/DeleteAllTest.java b/engine/src/test/java/com/arcadedb/engine/DeleteAllTest.java index e8f589899..0c03c14ed 100644 --- a/engine/src/test/java/com/arcadedb/engine/DeleteAllTest.java +++ b/engine/src/test/java/com/arcadedb/engine/DeleteAllTest.java @@ -3,6 +3,7 @@ import com.arcadedb.database.Database; import com.arcadedb.database.DatabaseFactory; import com.arcadedb.graph.MutableVertex; +import com.arcadedb.query.sql.executor.ResultSet; import com.arcadedb.utility.FileUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -14,7 +15,7 @@ public class DeleteAllTest { private final static int TOT_RECORDS = 100_000; private final static String VERTEX_TYPE = "Product"; private final static String EDGE_TYPE = "LinkedTo"; - private static final int CYCLES = 10; + private static final int CYCLES = 2; @Test public void testCreateAndDeleteGraph() { @@ -56,8 +57,11 @@ public void testCreateAndDeleteGraph() { Assertions.assertEquals(0, db.countType(EDGE_TYPE, true)); }); } + + final ResultSet result = db.command("sql", "check database"); + System.out.println(result.nextIfAvailable().toJSON()); } finally { - System.out.println(databaseFactory.getDatabasePath()); + if (databaseFactory.exists()) databaseFactory.open().drop(); } From e3d289ed37fe33459ade7b48bb8843999b85efb9 Mon Sep 17 00:00:00 2001 From: lvca Date: Thu, 4 Apr 2024 23:26:53 -0400 Subject: [PATCH 10/11] fix: vertex delete left space allocated The issue was with the missing deletion of the whole linked list segments. Only the head was deleted. This issue was found thanks to the CHECK DATABASE command. --- .../java/com/arcadedb/graph/GraphEngine.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/graph/GraphEngine.java b/engine/src/main/java/com/arcadedb/graph/GraphEngine.java index 9cdc6610c..ab42d747e 100644 --- a/engine/src/main/java/com/arcadedb/graph/GraphEngine.java +++ b/engine/src/main/java/com/arcadedb/graph/GraphEngine.java @@ -352,13 +352,16 @@ public void deleteEdge(final Edge edge) { public void deleteVertex(final VertexInternal vertex) { // RETRIEVE ALL THE EDGES TO DELETE AT THE END + final Set edgeChunkToDelete = new HashSet<>(); final List edgesToDelete = new ArrayList<>(); final EdgeLinkedList outEdges = getEdgeHeadChunk(vertex, Vertex.DIRECTION.OUT); if (outEdges != null) { - final Iterator outIterator = outEdges.edgeIterator(); + final EdgeIterator outIterator = (EdgeIterator) outEdges.edgeIterator(); while (outIterator.hasNext()) { + edgeChunkToDelete.add(outIterator.currentContainer.getIdentity()); + RID inV = null; try { final Edge nextEdge = outIterator.next(); @@ -371,16 +374,15 @@ public void deleteVertex(final VertexInternal vertex) { vertex.getIdentity()); } } - - final RID outRID = vertex.getOutEdgesHeadChunk(); - outRID.getRecord(false).delete(); } final EdgeLinkedList inEdges = getEdgeHeadChunk(vertex, Vertex.DIRECTION.IN); if (inEdges != null) { - final Iterator inIterator = inEdges.edgeIterator(); + final EdgeIterator inIterator = (EdgeIterator) inEdges.edgeIterator(); while (inIterator.hasNext()) { + edgeChunkToDelete.add(inIterator.currentContainer.getIdentity()); + RID outV = null; try { final Edge nextEdge = inIterator.next(); @@ -397,6 +399,10 @@ public void deleteVertex(final VertexInternal vertex) { inRID.getRecord(false).delete(); } + // DELETE CHUNKS FIRST (SPEED UP FOLLOWING EDGE DELETION) + for (Identifiable chunk : edgeChunkToDelete) + chunk.getRecord().delete(); + for (Identifiable edge : edgesToDelete) edge.asEdge().delete(); From 95c44f1f0dc4f8d67fa887c1ed6bbb82202ea52c Mon Sep 17 00:00:00 2001 From: lvca Date: Thu, 11 Apr 2024 19:02:30 -0400 Subject: [PATCH 11/11] fix: fixed issue after merge --- engine/src/main/java/com/arcadedb/graph/GraphEngine.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/graph/GraphEngine.java b/engine/src/main/java/com/arcadedb/graph/GraphEngine.java index d95dfebb2..021ce479b 100644 --- a/engine/src/main/java/com/arcadedb/graph/GraphEngine.java +++ b/engine/src/main/java/com/arcadedb/graph/GraphEngine.java @@ -396,10 +396,6 @@ public void deleteVertex(final VertexInternal vertex) { } } - // DELETE CHUNKS FIRST (SPEED UP FOLLOWING EDGE DELETION) - for (Identifiable chunk : edgeChunkToDelete) - chunk.getRecord().delete(); - for (Identifiable edge : edgesToDelete) edge.asEdge().delete();