diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/LocalIndexService.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/LocalIndexService.java index 9328f638cf..6af6a82bef 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/LocalIndexService.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/LocalIndexService.java @@ -495,6 +495,10 @@ public boolean purgeIndexEntries(long tableId, long indexId, ListFor better performance, use consistent numRowIds when calling this method. * @param tableId the table id of single point index * @param numRowIds the number of row ids to allocate @@ -136,7 +136,8 @@ public boolean equals(Scheme other) /** * Delete a range of row ids from the main index. This method only has effect on the persistent storage - * of the main index. + * of the main index. {@link #flushCache(long fileId)} should be called before this method + * delete the row ids from both cache and persistent storage. * {@link #getLocation(long)} of a row id within a deleted range returns null. * @param rowIdRange the row id range to be deleted * @return true on success @@ -154,7 +155,7 @@ public boolean equals(Scheme other) boolean flushCache(long fileId) throws MainIndexException; /** - * Flush the main index cache if exists and close the main index instance. + * Flush the main index cache (if exists) and close the main index instance. * This method is to be used by the main index factory to close the * managed main index instances when the process is shutting down. *

diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/MainIndexBuffer.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/MainIndexBuffer.java index 60ae77903a..5ee71ba582 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/MainIndexBuffer.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/MainIndexBuffer.java @@ -125,8 +125,7 @@ protected IndexProto.RowLocation lookup(long fileId, long rowId) throws MainInde */ public IndexProto.RowLocation lookup(long rowId) throws MainIndexException { - IndexProto.RowLocation location = null; - location = this.indexCache.lookup(rowId); + IndexProto.RowLocation location = this.indexCache.lookup(rowId); if (location == null) { for (Map.Entry> entry : this.indexBuffer.entrySet()) diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/MainIndexCache.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/MainIndexCache.java index b481b6a7b0..a089b03447 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/MainIndexCache.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/MainIndexCache.java @@ -52,6 +52,11 @@ public class MainIndexCache implements Closeable */ private final List> entryCacheBuckets; + /** + * Caches the ranges of row ids and the corresponding locations. The cache elements are of the {@link RowIdRange} + * type. However, we use {@link Object} to support looking up single row ids in this range cache. + * Ranges in this cache may overlap but not duplicate. + */ private final TreeSet rangeCache; public MainIndexCache() @@ -124,13 +129,17 @@ public void evictAllEntries() } } + /** + * Admitting a row id range that is overlapping but not duplicate with existing ranges in this cache is acceptable. + * @param range the range to admit into the cache. + */ public void admitRange(RowIdRange range) { this.rangeCache.add(range); } /** - * @param range the range to delete + * @param range the range to delete from the cache * @return true if the range is found and deleted */ public boolean evictRange(RowIdRange range) diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/vector/VectorColumnVector.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/vector/VectorColumnVector.java index d05380ce9f..a489f8ca2f 100644 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/vector/VectorColumnVector.java +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/vector/VectorColumnVector.java @@ -1,3 +1,22 @@ +/* + * Copyright 2023 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ package io.pixelsdb.pixels.core.vector; import com.google.flatbuffers.FlatBufferBuilder; diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/index/IndexServiceImpl.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/index/IndexServiceImpl.java index 48acc5630c..3b6b32967f 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/index/IndexServiceImpl.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/index/IndexServiceImpl.java @@ -612,6 +612,10 @@ public void purgeIndexEntries(IndexProto.PurgeIndexEntriesRequest request, rowLocationFirst.getRgId(), rowLocationFirst.getRgRowOffset(), rowLocationLast.getRgRowOffset()); + if (mainIndex.hasCache()) + { + mainIndex.flushCache(rowLocationFirst.getFileId()); + } mainIndex.deleteRowIdRange(rowIdRange); } else diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/transaction/TransServiceImpl.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/transaction/TransServiceImpl.java index 2c2bdd3592..94a7d7b958 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/transaction/TransServiceImpl.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/transaction/TransServiceImpl.java @@ -451,7 +451,7 @@ private static void pushWatermarks(boolean readOnly) logger.error("new running transactions obtained backward timestamps, this is illegal"); return; } - // ush watermark to the new min running transaction timestamp + // push watermark to the new min running transaction timestamp timestamp = minRunningTransTimestamp; } if (timestamp < 0) diff --git a/pixels-index/pixels-index-main-sqlite/src/main/java/io/pixelsdb/pixels/index/main/sqlite/SqliteMainIndex.java b/pixels-index/pixels-index-main-sqlite/src/main/java/io/pixelsdb/pixels/index/main/sqlite/SqliteMainIndex.java index 3e15ba1bcc..be60cbf016 100644 --- a/pixels-index/pixels-index-main-sqlite/src/main/java/io/pixelsdb/pixels/index/main/sqlite/SqliteMainIndex.java +++ b/pixels-index/pixels-index-main-sqlite/src/main/java/io/pixelsdb/pixels/index/main/sqlite/SqliteMainIndex.java @@ -248,10 +248,11 @@ private IndexProto.RowLocation getRowLocationFromSqlite(long rowId) throws MainI try { RowIdRange rowIdRange = getRowIdRangeFromSqlite(rowId); - // Issue #1150: add the range to cache to accelerate main index lookups - this.indexCache.admitRange(rowIdRange); if (rowIdRange != null) { + // Issue #1150: add the range to cache to accelerate main index lookups + this.indexCache.admitRange(rowIdRange); + long rowIdStart = rowIdRange.getRowIdStart(); long fileId = rowIdRange.getFileId(); int rgId = rowIdRange.getRgId();