From 4423c116e9acff279f91f50d7bb9e97c3906542e Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 7 Sep 2022 12:50:09 +0800 Subject: [PATCH 01/15] lsm --- lsm/pom.xml | 30 +++ .../org/apache/iotdb/lsm/context/Context.java | 101 +++++++++ .../apache/iotdb/lsm/context/ContextType.java | 27 +++ .../iotdb/lsm/context/DeleteContext.java | 43 ++++ .../iotdb/lsm/context/FlushContext.java | 43 ++++ .../iotdb/lsm/context/InsertContext.java | 50 +++++ .../iotdb/lsm/context/QueryContext.java | 43 ++++ .../org/apache/iotdb/lsm/example/Main.java | 209 ++++++++++++++++++ .../apache/iotdb/lsm/example/MemChunk.java | 44 ++++ .../apache/iotdb/lsm/example/MemGroup.java | 45 ++++ .../apache/iotdb/lsm/example/MemTable.java | 45 ++++ .../iotdb/lsm/example/MemTableManager.java | 75 +++++++ .../lsm/levelProcess/BasicLevelProcess.java | 52 +++++ .../lsm/levelProcess/DeleteLevelProcess.java | 31 +++ .../lsm/levelProcess/FlushLevelProcess.java | 30 +++ .../lsm/levelProcess/InsertLevelProcess.java | 31 +++ .../iotdb/lsm/levelProcess/LevelProcess.java | 29 +++ .../lsm/levelProcess/QueryLevelProcess.java | 31 +++ .../iotdb/lsm/manager/BasicLsmManager.java | 46 ++++ .../apache/iotdb/lsm/manager/LsmManager.java | 30 +++ .../iotdb/lsm/strategy/AccessStrategy.java | 35 +++ .../iotdb/lsm/strategy/BFSAccessStrategy.java | 56 +++++ .../lsm/strategy/PostOrderAccessStrategy.java | 47 ++++ .../lsm/strategy/PreOrderAccessStrategy.java | 44 ++++ .../lsm/strategy/RBFSAccessStrategy.java | 74 +++++++ 25 files changed, 1291 insertions(+) create mode 100644 lsm/pom.xml create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/context/ContextType.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java diff --git a/lsm/pom.xml b/lsm/pom.xml new file mode 100644 index 0000000000000..1bbf055742d06 --- /dev/null +++ b/lsm/pom.xml @@ -0,0 +1,30 @@ + + + + + iotdb-parent + org.apache.iotdb + 0.14.0-SNAPSHOT + + 4.0.0 + lsm + + 8 + 8 + + diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java new file mode 100644 index 0000000000000..381586e7c8294 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import org.apache.iotdb.lsm.strategy.AccessStrategy; +import org.apache.iotdb.lsm.strategy.PreOrderAccessStrategy; + +public class Context { + + // 类型 + ContextType type; + + // 访问策略 + AccessStrategy accessStrategy; + + // 所处的树深度 + int level; + + // 多少个线程处理该节点的子节点 + int threadNums; + + // 返回值 + Object result; + + public Context() { + accessStrategy = new PreOrderAccessStrategy(); + type = ContextType.NONE; + level = 0; + threadNums = 1; + } + + public Context( + ContextType type, + AccessStrategy accessStrategy, + int level, + boolean sync, + int threadNums, + Object result) { + this.type = type; + this.accessStrategy = accessStrategy; + this.level = level; + this.threadNums = threadNums; + this.result = result; + } + + public void setLevel(int level) { + this.level = level; + } + + public int getLevel() { + return level; + } + + public ContextType getType() { + return type; + } + + public int getThreadNums() { + return threadNums; + } + + public void setType(ContextType type) { + this.type = type; + } + + public void setThreadNums(int threadNums) { + this.threadNums = threadNums; + } + + public Object getResult() { + return result; + } + + public void setResult(Object result) { + this.result = result; + } + + public AccessStrategy getAccessStrategy() { + return accessStrategy; + } + + public void setAccessStrategy(AccessStrategy accessStrategy) { + this.accessStrategy = accessStrategy; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/ContextType.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/ContextType.java new file mode 100644 index 0000000000000..3ff2e3509afb1 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/ContextType.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +public enum ContextType { + NONE, + INSERT, + QUERY, + DELETE, + FLUSH; +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java new file mode 100644 index 0000000000000..e00db284e7bce --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class DeleteContext extends Context { + + List keys; + + public DeleteContext(Object... ks) { + super(); + keys = new ArrayList<>(); + keys.addAll(Arrays.asList(ks)); + type = ContextType.DELETE; + } + + public Object getKey() { + return keys.get(level); + } + + public int size() { + return keys.size(); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java new file mode 100644 index 0000000000000..ceacd2f636e5f --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import org.apache.iotdb.lsm.strategy.BFSAccessStrategy; +import org.apache.iotdb.lsm.strategy.RBFSAccessStrategy; + +public class FlushContext extends Context { + + // 最小的已经flush的层级 + int minimumFlushedLevel; + + public FlushContext() { + super(); + type = ContextType.FLUSH; + accessStrategy = new RBFSAccessStrategy(); + minimumFlushedLevel = Integer.MAX_VALUE; + } + + public int getMinimumFlushedLevel() { + return minimumFlushedLevel; + } + + public void setMinimumFlushedLevel(int minimumFlushedLevel) { + this.minimumFlushedLevel = minimumFlushedLevel; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java new file mode 100644 index 0000000000000..a34c9d5814ce8 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import org.apache.iotdb.lsm.strategy.PreOrderAccessStrategy; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class InsertContext extends Context { + + List keys; + + Object value; + + public InsertContext(Object value, Object... keys) { + super(); + this.value = value; + this.keys = new ArrayList<>(); + this.keys.addAll(Arrays.asList(keys)); + level = 0; + type = ContextType.INSERT; + accessStrategy = new PreOrderAccessStrategy(); + } + + public Object getKey() { + return keys.get(level); + } + + public Object getValue() { + return value; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java new file mode 100644 index 0000000000000..9b8d440106b48 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class QueryContext extends Context { + + List keys; + + public QueryContext(Object... ks) { + super(); + keys = new ArrayList<>(); + keys.addAll(Arrays.asList(ks)); + type = ContextType.QUERY; + } + + public Object getKey() { + return keys.get(level); + } + + public int size() { + return keys.size(); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java new file mode 100644 index 0000000000000..015963e8af134 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.example; + +import org.apache.iotdb.lsm.context.FlushContext; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.levelProcess.FlushLevelProcess; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; +import org.apache.iotdb.lsm.manager.BasicLsmManager; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Main { + public static void main(String[] args) { + MemTableManager memTableManager = new MemTableManager(); + System.out.println("-------------insert--------------"); + insertionExample(memTableManager); + System.out.println("-------------flush--------------"); + flushExample(memTableManager); + } + + public static void insertionExample(MemTableManager memTableManager) { + + BasicLsmManager baseLsmManager = + new BasicLsmManager().manager(memTableManager); + baseLsmManager + .nextLevel( + new InsertLevelProcess() { + @Override + public List getChildren(MemTableManager memNode, InsertContext context) { + Integer deviceID = (Integer) context.getValue(); + int maxDeviceID = memNode.getMaxDeviceID(); + List children = new ArrayList<>(); + if (deviceID / 65536 == maxDeviceID / 65536) { + children.add(memNode.getWorking()); + } else { + children.add(memNode.getImmutables().get(deviceID / 65536)); + } + return children; + } + + @Override + public void insert(MemTableManager memNode, InsertContext context) { + Integer deviceID = (Integer) context.getValue(); + int maxDeviceID = memNode.getMaxDeviceID(); + if (deviceID / 65536 == maxDeviceID / 65536) { + if (memNode.getWorking() == null) { + memNode.setWorking(new MemTable()); + } + } else if (deviceID > maxDeviceID) { + memNode + .getImmutables() + .put(memNode.getMaxDeviceID() / 65536, memNode.getWorking()); + memNode.setWorking(new MemTable()); + } + if (deviceID > maxDeviceID) { + memNode.setMaxDeviceID(deviceID); + } + } + }) + .nextLevel( + new InsertLevelProcess() { + @Override + public List getChildren(MemTable memNode, InsertContext context) { + String key = (String) context.getKey(); + List children = new ArrayList<>(); + children.add(memNode.getMap().get(key)); + return children; + } + + @Override + public void insert(MemTable memNode, InsertContext context) { + String key = (String) context.getKey(); + Map map = memNode.getMap(); + if (map.containsKey(key)) return; + map.put(key, new MemGroup()); + } + }) + .nextLevel( + new InsertLevelProcess() { + @Override + public List getChildren(MemGroup memNode, InsertContext context) { + String key = (String) context.getKey(); + List children = new ArrayList<>(); + children.add(memNode.getMap().get(key)); + return children; + } + + @Override + public void insert(MemGroup memNode, InsertContext context) { + String key = (String) context.getKey(); + Map map = memNode.getMap(); + if (map.containsKey(key)) return; + map.put(key, new MemChunk()); + } + }) + .nextLevel( + new InsertLevelProcess() { + @Override + public List getChildren(MemChunk memNode, InsertContext context) { + return null; + } + + @Override + public void insert(MemChunk memNode, InsertContext context) { + Integer deviceID = (Integer) context.getValue(); + List deviceIDs = memNode.getDeviceIDS(); + deviceIDs.add(deviceID); + } + }); + + baseLsmManager.process(new InsertContext(1, null, "a", "b")); + baseLsmManager.process(new InsertContext(2, null, "a", "d")); + baseLsmManager.process(new InsertContext(3, null, "a", "e")); + baseLsmManager.process(new InsertContext(4, null, "a", "b")); + baseLsmManager.process(new InsertContext(5, null, "a1", "b")); + baseLsmManager.process(new InsertContext(6, null, "a2", "b")); + baseLsmManager.process(new InsertContext(65535, null, "a", "b")); + baseLsmManager.process(new InsertContext(65536, null, "a", "b")); + baseLsmManager.process(new InsertContext(2, null, "a", "d")); + baseLsmManager.process(new InsertContext(3, null, "a", "e")); + baseLsmManager.process(new InsertContext(4, null, "a", "b")); + baseLsmManager.process(new InsertContext(5, null, "a1", "b")); + baseLsmManager.process(new InsertContext(6, null, "a2", "b")); + System.out.println(memTableManager); + } + + public static void flushExample(MemTableManager memTableManager) { + BasicLsmManager flushManager = + new BasicLsmManager().manager(memTableManager); + + flushManager + .nextLevel( + new FlushLevelProcess() { + @Override + public void flush(MemTableManager memNode, FlushContext context) { + System.out.println(memNode); + } + + @Override + public List getChildren(MemTableManager memNode, FlushContext context) { + List memTables = new ArrayList<>(); + memTables.addAll(memNode.getImmutables().values()); + if (memNode.getWorking() != null) memTables.add(memNode.getWorking()); + return memTables; + } + }) + .nextLevel( + new FlushLevelProcess() { + @Override + public void flush(MemTable memNode, FlushContext context) { + System.out.println(memNode); + } + + @Override + public List getChildren(MemTable memNode, FlushContext context) { + List memGroups = new ArrayList<>(); + memGroups.addAll(memNode.getMap().values()); + return memGroups; + } + }) + .nextLevel( + new FlushLevelProcess() { + @Override + public void flush(MemGroup memNode, FlushContext context) { + System.out.println(memNode); + } + + @Override + public List getChildren(MemGroup memNode, FlushContext context) { + List memChunk = new ArrayList<>(); + memChunk.addAll(memNode.getMap().values()); + return memChunk; + } + }) + .nextLevel( + new FlushLevelProcess() { + @Override + public void flush(MemChunk memNode, FlushContext context) { + System.out.println(memNode); + } + + @Override + public List getChildren(MemChunk memNode, FlushContext context) { + return new ArrayList<>(); + } + }); + + flushManager.process(new FlushContext()); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java new file mode 100644 index 0000000000000..c6e97fd52d9d2 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.example; + +import java.util.ArrayList; +import java.util.List; + +// device list +public class MemChunk { + List deviceIDS; + + public MemChunk() { + deviceIDS = new ArrayList<>(); + } + + public List getDeviceIDS() { + return deviceIDS; + } + + public void setDeviceIDS(List deviceIDS) { + this.deviceIDS = deviceIDS; + } + + @Override + public String toString() { + return "MemChunk{" + deviceIDS.toString() + '}'; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java new file mode 100644 index 0000000000000..69f61d5d451fa --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.example; + +import java.util.HashMap; +import java.util.Map; + +// tagvalue -> memChunk +public class MemGroup { + + Map map; + + public MemGroup() { + map = new HashMap<>(); + } + + public Map getMap() { + return map; + } + + public void setMap(Map map) { + this.map = map; + } + + @Override + public String toString() { + return "MemGroup{" + map.toString() + '}'; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java new file mode 100644 index 0000000000000..7cc82f19ecab7 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.example; + +import java.util.HashMap; +import java.util.Map; + +// tagkey - > MemGroup +public class MemTable { + + private Map map; + + public MemTable() { + map = new HashMap<>(); + } + + public Map getMap() { + return map; + } + + public void setMap(Map map) { + this.map = map; + } + + @Override + public String toString() { + return "MemTable{" + map.toString() + '}'; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java new file mode 100644 index 0000000000000..d0f50071747c1 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.example; + +import java.util.HashMap; +import java.util.Map; + +// 管理working memtable , immutable memtables,框架用户自定义 +public class MemTableManager { + + // 可写的memtable + private MemTable working; + + // 只读的memtables + private Map immutables; + + // 记录已插入的最大的deviceid + private int maxDeviceID; + + public MemTableManager() { + working = new MemTable(); + immutables = new HashMap<>(); + maxDeviceID = 0; + } + + public MemTable getWorking() { + return working; + } + + public void setWorking(MemTable working) { + this.working = working; + } + + public Map getImmutables() { + return immutables; + } + + public void setImmutables(Map immutables) { + this.immutables = immutables; + } + + public int getMaxDeviceID() { + return maxDeviceID; + } + + public void setMaxDeviceID(int maxDeviceID) { + this.maxDeviceID = maxDeviceID; + } + + @Override + public String toString() { + return "MemTableManager{" + + "working=" + + working.toString() + + ", immutables=" + + immutables.toString() + + '}'; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java new file mode 100644 index 0000000000000..9a6359eab1714 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.Context; + +import java.util.List; + +public abstract class BasicLevelProcess implements LevelProcess { + LevelProcess next; + + public abstract void handle(I memNode, C context); + + public abstract List getChildren(I memNode, C context); + + @Override + public LevelProcess nextLevel(LevelProcess next) { + this.next = next; + return next; + } + + @Override + public void process(I memNode, C context) { + int currentLevel = context.getLevel(); + context.getAccessStrategy().execute(this, memNode, context); + context.setLevel(currentLevel); + } + + public boolean hasNext() { + return next != null; + } + + public LevelProcess getNext() { + return next; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java new file mode 100644 index 0000000000000..f4a4c97cd0f5c --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.DeleteContext; + +public abstract class DeleteLevelProcess extends BasicLevelProcess { + + public abstract void delete(I memNode, DeleteContext context); + + @Override + public void handle(I memNode, DeleteContext context) { + delete(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java new file mode 100644 index 0000000000000..00363c7683b75 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.FlushContext; + +public abstract class FlushLevelProcess extends BasicLevelProcess { + + public abstract void flush(I memNode, FlushContext context); + + public void handle(I memNode, FlushContext context) { + flush(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java new file mode 100644 index 0000000000000..91f7e24c1f250 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.InsertContext; + +public abstract class InsertLevelProcess extends BasicLevelProcess { + + public abstract void insert(I memNode, InsertContext context); + + @Override + public void handle(I memNode, InsertContext context) { + insert(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java new file mode 100644 index 0000000000000..aa04c2bf172b6 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.Context; + +public interface LevelProcess { + LevelProcess nextLevel(LevelProcess next); + + void process(I memNode, C context); + + void handle(I memNode, C context); +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java new file mode 100644 index 0000000000000..5b3fc0e9ebb32 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.QueryContext; + +public abstract class QueryLevelProcess extends BasicLevelProcess { + + public abstract void query(I memNode, QueryContext context); + + @Override + public void handle(I memNode, QueryContext context) { + query(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java new file mode 100644 index 0000000000000..3d33dd1712864 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.manager; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.levelProcess.LevelProcess; + +public class BasicLsmManager implements LsmManager { + + T root; + + LevelProcess levelProcess; + + @Override + public BasicLsmManager manager(T memNode) { + root = memNode; + return this; + } + + @Override + public void process(C context) { + levelProcess.process(root, context); + } + + @Override + public LevelProcess nextLevel(LevelProcess levelProcess) { + this.levelProcess = levelProcess; + return levelProcess; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java new file mode 100644 index 0000000000000..3d60af6f69f8e --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.manager; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.levelProcess.LevelProcess; + +public interface LsmManager { + LsmManager manager(T memNode); + + void process(C context); + + LevelProcess nextLevel(LevelProcess next); +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java new file mode 100644 index 0000000000000..d7780a5019877 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +// 表示内存节点访问策略（先序，后序） +public interface AccessStrategy{ + + /** + * + * @param levelProcess 保存当前节点和子节点的处理方法 + * @param memNode 当前待处理的节点 + * @param context 上下文信息 + */ + void execute( + BasicLevelProcess levelProcess, I memNode, C context); +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java new file mode 100644 index 0000000000000..cf794b7097e61 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; + +public class BFSAccessStrategy implements AccessStrategy { + + Queue sameLevelMemNodes; + + @Override + public void execute( + BasicLevelProcess levelProcess, I memNode, C context) { + List children = new ArrayList<>(); + int currentLevel = context.getLevel(); + // 第一个使用bfs策略的节点 + if (sameLevelMemNodes == null) { + sameLevelMemNodes = new LinkedList<>(); + levelProcess.handle(memNode, context); + children = levelProcess.getChildren(memNode, context); + } else { + while (!sameLevelMemNodes.isEmpty()) { + I node = (I) sameLevelMemNodes.poll(); + levelProcess.handle(node, context); + children.addAll(levelProcess.getChildren(node, context)); + } + } + sameLevelMemNodes.addAll(children); + context.setLevel(currentLevel + 1); + if (levelProcess.hasNext() && !sameLevelMemNodes.isEmpty()) { + levelProcess.getNext().process(null, context); + } + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java new file mode 100644 index 0000000000000..3079041763701 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +import java.util.List; + +public class PostOrderAccessStrategy implements AccessStrategy { + + @Override + public void execute( + BasicLevelProcess levelProcess, I memNode, C context) { + int currentLevel = context.getLevel(); + AccessStrategy accessStrategy = context.getAccessStrategy(); + List children = levelProcess.getChildren(memNode, context); + // 处理子节点 + if (levelProcess.hasNext()) { + context.setLevel(currentLevel + 1); + for (O child : children) { + levelProcess.getNext().process(child, context); + } + } + + context.setLevel(currentLevel); + context.setAccessStrategy(accessStrategy); + // 处理该节点 + levelProcess.handle(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java new file mode 100644 index 0000000000000..11f8be0844744 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +import java.util.List; + +public class PreOrderAccessStrategy implements AccessStrategy { + + @Override + public void execute( + BasicLevelProcess levelProcess, I memNode, C context) { + int currentLevel = context.getLevel(); + // 处理该节点 + levelProcess.handle(memNode, context); + List children = levelProcess.getChildren(memNode, context); + + // 处理子节点 + if (levelProcess.hasNext()) { + context.setLevel(currentLevel + 1); + for (O child : children) { + levelProcess.getNext().process(child, context); + } + } + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java new file mode 100644 index 0000000000000..e6b315eb145af --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.FlushContext; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +import java.util.List; + +public class RBFSAccessStrategy implements AccessStrategy{ + @Override + public void execute(BasicLevelProcess levelProcess, I memNode, C context) { + FlushContext flushContext = (FlushContext) context; + int currentLevel = context.getLevel(); + // 如果当前节点是最深的一层节点 + if(!levelProcess.hasNext()){ + flushContext.setMinimumFlushedLevel(currentLevel); + } + // 如果是根节点 + if(currentLevel == 0){ + while(flushContext.getMinimumFlushedLevel() != currentLevel){ + List children = levelProcess.getChildren(memNode, context); + for(O child : children){ + // 处理子节点 + flushContext.setLevel(currentLevel+1); + levelProcess.getNext().process(child,context); + flushContext.setLevel(currentLevel); + } + // 每次处理完-1 + flushContext.setMinimumFlushedLevel(flushContext.getMinimumFlushedLevel() - 1); + } + // 处理root节点 + levelProcess.handle(memNode, context); + return; + } + + // 后序遍历，处理level == minimumFlushedLevel的节点 + + // 已经处理过，直接return + if(currentLevel > flushContext.getMinimumFlushedLevel()) return; + + + // 处理子节点 + if(currentLevel == flushContext.getMinimumFlushedLevel()){ + levelProcess.handle(memNode, context); + return; + } + List children = levelProcess.getChildren(memNode,context); + for(O child : children){ + flushContext.setLevel(currentLevel+1); + levelProcess.getNext().process(child,context); + flushContext.setLevel(currentLevel); + } + + } + +} From 07052643e1b5e96672ae02b9b4ad45b8b14f0c84 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 7 Sep 2022 14:14:14 +0800 Subject: [PATCH 02/15] parent pom add lsm --- lsm/pom.xml | 4 ++- .../iotdb/lsm/context/FlushContext.java | 1 - .../iotdb/lsm/strategy/AccessStrategy.java | 5 ++- .../lsm/strategy/RBFSAccessStrategy.java | 32 +++++++++---------- pom.xml | 1 + 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/lsm/pom.xml b/lsm/pom.xml index 1bbf055742d06..d6ff4cf992ad9 100644 --- a/lsm/pom.xml +++ b/lsm/pom.xml @@ -20,9 +20,11 @@ iotdb-parent org.apache.iotdb 0.14.0-SNAPSHOT + ../pom.xml 4.0.0 - lsm + iotdb-lsm + IoTDB lsm 8 8 diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java index ceacd2f636e5f..cb8c938175811 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.lsm.context; -import org.apache.iotdb.lsm.strategy.BFSAccessStrategy; import org.apache.iotdb.lsm.strategy.RBFSAccessStrategy; public class FlushContext extends Context { diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java index d7780a5019877..9ff3f0f0a03e3 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java @@ -22,11 +22,10 @@ import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; // 表示内存节点访问策略（先序，后序） -public interface AccessStrategy{ +public interface AccessStrategy { /** - * - * @param levelProcess 保存当前节点和子节点的处理方法 + * @param levelProcess 保存当前节点和子节点的处理方法 * @param memNode 当前待处理的节点 * @param context 上下文信息 */ diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java index e6b315eb145af..d9f4fa19004c2 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java @@ -24,23 +24,24 @@ import java.util.List; -public class RBFSAccessStrategy implements AccessStrategy{ +public class RBFSAccessStrategy implements AccessStrategy { @Override - public void execute(BasicLevelProcess levelProcess, I memNode, C context) { + public void execute( + BasicLevelProcess levelProcess, I memNode, C context) { FlushContext flushContext = (FlushContext) context; int currentLevel = context.getLevel(); // 如果当前节点是最深的一层节点 - if(!levelProcess.hasNext()){ + if (!levelProcess.hasNext()) { flushContext.setMinimumFlushedLevel(currentLevel); } // 如果是根节点 - if(currentLevel == 0){ - while(flushContext.getMinimumFlushedLevel() != currentLevel){ + if (currentLevel == 0) { + while (flushContext.getMinimumFlushedLevel() != currentLevel) { List children = levelProcess.getChildren(memNode, context); - for(O child : children){ + for (O child : children) { // 处理子节点 - flushContext.setLevel(currentLevel+1); - levelProcess.getNext().process(child,context); + flushContext.setLevel(currentLevel + 1); + levelProcess.getNext().process(child, context); flushContext.setLevel(currentLevel); } // 每次处理完-1 @@ -54,21 +55,18 @@ public void execute(BasicLevelProcess levelPr // 后序遍历，处理level == minimumFlushedLevel的节点 // 已经处理过，直接return - if(currentLevel > flushContext.getMinimumFlushedLevel()) return; - + if (currentLevel > flushContext.getMinimumFlushedLevel()) return; // 处理子节点 - if(currentLevel == flushContext.getMinimumFlushedLevel()){ + if (currentLevel == flushContext.getMinimumFlushedLevel()) { levelProcess.handle(memNode, context); return; } - List children = levelProcess.getChildren(memNode,context); - for(O child : children){ - flushContext.setLevel(currentLevel+1); - levelProcess.getNext().process(child,context); + List children = levelProcess.getChildren(memNode, context); + for (O child : children) { + flushContext.setLevel(currentLevel + 1); + levelProcess.getNext().process(child, context); flushContext.setLevel(currentLevel); } - } - } diff --git a/pom.xml b/pom.xml index f43d7aa3132d2..f3a13adb107e5 100644 --- a/pom.xml +++ b/pom.xml @@ -120,6 +120,7 @@ udf-api rewrite-tsfile-tool external-api + lsm From 66cf428e1b3778fc6401ef63091f15e015f737bf Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 7 Sep 2022 18:05:02 +0800 Subject: [PATCH 03/15] lsm preOrder level fix --- lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java | 8 ++++---- .../apache/iotdb/lsm/levelProcess/BasicLevelProcess.java | 2 -- .../apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java | 1 + 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java index 015963e8af134..21796877734da 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java @@ -152,7 +152,7 @@ public static void flushExample(MemTableManager memTableManager) { new FlushLevelProcess() { @Override public void flush(MemTableManager memNode, FlushContext context) { - System.out.println(memNode); + System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); } @Override @@ -167,7 +167,7 @@ public List getChildren(MemTableManager memNode, FlushContext context) new FlushLevelProcess() { @Override public void flush(MemTable memNode, FlushContext context) { - System.out.println(memNode); + System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); } @Override @@ -181,7 +181,7 @@ public List getChildren(MemTable memNode, FlushContext context) { new FlushLevelProcess() { @Override public void flush(MemGroup memNode, FlushContext context) { - System.out.println(memNode); + System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); } @Override @@ -195,7 +195,7 @@ public List getChildren(MemGroup memNode, FlushContext context) { new FlushLevelProcess() { @Override public void flush(MemChunk memNode, FlushContext context) { - System.out.println(memNode); + System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); } @Override diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java index 9a6359eab1714..f4ced68ec5a08 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java @@ -37,9 +37,7 @@ public LevelProcess nextLevel(LevelProcess next) { @Override public void process(I memNode, C context) { - int currentLevel = context.getLevel(); context.getAccessStrategy().execute(this, memNode, context); - context.setLevel(currentLevel); } public boolean hasNext() { diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java index 11f8be0844744..2c0b1b7b03a6c 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java @@ -40,5 +40,6 @@ public void execute( levelProcess.getNext().process(child, context); } } + context.setLevel(currentLevel); } } From a9f3d8343f391d389465e2e2805e188d0d08d26d Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 7 Sep 2022 21:32:39 +0800 Subject: [PATCH 04/15] lsm preOrder level fix --- .../java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java index d9f4fa19004c2..1505361947793 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java @@ -52,8 +52,6 @@ public void execute( return; } - // 后序遍历，处理level == minimumFlushedLevel的节点 - // 已经处理过，直接return if (currentLevel > flushContext.getMinimumFlushedLevel()) return; From b100777f760d6e7ca7650504025e20f9747672ee Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Thu, 8 Sep 2022 19:29:37 +0800 Subject: [PATCH 05/15] lsm add levelUpperBound --- .../org/apache/iotdb/lsm/context/Context.java | 26 +++++++++---------- .../iotdb/lsm/context/FlushContext.java | 15 ----------- .../org/apache/iotdb/lsm/example/Main.java | 8 +++--- .../lsm/strategy/RBFSAccessStrategy.java | 23 +++++++--------- 4 files changed, 26 insertions(+), 46 deletions(-) diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java index 381586e7c8294..305b31747f6c3 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java @@ -35,6 +35,9 @@ public class Context { // 多少个线程处理该节点的子节点 int threadNums; + // 上界,大于该值的层级不会被处理 + int levelUpperBound; + // 返回值 Object result; @@ -43,20 +46,7 @@ public Context() { type = ContextType.NONE; level = 0; threadNums = 1; - } - - public Context( - ContextType type, - AccessStrategy accessStrategy, - int level, - boolean sync, - int threadNums, - Object result) { - this.type = type; - this.accessStrategy = accessStrategy; - this.level = level; - this.threadNums = threadNums; - this.result = result; + levelUpperBound = Integer.MAX_VALUE; } public void setLevel(int level) { @@ -98,4 +88,12 @@ public AccessStrategy getAccessStrategy() { public void setAccessStrategy(AccessStrategy accessStrategy) { this.accessStrategy = accessStrategy; } + + public int getLevelUpperBound() { + return levelUpperBound; + } + + public void setLevelUpperBound(int levelUpperBound) { + this.levelUpperBound = levelUpperBound; + } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java index cb8c938175811..2fd266396c8f8 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java @@ -21,22 +21,7 @@ import org.apache.iotdb.lsm.strategy.RBFSAccessStrategy; public class FlushContext extends Context { - - // 最小的已经flush的层级 - int minimumFlushedLevel; - public FlushContext() { - super(); - type = ContextType.FLUSH; accessStrategy = new RBFSAccessStrategy(); - minimumFlushedLevel = Integer.MAX_VALUE; - } - - public int getMinimumFlushedLevel() { - return minimumFlushedLevel; - } - - public void setMinimumFlushedLevel(int minimumFlushedLevel) { - this.minimumFlushedLevel = minimumFlushedLevel; } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java index 21796877734da..bea5c80e15873 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java @@ -152,7 +152,7 @@ public static void flushExample(MemTableManager memTableManager) { new FlushLevelProcess() { @Override public void flush(MemTableManager memNode, FlushContext context) { - System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); } @Override @@ -167,7 +167,7 @@ public List getChildren(MemTableManager memNode, FlushContext context) new FlushLevelProcess() { @Override public void flush(MemTable memNode, FlushContext context) { - System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); } @Override @@ -181,7 +181,7 @@ public List getChildren(MemTable memNode, FlushContext context) { new FlushLevelProcess() { @Override public void flush(MemGroup memNode, FlushContext context) { - System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); } @Override @@ -195,7 +195,7 @@ public List getChildren(MemGroup memNode, FlushContext context) { new FlushLevelProcess() { @Override public void flush(MemChunk memNode, FlushContext context) { - System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); } @Override diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java index 1505361947793..62d62295786fc 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java @@ -19,7 +19,6 @@ package org.apache.iotdb.lsm.strategy; import org.apache.iotdb.lsm.context.Context; -import org.apache.iotdb.lsm.context.FlushContext; import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; import java.util.List; @@ -28,24 +27,22 @@ public class RBFSAccessStrategy implements AccessStrategy { @Override public void execute( BasicLevelProcess levelProcess, I memNode, C context) { - FlushContext flushContext = (FlushContext) context; int currentLevel = context.getLevel(); - // 如果当前节点是最深的一层节点 - if (!levelProcess.hasNext()) { - flushContext.setMinimumFlushedLevel(currentLevel); + if (Integer.MAX_VALUE == context.getLevelUpperBound() && !levelProcess.hasNext()) { + context.setLevelUpperBound(context.getLevel()); } // 如果是根节点 if (currentLevel == 0) { - while (flushContext.getMinimumFlushedLevel() != currentLevel) { + while (context.getLevelUpperBound() != currentLevel) { List children = levelProcess.getChildren(memNode, context); for (O child : children) { // 处理子节点 - flushContext.setLevel(currentLevel + 1); + context.setLevel(currentLevel + 1); levelProcess.getNext().process(child, context); - flushContext.setLevel(currentLevel); + context.setLevel(currentLevel); } // 每次处理完-1 - flushContext.setMinimumFlushedLevel(flushContext.getMinimumFlushedLevel() - 1); + context.setLevelUpperBound(context.getLevelUpperBound() - 1); } // 处理root节点 levelProcess.handle(memNode, context); @@ -53,18 +50,18 @@ public void execute( } // 已经处理过，直接return - if (currentLevel > flushContext.getMinimumFlushedLevel()) return; + if (currentLevel > context.getLevelUpperBound()) return; // 处理子节点 - if (currentLevel == flushContext.getMinimumFlushedLevel()) { + if (currentLevel == context.getLevelUpperBound()) { levelProcess.handle(memNode, context); return; } List children = levelProcess.getChildren(memNode, context); for (O child : children) { - flushContext.setLevel(currentLevel + 1); + context.setLevel(currentLevel + 1); levelProcess.getNext().process(child, context); - flushContext.setLevel(currentLevel); + context.setLevel(currentLevel); } } } From cb51763c5d853fcd4a41928b26494970503e02c1 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Fri, 9 Sep 2022 19:50:47 +0800 Subject: [PATCH 06/15] add TagInvertedIndex --- .../tagIndex/ITagInvertedIndex.java | 31 +++++++++++++++++++ .../tagIndex/TagInvertedIndex.java | 23 ++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java create mode 100644 lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java diff --git a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java new file mode 100644 index 0000000000000..a6798ab0096c3 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; + +import java.io.File; +import java.util.List; +import java.util.Map; + +public interface ITagInvertedIndex { + void addTag(String tagKey,String tagValue,int id); + void addTags(Map tags,int id); + void removeTag(String tagKey,String tagValue,int id); + void removeTags(Map tags,int id); + List getMatchedIDs(); +} diff --git a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java new file mode 100644 index 0000000000000..cbfbb9ee58983 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; + +public class TagInvertedIndex { + +} From 58e5eb4d0cb4398c90f9995a54bf17dcc341ad7a Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Fri, 9 Sep 2022 19:52:08 +0800 Subject: [PATCH 07/15] add TagInvertedIndex --- .../tagIndex/ITagInvertedIndex.java | 15 +++++++----- .../tagIndex/TagInvertedIndex.java | 23 ------------------- 2 files changed, 9 insertions(+), 29 deletions(-) delete mode 100644 lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java diff --git a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java index a6798ab0096c3..3934a6b680d8a 100644 --- a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java +++ b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java @@ -18,14 +18,17 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; -import java.io.File; import java.util.List; import java.util.Map; public interface ITagInvertedIndex { - void addTag(String tagKey,String tagValue,int id); - void addTags(Map tags,int id); - void removeTag(String tagKey,String tagValue,int id); - void removeTags(Map tags,int id); - List getMatchedIDs(); + void addTag(String tagKey, String tagValue, int id); + + void addTags(Map tags, int id); + + void removeTag(String tagKey, String tagValue, int id); + + void removeTags(Map tags, int id); + + List getMatchedIDs(Map tags); } diff --git a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java deleted file mode 100644 index cbfbb9ee58983..0000000000000 --- a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; - -public class TagInvertedIndex { - -} From 3c49cb9351c42831a6ebd431534debb0102bf07d Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Tue, 13 Sep 2022 15:30:30 +0800 Subject: [PATCH 08/15] impl TagInvertedIndex --- lsm/pom.xml | 4 - .../iotdb/lsm/context/DeleteContext.java | 12 +- .../iotdb/lsm/context/QueryContext.java | 3 + pom.xml | 1 + schema-engine-tag/pom.xml | 51 ++++++ .../tagIndex/ITagInvertedIndex.java | 4 - .../tagIndex/TagInvertedIndex.java | 153 ++++++++++++++++++ .../tagIndex/deletion/DeletionManager.java | 41 +++++ .../tagIndex/deletion/MemChunkDeletion.java | 38 +++++ .../deletion/MemChunkGroupDeletion.java | 47 ++++++ .../tagIndex/deletion/MemTableDeletion.java | 56 +++++++ .../tagIndex/insertion/InsertionManager.java | 42 +++++ .../insertion/MemChunkGroupInsertion.java | 44 +++++ .../tagIndex/insertion/MemChunkInsertion.java | 38 +++++ .../tagIndex/insertion/MemTableInsertion.java | 48 ++++++ .../tagIndex/memtable/MemChunk.java | 52 ++++++ .../tagIndex/memtable/MemChunkGroup.java | 53 ++++++ .../tagIndex/memtable/MemTable.java | 84 ++++++++++ .../tagIndex/query/MemChunkGroupQuery.java | 41 +++++ .../tagIndex/query/MemChunkQuery.java | 42 +++++ .../tagIndex/query/MemTableQuery.java | 54 +++++++ .../tagIndex/query/QueryManager.java | 41 +++++ .../tagIndex/TagInvertedIndexTest.java | 148 +++++++++++++++++ .../org/apache/iotdb/db/conf/IoTDBConfig.java | 10 ++ 24 files changed, 1098 insertions(+), 9 deletions(-) create mode 100644 schema-engine-tag/pom.xml rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java (90%) create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java create mode 100644 schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java diff --git a/lsm/pom.xml b/lsm/pom.xml index d6ff4cf992ad9..081c60b0abc73 100644 --- a/lsm/pom.xml +++ b/lsm/pom.xml @@ -25,8 +25,4 @@ 4.0.0 iotdb-lsm IoTDB lsm - - 8 - 8 - diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java index e00db284e7bce..46d78b9ed5131 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java @@ -18,6 +18,8 @@ */ package org.apache.iotdb.lsm.context; +import org.apache.iotdb.lsm.strategy.PostOrderAccessStrategy; + import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -26,17 +28,25 @@ public class DeleteContext extends Context { List keys; - public DeleteContext(Object... ks) { + Object value; + + public DeleteContext(Object value, Object... ks) { super(); + this.value = value; keys = new ArrayList<>(); keys.addAll(Arrays.asList(ks)); type = ContextType.DELETE; + accessStrategy = new PostOrderAccessStrategy(); } public Object getKey() { return keys.get(level); } + public Object getValue() { + return value; + } + public int size() { return keys.size(); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java index 9b8d440106b48..7221b643f7fac 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java @@ -18,6 +18,8 @@ */ package org.apache.iotdb.lsm.context; +import org.apache.iotdb.lsm.strategy.PostOrderAccessStrategy; + import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -31,6 +33,7 @@ public QueryContext(Object... ks) { keys = new ArrayList<>(); keys.addAll(Arrays.asList(ks)); type = ContextType.QUERY; + accessStrategy = new PostOrderAccessStrategy(); } public Object getKey() { diff --git a/pom.xml b/pom.xml index f3a13adb107e5..f5b6c61ec7a97 100644 --- a/pom.xml +++ b/pom.xml @@ -121,6 +121,7 @@ rewrite-tsfile-tool external-api lsm + schema-engine-tag diff --git a/schema-engine-tag/pom.xml b/schema-engine-tag/pom.xml new file mode 100644 index 0000000000000..e44c73f76f055 --- /dev/null +++ b/schema-engine-tag/pom.xml @@ -0,0 +1,51 @@ + + + + + iotdb-parent + org.apache.iotdb + 0.14.0-SNAPSHOT + ../pom.xml + + 4.0.0 + schema-engine-tag + schema-engine-tag + + + + org.roaringbitmap + RoaringBitmap + 0.9.32 + + + + org.apache.iotdb + iotdb-lsm + ${project.version} + provided + + + org.apache.iotdb + iotdb-server + ${project.version} + + + + \ No newline at end of file diff --git a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java similarity index 90% rename from lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java rename to schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java index 3934a6b680d8a..cf5e1a19c0ea7 100644 --- a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java @@ -22,12 +22,8 @@ import java.util.Map; public interface ITagInvertedIndex { - void addTag(String tagKey, String tagValue, int id); - void addTags(Map tags, int id); - void removeTag(String tagKey, String tagValue, int id); - void removeTags(Map tags, int id); List getMatchedIDs(Map tags); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java new file mode 100644 index 0000000000000..89c3260ed0e55 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; + +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.DeletionManager; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.InsertionManager; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.QueryManager; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.QueryContext; + +import org.roaringbitmap.RoaringBitmap; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class TagInvertedIndex implements ITagInvertedIndex { + + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + private int numOfDeviceIdsInMemTable; + + private MemTable workingMemTable; + + private MemTable unsequenceMemTable; + + private Map immutableMemTables; + + private int maxDeviceID; + + public TagInvertedIndex() { + workingMemTable = new MemTable(MemTable.WORKING); + unsequenceMemTable = new MemTable(MemTable.UNSEQUENCE); + immutableMemTables = new HashMap<>(); + numOfDeviceIdsInMemTable = config.getNumOfDeviceIdsInMemTable(); + maxDeviceID = 0; + } + + @Override + public synchronized void addTags(Map tags, int id) { + MemTable memTable = null; + // 出现乱序 + if (id < maxDeviceID) { + memTable = unsequenceMemTable; + } else { + if (!inWorkingMemTable(id)) { + workingMemTable.setStatus(MemTable.IMMUTABLE); + immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); + workingMemTable = new MemTable(MemTable.WORKING); + } + memTable = workingMemTable; + maxDeviceID = id; + } + for (Map.Entry tag : tags.entrySet()) { + addTag(memTable, tag.getKey(), tag.getValue(), id); + } + } + + @Override + public synchronized void removeTags(Map tags, int id) { + List memTables = new ArrayList<>(); + // 出现乱序 + if (inWorkingMemTable(id)) { + memTables.add(workingMemTable); + } else { + memTables.add(unsequenceMemTable); + memTables.add(immutableMemTables.get(id / numOfDeviceIdsInMemTable)); + } + for (Map.Entry tag : tags.entrySet()) { + removeTag(memTables, tag.getKey(), tag.getValue(), id); + } + } + + @Override + public synchronized List getMatchedIDs(Map tags) { + List memTables = new ArrayList<>(); + memTables.add(workingMemTable); + memTables.add(unsequenceMemTable); + memTables.addAll(immutableMemTables.values()); + RoaringBitmap roaringBitmap = new RoaringBitmap(); + int i = 0; + for (Map.Entry tag : tags.entrySet()) { + RoaringBitmap rb = getMatchedIDs(memTables, tag.getKey(), tag.getValue()); + if (i == 0) roaringBitmap = rb; + else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb); + i++; + } + return Arrays.stream(roaringBitmap.toArray()).boxed().collect(Collectors.toList()); + } + + @Override + public String toString() { + return "TagInvertedIndex{" + + "numOfDeviceIdsInMemTable=" + + numOfDeviceIdsInMemTable + + ", workingMemTable=" + + workingMemTable + + ", unsequenceMemTable=" + + unsequenceMemTable + + ", immutableMemTables=" + + immutableMemTables + + ", maxDeviceID=" + + maxDeviceID + + '}'; + } + + private synchronized boolean inWorkingMemTable(int id) { + return id / numOfDeviceIdsInMemTable == maxDeviceID / numOfDeviceIdsInMemTable; + } + + private void addTag(MemTable memTable, String tagKey, String tagValue, int id) { + InsertContext insertContext = new InsertContext(id, tagKey, tagValue); + InsertionManager.getInstance().manager(memTable).process(insertContext); + } + + private void removeTag(List memTables, String tagKey, String tagValue, int id) { + DeleteContext deleteContext = new DeleteContext(id, tagKey, tagValue); + for (MemTable memTable : memTables) { + DeletionManager.getInstance().manager(memTable).process(deleteContext); + } + } + + private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue) { + QueryContext queryContext = new QueryContext(tagKey, tagValue); + for (MemTable memTable : memTables) { + QueryManager.getInstance().manager(memTable).process(queryContext); + } + return (RoaringBitmap) queryContext.getResult(); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java new file mode 100644 index 0000000000000..4125a1097db18 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.manager.BasicLsmManager; + +public class DeletionManager extends BasicLsmManager { + private DeletionManager() { + this.nextLevel(new MemTableDeletion()) + .nextLevel(new MemChunkGroupDeletion()) + .nextLevel(new MemChunkDeletion()); + } + + public static DeletionManager getInstance() { + return DeletionManagerHolder.INSTANCE; + } + + private static class DeletionManagerHolder { + private static final DeletionManager INSTANCE = new DeletionManager(); + + private DeletionManagerHolder() {} + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java new file mode 100644 index 0000000000000..c95a617765b66 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess; + +import java.util.List; + +public class MemChunkDeletion extends DeleteLevelProcess { + @Override + public List getChildren(MemChunk memNode, DeleteContext context) { + return null; + } + + @Override + public void delete(MemChunk memNode, DeleteContext context) { + Integer deviceID = (Integer) context.getValue(); + memNode.remove(deviceID); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java new file mode 100644 index 0000000000000..651553fd86032 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess; + +import java.util.ArrayList; +import java.util.List; + +public class MemChunkGroupDeletion extends DeleteLevelProcess { + @Override + public List getChildren(MemChunkGroup memNode, DeleteContext context) { + List memChunks = new ArrayList<>(); + String tagValue = (String) context.getKey(); + MemChunk child = memNode.get(tagValue); + if (child != null) memChunks.add(child); + return memChunks; + } + + @Override + public void delete(MemChunkGroup memNode, DeleteContext context) { + String tagValue = (String) context.getKey(); + MemChunk child = memNode.get(tagValue); + if (child == null || child.isEmpty()) { + memNode.remove(tagValue); + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java new file mode 100644 index 0000000000000..0a1e5b3d05715 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +public class MemTableDeletion extends DeleteLevelProcess { + @Override + public List getChildren(MemTable memNode, DeleteContext context) { + if (memNode.isImmutable()) return new ArrayList<>(); + List memChunkGroups = new ArrayList<>(); + String tagKey = (String) context.getKey(); + MemChunkGroup child = memNode.get(tagKey); + if (child != null) memChunkGroups.add(child); + return memChunkGroups; + } + + @Override + public void delete(MemTable memNode, DeleteContext context) { + if (memNode.isImmutable()) { + Set deletionList = memNode.getDeletionList(); + if (!deletionList.contains(context.getValue())) { + deletionList.add((Integer) context.getValue()); + } + return; + } + String tagKey = (String) context.getKey(); + MemChunkGroup child = memNode.get(tagKey); + if (child == null || child.isEmpty()) { + memNode.remove(tagKey); + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java new file mode 100644 index 0000000000000..3e6d71493bd30 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.manager.BasicLsmManager; + +public class InsertionManager extends BasicLsmManager { + + private InsertionManager() { + this.nextLevel(new MemTableInsertion()) + .nextLevel(new MemChunkGroupInsertion()) + .nextLevel(new MemChunkInsertion()); + } + + public static InsertionManager getInstance() { + return InsertionManagerHolder.INSTANCE; + } + + private static class InsertionManagerHolder { + private static final InsertionManager INSTANCE = new InsertionManager(); + + private InsertionManagerHolder() {} + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java new file mode 100644 index 0000000000000..843084223f4ea --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; + +import java.util.ArrayList; +import java.util.List; + +public class MemChunkGroupInsertion extends InsertLevelProcess { + @Override + public List getChildren(MemChunkGroup memNode, InsertContext context) { + List memChunks = new ArrayList<>(); + String tagValue = (String) context.getKey(); + MemChunk child = memNode.get(tagValue); + if (child != null) memChunks.add(child); + return memChunks; + } + + @Override + public void insert(MemChunkGroup memNode, InsertContext context) { + String tagValue = (String) context.getKey(); + memNode.put(tagValue); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java new file mode 100644 index 0000000000000..5dec23dcb7d1b --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; + +import java.util.List; + +public class MemChunkInsertion extends InsertLevelProcess { + @Override + public List getChildren(MemChunk memNode, InsertContext context) { + return null; + } + + @Override + public void insert(MemChunk memNode, InsertContext context) { + Integer deviceID = (Integer) context.getValue(); + memNode.put(deviceID); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java new file mode 100644 index 0000000000000..ef88f0591bc03 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; + +import java.util.ArrayList; +import java.util.List; + +// memtable的insert操作 +public class MemTableInsertion extends InsertLevelProcess { + + @Override + public List getChildren(MemTable memNode, InsertContext context) { + if (memNode.isImmutable()) return new ArrayList<>(); + List memChunkGroups = new ArrayList<>(); + String tagKey = (String) context.getKey(); + MemChunkGroup child = memNode.get(tagKey); + if (child != null) memChunkGroups.add(child); + return memChunkGroups; + } + + @Override + public void insert(MemTable memNode, InsertContext context) { + if (memNode.isImmutable()) return; + String tagKey = (String) context.getKey(); + memNode.put(tagKey); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java new file mode 100644 index 0000000000000..b46a770ca97e3 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable; + +import org.roaringbitmap.RoaringBitmap; + +// 管理设备id集合 +public class MemChunk { + private RoaringBitmap roaringBitmap; + + public MemChunk() { + roaringBitmap = new RoaringBitmap(); + } + + public boolean isEmpty() { + if (roaringBitmap == null) return true; + return roaringBitmap.isEmpty(); + } + + @Override + public String toString() { + return roaringBitmap.toString(); + } + + public void put(int id) { + roaringBitmap.add(id); + } + + public void remove(int id) { + roaringBitmap.remove(id); + } + + public RoaringBitmap getRoaringBitmap() { + return this.roaringBitmap; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java new file mode 100644 index 0000000000000..f6765030183a5 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable; + +import java.util.HashMap; +import java.util.Map; + +public class MemChunkGroup { + private Map memChunkMap; + + public MemChunkGroup() { + memChunkMap = new HashMap<>(); + } + + public void put(String tagValue) { + if (!memChunkMap.containsKey(tagValue)) { + memChunkMap.put(tagValue, new MemChunk()); + } + } + + @Override + public String toString() { + return memChunkMap.toString(); + } + + public MemChunk get(String tagValue) { + return memChunkMap.get(tagValue); + } + + public void remove(String tagValue) { + memChunkMap.remove(tagValue); + } + + public boolean isEmpty() { + return memChunkMap.isEmpty(); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java new file mode 100644 index 0000000000000..af47581ac9c3d --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class MemTable { + public static final String WORKING = "working"; + + public static final String IMMUTABLE = "immutable"; + + public static final String UNSEQUENCE = "unsequence"; + + private Map memChunkGroupMap; + + private String status; + + private Set deletionList; + + public MemTable(String status) { + memChunkGroupMap = new HashMap<>(); + this.status = status; + deletionList = new HashSet<>(); + } + + public void put(String tagKey) { + if (this.status.equals(IMMUTABLE)) return; + if (!memChunkGroupMap.containsKey(tagKey)) { + memChunkGroupMap.put(tagKey, new MemChunkGroup()); + } + } + + @Override + public String toString() { + return "MemTable{" + + "memChunkGroupMap=" + + memChunkGroupMap + + ", status='" + + status + + '\'' + + ", deletionList=" + + deletionList + + '}'; + } + + public MemChunkGroup get(String tagKey) { + return memChunkGroupMap.get(tagKey); + } + + public void remove(String tagKey) { + memChunkGroupMap.remove(tagKey); + } + + public boolean isImmutable() { + return status.equals(IMMUTABLE); + } + + public Set getDeletionList() { + return deletionList; + } + + public void setStatus(String status) { + this.status = status; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java new file mode 100644 index 0000000000000..bf9656e82205d --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess; + +import java.util.ArrayList; +import java.util.List; + +public class MemChunkGroupQuery extends QueryLevelProcess { + @Override + public List getChildren(MemChunkGroup memNode, QueryContext context) { + List memChunks = new ArrayList<>(); + String tagValue = (String) context.getKey(); + MemChunk child = memNode.get(tagValue); + if (child != null) memChunks.add(child); + return memChunks; + } + + @Override + public void query(MemChunkGroup memNode, QueryContext context) {} +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java new file mode 100644 index 0000000000000..d67de88f62bb4 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess; + +import org.roaringbitmap.RoaringBitmap; + +import java.util.List; + +public class MemChunkQuery extends QueryLevelProcess { + @Override + public List getChildren(MemChunk memNode, QueryContext context) { + return null; + } + + @Override + public void query(MemChunk memNode, QueryContext context) { + RoaringBitmap roaringBitmap = (RoaringBitmap) context.getResult(); + if (roaringBitmap == null) roaringBitmap = new RoaringBitmap(); + RoaringBitmap now = RoaringBitmap.or(roaringBitmap, memNode.getRoaringBitmap()); + context.setResult(now); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java new file mode 100644 index 0000000000000..b1acbf845b14e --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess; + +import org.roaringbitmap.RoaringBitmap; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +public class MemTableQuery extends QueryLevelProcess { + + @Override + public List getChildren(MemTable memNode, QueryContext context) { + List memChunkGroups = new ArrayList<>(); + String tagKey = (String) context.getKey(); + MemChunkGroup child = memNode.get(tagKey); + if (child != null) memChunkGroups.add(child); + return memChunkGroups; + } + + @Override + public void query(MemTable memNode, QueryContext context) { + // 如果是immutable，则需要在查询结果中删除deletionList中的id + if (memNode.isImmutable()) { + RoaringBitmap roaringBitmap = (RoaringBitmap) context.getResult(); + Set deletionList = memNode.getDeletionList(); + for (Integer id : deletionList) { + roaringBitmap.remove(id); + } + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java new file mode 100644 index 0000000000000..8987c49801f36 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.manager.BasicLsmManager; + +public class QueryManager extends BasicLsmManager { + private QueryManager() { + this.nextLevel(new MemTableQuery()) + .nextLevel(new MemChunkGroupQuery()) + .nextLevel(new MemChunkQuery()); + } + + public static QueryManager getInstance() { + return QueryManagerHolder.INSTANCE; + } + + private static class QueryManagerHolder { + private static final QueryManager INSTANCE = new QueryManager(); + + private QueryManagerHolder() {} + } +} diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java new file mode 100644 index 0000000000000..4a17213c77ba8 --- /dev/null +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.tsfile.utils.Pair; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +public class TagInvertedIndexTest { + private String[][] record = + new String[][] { + {"tag1=q", "tag2=a", "1"}, + {"tag1=q", "tag2=s", "2"}, + {"tag1=q", "tag2=a", "tag3=z", "3"}, + {"tag1=q", "tag2=s", "5"}, + {"tag1=w", "tag2=d", "6"}, + {"tag1=q", "tag2=d", "tag3=e", "7"}, + {"tag1=q", "tag3=v", "4"}, + {"tag1=r", "tag2=d", "9"}, + {"tag1=t", "tag2=f", "10"}, + {"tag1=t", "tag2=g", "8"}, + {"tag1=t", "tag2=h", "11"}, + {"tag1=y", "tag2=j", "13"}, + {"tag1=u", "tag2=k", "14"}, + {"tag1=q", "tag2=a", "tag3=l", "12"}, + {"tag1=q", "tag2=a", "tag3=x", "15"}, + {"tag1=q", "tag2=b", "tag3=x", "18"}, + {"tag1=y", "tag2=a", "tag4=z", "17"}, + {"tag1=q", "tag2=a", "tag4=z", "16"}, + }; + + private int numOfDeviceIdsInMemTable; + + private TagInvertedIndex tagInvertedIndex; + + @Before + public void setUp() throws Exception { + numOfDeviceIdsInMemTable = + IoTDBDescriptor.getInstance().getConfig().getNumOfDeviceIdsInMemTable(); + IoTDBDescriptor.getInstance().getConfig().setNumOfDeviceIdsInMemTable(3); + tagInvertedIndex = new TagInvertedIndex(); + } + + @After + public void tearDown() throws Exception { + IoTDBDescriptor.getInstance().getConfig().setNumOfDeviceIdsInMemTable(numOfDeviceIdsInMemTable); + tagInvertedIndex = null; + } + + public void addTags() { + List, Integer>> records = generateTags(); + for (Pair, Integer> pair : records) { + tagInvertedIndex.addTags(pair.left, pair.right); + } + } + + public void removeTags() { + Pair, Integer> tags = generateTag(record[0]); + tagInvertedIndex.removeTags(tags.left, tags.right); + tags = generateTag(record[1]); + tagInvertedIndex.removeTags(tags.left, tags.right); + tags = generateTag(record[6]); + tagInvertedIndex.removeTags(tags.left, tags.right); + tags = generateTag(record[13]); + tagInvertedIndex.removeTags(tags.left, tags.right); + } + + @Test + public void getMatchedIDs() { + addTags(); + System.out.println("------------addTags------------"); + System.out.println(tagInvertedIndex); + Map tags1 = new HashMap<>(); + tags1.put("tag1", "q"); + + Map tags2 = new HashMap<>(); + tags2.put("tag1", "q"); + tags2.put("tag2", "a"); + + List ids = tagInvertedIndex.getMatchedIDs(tags1); + List verify = Arrays.asList(1, 2, 3, 4, 5, 7, 12, 15, 16, 18); + assertEquals(verify, ids); + + ids = tagInvertedIndex.getMatchedIDs(tags2); + verify = Arrays.asList(1, 3, 12, 15, 16); + assertEquals(verify, ids); + + removeTags(); + System.out.println("------------removeTags------------"); + System.out.println(tagInvertedIndex); + ids = tagInvertedIndex.getMatchedIDs(tags1); + verify = Arrays.asList(3, 5, 7, 15, 16, 18); + assertEquals(verify, ids); + + ids = tagInvertedIndex.getMatchedIDs(tags2); + verify = Arrays.asList(3, 15, 16); + assertEquals(verify, ids); + + System.out.println("------------query------------"); + System.out.println(tagInvertedIndex); + } + + private List, Integer>> generateTags() { + List, Integer>> pairs = new ArrayList<>(); + for (String[] strings : record) { + pairs.add(generateTag(strings)); + } + return pairs; + } + + private Pair, Integer> generateTag(String[] strings) { + Map tags = new HashMap<>(); + int i = 0; + for (; i < strings.length - 1; i++) { + String[] str = strings[i].split("="); + tags.put(str[0], str[1]); + } + Pair, Integer> pair = new Pair<>(tags, Integer.valueOf(strings[i])); + return pair; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 750d10ec3d787..495a7bb7bcac2 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -839,6 +839,8 @@ public class IoTDBConfig { */ private boolean enableIDTableLogFile = false; + private int numOfDeviceIdsInMemTable = 65536; + /** whether to use persistent schema mode */ private String schemaEngineMode = "Memory"; @@ -2751,6 +2753,14 @@ public void setEnableIDTableLogFile(boolean enableIDTableLogFile) { this.enableIDTableLogFile = enableIDTableLogFile; } + public int getNumOfDeviceIdsInMemTable() { + return numOfDeviceIdsInMemTable; + } + + public void setNumOfDeviceIdsInMemTable(int numOfDeviceIdsInMemTable) { + this.numOfDeviceIdsInMemTable = numOfDeviceIdsInMemTable; + } + public String getSchemaEngineMode() { return schemaEngineMode; } From 7230ebfa7a713e342bb7ac5feb01b43e0121159f Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Tue, 13 Sep 2022 18:58:32 +0800 Subject: [PATCH 09/15] impl TagInvertedIndex --- schema-engine-tag/pom.xml | 11 +- .../tagSchemaRegion/MockTagSchemaRegion.java | 325 ++++++++++++++++++ .../tagSchemaRegion/TagSchemaRegion.java | 21 ++ .../tagIndex/TagInvertedIndexTest.java | 40 +-- 4 files changed, 368 insertions(+), 29 deletions(-) create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java diff --git a/schema-engine-tag/pom.xml b/schema-engine-tag/pom.xml index e44c73f76f055..c5b8f565525a0 100644 --- a/schema-engine-tag/pom.xml +++ b/schema-engine-tag/pom.xml @@ -15,31 +15,25 @@ See the License for the specific language governing permissions and limitations under the License. --> - + iotdb-parent org.apache.iotdb 0.14.0-SNAPSHOT - ../pom.xml 4.0.0 schema-engine-tag schema-engine-tag - org.roaringbitmap RoaringBitmap 0.9.32 - org.apache.iotdb iotdb-lsm ${project.version} - provided org.apache.iotdb @@ -47,5 +41,4 @@ ${project.version} - - \ No newline at end of file + diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java new file mode 100644 index 0000000000000..184ce789b99cd --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java @@ -0,0 +1,325 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion; + +import org.apache.iotdb.common.rpc.thrift.TSchemaNode; +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.metadata.LocalSchemaProcessor; +import org.apache.iotdb.db.metadata.mnode.IMNode; +import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; +import org.apache.iotdb.db.metadata.path.MeasurementPath; +import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.metadata.template.Template; +import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.qp.physical.crud.InsertPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan; +import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan; +import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan; +import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.dataset.ShowDevicesResult; +import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.Pair; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +public class MockTagSchemaRegion implements ISchemaRegion { + + @Override + public void init() throws MetadataException {} + + @Override + public void clear() {} + + @Override + public void forceMlog() {} + + @Override + public SchemaRegionId getSchemaRegionId() { + return null; + } + + @Override + public String getStorageGroupFullPath() { + return null; + } + + @Override + public void deleteSchemaRegion() throws MetadataException {} + + @Override + public boolean createSnapshot(File snapshotDir) { + return false; + } + + @Override + public void loadSnapshot(File latestSnapshotRootDir) {} + + @Override + public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException {} + + @Override + public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException {} + + @Override + public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException {} + + @Override + public boolean isPathExist(PartialPath path) throws MetadataException { + return false; + } + + @Override + public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, Map templateMap, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains) + throws MetadataException { + return 0; + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { + return null; + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, + int level, + boolean isPrefixMatch, + String key, + String value, + boolean isContains) + throws MetadataException { + return null; + } + + @Override + public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public List getNodesListInGivenLevel( + PartialPath pathPattern, + int nodeLevel, + boolean isPrefixMatch, + LocalSchemaProcessor.StorageGroupFilter filter) + throws MetadataException { + return null; + } + + @Override + public Set getChildNodePathInNextLevel(PartialPath pathPattern) + throws MetadataException { + return null; + } + + @Override + public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException { + return null; + } + + @Override + public Set getBelongedDevices(PartialPath timeseries) throws MetadataException { + return null; + } + + @Override + public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan) + throws MetadataException { + return null; + } + + @Override + public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public Pair, Integer> getMeasurementPathsWithAlias( + PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public List fetchSchema( + PartialPath pathPattern, Map templateMap) throws MetadataException { + return null; + } + + @Override + public Pair, Integer> showTimeseries( + ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { + return null; + } + + @Override + public List getAllMeasurementByDevicePath(PartialPath devicePath) + throws PathNotExistException { + return null; + } + + @Override + public IMNode getDeviceNode(PartialPath path) throws MetadataException { + return null; + } + + @Override + public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException { + return null; + } + + @Override + public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException {} + + @Override + public void upsertTagsAndAttributes( + String alias, + Map tagsMap, + Map attributesMap, + PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void addAttributes(Map attributesMap, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void addTags(Map tagsMap, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void dropTagsOrAttributes(Set keySet, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) + throws MetadataException, IOException { + return null; + } + + @Override + public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( + PartialPath devicePath, + String[] measurements, + Function getDataType, + boolean aligned) + throws MetadataException { + return null; + } + + @Override + public Set getPathsSetTemplate(String templateName) throws MetadataException { + return null; + } + + @Override + public Set getPathsUsingTemplate(String templateName) throws MetadataException { + return null; + } + + @Override + public boolean isTemplateAppendable(Template template, List measurements) + throws MetadataException { + return false; + } + + @Override + public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException {} + + @Override + public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException {} + + @Override + public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException {} + + @Override + public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template) + throws MetadataException {} + + @Override + public List getPathsUsingTemplate(int templateId) throws MetadataException { + return null; + } + + @Override + public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException { + return null; + } + + @Override + public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException {} +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java new file mode 100644 index 0000000000000..2f04d7c095a73 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion; + +public class TagSchemaRegion {} diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java index 4a17213c77ba8..a927c2ebbce79 100644 --- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java @@ -36,24 +36,24 @@ public class TagInvertedIndexTest { private String[][] record = new String[][] { - {"tag1=q", "tag2=a", "1"}, - {"tag1=q", "tag2=s", "2"}, - {"tag1=q", "tag2=a", "tag3=z", "3"}, - {"tag1=q", "tag2=s", "5"}, - {"tag1=w", "tag2=d", "6"}, - {"tag1=q", "tag2=d", "tag3=e", "7"}, - {"tag1=q", "tag3=v", "4"}, - {"tag1=r", "tag2=d", "9"}, - {"tag1=t", "tag2=f", "10"}, - {"tag1=t", "tag2=g", "8"}, - {"tag1=t", "tag2=h", "11"}, - {"tag1=y", "tag2=j", "13"}, - {"tag1=u", "tag2=k", "14"}, - {"tag1=q", "tag2=a", "tag3=l", "12"}, - {"tag1=q", "tag2=a", "tag3=x", "15"}, - {"tag1=q", "tag2=b", "tag3=x", "18"}, - {"tag1=y", "tag2=a", "tag4=z", "17"}, - {"tag1=q", "tag2=a", "tag4=z", "16"}, + {"tag1=q", "tag2=a", "1"}, + {"tag1=q", "tag2=s", "2"}, + {"tag1=q", "tag2=a", "tag3=z", "3"}, + {"tag1=q", "tag3=v", "4"}, + {"tag1=q", "tag2=s", "5"}, + {"tag1=w", "tag2=d", "6"}, + {"tag1=q", "tag2=d", "tag3=e", "7"}, + {"tag1=t", "tag2=g", "8"}, + {"tag1=r", "tag2=d", "9"}, + {"tag1=t", "tag2=f", "10"}, + {"tag1=t", "tag2=h", "11"}, + {"tag1=q", "tag2=a", "tag3=l", "12"}, + {"tag1=y", "tag2=j", "13"}, + {"tag1=u", "tag2=k", "14"}, + {"tag1=q", "tag2=a", "tag3=x", "15"}, + {"tag1=q", "tag2=a", "tag4=z", "16"}, + {"tag1=y", "tag2=a", "tag4=z", "17"}, + {"tag1=q", "tag2=b", "tag3=x", "18"}, }; private int numOfDeviceIdsInMemTable; @@ -86,9 +86,9 @@ public void removeTags() { tagInvertedIndex.removeTags(tags.left, tags.right); tags = generateTag(record[1]); tagInvertedIndex.removeTags(tags.left, tags.right); - tags = generateTag(record[6]); + tags = generateTag(record[3]); tagInvertedIndex.removeTags(tags.left, tags.right); - tags = generateTag(record[13]); + tags = generateTag(record[11]); tagInvertedIndex.removeTags(tags.left, tags.right); } From 7da09589521762ed75c68ddebb5e923bbe61ce25 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 14 Sep 2022 16:40:41 +0800 Subject: [PATCH 10/15] impl wal --- .../iotdb/lsm/context/DeleteContext.java | 9 ++ .../iotdb/lsm/context/InsertContext.java | 9 ++ .../org/apache/iotdb/lsm/example/Main.java | 38 +++--- .../iotdb/lsm/manager/BasicLsmManager.java | 16 +-- .../apache/iotdb/lsm/manager/LsmManager.java | 3 +- .../org/apache/iotdb/lsm/wal/IWALReader.java | 31 +++++ .../org/apache/iotdb/lsm/wal/IWALWriter.java | 30 +++++ .../org/apache/iotdb/lsm/wal/WALReader.java | 92 ++++++++++++++ .../org/apache/iotdb/lsm/wal/WALRecord.java | 38 ++++++ .../org/apache/iotdb/lsm/wal/WALWriter.java | 98 +++++++++++++++ .../tagIndex/TagInvertedIndex.java | 105 ++++++++++------ .../tagIndex/deletion/DeletionManager.java | 23 ++-- .../tagIndex/insertion/InsertionManager.java | 26 ++-- .../tagIndex/memtable/MemTable.java | 2 - .../tagIndex/query/QueryManager.java | 17 +-- .../tagIndex/recover/RecoverManager.java | 33 +++++ .../tagIndex/wal/WALEntry.java | 95 ++++++++++++++ .../tagIndex/wal/WALManager.java | 117 ++++++++++++++++++ .../tagIndex/TagInvertedIndexTest.java | 53 +++++--- 19 files changed, 717 insertions(+), 118 deletions(-) create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java index 46d78b9ed5131..2b99196b0c3aa 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java @@ -39,10 +39,19 @@ public DeleteContext(Object value, Object... ks) { accessStrategy = new PostOrderAccessStrategy(); } + public DeleteContext(Object value,List keys){ + this.value = value; + this.keys = keys; + } + public Object getKey() { return keys.get(level); } + public List getKeys() { + return keys; + } + public Object getValue() { return value; } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java index a34c9d5814ce8..e8d00da4979d7 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java @@ -40,10 +40,19 @@ public InsertContext(Object value, Object... keys) { accessStrategy = new PreOrderAccessStrategy(); } + public InsertContext(Object value,List keys){ + this.value = value; + this.keys = keys; + } + public Object getKey() { return keys.get(level); } + public List getKeys() { + return keys; + } + public Object getValue() { return value; } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java index bea5c80e15873..7dbe793ac5fcd 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java @@ -29,7 +29,7 @@ import java.util.Map; public class Main { - public static void main(String[] args) { + public static void main(String[] args) throws Exception { MemTableManager memTableManager = new MemTableManager(); System.out.println("-------------insert--------------"); insertionExample(memTableManager); @@ -37,10 +37,10 @@ public static void main(String[] args) { flushExample(memTableManager); } - public static void insertionExample(MemTableManager memTableManager) { + public static void insertionExample(MemTableManager memTableManager) throws Exception { BasicLsmManager baseLsmManager = - new BasicLsmManager().manager(memTableManager); + new BasicLsmManager(); baseLsmManager .nextLevel( new InsertLevelProcess() { @@ -127,25 +127,25 @@ public void insert(MemChunk memNode, InsertContext context) { } }); - baseLsmManager.process(new InsertContext(1, null, "a", "b")); - baseLsmManager.process(new InsertContext(2, null, "a", "d")); - baseLsmManager.process(new InsertContext(3, null, "a", "e")); - baseLsmManager.process(new InsertContext(4, null, "a", "b")); - baseLsmManager.process(new InsertContext(5, null, "a1", "b")); - baseLsmManager.process(new InsertContext(6, null, "a2", "b")); - baseLsmManager.process(new InsertContext(65535, null, "a", "b")); - baseLsmManager.process(new InsertContext(65536, null, "a", "b")); - baseLsmManager.process(new InsertContext(2, null, "a", "d")); - baseLsmManager.process(new InsertContext(3, null, "a", "e")); - baseLsmManager.process(new InsertContext(4, null, "a", "b")); - baseLsmManager.process(new InsertContext(5, null, "a1", "b")); - baseLsmManager.process(new InsertContext(6, null, "a2", "b")); + baseLsmManager.process(memTableManager, new InsertContext(1, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertContext(2, null, "a", "d")); + baseLsmManager.process(memTableManager, new InsertContext(3, null, "a", "e")); + baseLsmManager.process(memTableManager, new InsertContext(4, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertContext(5, null, "a1", "b")); + baseLsmManager.process(memTableManager, new InsertContext(6, null, "a2", "b")); + baseLsmManager.process(memTableManager, new InsertContext(65535, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertContext(65536, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertContext(2, null, "a", "d")); + baseLsmManager.process(memTableManager, new InsertContext(3, null, "a", "e")); + baseLsmManager.process(memTableManager, new InsertContext(4, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertContext(5, null, "a1", "b")); + baseLsmManager.process(memTableManager, new InsertContext(6, null, "a2", "b")); System.out.println(memTableManager); } - public static void flushExample(MemTableManager memTableManager) { + public static void flushExample(MemTableManager memTableManager) throws Exception { BasicLsmManager flushManager = - new BasicLsmManager().manager(memTableManager); + new BasicLsmManager(); flushManager .nextLevel( @@ -204,6 +204,6 @@ public List getChildren(MemChunk memNode, FlushContext context) { } }); - flushManager.process(new FlushContext()); + flushManager.process(memTableManager, new FlushContext()); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java index 3d33dd1712864..a5b91439662aa 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java @@ -21,21 +21,21 @@ import org.apache.iotdb.lsm.context.Context; import org.apache.iotdb.lsm.levelProcess.LevelProcess; -public class BasicLsmManager implements LsmManager { +import java.io.IOException; - T root; +public class BasicLsmManager implements LsmManager { LevelProcess levelProcess; - @Override - public BasicLsmManager manager(T memNode) { - root = memNode; - return this; - } + public void preProcess(T root, C context) throws Exception {} + + public void postProcess(T root, C context) throws Exception{} @Override - public void process(C context) { + public void process(T root, C context) throws Exception{ + preProcess(root, context); levelProcess.process(root, context); + postProcess(root, context); } @Override diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java index 3d60af6f69f8e..2940ad60b1751 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java @@ -22,9 +22,8 @@ import org.apache.iotdb.lsm.levelProcess.LevelProcess; public interface LsmManager { - LsmManager manager(T memNode); - void process(C context); + void process(T memNode, C context) throws Exception; LevelProcess nextLevel(LevelProcess next); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java new file mode 100644 index 0000000000000..6d52ef1577fcd --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import java.io.FileNotFoundException; +import java.io.IOException; + +public interface IWALReader { + + void close() throws IOException; + + boolean hasNext() throws FileNotFoundException; + + WALRecord next() throws FileNotFoundException; +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java new file mode 100644 index 0000000000000..6d3fecb21e2eb --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import java.io.IOException; + +public interface IWALWriter { + + void write(WALRecord walRecord) throws IOException; + + void force() throws IOException; + + void close() throws IOException; +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java new file mode 100644 index 0000000000000..a3fef56b0860d --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.NoSuchElementException; + +public class WALReader implements IWALReader { + private static final Logger logger = LoggerFactory.getLogger(WALReader.class); + private final File logFile; + private final WALRecord prototype; + private final DataInputStream logStream; + private WALRecord nextRecord; + private boolean fileCorrupted = false; + + public WALReader(File logFile, WALRecord prototype) throws IOException { + this.logFile = logFile; + this.logStream = + new DataInputStream(new BufferedInputStream(Files.newInputStream(logFile.toPath()))); + this.prototype = prototype; + } + + @Override + public void close() throws IOException { + logStream.close(); + } + + @Override + public boolean hasNext() { + if (nextRecord != null) { + return true; + } + try { + if (fileCorrupted) { + return false; + } + int logSize = logStream.readInt(); + if (logSize <= 0) { + return false; + } + nextRecord = prototype.clone(); + nextRecord.deserialize(logStream); + } catch (EOFException e) { + logger.info(""); + return false; + } catch (IOException e) { + logger.warn(""); + fileCorrupted = true; + return false; + } + return true; + } + + @Override + public WALRecord next() { + if (nextRecord == null) { + throw new NoSuchElementException(); + } + WALRecord walRecord = nextRecord; + nextRecord = null; + return walRecord; + } + + @Override + public String toString() { + return "WALReader{" + "logFile=" + logFile + '}'; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java new file mode 100644 index 0000000000000..a44b8f9d243c5 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public abstract class WALRecord implements Cloneable { + public abstract void serialize(ByteBuffer buffer); + + public abstract void deserialize(DataInputStream stream) throws IOException; + + @Override + public WALRecord clone() { + try { + return (WALRecord) super.clone(); + } catch (CloneNotSupportedException e) { + throw new AssertionError(); + } + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java new file mode 100644 index 0000000000000..dccd8a92233db --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.FileChannel; + +public class WALWriter implements IWALWriter { + private static final Logger logger = LoggerFactory.getLogger(WALWriter.class); + private File logFile; + private FileOutputStream fileOutputStream; + private FileChannel channel; + private final ByteBuffer lengthBuffer = ByteBuffer.allocate(4); + private final ByteBuffer walBuffer = ByteBuffer.allocate(10000); + private final boolean forceEachWrite; + + public WALWriter(File logFile, boolean forceEachWrite) throws FileNotFoundException { + this.logFile = logFile; + this.forceEachWrite = forceEachWrite; + fileOutputStream = new FileOutputStream(logFile, true); + channel = fileOutputStream.getChannel(); + } + + @Override + public void write(WALRecord walRecord) throws IOException { + if (channel == null) { + fileOutputStream = new FileOutputStream(logFile, true); + channel = fileOutputStream.getChannel(); + } + walBuffer.clear(); + walRecord.serialize(walBuffer); + walBuffer.flip(); + int logSize = walBuffer.limit(); + lengthBuffer.clear(); + lengthBuffer.putInt(logSize); + lengthBuffer.flip(); + + try { + channel.write(lengthBuffer); + channel.write(walBuffer); + + if (this.forceEachWrite) { + channel.force(true); + } + } catch (ClosedChannelException ignored) { + logger.warn("someone interrupt current thread, so no need to do write for io safety"); + } + } + + @Override + public void force() throws IOException { + if (channel != null && channel.isOpen()) { + channel.force(true); + } + } + + @Override + public void close() throws IOException { + if (channel != null) { + if (channel.isOpen()) { + channel.force(true); + } + fileOutputStream.close(); + fileOutputStream = null; + channel.close(); + channel = null; + } + } + + @Override + public String toString() { + return "WALLogWriter{" + "logFile=" + logFile + '}'; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java index 89c3260ed0e55..71c2ee923106a 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -18,18 +18,25 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.DeletionManager; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.InsertionManager; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.QueryManager; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.recover.RecoverManager; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; import org.apache.iotdb.lsm.context.DeleteContext; import org.apache.iotdb.lsm.context.InsertContext; import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.wal.WALWriter; import org.roaringbitmap.RoaringBitmap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -38,44 +45,61 @@ import java.util.stream.Collectors; public class TagInvertedIndex implements ITagInvertedIndex { + private static final Logger logger = LoggerFactory.getLogger(TagInvertedIndex.class); private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private int numOfDeviceIdsInMemTable; + private final InsertionManager insertionManager; - private MemTable workingMemTable; + private final DeletionManager deletionManager; + + private final QueryManager queryManager; + + private final WALManager walManager; + + private final RecoverManager recoverManager; - private MemTable unsequenceMemTable; + private final int numOfDeviceIdsInMemTable; - private Map immutableMemTables; + private final Map immutableMemTables; + + private MemTable workingMemTable; private int maxDeviceID; - public TagInvertedIndex() { + public TagInvertedIndex(String schemaDirPath) throws IOException { + walManager = new WALManager(schemaDirPath); + insertionManager = new InsertionManager(walManager); + deletionManager = new DeletionManager(walManager); + recoverManager = new RecoverManager(walManager); + queryManager = new QueryManager(); workingMemTable = new MemTable(MemTable.WORKING); - unsequenceMemTable = new MemTable(MemTable.UNSEQUENCE); immutableMemTables = new HashMap<>(); numOfDeviceIdsInMemTable = config.getNumOfDeviceIdsInMemTable(); maxDeviceID = 0; + recover(); + } + + public synchronized void recover(){ + recoverManager.recover(this); } @Override public synchronized void addTags(Map tags, int id) { MemTable memTable = null; - // 出现乱序 - if (id < maxDeviceID) { - memTable = unsequenceMemTable; - } else { - if (!inWorkingMemTable(id)) { - workingMemTable.setStatus(MemTable.IMMUTABLE); - immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); - workingMemTable = new MemTable(MemTable.WORKING); - } - memTable = workingMemTable; - maxDeviceID = id; + if (!inWorkingMemTable(id)) { + workingMemTable.setStatus(MemTable.IMMUTABLE); + immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); + workingMemTable = new MemTable(MemTable.WORKING); } - for (Map.Entry tag : tags.entrySet()) { - addTag(memTable, tag.getKey(), tag.getValue(), id); + memTable = workingMemTable; + maxDeviceID = id; + try { + for (Map.Entry tag : tags.entrySet()) { + addTag(memTable, tag.getKey(), tag.getValue(), id); + } + }catch (Exception e){ + logger.error(e.getMessage()); } } @@ -86,11 +110,14 @@ public synchronized void removeTags(Map tags, int id) { if (inWorkingMemTable(id)) { memTables.add(workingMemTable); } else { - memTables.add(unsequenceMemTable); memTables.add(immutableMemTables.get(id / numOfDeviceIdsInMemTable)); } - for (Map.Entry tag : tags.entrySet()) { - removeTag(memTables, tag.getKey(), tag.getValue(), id); + try { + for (Map.Entry tag : tags.entrySet()) { + removeTag(memTables, tag.getKey(), tag.getValue(), id); + } + }catch (Exception e){ + logger.error(e.getMessage()); } } @@ -98,15 +125,18 @@ public synchronized void removeTags(Map tags, int id) { public synchronized List getMatchedIDs(Map tags) { List memTables = new ArrayList<>(); memTables.add(workingMemTable); - memTables.add(unsequenceMemTable); memTables.addAll(immutableMemTables.values()); RoaringBitmap roaringBitmap = new RoaringBitmap(); int i = 0; - for (Map.Entry tag : tags.entrySet()) { - RoaringBitmap rb = getMatchedIDs(memTables, tag.getKey(), tag.getValue()); - if (i == 0) roaringBitmap = rb; - else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb); - i++; + try { + for (Map.Entry tag : tags.entrySet()) { + RoaringBitmap rb = getMatchedIDs(memTables, tag.getKey(), tag.getValue()); + if (i == 0) roaringBitmap = rb; + else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb); + i++; + } + }catch (Exception e){ + logger.error(e.getMessage()); } return Arrays.stream(roaringBitmap.toArray()).boxed().collect(Collectors.toList()); } @@ -118,8 +148,6 @@ public String toString() { + numOfDeviceIdsInMemTable + ", workingMemTable=" + workingMemTable - + ", unsequenceMemTable=" - + unsequenceMemTable + ", immutableMemTables=" + immutableMemTables + ", maxDeviceID=" @@ -131,23 +159,28 @@ private synchronized boolean inWorkingMemTable(int id) { return id / numOfDeviceIdsInMemTable == maxDeviceID / numOfDeviceIdsInMemTable; } - private void addTag(MemTable memTable, String tagKey, String tagValue, int id) { + private void addTag(MemTable memTable, String tagKey, String tagValue, int id) throws Exception { InsertContext insertContext = new InsertContext(id, tagKey, tagValue); - InsertionManager.getInstance().manager(memTable).process(insertContext); + insertionManager.process(memTable, insertContext); } - private void removeTag(List memTables, String tagKey, String tagValue, int id) { + private void removeTag(List memTables, String tagKey, String tagValue, int id) throws Exception { DeleteContext deleteContext = new DeleteContext(id, tagKey, tagValue); for (MemTable memTable : memTables) { - DeletionManager.getInstance().manager(memTable).process(deleteContext); + deletionManager.process(memTable, deleteContext); } } - private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue) { + private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue) throws Exception { QueryContext queryContext = new QueryContext(tagKey, tagValue); for (MemTable memTable : memTables) { - QueryManager.getInstance().manager(memTable).process(queryContext); + queryManager.process(memTable, queryContext); } return (RoaringBitmap) queryContext.getResult(); } + + @TestOnly + public void clear() throws IOException { + walManager.close(); + } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java index 4125a1097db18..0d8e62a769f67 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java @@ -19,23 +19,24 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; import org.apache.iotdb.lsm.context.DeleteContext; import org.apache.iotdb.lsm.manager.BasicLsmManager; +import java.io.IOException; + public class DeletionManager extends BasicLsmManager { - private DeletionManager() { - this.nextLevel(new MemTableDeletion()) - .nextLevel(new MemChunkGroupDeletion()) - .nextLevel(new MemChunkDeletion()); - } - public static DeletionManager getInstance() { - return DeletionManagerHolder.INSTANCE; - } + WALManager walManager; - private static class DeletionManagerHolder { - private static final DeletionManager INSTANCE = new DeletionManager(); + public DeletionManager(WALManager walManager) { + this.walManager = walManager; + initLevelProcess(); + } - private DeletionManagerHolder() {} + private void initLevelProcess() { + this.nextLevel(new MemTableDeletion()) + .nextLevel(new MemChunkGroupDeletion()) + .nextLevel(new MemChunkDeletion()); } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java index 3e6d71493bd30..4b4dd29ee8c62 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java @@ -19,24 +19,30 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALEntry; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; import org.apache.iotdb.lsm.context.InsertContext; import org.apache.iotdb.lsm.manager.BasicLsmManager; +import java.io.IOException; + public class InsertionManager extends BasicLsmManager { - private InsertionManager() { - this.nextLevel(new MemTableInsertion()) - .nextLevel(new MemChunkGroupInsertion()) - .nextLevel(new MemChunkInsertion()); - } + private WALManager walManager; - public static InsertionManager getInstance() { - return InsertionManagerHolder.INSTANCE; + public InsertionManager(WALManager walManager){ + this.walManager = walManager; + initLevelProcess(); } - private static class InsertionManagerHolder { - private static final InsertionManager INSTANCE = new InsertionManager(); + @Override + public void preProcess(MemTable root, InsertContext context) throws IOException { + walManager.write(context); + } - private InsertionManagerHolder() {} + private void initLevelProcess() { + this.nextLevel(new MemTableInsertion()) + .nextLevel(new MemChunkGroupInsertion()) + .nextLevel(new MemChunkInsertion()); } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java index af47581ac9c3d..f0730e162df1c 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java @@ -28,8 +28,6 @@ public class MemTable { public static final String IMMUTABLE = "immutable"; - public static final String UNSEQUENCE = "unsequence"; - private Map memChunkGroupMap; private String status; diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java index 8987c49801f36..4a8e3fdbcb0eb 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java @@ -23,19 +23,14 @@ import org.apache.iotdb.lsm.manager.BasicLsmManager; public class QueryManager extends BasicLsmManager { - private QueryManager() { - this.nextLevel(new MemTableQuery()) - .nextLevel(new MemChunkGroupQuery()) - .nextLevel(new MemChunkQuery()); - } - public static QueryManager getInstance() { - return QueryManagerHolder.INSTANCE; + public QueryManager() { + initLevelProcess(); } - private static class QueryManagerHolder { - private static final QueryManager INSTANCE = new QueryManager(); - - private QueryManagerHolder() {} + private void initLevelProcess() { + this.nextLevel(new MemTableQuery()) + .nextLevel(new MemChunkGroupQuery()) + .nextLevel(new MemChunkQuery()); } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java new file mode 100644 index 0000000000000..2c3d38bc06278 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.recover; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.TagInvertedIndex; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; + +public class RecoverManager { + + WALManager walManager; + + public RecoverManager(WALManager walManager) { + this.walManager = walManager; + } + + public void recover(TagInvertedIndex tagInvertedIndex) {} +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java new file mode 100644 index 0000000000000..e70ad62100b10 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal; + +import org.apache.iotdb.lsm.wal.WALRecord; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class WALEntry extends WALRecord { + private int type; + + private List keys; + + private int deviceID; + + public WALEntry() {} + + public WALEntry(int type, List keys, int deviceID) { + this.type = type; + this.keys = keys; + this.deviceID = deviceID; + } + + @Override + public void serialize(ByteBuffer buffer) { + ReadWriteIOUtils.write(type, buffer); + ReadWriteIOUtils.write(deviceID, buffer); + ReadWriteIOUtils.write(keys.size(), buffer); + for (String key : keys) { + ReadWriteIOUtils.write(key, buffer); + } + } + + @Override + public void deserialize(DataInputStream stream) throws IOException { + this.type = stream.readInt(); + this.deviceID = stream.readInt(); + int length = stream.readInt(); + this.keys = new ArrayList<>(); + for (int i = 0; i < length; i++) { + String key = ReadWriteIOUtils.readString(stream); + keys.add(key); + } + } + + public int getType() { + return type; + } + + public void setType(int type) { + this.type = type; + } + + public List getKeys() { + return keys; + } + + public void setKeys(List keys) { + this.keys = keys; + } + + public int getDeviceID() { + return deviceID; + } + + public void setDeviceID(int deviceID) { + this.deviceID = deviceID; + } + + @Override + public String toString() { + return "WALEntry{" + "type=" + type + ", keys=" + keys + ", deviceID=" + deviceID + '}'; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java new file mode 100644 index 0000000000000..88bdf68dd3143 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.wal.WALReader; +import org.apache.iotdb.lsm.wal.WALWriter; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class WALManager { + private static final String WAL_FILE_NAME = "tagInvertedIndex.log"; + private static final int INSERT = 1; + private static final int DELETE = 2; + private final String schemaDirPath; + private File walFile; + private WALWriter walWriter; + private WALReader walReader; + + public WALManager(String schemaDirPath) throws IOException { + this.schemaDirPath = schemaDirPath; + initFile(schemaDirPath); + walWriter = new WALWriter(walFile, false); + walReader = new WALReader(walFile, new WALEntry()); + } + + private void initFile(String schemaDirPath) throws IOException { + File schemaDir = new File(schemaDirPath); + schemaDir.mkdirs(); + walFile = new File(this.schemaDirPath, WAL_FILE_NAME); + if (!walFile.exists()) { + walFile.createNewFile(); + } + } + + public synchronized void write(Context context) throws IOException { + switch (context.getType()) { + case INSERT: + process((InsertContext) context); + break; + case DELETE: + process((DeleteContext) context); + break; + default: + break; + } + } + + public synchronized Context read() { + if (walReader.hasNext()) { + WALEntry walEntry = (WALEntry) walReader.next(); + if (walEntry.getType() == INSERT) { + return generateInsertContext(walEntry); + } + if (walEntry.getType() == DELETE) { + return generateDeleteContext(walEntry); + } + } + return new Context(); + } + + private InsertContext generateInsertContext(WALEntry walEntry) { + InsertContext insertContext = new InsertContext(walEntry.getDeviceID(), walEntry.getKeys()); + return insertContext; + } + + private DeleteContext generateDeleteContext(WALEntry walEntry) { + DeleteContext deleteContext = new DeleteContext(walEntry.getDeviceID(), walEntry.getKeys()); + return deleteContext; + } + + private void process(InsertContext insertContext) throws IOException { + List objects = insertContext.getKeys(); + List keys = new ArrayList<>(); + for (Object o : objects) { + keys.add((String) o); + } + WALEntry walEntry = new WALEntry(INSERT, keys, (Integer) insertContext.getValue()); + walWriter.write(walEntry); + } + + private void process(DeleteContext deleteContext) throws IOException { + List objects = deleteContext.getKeys(); + List keys = new ArrayList<>(); + for (Object o : objects) { + keys.add((String) o); + } + WALEntry walEntry = new WALEntry(DELETE, keys, (Integer) deleteContext.getValue()); + walWriter.write(walEntry); + } + + public void close() throws IOException { + walWriter.close(); + walReader.close(); + } +} diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java index a927c2ebbce79..eba4c1067d3f3 100644 --- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; +import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.tsfile.utils.Pair; @@ -25,6 +26,7 @@ import org.junit.Before; import org.junit.Test; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -36,42 +38,55 @@ public class TagInvertedIndexTest { private String[][] record = new String[][] { - {"tag1=q", "tag2=a", "1"}, - {"tag1=q", "tag2=s", "2"}, - {"tag1=q", "tag2=a", "tag3=z", "3"}, - {"tag1=q", "tag3=v", "4"}, - {"tag1=q", "tag2=s", "5"}, - {"tag1=w", "tag2=d", "6"}, - {"tag1=q", "tag2=d", "tag3=e", "7"}, - {"tag1=t", "tag2=g", "8"}, - {"tag1=r", "tag2=d", "9"}, - {"tag1=t", "tag2=f", "10"}, - {"tag1=t", "tag2=h", "11"}, - {"tag1=q", "tag2=a", "tag3=l", "12"}, - {"tag1=y", "tag2=j", "13"}, - {"tag1=u", "tag2=k", "14"}, - {"tag1=q", "tag2=a", "tag3=x", "15"}, - {"tag1=q", "tag2=a", "tag4=z", "16"}, - {"tag1=y", "tag2=a", "tag4=z", "17"}, - {"tag1=q", "tag2=b", "tag3=x", "18"}, + {"tag1=q", "tag2=a", "1"}, + {"tag1=q", "tag2=s", "2"}, + {"tag1=q", "tag2=a", "tag3=z", "3"}, + {"tag1=q", "tag3=v", "4"}, + {"tag1=q", "tag2=s", "5"}, + {"tag1=w", "tag2=d", "6"}, + {"tag1=q", "tag2=d", "tag3=e", "7"}, + {"tag1=t", "tag2=g", "8"}, + {"tag1=r", "tag2=d", "9"}, + {"tag1=t", "tag2=f", "10"}, + {"tag1=t", "tag2=h", "11"}, + {"tag1=q", "tag2=a", "tag3=l", "12"}, + {"tag1=y", "tag2=j", "13"}, + {"tag1=u", "tag2=k", "14"}, + {"tag1=q", "tag2=a", "tag3=x", "15"}, + {"tag1=q", "tag2=a", "tag4=z", "16"}, + {"tag1=y", "tag2=a", "tag4=z", "17"}, + {"tag1=q", "tag2=b", "tag3=x", "18"}, }; private int numOfDeviceIdsInMemTable; private TagInvertedIndex tagInvertedIndex; + private String storageGroupDirPath; + + private String schemaRegionDirPath; + + private String storageGroupFullPath = "root/testTagIndex"; + + private String schemaDir; + @Before public void setUp() throws Exception { numOfDeviceIdsInMemTable = IoTDBDescriptor.getInstance().getConfig().getNumOfDeviceIdsInMemTable(); IoTDBDescriptor.getInstance().getConfig().setNumOfDeviceIdsInMemTable(3); - tagInvertedIndex = new TagInvertedIndex(); + schemaDir = IoTDBDescriptor.getInstance().getConfig().getSchemaDir(); + storageGroupDirPath = schemaDir + File.separator + storageGroupFullPath; + schemaRegionDirPath = storageGroupDirPath + File.separator + 0; + tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath); } @After public void tearDown() throws Exception { IoTDBDescriptor.getInstance().getConfig().setNumOfDeviceIdsInMemTable(numOfDeviceIdsInMemTable); + tagInvertedIndex.clear(); tagInvertedIndex = null; + FileUtils.deleteDirectoryAndEmptyParent(new File(schemaDir)); } public void addTags() { From ff320a8514435ed56a166ebfdcb5f0e40e8eee83 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 14 Sep 2022 20:13:35 +0800 Subject: [PATCH 11/15] impl RecoverManager --- .../org/apache/iotdb/lsm/context/Context.java | 12 +++++ .../iotdb/lsm/context/DeleteContext.java | 13 ++++- .../iotdb/lsm/context/InsertContext.java | 18 +++++-- .../iotdb/lsm/manager/BasicLsmManager.java | 6 +-- .../tagIndex/ITagInvertedIndex.java | 8 +++ .../tagIndex/TagInvertedIndex.java | 52 +++++++++++++++---- .../tagIndex/deletion/DeletionManager.java | 9 +++- .../tagIndex/insertion/InsertionManager.java | 7 +-- .../tagIndex/recover/RecoverManager.java | 19 ++++++- .../tagIndex/wal/WALManager.java | 13 ++++- .../tagIndex/TagInvertedIndexTest.java | 30 ++++++++--- 11 files changed, 153 insertions(+), 34 deletions(-) diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java index 305b31747f6c3..3f369b3017734 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java @@ -41,12 +41,16 @@ public class Context { // 返回值 Object result; + // 是否正在recover + boolean recover; + public Context() { accessStrategy = new PreOrderAccessStrategy(); type = ContextType.NONE; level = 0; threadNums = 1; levelUpperBound = Integer.MAX_VALUE; + recover = false; } public void setLevel(int level) { @@ -96,4 +100,12 @@ public int getLevelUpperBound() { public void setLevelUpperBound(int levelUpperBound) { this.levelUpperBound = levelUpperBound; } + + public boolean isRecover() { + return recover; + } + + public void setRecover(boolean recover) { + this.recover = recover; + } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java index 2b99196b0c3aa..9b4b30ac41a75 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java @@ -30,6 +30,12 @@ public class DeleteContext extends Context { Object value; + public DeleteContext() { + super(); + type = ContextType.DELETE; + accessStrategy = new PostOrderAccessStrategy(); + } + public DeleteContext(Object value, Object... ks) { super(); this.value = value; @@ -39,11 +45,14 @@ public DeleteContext(Object value, Object... ks) { accessStrategy = new PostOrderAccessStrategy(); } - public DeleteContext(Object value,List keys){ - this.value = value; + public void setKeys(List keys) { this.keys = keys; } + public void setValue(Object value) { + this.value = value; + } + public Object getKey() { return keys.get(level); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java index e8d00da4979d7..b09d1540cbc47 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java @@ -30,23 +30,31 @@ public class InsertContext extends Context { Object value; + public InsertContext() { + super(); + type = ContextType.INSERT; + accessStrategy = new PreOrderAccessStrategy(); + } + public InsertContext(Object value, Object... keys) { super(); this.value = value; this.keys = new ArrayList<>(); this.keys.addAll(Arrays.asList(keys)); - level = 0; type = ContextType.INSERT; accessStrategy = new PreOrderAccessStrategy(); } - public InsertContext(Object value,List keys){ - this.value = value; + public Object getKey() { + return keys.get(level); + } + + public void setKeys(List keys) { this.keys = keys; } - public Object getKey() { - return keys.get(level); + public void setValue(Object value) { + this.value = value; } public List getKeys() { diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java index a5b91439662aa..1855f04c0fe48 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java @@ -21,18 +21,16 @@ import org.apache.iotdb.lsm.context.Context; import org.apache.iotdb.lsm.levelProcess.LevelProcess; -import java.io.IOException; - public class BasicLsmManager implements LsmManager { LevelProcess levelProcess; public void preProcess(T root, C context) throws Exception {} - public void postProcess(T root, C context) throws Exception{} + public void postProcess(T root, C context) throws Exception {} @Override - public void process(T root, C context) throws Exception{ + public void process(T root, C context) throws Exception { preProcess(root, context); levelProcess.process(root, context); postProcess(root, context); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java index cf5e1a19c0ea7..d5269bbce987e 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java @@ -18,12 +18,20 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.InsertContext; + import java.util.List; import java.util.Map; public interface ITagInvertedIndex { + + void addTags(InsertContext context); + void addTags(Map tags, int id); + void removeTags(DeleteContext context); + void removeTags(Map tags, int id); List getMatchedIDs(Map tags); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java index 71c2ee923106a..b76ed104d85fa 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -31,7 +31,6 @@ import org.apache.iotdb.lsm.context.InsertContext; import org.apache.iotdb.lsm.context.QueryContext; -import org.apache.iotdb.lsm.wal.WALWriter; import org.roaringbitmap.RoaringBitmap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,25 +79,57 @@ public TagInvertedIndex(String schemaDirPath) throws IOException { recover(); } - public synchronized void recover(){ + public synchronized void recover() { recoverManager.recover(this); } + @Override + public synchronized void addTags(InsertContext context) { + int id = (int) context.getValue(); + if (!inWorkingMemTable(id)) { + workingMemTable.setStatus(MemTable.IMMUTABLE); + immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); + workingMemTable = new MemTable(MemTable.WORKING); + } + MemTable memTable = workingMemTable; + maxDeviceID = id; + try { + insertionManager.process(memTable, context); + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + @Override public synchronized void addTags(Map tags, int id) { - MemTable memTable = null; if (!inWorkingMemTable(id)) { workingMemTable.setStatus(MemTable.IMMUTABLE); immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); workingMemTable = new MemTable(MemTable.WORKING); } - memTable = workingMemTable; + MemTable memTable = workingMemTable; maxDeviceID = id; try { for (Map.Entry tag : tags.entrySet()) { addTag(memTable, tag.getKey(), tag.getValue(), id); } - }catch (Exception e){ + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + + @Override + public void removeTags(DeleteContext context) { + int id = (int) context.getValue(); + MemTable memTable = null; + if (inWorkingMemTable(id)) { + memTable = workingMemTable; + } else { + memTable = immutableMemTables.get(id / numOfDeviceIdsInMemTable); + } + try { + deletionManager.process(memTable, context); + } catch (Exception e) { logger.error(e.getMessage()); } } @@ -106,7 +137,6 @@ public synchronized void addTags(Map tags, int id) { @Override public synchronized void removeTags(Map tags, int id) { List memTables = new ArrayList<>(); - // 出现乱序 if (inWorkingMemTable(id)) { memTables.add(workingMemTable); } else { @@ -116,7 +146,7 @@ public synchronized void removeTags(Map tags, int id) { for (Map.Entry tag : tags.entrySet()) { removeTag(memTables, tag.getKey(), tag.getValue(), id); } - }catch (Exception e){ + } catch (Exception e) { logger.error(e.getMessage()); } } @@ -135,7 +165,7 @@ public synchronized List getMatchedIDs(Map tags) { else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb); i++; } - }catch (Exception e){ + } catch (Exception e) { logger.error(e.getMessage()); } return Arrays.stream(roaringBitmap.toArray()).boxed().collect(Collectors.toList()); @@ -164,14 +194,16 @@ private void addTag(MemTable memTable, String tagKey, String tagValue, int id) t insertionManager.process(memTable, insertContext); } - private void removeTag(List memTables, String tagKey, String tagValue, int id) throws Exception { + private void removeTag(List memTables, String tagKey, String tagValue, int id) + throws Exception { DeleteContext deleteContext = new DeleteContext(id, tagKey, tagValue); for (MemTable memTable : memTables) { deletionManager.process(memTable, deleteContext); } } - private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue) throws Exception { + private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue) + throws Exception { QueryContext queryContext = new QueryContext(tagKey, tagValue); for (MemTable memTable : memTables) { queryManager.process(memTable, queryContext); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java index 0d8e62a769f67..f31b6461b2ca3 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java @@ -23,8 +23,6 @@ import org.apache.iotdb.lsm.context.DeleteContext; import org.apache.iotdb.lsm.manager.BasicLsmManager; -import java.io.IOException; - public class DeletionManager extends BasicLsmManager { WALManager walManager; @@ -34,6 +32,13 @@ public DeletionManager(WALManager walManager) { initLevelProcess(); } + @Override + public void preProcess(MemTable root, DeleteContext context) throws Exception { + if (!context.isRecover()) { + walManager.write(context); + } + } + private void initLevelProcess() { this.nextLevel(new MemTableDeletion()) .nextLevel(new MemChunkGroupDeletion()) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java index 4b4dd29ee8c62..0e203a3d646ba 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; -import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALEntry; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; import org.apache.iotdb.lsm.context.InsertContext; import org.apache.iotdb.lsm.manager.BasicLsmManager; @@ -30,14 +29,16 @@ public class InsertionManager extends BasicLsmManager { private WALManager walManager; - public InsertionManager(WALManager walManager){ + public InsertionManager(WALManager walManager) { this.walManager = walManager; initLevelProcess(); } @Override public void preProcess(MemTable root, InsertContext context) throws IOException { - walManager.write(context); + if (!context.isRecover()) { + walManager.write(context); + } } private void initLevelProcess() { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java index 2c3d38bc06278..98b4ee5745059 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java @@ -20,6 +20,9 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.TagInvertedIndex; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.InsertContext; public class RecoverManager { @@ -29,5 +32,19 @@ public RecoverManager(WALManager walManager) { this.walManager = walManager; } - public void recover(TagInvertedIndex tagInvertedIndex) {} + public void recover(TagInvertedIndex tagInvertedIndex) { + while (true) { + Context context = walManager.read(); + switch (context.getType()) { + case INSERT: + tagInvertedIndex.addTags((InsertContext) context); + break; + case DELETE: + tagInvertedIndex.removeTags((DeleteContext) context); + break; + default: + return; + } + } + } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java index 88bdf68dd3143..b0ac296f6c15f 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java @@ -67,6 +67,7 @@ public synchronized void write(Context context) throws IOException { } } + // 用于recover public synchronized Context read() { if (walReader.hasNext()) { WALEntry walEntry = (WALEntry) walReader.next(); @@ -81,12 +82,22 @@ public synchronized Context read() { } private InsertContext generateInsertContext(WALEntry walEntry) { - InsertContext insertContext = new InsertContext(walEntry.getDeviceID(), walEntry.getKeys()); + InsertContext insertContext = new InsertContext(); + List objects = new ArrayList<>(); + objects.addAll(walEntry.getKeys()); + insertContext.setKeys(objects); + insertContext.setValue(walEntry.getDeviceID()); + insertContext.setRecover(true); return insertContext; } private DeleteContext generateDeleteContext(WALEntry walEntry) { DeleteContext deleteContext = new DeleteContext(walEntry.getDeviceID(), walEntry.getKeys()); + List objects = new ArrayList<>(); + objects.addAll(walEntry.getKeys()); + deleteContext.setKeys(objects); + deleteContext.setValue(walEntry.getDeviceID()); + deleteContext.setRecover(true); return deleteContext; } diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java index eba4c1067d3f3..f193862a68592 100644 --- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java @@ -27,6 +27,7 @@ import org.junit.Test; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -110,8 +111,6 @@ public void removeTags() { @Test public void getMatchedIDs() { addTags(); - System.out.println("------------addTags------------"); - System.out.println(tagInvertedIndex); Map tags1 = new HashMap<>(); tags1.put("tag1", "q"); @@ -128,8 +127,7 @@ public void getMatchedIDs() { assertEquals(verify, ids); removeTags(); - System.out.println("------------removeTags------------"); - System.out.println(tagInvertedIndex); + ids = tagInvertedIndex.getMatchedIDs(tags1); verify = Arrays.asList(3, 5, 7, 15, 16, 18); assertEquals(verify, ids); @@ -137,9 +135,29 @@ public void getMatchedIDs() { ids = tagInvertedIndex.getMatchedIDs(tags2); verify = Arrays.asList(3, 15, 16); assertEquals(verify, ids); + } + + @Test + public void testRecover() throws IOException { + Map tags1 = new HashMap<>(); + tags1.put("tag1", "q"); - System.out.println("------------query------------"); - System.out.println(tagInvertedIndex); + Map tags2 = new HashMap<>(); + tags2.put("tag1", "q"); + tags2.put("tag2", "a"); + addTags(); + removeTags(); + + tagInvertedIndex.clear(); + tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath); + + List ids = tagInvertedIndex.getMatchedIDs(tags1); + List verify = Arrays.asList(3, 5, 7, 15, 16, 18); + assertEquals(verify, ids); + + ids = tagInvertedIndex.getMatchedIDs(tags2); + verify = Arrays.asList(3, 15, 16); + assertEquals(verify, ids); } private List, Integer>> generateTags() { From 9ca86d89649e9ab86070797c760c21e19f385eb2 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 14 Sep 2022 20:14:37 +0800 Subject: [PATCH 12/15] impl RecoverManager --- .../db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java index b76ed104d85fa..9ae8b0ddb2dce 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -185,7 +185,7 @@ public String toString() { + '}'; } - private synchronized boolean inWorkingMemTable(int id) { + private boolean inWorkingMemTable(int id) { return id / numOfDeviceIdsInMemTable == maxDeviceID / numOfDeviceIdsInMemTable; } From b4758ae45d894abbadea43af8edb1927f54a3ccb Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 14 Sep 2022 20:15:47 +0800 Subject: [PATCH 13/15] impl RecoverManager --- .../tagSchemaRegion/tagIndex/recover/RecoverManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java index 98b4ee5745059..906fafb25503b 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.recover; -import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.TagInvertedIndex; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.ITagInvertedIndex; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; import org.apache.iotdb.lsm.context.Context; import org.apache.iotdb.lsm.context.DeleteContext; @@ -32,7 +32,7 @@ public RecoverManager(WALManager walManager) { this.walManager = walManager; } - public void recover(TagInvertedIndex tagInvertedIndex) { + public void recover(ITagInvertedIndex tagInvertedIndex) { while (true) { Context context = walManager.read(); switch (context.getType()) { From dbabba977290249da4f96194f195ddccdcccef09 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 14 Sep 2022 20:20:05 +0800 Subject: [PATCH 14/15] impl RecoverManager --- .../tagSchemaRegion/tagIndex/deletion/DeletionManager.java | 2 +- .../tagSchemaRegion/tagIndex/recover/RecoverManager.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java index f31b6461b2ca3..211eed0bbe944 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java @@ -25,7 +25,7 @@ public class DeletionManager extends BasicLsmManager { - WALManager walManager; + private WALManager walManager; public DeletionManager(WALManager walManager) { this.walManager = walManager; diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java index 906fafb25503b..ec9fd74729565 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java @@ -26,7 +26,7 @@ public class RecoverManager { - WALManager walManager; + private WALManager walManager; public RecoverManager(WALManager walManager) { this.walManager = walManager; From 1f9338ae7e83dd6e740e8aafbd279221c58f98df Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 14 Sep 2022 20:36:38 +0800 Subject: [PATCH 15/15] impl RecoverManager --- lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java | 1 + .../db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java index a44b8f9d243c5..4410b6f8a2007 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java @@ -23,6 +23,7 @@ import java.nio.ByteBuffer; public abstract class WALRecord implements Cloneable { + public abstract void serialize(ByteBuffer buffer); public abstract void deserialize(DataInputStream stream) throws IOException; diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java index e70ad62100b10..6cf24548492a2 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java @@ -34,9 +34,10 @@ public class WALEntry extends WALRecord { private int deviceID; - public WALEntry() {} + public WALEntry() {super();} public WALEntry(int type, List keys, int deviceID) { + super(); this.type = type; this.keys = keys; this.deviceID = deviceID;