From 5aed5adff34e36fc1d7ec2c825be2a472ecbf388 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Sun, 18 Sep 2022 16:42:09 +0800 Subject: [PATCH 01/30] Influxdb protocol adapts tag schema region --- .../iotdb/influxdb/InfluxDBExample.java | 2 +- .../influxdb/integration/IoTDBInfluxDBIT.java | 7 +- .../db/protocol/influxdb/dto/IoTDBPoint.java | 7 +- .../handler/AbstractQueryHandler.java | 66 ++---- .../influxdb/handler/NewQueryHandler.java | 83 +++----- .../influxdb/handler/QueryHandler.java | 56 +----- .../influxdb/handler/QueryHandlerFactory.java | 42 ++++ .../influxdb/handler/TagQueryHandler.java | 120 +++++++++++ .../meta/AbstractInfluxDBMetaManager.java | 15 +- .../influxdb/meta/IInfluxDBMetaManager.java | 39 ++++ .../influxdb/meta/InfluxDBMetaManager.java | 50 +++++ .../meta/InfluxDBMetaManagerFactory.java | 42 ++++ .../influxdb/meta/NewInfluxDBMetaManager.java | 32 ++- .../influxdb/meta/TagInfluxDBMetaManager.java | 189 ++++++++++++++++++ .../influxdb/util/QueryResultUtils.java | 102 +++++++++- .../thrift/impl/InfluxDBServiceImpl.java | 3 +- .../thrift/impl/NewInfluxDBServiceImpl.java | 32 ++- 17 files changed, 706 insertions(+), 181 deletions(-) create mode 100644 server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandlerFactory.java create mode 100644 server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java create mode 100644 server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/IInfluxDBMetaManager.java create mode 100644 server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManagerFactory.java create mode 100644 server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/TagInfluxDBMetaManager.java diff --git a/example/influxdb-protocol-example/src/main/java/org/apache/iotdb/influxdb/InfluxDBExample.java b/example/influxdb-protocol-example/src/main/java/org/apache/iotdb/influxdb/InfluxDBExample.java index 64a5d178226a7..cdb884dfc1b27 100644 --- a/example/influxdb-protocol-example/src/main/java/org/apache/iotdb/influxdb/InfluxDBExample.java +++ b/example/influxdb-protocol-example/src/main/java/org/apache/iotdb/influxdb/InfluxDBExample.java @@ -102,7 +102,7 @@ private static void queryData() { query = new Query( - "select count(temperature),first(temperature),last(temperature),max(temperature),mean(temperature),median(temperature),min(temperature),mode(temperature),spread(temperature),stddev(temperature),sum(temperature) from student where ((workshop=\"A1\" and production=\"B1\" and cell =\"C1\" ) or temperature< 15 )", + "select count(temperature),first(temperature),last(temperature),max(temperature),mean(temperature),median(temperature),min(temperature),mode(temperature),spread(temperature),stddev(temperature),sum(temperature) from factory where ((workshop=\"A1\" and production=\"B1\" and cell =\"C1\" ) or temperature< 15 )", database); result = influxDB.query(query); System.out.println("query2 result:" + result.getResults().get(0).getSeries().get(0).toString()); diff --git a/influxdb-protocol/src/test/java/org/apache/iotdb/influxdb/integration/IoTDBInfluxDBIT.java b/influxdb-protocol/src/test/java/org/apache/iotdb/influxdb/integration/IoTDBInfluxDBIT.java index d8c97dc2b4ab5..047cada0f145d 100644 --- a/influxdb-protocol/src/test/java/org/apache/iotdb/influxdb/integration/IoTDBInfluxDBIT.java +++ b/influxdb-protocol/src/test/java/org/apache/iotdb/influxdb/integration/IoTDBInfluxDBIT.java @@ -33,11 +33,15 @@ import org.testcontainers.containers.GenericContainer; import org.testcontainers.utility.DockerImageName; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class IoTDBInfluxDBIT { @@ -164,8 +168,9 @@ public void testCommonQueryColumn() { QueryResult.Series series = result.getResults().get(0).getSeries().get(0); String[] retArray = new String[] {"time", "name", "sex", "province", "country", "score", "tel"}; + Set columnNames = new HashSet<>(Arrays.asList(retArray)); for (int i = 0; i < series.getColumns().size(); i++) { - assertEquals(retArray[i], series.getColumns().get(i)); + assertTrue(columnNames.contains(series.getColumns().get(i))); } } diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java index ac7c1ce9842e1..9f5aea7983839 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.protocol.influxdb.meta.AbstractInfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.IInfluxDBMetaManager; import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; import org.apache.iotdb.db.utils.DataTypeUtils; import org.apache.iotdb.db.utils.ParameterUtils; @@ -59,7 +59,7 @@ public IoTDBPoint( } public IoTDBPoint( - String database, Point point, AbstractInfluxDBMetaManager metaManager, long sessionID) { + String database, Point point, IInfluxDBMetaManager influxDBMetaManager, long sessionID) { String measurement = null; Map tags = new HashMap<>(); Map fields = new HashMap<>(); @@ -105,7 +105,8 @@ public IoTDBPoint( } ParameterUtils.checkNonEmptyString(database, "database"); ParameterUtils.checkNonEmptyString(measurement, "measurement name"); - String path = metaManager.generatePath(database, measurement, tags, sessionID); + String path = + influxDBMetaManager.generatePath(database, measurement, tags, fields.keySet(), sessionID); List measurements = new ArrayList<>(); List types = new ArrayList<>(); List values = new ArrayList<>(); diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java index 78899f7e34bb3..10bf4dced9281 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java @@ -29,7 +29,7 @@ import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; import org.apache.iotdb.db.protocol.influxdb.function.aggregator.InfluxAggregator; import org.apache.iotdb.db.protocol.influxdb.function.selector.InfluxSelector; -import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManagerFactory; import org.apache.iotdb.db.protocol.influxdb.operator.InfluxQueryOperator; import org.apache.iotdb.db.protocol.influxdb.operator.InfluxSelectComponent; import org.apache.iotdb.db.protocol.influxdb.util.FilterUtils; @@ -40,7 +40,6 @@ import org.apache.iotdb.db.qp.logical.Operator; import org.apache.iotdb.db.qp.logical.crud.BasicFunctionOperator; import org.apache.iotdb.db.qp.logical.crud.FilterOperator; -import org.apache.iotdb.db.service.basic.ServiceProvider; import org.apache.iotdb.protocol.influxdb.rpc.thrift.InfluxQueryResultRsp; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -57,30 +56,24 @@ public abstract class AbstractQueryHandler { - abstract Map getFieldOrders( - String database, String measurement, ServiceProvider serviceProvider, long sessionId); - abstract InfluxFunctionValue updateByIoTDBFunc( - InfluxFunction function, ServiceProvider serviceProvider, String path, long sessionid); + String database, String measurement, InfluxFunction function, long sessionid); abstract QueryResult queryByConditions( String querySql, String database, String measurement, - ServiceProvider serviceProvider, + Map tagOrders, Map fieldOrders, long sessionId) throws AuthException; public final InfluxQueryResultRsp queryInfluxDB( - String database, - InfluxQueryOperator queryOperator, - long sessionId, - ServiceProvider serviceProvider) { + String database, InfluxQueryOperator queryOperator, long sessionId) { String measurement = queryOperator.getFromComponent().getPrefixPaths().get(0).getFullPath(); // The list of fields under the current measurement and the order of the specified rules Map fieldOrders = - getFieldOrders(database, measurement, serviceProvider, sessionId); + InfluxDBMetaManagerFactory.getInstance().getFieldOrders(database, measurement, sessionId); QueryResult queryResult; InfluxQueryResultRsp tsQueryResultRsp = new InfluxQueryResultRsp(); try { @@ -96,7 +89,6 @@ public final InfluxQueryResultRsp queryInfluxDB( : null, database, measurement, - serviceProvider, fieldOrders, sessionId); // step2 : select filter @@ -106,11 +98,7 @@ public final InfluxQueryResultRsp queryInfluxDB( else { queryResult = queryFuncWithoutFilter( - queryOperator.getSelectComponent(), - database, - measurement, - serviceProvider, - sessionId); + queryOperator.getSelectComponent(), database, measurement, sessionId); } return tsQueryResultRsp .setResultJsonString(JacksonUtils.bean2Json(queryResult)) @@ -274,18 +262,14 @@ else if (selectComponent.isHasCommonQuery()) { * @param selectComponent select data to query * @return select query result */ - public final QueryResult queryFuncWithoutFilter( - InfluxSelectComponent selectComponent, - String database, - String measurement, - ServiceProvider serviceProvider, - long sessionid) { + public QueryResult queryFuncWithoutFilter( + InfluxSelectComponent selectComponent, String database, String measurement, long sessionid) { // columns List columns = new ArrayList<>(); columns.add(InfluxSQLConstant.RESERVED_TIME); List functions = new ArrayList<>(); - String path = "root." + database + "." + measurement; + for (ResultColumn resultColumn : selectComponent.getResultColumns()) { Expression expression = resultColumn.getExpression(); if (expression instanceof FunctionExpression) { @@ -300,7 +284,7 @@ public final QueryResult queryFuncWithoutFilter( List> values = new ArrayList<>(); for (InfluxFunction function : functions) { InfluxFunctionValue functionValue = - updateByIoTDBFunc(function, serviceProvider, path, sessionid); + updateByIoTDBFunc(database, measurement, function, sessionid); // InfluxFunctionValue functionValue = function.calculateByIoTDBFunc(); if (value.size() == 0) { value.add(functionValue.getTimestamp()); @@ -330,40 +314,33 @@ public QueryResult queryExpr( FilterOperator operator, String database, String measurement, - ServiceProvider serviceProvider, Map fieldOrders, Long sessionId) throws AuthException { if (operator == null) { List expressions = new ArrayList<>(); - return queryByConditions( - expressions, database, measurement, serviceProvider, fieldOrders, sessionId); + return queryByConditions(expressions, database, measurement, fieldOrders, sessionId); } else if (operator instanceof BasicFunctionOperator) { List iExpressions = new ArrayList<>(); iExpressions.add(getIExpressionForBasicFunctionOperator((BasicFunctionOperator) operator)); - return queryByConditions( - iExpressions, database, measurement, serviceProvider, fieldOrders, sessionId); + return queryByConditions(iExpressions, database, measurement, fieldOrders, sessionId); } else { FilterOperator leftOperator = operator.getChildren().get(0); FilterOperator rightOperator = operator.getChildren().get(1); if (operator.getFilterType() == FilterConstant.FilterType.KW_OR) { return QueryResultUtils.orQueryResultProcess( - queryExpr(leftOperator, database, measurement, serviceProvider, fieldOrders, sessionId), - queryExpr( - rightOperator, database, measurement, serviceProvider, fieldOrders, sessionId)); + queryExpr(leftOperator, database, measurement, fieldOrders, sessionId), + queryExpr(rightOperator, database, measurement, fieldOrders, sessionId)); } else if (operator.getFilterType() == FilterConstant.FilterType.KW_AND) { if (canMergeOperator(leftOperator) && canMergeOperator(rightOperator)) { List iExpressions1 = getIExpressionByFilterOperatorOperator(leftOperator); List iExpressions2 = getIExpressionByFilterOperatorOperator(rightOperator); iExpressions1.addAll(iExpressions2); - return queryByConditions( - iExpressions1, database, measurement, serviceProvider, fieldOrders, sessionId); + return queryByConditions(iExpressions1, database, measurement, fieldOrders, sessionId); } else { return QueryResultUtils.andQueryResultProcess( - queryExpr( - leftOperator, database, measurement, serviceProvider, fieldOrders, sessionId), - queryExpr( - rightOperator, database, measurement, serviceProvider, fieldOrders, sessionId)); + queryExpr(leftOperator, database, measurement, fieldOrders, sessionId), + queryExpr(rightOperator, database, measurement, fieldOrders, sessionId)); } } } @@ -376,11 +353,10 @@ public QueryResult queryExpr( * @param expressions list of conditions, including tag and field condition * @return returns the results of the influxdb query */ - private QueryResult queryByConditions( + public QueryResult queryByConditions( List expressions, String database, String measurement, - ServiceProvider serviceProvider, Map fieldOrders, Long sessionId) throws AuthException { @@ -390,7 +366,8 @@ private QueryResult queryByConditions( List fieldExpressions = new ArrayList<>(); // maximum number of tags in the current query criteria int currentQueryMaxTagNum = 0; - Map tagOrders = InfluxDBMetaManager.getTagOrders(database, measurement); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, sessionId); for (IExpression expression : expressions) { SingleSeriesExpression singleSeriesExpression = ((SingleSeriesExpression) expression); // the current condition is in tag @@ -445,8 +422,7 @@ private QueryResult queryByConditions( realQuerySql += " where " + realIotDBCondition; } realQuerySql += " align by device"; - return queryByConditions( - realQuerySql, database, measurement, serviceProvider, fieldOrders, sessionId); + return queryByConditions(realQuerySql, database, measurement, null, fieldOrders, sessionId); } /** diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java index ee8d0db5c06b8..3d536df10a326 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java @@ -18,77 +18,31 @@ */ package org.apache.iotdb.db.protocol.influxdb.handler; -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.db.protocol.influxdb.constant.InfluxConstant; import org.apache.iotdb.db.protocol.influxdb.constant.InfluxSQLConstant; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunction; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; -import org.apache.iotdb.db.protocol.influxdb.meta.NewInfluxDBMetaManager; import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; -import org.apache.iotdb.db.service.basic.ServiceProvider; import org.apache.iotdb.db.service.thrift.impl.NewInfluxDBServiceImpl; -import org.apache.iotdb.rpc.TSStatusCode; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; -import org.influxdb.InfluxDBException; import org.influxdb.dto.QueryResult; -import java.util.HashMap; import java.util.List; import java.util.Map; public class NewQueryHandler extends AbstractQueryHandler { - public static TSExecuteStatementResp executeStatement(String sql, long sessionId) { - TSExecuteStatementReq tsExecuteStatementReq = new TSExecuteStatementReq(); - tsExecuteStatementReq.setStatement(sql); - tsExecuteStatementReq.setSessionId(sessionId); - tsExecuteStatementReq.setStatementId( - NewInfluxDBServiceImpl.getClientRPCService().requestStatementId(sessionId)); - tsExecuteStatementReq.setFetchSize(InfluxConstant.DEFAULT_FETCH_SIZE); - TSExecuteStatementResp executeStatementResp = - NewInfluxDBServiceImpl.getClientRPCService().executeStatement(tsExecuteStatementReq); - TSStatus tsStatus = executeStatementResp.getStatus(); - if (tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - throw new InfluxDBException(tsStatus.getMessage()); - } - return executeStatementResp; - } - - @Override - public Map getFieldOrders( - String database, String measurement, ServiceProvider serviceProvider, long sessionID) { - Map fieldOrders = new HashMap<>(); - String showTimeseriesSql = "show timeseries root." + database + '.' + measurement + ".**"; - TSExecuteStatementResp executeStatementResp = executeStatement(showTimeseriesSql, sessionID); - List paths = QueryResultUtils.getFullPaths(executeStatementResp); - Map tagOrders = NewInfluxDBMetaManager.getTagOrders(database, measurement); - int tagOrderNums = tagOrders.size(); - int fieldNums = 0; - for (String path : paths) { - String filed = StringUtils.getFieldByPath(path); - if (!fieldOrders.containsKey(filed)) { - // The corresponding order of fields is 1 + tagNum (the first is timestamp, then all tags, - // and finally all fields) - fieldOrders.put(filed, tagOrderNums + fieldNums + 1); - fieldNums++; - } - } - return fieldOrders; - } - - @Override - public InfluxFunctionValue updateByIoTDBFunc( - InfluxFunction function, ServiceProvider serviceProvider, String path, long sessionid) { + public final InfluxFunctionValue updateByIoTDBFunc( + String path, InfluxFunction function, long sessionid) { switch (function.getFunctionName()) { case InfluxSQLConstant.COUNT: { String functionSql = StringUtils.generateFunctionSql( function.getFunctionName(), function.getParmaName(), path); - TSExecuteStatementResp tsExecuteStatementResp = executeStatement(functionSql, sessionid); + TSExecuteStatementResp tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSql, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { @@ -101,7 +55,7 @@ public InfluxFunctionValue updateByIoTDBFunc( String functionSqlCount = StringUtils.generateFunctionSql("count", function.getParmaName(), path); TSExecuteStatementResp tsExecuteStatementResp = - executeStatement(functionSqlCount, sessionid); + NewInfluxDBServiceImpl.executeStatement(functionSqlCount, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { @@ -109,7 +63,8 @@ public InfluxFunctionValue updateByIoTDBFunc( } String functionSqlSum = StringUtils.generateFunctionSql("sum", function.getParmaName(), path); - tsExecuteStatementResp = executeStatement(functionSqlSum, sessionid); + tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSqlSum, sessionid); list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { function.updateValueIoTDBFunc(null, influxFunctionValue); @@ -120,7 +75,8 @@ public InfluxFunctionValue updateByIoTDBFunc( { String functionSql = StringUtils.generateFunctionSql("sum", function.getParmaName(), path); - TSExecuteStatementResp tsExecuteStatementResp = executeStatement(functionSql, sessionid); + TSExecuteStatementResp tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSql, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { @@ -142,7 +98,8 @@ public InfluxFunctionValue updateByIoTDBFunc( StringUtils.generateFunctionSql("last_value", function.getParmaName(), path); functionName = "last_value"; } - TSExecuteStatementResp tsExecuteStatementResp = executeStatement(functionSql, sessionid); + TSExecuteStatementResp tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSql, sessionid); Map map = QueryResultUtils.getColumnNameAndValue(tsExecuteStatementResp); for (String colume : map.keySet()) { Object o = map.get(colume); @@ -152,7 +109,8 @@ public InfluxFunctionValue updateByIoTDBFunc( String.format( "select %s from %s where %s=%s", function.getParmaName(), devicePath, fullPath, o); - TSExecuteStatementResp resp = executeStatement(specificSql, sessionid); + TSExecuteStatementResp resp = + NewInfluxDBServiceImpl.executeStatement(specificSql, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(resp); for (InfluxFunctionValue influxFunctionValue : list) { function.updateValueIoTDBFunc(influxFunctionValue); @@ -171,7 +129,8 @@ public InfluxFunctionValue updateByIoTDBFunc( functionSql = StringUtils.generateFunctionSql("min_value", function.getParmaName(), path); } - TSExecuteStatementResp tsExecuteStatementResp = executeStatement(functionSql, sessionid); + TSExecuteStatementResp tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSql, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { @@ -185,15 +144,23 @@ public InfluxFunctionValue updateByIoTDBFunc( return function.calculateByIoTDBFunc(); } + @Override + public InfluxFunctionValue updateByIoTDBFunc( + String database, String measurement, InfluxFunction function, long sessionid) { + String path = "root." + database + "." + measurement; + return updateByIoTDBFunc(path, function, sessionid); + } + @Override public QueryResult queryByConditions( String querySql, String database, String measurement, - ServiceProvider serviceProvider, + Map tagOrders, Map fieldOrders, long sessionId) { - TSExecuteStatementResp executeStatementResp = executeStatement(querySql, sessionId); + TSExecuteStatementResp executeStatementResp = + NewInfluxDBServiceImpl.executeStatement(querySql, sessionId); return QueryResultUtils.iotdbResultConvertInfluxResult( executeStatementResp, database, measurement, fieldOrders); } diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java index b58b65f6bf5c8..3a895bcb9a2bb 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java @@ -27,14 +27,13 @@ import org.apache.iotdb.db.protocol.influxdb.constant.InfluxSQLConstant; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunction; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; -import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManager; import org.apache.iotdb.db.protocol.influxdb.util.FieldUtils; import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; -import org.apache.iotdb.db.qp.physical.PhysicalPlan; import org.apache.iotdb.db.qp.physical.crud.QueryPlan; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.control.SessionManager; +import org.apache.iotdb.db.service.IoTDB; import org.apache.iotdb.db.service.basic.ServiceProvider; import org.apache.iotdb.tsfile.exception.filter.QueryFilterOptimizationException; import org.apache.iotdb.tsfile.read.common.Field; @@ -49,62 +48,17 @@ import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; public class QueryHandler extends AbstractQueryHandler { - @Override - public Map getFieldOrders( - String database, String measurement, ServiceProvider serviceProvider, long sessionID) { - Map fieldOrders = new HashMap<>(); - long queryId = ServiceProvider.SESSION_MANAGER.requestQueryId(true); - try { - String showTimeseriesSql = "show timeseries root." + database + '.' + measurement + ".**"; - PhysicalPlan physicalPlan = - serviceProvider.getPlanner().parseSQLToPhysicalPlan(showTimeseriesSql); - QueryContext queryContext = - serviceProvider.genQueryContext( - queryId, - true, - System.currentTimeMillis(), - showTimeseriesSql, - InfluxConstant.DEFAULT_CONNECTION_TIMEOUT_MS); - QueryDataSet queryDataSet = - serviceProvider.createQueryDataSet( - queryContext, physicalPlan, InfluxConstant.DEFAULT_FETCH_SIZE); - int fieldNums = 0; - Map tagOrders = InfluxDBMetaManager.getTagOrders(database, measurement); - int tagOrderNums = tagOrders.size(); - while (queryDataSet.hasNext()) { - List fields = queryDataSet.next().getFields(); - String filed = StringUtils.getFieldByPath(fields.get(0).getStringValue()); - if (!fieldOrders.containsKey(filed)) { - // The corresponding order of fields is 1 + tagNum (the first is timestamp, then all tags, - // and finally all fields) - fieldOrders.put(filed, tagOrderNums + fieldNums + 1); - fieldNums++; - } - } - } catch (QueryProcessException - | TException - | StorageEngineException - | SQLException - | IOException - | InterruptedException - | QueryFilterOptimizationException - | MetadataException e) { - throw new InfluxDBException(e.getMessage()); - } finally { - ServiceProvider.SESSION_MANAGER.releaseQueryResourceNoExceptions(queryId); - } - return fieldOrders; - } + ServiceProvider serviceProvider = IoTDB.serviceProvider; @Override public InfluxFunctionValue updateByIoTDBFunc( - InfluxFunction function, ServiceProvider serviceProvider, String path, long sessionid) { + String database, String measurement, InfluxFunction function, long sessionid) { + String path = "root." + database + "." + measurement; switch (function.getFunctionName()) { case InfluxSQLConstant.COUNT: { @@ -481,7 +435,7 @@ public QueryResult queryByConditions( String querySql, String database, String measurement, - ServiceProvider serviceProvider, + Map tagOrders, Map fieldOrders, long sessionId) throws AuthException { diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandlerFactory.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandlerFactory.java new file mode 100644 index 0000000000000..bfdf413a9c30b --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandlerFactory.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.handler; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.schemaregion.SchemaEngineMode; +import org.apache.iotdb.db.service.thrift.impl.ClientRPCServiceImpl; + +public class QueryHandlerFactory { + public static AbstractQueryHandler getInstance() { + if (IoTDBDescriptor.getInstance() + .getConfig() + .getRpcImplClassName() + .equals(ClientRPCServiceImpl.class.getName())) { + switch (SchemaEngineMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { + case Tag: + return new TagQueryHandler(); + default: + return new NewQueryHandler(); + } + } else { + return new QueryHandler(); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java new file mode 100644 index 0000000000000..0f48caa5409f3 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.handler; + +import org.apache.iotdb.commons.auth.AuthException; +import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunction; +import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; +import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManagerFactory; +import org.apache.iotdb.db.protocol.influxdb.util.FilterUtils; +import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; +import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; +import org.apache.iotdb.db.service.thrift.impl.NewInfluxDBServiceImpl; +import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; +import org.apache.iotdb.tsfile.read.expression.IExpression; +import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression; + +import org.influxdb.dto.QueryResult; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** use in tag schema region */ +public class TagQueryHandler extends NewQueryHandler { + + @Override + public InfluxFunctionValue updateByIoTDBFunc( + String database, String measurement, InfluxFunction function, long sessionid) { + String path = "root." + database + ".measurement." + measurement; + return updateByIoTDBFunc(path, function, sessionid); + } + + @Override + public QueryResult queryByConditions( + String querySql, + String database, + String measurement, + Map tagOrders, + Map fieldOrders, + long sessionId) { + TSExecuteStatementResp executeStatementResp = + NewInfluxDBServiceImpl.executeStatement(querySql, sessionId); + return QueryResultUtils.iotdbResultConvertInfluxResult( + executeStatementResp, database, measurement, tagOrders, fieldOrders); + } + + @Override + public QueryResult queryByConditions( + List expressions, + String database, + String measurement, + Map fieldOrders, + Long sessionId) { + List fieldExpressions = new ArrayList<>(); + List tagExpressions = new ArrayList<>(); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, sessionId); + for (IExpression expression : expressions) { + SingleSeriesExpression singleSeriesExpression = ((SingleSeriesExpression) expression); + // the current condition is in tag + if (tagOrders.containsKey(singleSeriesExpression.getSeriesPath().getFullPath())) { + tagExpressions.add(singleSeriesExpression); + } else { + fieldExpressions.add(singleSeriesExpression); + } + } + // construct the actual query path + StringBuilder curQueryPath = + new StringBuilder("root." + database + ".measurement." + measurement); + for (SingleSeriesExpression singleSeriesExpression : tagExpressions) { + String tagKey = singleSeriesExpression.getSeriesPath().getFullPath(); + String tagValue = + StringUtils.removeQuotation( + FilterUtils.getFilterStringValue(singleSeriesExpression.getFilter())); + curQueryPath.append(".").append(tagKey).append(".").append(tagValue); + } + curQueryPath.append(".**"); + + // construct actual query condition + StringBuilder realIotDBCondition = new StringBuilder(); + for (int i = 0; i < fieldExpressions.size(); i++) { + SingleSeriesExpression singleSeriesExpression = fieldExpressions.get(i); + if (i != 0) { + realIotDBCondition.append(" and "); + } + realIotDBCondition + .append(singleSeriesExpression.getSeriesPath().getFullPath()) + .append(" ") + .append((FilterUtils.getFilerSymbol(singleSeriesExpression.getFilter()))) + .append(" ") + .append(FilterUtils.getFilterStringValue(singleSeriesExpression.getFilter())); + } + // actual query SQL statement + String realQuerySql; + + realQuerySql = "select * from " + curQueryPath; + if (!(realIotDBCondition.length() == 0)) { + realQuerySql += " where " + realIotDBCondition; + } + realQuerySql += " align by device"; + return queryByConditions( + realQuerySql, database, measurement, tagOrders, fieldOrders, sessionId); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java index 513b06e59f7b6..176230f39d91d 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java @@ -22,8 +22,9 @@ import java.util.HashMap; import java.util.Map; +import java.util.Set; -public abstract class AbstractInfluxDBMetaManager { +public abstract class AbstractInfluxDBMetaManager implements IInfluxDBMetaManager { protected static final String SELECT_TAG_INFO_SQL = "select database_name,measurement_name,tag_name,tag_order from root.TAG_INFO "; @@ -32,7 +33,8 @@ public abstract class AbstractInfluxDBMetaManager { protected static Map>> database2Measurement2TagOrders = new HashMap<>(); - public static Map getTagOrders(String database, String measurement) { + @Override + public Map getTagOrders(String database, String measurement, long sessionID) { Map tagOrders = new HashMap<>(); Map> measurement2TagOrders = database2Measurement2TagOrders.get(database); @@ -45,8 +47,6 @@ public static Map getTagOrders(String database, String measurem return tagOrders; } - abstract void recover(); - abstract void setStorageGroup(String database, long sessionID); abstract void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID); @@ -69,8 +69,13 @@ public final synchronized Map getTagOrdersWithAutoCreatingSchem return createDatabase(database, sessionID).computeIfAbsent(measurement, m -> new HashMap<>()); } + @Override public final synchronized String generatePath( - String database, String measurement, Map tags, long sessionID) { + String database, + String measurement, + Map tags, + Set fields, + long sessionID) { Map tagKeyToLayerOrders = getTagOrdersWithAutoCreatingSchema(database, measurement, sessionID); // to support rollback if fails to persisting new tag info diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/IInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/IInfluxDBMetaManager.java new file mode 100644 index 0000000000000..e3804ca5e76b5 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/IInfluxDBMetaManager.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.meta; + +import java.util.Map; +import java.util.Set; + +/** used to manage influxdb metadata */ +public interface IInfluxDBMetaManager { + + void recover(); + + Map getFieldOrders(String database, String measurement, long sessionId); + + String generatePath( + String database, + String measurement, + Map tags, + Set fields, + long sessionID); + + Map getTagOrders(String database, String measurement, long sessionID); +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java index f2e58de977ede..e59d96dfbee3c 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java @@ -25,7 +25,10 @@ import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.StorageGroupNotSetException; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.protocol.influxdb.constant.InfluxConstant; +import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; import org.apache.iotdb.db.qp.Planner; +import org.apache.iotdb.db.qp.physical.PhysicalPlan; import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; import org.apache.iotdb.db.qp.physical.crud.QueryPlan; import org.apache.iotdb.db.qp.physical.sys.SetStorageGroupPlan; @@ -142,6 +145,53 @@ public void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID) } } + @Override + public Map getFieldOrders(String database, String measurement, long sessionID) { + Map fieldOrders = new HashMap<>(); + long queryId = ServiceProvider.SESSION_MANAGER.requestQueryId(true); + try { + String showTimeseriesSql = "show timeseries root." + database + '.' + measurement + ".**"; + PhysicalPlan physicalPlan = + serviceProvider.getPlanner().parseSQLToPhysicalPlan(showTimeseriesSql); + QueryContext queryContext = + serviceProvider.genQueryContext( + queryId, + true, + System.currentTimeMillis(), + showTimeseriesSql, + InfluxConstant.DEFAULT_CONNECTION_TIMEOUT_MS); + QueryDataSet queryDataSet = + serviceProvider.createQueryDataSet( + queryContext, physicalPlan, InfluxConstant.DEFAULT_FETCH_SIZE); + int fieldNums = 0; + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, sessionID); + int tagOrderNums = tagOrders.size(); + while (queryDataSet.hasNext()) { + List fields = queryDataSet.next().getFields(); + String filed = StringUtils.getFieldByPath(fields.get(0).getStringValue()); + if (!fieldOrders.containsKey(filed)) { + // The corresponding order of fields is 1 + tagNum (the first is timestamp, then all tags, + // and finally all fields) + fieldOrders.put(filed, tagOrderNums + fieldNums + 1); + fieldNums++; + } + } + } catch (QueryProcessException + | TException + | StorageEngineException + | SQLException + | IOException + | InterruptedException + | QueryFilterOptimizationException + | MetadataException e) { + throw new InfluxDBException(e.getMessage()); + } finally { + ServiceProvider.SESSION_MANAGER.releaseQueryResourceNoExceptions(queryId); + } + return fieldOrders; + } + private static class InfluxDBMetaManagerHolder { private static final InfluxDBMetaManager INSTANCE = new InfluxDBMetaManager(); diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManagerFactory.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManagerFactory.java new file mode 100644 index 0000000000000..abad6b3e0ad08 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManagerFactory.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.meta; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.schemaregion.SchemaEngineMode; +import org.apache.iotdb.db.service.thrift.impl.ClientRPCServiceImpl; + +public class InfluxDBMetaManagerFactory { + public static IInfluxDBMetaManager getInstance() { + if (IoTDBDescriptor.getInstance() + .getConfig() + .getRpcImplClassName() + .equals(ClientRPCServiceImpl.class.getName())) { + switch (SchemaEngineMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { + case Tag: + return TagInfluxDBMetaManager.getInstance(); + default: + return NewInfluxDBMetaManager.getInstance(); + } + } else { + return InfluxDBMetaManager.getInstance(); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java index 5269a2bf443bf..10685303c6750 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java @@ -19,8 +19,8 @@ package org.apache.iotdb.db.protocol.influxdb.meta; import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.db.protocol.influxdb.handler.NewQueryHandler; import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; +import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; import org.apache.iotdb.db.service.thrift.impl.ClientRPCServiceImpl; import org.apache.iotdb.db.service.thrift.impl.NewInfluxDBServiceImpl; import org.apache.iotdb.rpc.IoTDBConnectionException; @@ -57,10 +57,13 @@ public void recover() { try { TSOpenSessionResp tsOpenSessionResp = clientRPCService.openSession( - new TSOpenSessionReq().setUsername("root").setPassword("root")); + new TSOpenSessionReq() + .setUsername("root") + .setPassword("root") + .setZoneId("Asia/Shanghai")); sessionID = tsOpenSessionResp.getSessionId(); TSExecuteStatementResp resp = - NewQueryHandler.executeStatement(SELECT_TAG_INFO_SQL, sessionID); + NewInfluxDBServiceImpl.executeStatement(SELECT_TAG_INFO_SQL, sessionID); IoTDBJDBCDataSet dataSet = QueryResultUtils.creatIoTJDBCDataset(resp); try { Map> measurement2TagOrders; @@ -121,6 +124,29 @@ public void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID) } } + @Override + public Map getFieldOrders(String database, String measurement, long sessionID) { + Map fieldOrders = new HashMap<>(); + String showTimeseriesSql = "show timeseries root." + database + '.' + measurement + ".**"; + TSExecuteStatementResp executeStatementResp = + NewInfluxDBServiceImpl.executeStatement(showTimeseriesSql, sessionID); + List paths = QueryResultUtils.getFullPaths(executeStatementResp); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, sessionID); + int tagOrderNums = tagOrders.size(); + int fieldNums = 0; + for (String path : paths) { + String filed = StringUtils.getFieldByPath(path); + if (!fieldOrders.containsKey(filed)) { + // The corresponding order of fields is 1 + tagNum (the first is timestamp, then all tags, + // and finally all fields) + fieldOrders.put(filed, tagOrderNums + fieldNums + 1); + fieldNums++; + } + } + return fieldOrders; + } + private static class InfluxDBMetaManagerHolder { private static final NewInfluxDBMetaManager INSTANCE = new NewInfluxDBMetaManager(); diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/TagInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/TagInfluxDBMetaManager.java new file mode 100644 index 0000000000000..87fd7da44e7f5 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/TagInfluxDBMetaManager.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.meta; + +import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; +import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; +import org.apache.iotdb.db.service.thrift.impl.NewInfluxDBServiceImpl; +import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +/** use in tag schema region */ +public class TagInfluxDBMetaManager implements IInfluxDBMetaManager { + private static final String STORAGE_GROUP_PATH = "root.influxdbmate"; + + private static final String TAGS_SET = "set.tags"; + + private static final String FIELDS_SET = "set.fields"; + + private TagInfluxDBMetaManager() {} + + public static TagInfluxDBMetaManager getInstance() { + return TagInfluxDBMetaManagerHolder.INSTANCE; + } + + /** use tag schema region to save state information, no need to recover here */ + @Override + public void recover() {} + + /** + * get the fields information of influxdb corresponding database and measurement through tag + * schema region + * + * @param database influxdb database + * @param measurement influxdb measurement + * @param sessionId session id + * @return field information + */ + @Override + public Map getFieldOrders(String database, String measurement, long sessionId) { + return getTimeseriesFieldOrders(database, measurement, FIELDS_SET, sessionId); + } + + /** + * convert the database,measurement,and tags of influxdb to device path of IoTDB,and save the tags + * and fields information of the database and measurement to the tag schema region + * + * @param database influxdb database + * @param measurement influxdb measurement + * @param tags influxdb tags + * @param fields influxdb fields + * @param sessionID session id + * @return device path + */ + @Override + public String generatePath( + String database, + String measurement, + Map tags, + Set fields, + long sessionID) { + createInfluxDBMetaTimeseries(database, measurement, tags, fields, sessionID); + return generateDevicesPath(database, measurement, tags); + } + + private void createInfluxDBMetaTimeseries( + String database, + String measurement, + Map tags, + Set fields, + long sessionID) { + List fieldsList = new ArrayList<>(tags.keySet()); + createInfluxDBMetaTimeseries(database, measurement, TAGS_SET, fieldsList, sessionID); + fieldsList.clear(); + fieldsList.addAll(fields); + createInfluxDBMetaTimeseries(database, measurement, FIELDS_SET, fieldsList, sessionID); + } + + private void createInfluxDBMetaTimeseries( + String database, String measurement, String device, List fields, long sessionID) { + String statement = generateTimeseriesStatement(database, measurement, device, fields); + NewInfluxDBServiceImpl.executeStatement(statement, sessionID); + } + + private String generateTimeseriesStatement( + String database, String measurement, String device, List fields) { + StringBuilder timeseriesStatement = + new StringBuilder( + "create aligned timeseries " + + STORAGE_GROUP_PATH + + ".database." + + database + + ".measurement." + + measurement + + "." + + device + + "("); + for (int i = 0; i < fields.size() - 1; i++) { + String field = fields.get(i); + timeseriesStatement.append(field).append(" BOOLEAN, "); + } + timeseriesStatement.append(fields.get(fields.size() - 1)).append(" BOOLEAN)"); + return timeseriesStatement.toString(); + } + + /** + * get the tags information of influxdb corresponding database and measurement through tag schema + * region + * + * @param database influxdb database + * @param measurement influxdb measurement + * @param sessionID session id + * @return tags information + */ + @Override + public Map getTagOrders(String database, String measurement, long sessionID) { + return getTimeseriesFieldOrders(database, measurement, TAGS_SET, sessionID); + } + + private Map getTimeseriesFieldOrders( + String database, String measurement, String device, long sessionID) { + TSExecuteStatementResp statementResp = + NewInfluxDBServiceImpl.executeStatement( + "show timeseries " + + STORAGE_GROUP_PATH + + ".database." + + database + + ".measurement." + + measurement + + "." + + device, + sessionID); + List timeseriesPaths = QueryResultUtils.getFullPaths(statementResp); + Map fieldOrders = new HashMap<>(); + for (String timeseriesPath : timeseriesPaths) { + String field = StringUtils.getFieldByPath(timeseriesPath); + fieldOrders.put(field, fieldOrders.size()); + } + return fieldOrders; + } + + /** + * convert the database,measurement,and tags of influxdb to device path of IoTDB,ensure that + * influxdb records with the same semantics generate the same device path, so the device path is + * generated in order after sorting the tags + * + * @param database influxdb database + * @param measurement influxdb measurement + * @param tags influxdb tags + * @return device path + */ + private String generateDevicesPath( + String database, String measurement, Map tags) { + TreeMap tagsMap = new TreeMap<>(tags); + tagsMap.put("measurement", measurement); + StringBuilder devicePath = new StringBuilder("root." + database); + for (String tagKey : tagsMap.keySet()) { + devicePath.append(".").append(tagKey).append(".").append(tagsMap.get(tagKey)); + } + return devicePath.toString(); + } + + private static class TagInfluxDBMetaManagerHolder { + private static final TagInfluxDBMetaManager INSTANCE = new TagInfluxDBMetaManager(); + + private TagInfluxDBMetaManagerHolder() {} + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/util/QueryResultUtils.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/util/QueryResultUtils.java index 325971df2598c..25199f6429f3b 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/util/QueryResultUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/util/QueryResultUtils.java @@ -20,7 +20,7 @@ import org.apache.iotdb.db.protocol.influxdb.constant.InfluxConstant; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; -import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManagerFactory; import org.apache.iotdb.db.query.dataset.AlignByDeviceDataSet; import org.apache.iotdb.rpc.IoTDBJDBCDataSet; import org.apache.iotdb.rpc.StatementExecutionException; @@ -82,7 +82,8 @@ public static QueryResult iotdbResultConvertInfluxResult( QueryResult.Series series = new QueryResult.Series(); series.setName(measurement); // gets the reverse map of the tag - Map tagOrders = InfluxDBMetaManager.getTagOrders(database, measurement); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, -1); Map tagOrderReversed = tagOrders.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); @@ -294,6 +295,12 @@ public static boolean checkQueryResultNull(QueryResult queryResult) { return queryResult.getResults().get(0).getSeries() == null; } + /** + * parse time series paths from query results + * + * @param tsExecuteStatementResp query results + * @return time series paths + */ public static List getFullPaths(TSExecuteStatementResp tsExecuteStatementResp) { List res = new ArrayList<>(); IoTDBJDBCDataSet ioTDBJDBCDataSet = creatIoTJDBCDataset(tsExecuteStatementResp); @@ -309,6 +316,13 @@ public static List getFullPaths(TSExecuteStatementResp tsExecuteStatemen return res; } + /** + * Convert align by device query result of NewIoTDB to the query result of influxdb,used for + * Memory and schema_file schema region + * + * @param tsExecuteStatementResp NewIoTDB execute statement resp to be converted + * @return query results in influxdb format + */ public static QueryResult iotdbResultConvertInfluxResult( TSExecuteStatementResp tsExecuteStatementResp, String database, @@ -321,7 +335,8 @@ public static QueryResult iotdbResultConvertInfluxResult( QueryResult.Series series = new QueryResult.Series(); series.setName(measurement); // gets the reverse map of the tag - Map tagOrders = InfluxDBMetaManager.getTagOrders(database, measurement); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, -1); Map tagOrderReversed = tagOrders.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); @@ -381,6 +396,87 @@ public static QueryResult iotdbResultConvertInfluxResult( return queryResult; } + /** + * Convert align by device query result of NewIoTDB to the query result of influxdb,used for tag + * schema region + * + * @param tsExecuteStatementResp NewIoTDB execute statement resp to be converted + * @return query results in influxdb format + */ + public static QueryResult iotdbResultConvertInfluxResult( + TSExecuteStatementResp tsExecuteStatementResp, + String database, + String measurement, + Map tagOrders, + Map fieldOrders) { + if (tsExecuteStatementResp == null) { + return getNullQueryResult(); + } + // generate series + QueryResult.Series series = new QueryResult.Series(); + series.setName(measurement); + Map tagOrderReversed = + tagOrders.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + int tagSize = tagOrderReversed.size(); + Map fieldOrdersReversed = + fieldOrders.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + ArrayList tagList = new ArrayList<>(); + for (int i = 0; i < tagSize; i++) { + tagList.add(tagOrderReversed.get(i)); + } + + ArrayList fieldList = new ArrayList<>(); + for (int i = 0; i < fieldOrders.size(); i++) { + fieldList.add(fieldOrdersReversed.get(i)); + } + + ArrayList columns = new ArrayList<>(); + columns.add("time"); + columns.addAll(tagList); + columns.addAll(fieldList); + // insert columns into series + series.setColumns(columns); + List> values = new ArrayList<>(); + IoTDBJDBCDataSet ioTDBJDBCDataSet = creatIoTJDBCDataset(tsExecuteStatementResp); + try { + while (ioTDBJDBCDataSet.hasCachedResults()) { + Object[] value = new Object[columns.size()]; + ioTDBJDBCDataSet.constructOneRow(); + value[0] = Long.valueOf(ioTDBJDBCDataSet.getValueByName("Time")); + String deviceName = ioTDBJDBCDataSet.getValueByName("Device"); + String[] deviceNameList = deviceName.split("\\."); + for (int i = 2; i < deviceNameList.length; i += 2) { + if (tagOrders.containsKey(deviceNameList[i])) { + int position = tagOrders.get(deviceNameList[i]) + 1; + value[position] = deviceNameList[i + 1]; + } + } + for (int i = 3; i <= ioTDBJDBCDataSet.columnNameList.size(); i++) { + Object o = ioTDBJDBCDataSet.getObject(ioTDBJDBCDataSet.findColumnNameByIndex(i)); + if (o != null) { + // insert the value of filed into it + int position = fieldOrders.get(ioTDBJDBCDataSet.findColumnNameByIndex(i)) + tagSize + 1; + value[position] = o; + } + } + values.add(Arrays.asList(value)); + } + } catch (Exception e) { + e.printStackTrace(); + } + + series.setValues(values); + + QueryResult queryResult = new QueryResult(); + QueryResult.Result result = new QueryResult.Result(); + result.setSeries(new ArrayList<>(Arrays.asList(series))); + queryResult.setResults(new ArrayList<>(Arrays.asList(result))); + + return queryResult; + } + public static List getInfluxFunctionValues( TSExecuteStatementResp tsExecuteStatementResp) { IoTDBJDBCDataSet ioTDBJDBCDataSet = creatIoTJDBCDataset(tsExecuteStatementResp); diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InfluxDBServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InfluxDBServiceImpl.java index 89e5429dd19a4..8209b434876af 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InfluxDBServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InfluxDBServiceImpl.java @@ -151,8 +151,7 @@ public InfluxTSStatus createDatabase(InfluxCreateDatabaseReq req) { public InfluxQueryResultRsp query(InfluxQueryReq req) throws TException { Operator operator = InfluxDBLogicalGenerator.generate(req.command); queryHandler.checkInfluxDBQueryOperator(operator); - return queryHandler.queryInfluxDB( - req.database, (InfluxQueryOperator) operator, req.sessionId, IoTDB.serviceProvider); + return queryHandler.queryInfluxDB(req.database, (InfluxQueryOperator) operator, req.sessionId); } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java index 422bc27fd46af..1c40c94bc7d60 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java @@ -19,17 +19,17 @@ package org.apache.iotdb.db.service.thrift.impl; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.db.protocol.influxdb.constant.InfluxConstant; import org.apache.iotdb.db.protocol.influxdb.dto.IoTDBPoint; import org.apache.iotdb.db.protocol.influxdb.handler.AbstractQueryHandler; -import org.apache.iotdb.db.protocol.influxdb.handler.NewQueryHandler; +import org.apache.iotdb.db.protocol.influxdb.handler.QueryHandlerFactory; import org.apache.iotdb.db.protocol.influxdb.input.InfluxLineParser; -import org.apache.iotdb.db.protocol.influxdb.meta.AbstractInfluxDBMetaManager; -import org.apache.iotdb.db.protocol.influxdb.meta.NewInfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.IInfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManagerFactory; import org.apache.iotdb.db.protocol.influxdb.operator.InfluxQueryOperator; import org.apache.iotdb.db.protocol.influxdb.sql.InfluxDBLogicalGenerator; import org.apache.iotdb.db.protocol.influxdb.util.InfluxReqAndRespUtils; import org.apache.iotdb.db.qp.logical.Operator; -import org.apache.iotdb.db.service.IoTDB; import org.apache.iotdb.db.utils.DataTypeUtils; import org.apache.iotdb.protocol.influxdb.rpc.thrift.InfluxCloseSessionReq; import org.apache.iotdb.protocol.influxdb.rpc.thrift.InfluxCreateDatabaseReq; @@ -42,6 +42,8 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.TSCloseSessionReq; +import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; +import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; import org.apache.iotdb.service.rpc.thrift.TSInsertRecordReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp; @@ -57,13 +59,14 @@ public class NewInfluxDBServiceImpl implements IInfluxDBServiceWithHandler { private static final ClientRPCServiceImpl clientRPCService = new ClientRPCServiceImpl(); - private final AbstractInfluxDBMetaManager metaManager; + private final IInfluxDBMetaManager metaManager; private final AbstractQueryHandler queryHandler; public NewInfluxDBServiceImpl() { - metaManager = NewInfluxDBMetaManager.getInstance(); - queryHandler = new NewQueryHandler(); + metaManager = InfluxDBMetaManagerFactory.getInstance(); + metaManager.recover(); + queryHandler = QueryHandlerFactory.getInstance(); } public static ClientRPCServiceImpl getClientRPCService() { @@ -117,8 +120,19 @@ public InfluxTSStatus createDatabase(InfluxCreateDatabaseReq req) { public InfluxQueryResultRsp query(InfluxQueryReq req) throws TException { Operator operator = InfluxDBLogicalGenerator.generate(req.command); queryHandler.checkInfluxDBQueryOperator(operator); - return queryHandler.queryInfluxDB( - req.database, (InfluxQueryOperator) operator, req.sessionId, IoTDB.serviceProvider); + return queryHandler.queryInfluxDB(req.database, (InfluxQueryOperator) operator, req.sessionId); + } + + public static TSExecuteStatementResp executeStatement(String sql, long sessionId) { + TSExecuteStatementReq tsExecuteStatementReq = new TSExecuteStatementReq(); + tsExecuteStatementReq.setStatement(sql); + tsExecuteStatementReq.setSessionId(sessionId); + tsExecuteStatementReq.setStatementId( + NewInfluxDBServiceImpl.getClientRPCService().requestStatementId(sessionId)); + tsExecuteStatementReq.setFetchSize(InfluxConstant.DEFAULT_FETCH_SIZE); + TSExecuteStatementResp executeStatementResp = + NewInfluxDBServiceImpl.getClientRPCService().executeStatement(tsExecuteStatementReq); + return executeStatementResp; } @Override From 5460b77ebc619b115050f133d46d80dbb7f0a537 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Sun, 18 Sep 2022 16:51:53 +0800 Subject: [PATCH 02/30] Implement mock tag schema region --- .../metadata/schemaregion/SchemaEngine.java | 6 + .../schemaregion/SchemaEngineMode.java | 3 +- .../tagschemaregion/MockTagSchemaRegion.java | 927 ++++++++++++++++++ .../common/schematree/ClusterSchemaTree.java | 66 +- 4 files changed, 993 insertions(+), 9 deletions(-) create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/tagschemaregion/MockTagSchemaRegion.java diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java index 358d37f7b8bc2..7b3d243242dbd 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java @@ -35,6 +35,7 @@ import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; import org.apache.iotdb.db.metadata.mtree.ConfigMTree; import org.apache.iotdb.db.metadata.rescon.SchemaResourceManager; +import org.apache.iotdb.db.metadata.schemaregion.tagschemaregion.MockTagSchemaRegion; import org.apache.iotdb.db.metadata.visitor.SchemaExecutionVisitor; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; import org.apache.iotdb.external.api.ISeriesNumerLimiter; @@ -323,6 +324,11 @@ private ISchemaRegion createSchemaRegionWithoutExistenceCheck( new RSchemaRegionLoader() .loadRSchemaRegion(storageGroup, schemaRegionId, storageGroupMNode); break; + case Tag: + schemaRegion = + new MockTagSchemaRegion( + storageGroup, schemaRegionId, storageGroupMNode, seriesNumerLimiter); + break; default: throw new UnsupportedOperationException( String.format( diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngineMode.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngineMode.java index 9147b9374c018..6b3270e87612b 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngineMode.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngineMode.java @@ -22,5 +22,6 @@ public enum SchemaEngineMode { Memory, Schema_File, - Rocksdb_based + Rocksdb_based, + Tag } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/tagschemaregion/MockTagSchemaRegion.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/tagschemaregion/MockTagSchemaRegion.java new file mode 100644 index 0000000000000..e4122375469c5 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/tagschemaregion/MockTagSchemaRegion.java @@ -0,0 +1,927 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.metadata.schemaregion.tagschemaregion; + +import org.apache.iotdb.common.rpc.thrift.TSchemaNode; +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.AlignedTimeseriesException; +import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; +import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.metadata.LocalSchemaProcessor; +import org.apache.iotdb.db.metadata.idtable.IDTable; +import org.apache.iotdb.db.metadata.idtable.IDTableManager; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.InsertMeasurementMNode; +import org.apache.iotdb.db.metadata.idtable.entry.SHA256DeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.db.metadata.mnode.EntityMNode; +import org.apache.iotdb.db.metadata.mnode.IMNode; +import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; +import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; +import org.apache.iotdb.db.metadata.path.MeasurementPath; +import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.metadata.schemaregion.SchemaRegionUtils; +import org.apache.iotdb.db.metadata.template.Template; +import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.mpp.common.schematree.MeasurementSchemaInfo; +import org.apache.iotdb.db.qp.physical.crud.InsertPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan; +import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan; +import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan; +import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.dataset.ShowDevicesResult; +import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; +import org.apache.iotdb.external.api.ISeriesNumerLimiter; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; + +public class MockTagSchemaRegion implements ISchemaRegion { + + protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private final String TAIL = ".**"; + private final IStorageGroupMNode storageGroupMNode; + private String storageGroupFullPath; + private SchemaRegionId schemaRegionId; + + private Map>> tagInvertedIndex; + + private List deviceIDS; + + private IDTable idTable; + + private final ISeriesNumerLimiter seriesNumerLimiter; + + public MockTagSchemaRegion( + PartialPath storageGroup, + SchemaRegionId schemaRegionId, + IStorageGroupMNode storageGroupMNode, + ISeriesNumerLimiter seriesNumerLimiter) + throws MetadataException { + + storageGroupFullPath = storageGroup.getFullPath(); + this.schemaRegionId = schemaRegionId; + this.storageGroupMNode = storageGroupMNode; + this.deviceIDS = new ArrayList<>(); + this.seriesNumerLimiter = seriesNumerLimiter; + tagInvertedIndex = new ConcurrentHashMap<>(); + idTable = IDTableManager.getInstance().getIDTable(storageGroup); + init(); + } + + @NotNull + private Map pathToTags(String path) { + if (path.length() <= storageGroupFullPath.length()) return new TreeMap<>(); + String devicePath = path.substring(storageGroupFullPath.length() + 1); + String[] tags = devicePath.split("\\."); + Map tagsMap = new TreeMap<>(); + for (int i = 0; i < tags.length; i += 2) { + tagsMap.put(tags[i], tags[i + 1]); + } + return tagsMap; + } + + public String tagsToPath(Map tags) { + StringBuilder stringBuilder = new StringBuilder(storageGroupFullPath); + for (String tagKey : tags.keySet()) { + stringBuilder.append(".").append(tagKey).append(".").append(tags.get(tagKey)); + } + return stringBuilder.toString(); + } + + @Override + public void init() throws MetadataException { + if (!config.isEnableIDTableLogFile() + && config.getDeviceIDTransformationMethod().equals("SHA256")) { + throw new MetadataException( + "enableIDTableLogFile OR deviceIDTransformationMethod==\"Plain\""); + } + } + + @Override + public void clear() { + return; + } + + @Override + public void forceMlog() { + return; + } + + @Override + public SchemaRegionId getSchemaRegionId() { + return schemaRegionId; + } + + @Override + public String getStorageGroupFullPath() { + return storageGroupFullPath; + } + + @Override + public void deleteSchemaRegion() throws MetadataException { + return; + } + + @Override + public boolean createSnapshot(File snapshotDir) { + return false; + } + + @Override + public void loadSnapshot(File latestSnapshotRootDir) { + return; + } + + private void createTagInvertedIndex(PartialPath devicePath) { + IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath); + Map tagsMap = pathToTags(devicePath.getFullPath()); + + deviceIDS.add(deviceID); + + for (String tagkey : tagsMap.keySet()) { + String tagValue = tagsMap.get(tagkey); + Map> tagkeyMap = + tagInvertedIndex.computeIfAbsent(tagkey, key -> new HashMap<>()); + List ids = tagkeyMap.computeIfAbsent(tagValue, key -> new ArrayList<>()); + ids.add(deviceIDS.size() - 1); + } + } + + private void createTimeseries( + PartialPath path, + TSDataType dataType, + TSEncoding encoding, + CompressionType compressor, + Map props) + throws MetadataException { + createTimeseries( + new CreateTimeSeriesPlan(path, dataType, encoding, compressor, props, null, null, null), 0); + } + + private void createAlignedTimeSeries( + PartialPath prefixPath, + List measurements, + List dataTypes, + List encodings, + List compressors) + throws MetadataException { + createAlignedTimeSeries( + new CreateAlignedTimeSeriesPlan( + prefixPath, measurements, dataTypes, encodings, compressors, null, null, null)); + } + + @Override // [iotdb|newIotdb/创建非对齐时间序列] [newIotdb/insert 2自动创建时间序列] + public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException { + PartialPath devicePath = plan.getPath().getDevicePath(); + Map tags = pathToTags(devicePath.getFullPath()); + PartialPath path = new PartialPath(tagsToPath(tags) + "." + plan.getPath().getMeasurement()); + plan.setPath(path); + devicePath = plan.getPath().getDevicePath(); + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + if (deviceEntry != null) { + if (deviceEntry.isAligned()) { + throw new AlignedTimeseriesException( + "Timeseries under this entity is not aligned, please use createTimeseries or change entity.", + devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); + } else if (deviceEntry.getMeasurementMap().containsKey(plan.getPath().getMeasurement())) { + throw new PathAlreadyExistException( + devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); + } + } + idTable.createTimeseries(plan); + if (deviceEntry == null) { + createTagInvertedIndex(devicePath); + } + } + + @Override // [iotdb|newIotdb/对齐时间序列] [newIotdb/insert 2自动创建时间序列] + public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException { + PartialPath devicePath = plan.getPrefixPath(); + Map tags = pathToTags(devicePath.getFullPath()); + PartialPath path = new PartialPath(tagsToPath(tags)); + plan.setPrefixPath(path); + devicePath = plan.getPrefixPath(); + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + if (deviceEntry != null) { + if (!deviceEntry.isAligned()) { + throw new AlignedTimeseriesException( + "Timeseries under this entity is aligned, please use createAlignedTimeseries or change entity.", + devicePath.getFullPath()); + } else { + List measurements = plan.getMeasurements(); + List dataTypes = plan.getDataTypes(); + List encodings = plan.getEncodings(); + List compressors = plan.getCompressors(); + + List tmpMeasurements = new LinkedList<>(); + List tmpDataTypes = new LinkedList<>(); + List tmpEncodings = new LinkedList<>(); + List tmpCompressors = new LinkedList<>(); + for (int i = 0; i < measurements.size(); i++) { + String measurement = measurements.get(i); + if (!deviceEntry.getMeasurementMap().containsKey(measurement)) { + tmpMeasurements.add(measurements.get(i)); + tmpDataTypes.add(dataTypes.get(i)); + tmpEncodings.add(encodings.get(i)); + tmpCompressors.add(compressors.get(i)); + } + } + if (tmpMeasurements.size() == 0) + throw new PathAlreadyExistException(devicePath.getFullPath()); + plan.setMeasurements(tmpMeasurements); + plan.setDataTypes(tmpDataTypes); + plan.setEncodings(tmpEncodings); + plan.setCompressors(tmpCompressors); + } + } + idTable.createAlignedTimeseries(plan); + if (deviceEntry == null) { + createTagInvertedIndex(devicePath); + } + } + + @Override + public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public int constructSchemaBlackList(PathPatternTree patternTree) throws MetadataException { + return 0; + } + + @Override + public void rollbackSchemaBlackList(PathPatternTree patternTree) throws MetadataException {} + + @Override + public List fetchSchemaBlackList(PathPatternTree patternTree) + throws MetadataException { + return null; + } + + @Override + public void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws MetadataException {} + + @Override + public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public boolean isPathExist(PartialPath path) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + int res = 0; + List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); + for (IDeviceID deviceID : deviceIDs) { + res += idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap().keySet().size(); + } + return res; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, Map templateMap, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains) + throws MetadataException { + return 0; + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, + int level, + boolean isPrefixMatch, + String key, + String value, + boolean isContains) + throws MetadataException { + return null; + } + + @Override + public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + if (pathPattern.getFullPath().length() <= storageGroupFullPath.length()) { + return deviceIDS.size(); + } else { + return getDeviceIDsByInvertedIndex(pathPattern).size(); + } + } + + @Override + public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public List getNodesListInGivenLevel( + PartialPath pathPattern, + int nodeLevel, + boolean isPrefixMatch, + LocalSchemaProcessor.StorageGroupFilter filter) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getChildNodePathInNextLevel(PartialPath pathPattern) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getBelongedDevices(PartialPath timeseries) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override // [newIotdb/show timeseries] [newIotdb/count device] [newIotdb/count timeseries] + public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); + Set res = new HashSet<>(); + String devicePath = pathPattern.getFullPath(); + if (!devicePath.endsWith(TAIL) && !devicePath.equals(storageGroupFullPath)) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); + if (deviceEntry != null) { + res.add(pathPattern); + } + return res; + } + for (IDeviceID deviceID : deviceIDs) { + if (deviceID instanceof SHA256DeviceID) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map map = deviceEntry.getMeasurementMap(); + for (String m : map.keySet()) { + SchemaEntry schemaEntry = map.get(m); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + res.add( + new PartialPath( + diskSchemaEntry.seriesKey.substring( + 0, + diskSchemaEntry.seriesKey.length() + - diskSchemaEntry.measurementName.length() + - 1))); + break; + } + } else { + res.add(new PartialPath(deviceID.toStringID())); + } + } + return res; + } + + @Override + public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override // [newIotDB / insert1,3] [newIotDB/select] [newIotdb/select count()] [newIotdb/select + // .. groupby level] + public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + PartialPath devicePath = pathPattern.getDevicePath(); + // 批量查询.路径以".**"结尾,如： + // root.sg.tag1.a.** + // root.sg.tagx.c.tag2.v.** + // 点查询.路径不以".**",直接走IDTable,精确查询 + if (devicePath.getFullPath().endsWith(TAIL)) { + return getMeasurementPathsWithBatchQuery(devicePath, isPrefixMatch); + } else { + return getMeasurementPathsWithPointQuery(devicePath, isPrefixMatch); + } + } + + private List getMeasurementPathsWithPointQuery( + PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { + List res = new LinkedList<>(); + String path = devicePath.getFullPath(); + Map tags = pathToTags(path); + path = tagsToPath(tags); + DeviceEntry deviceEntry = idTable.getDeviceEntry(path); + if (deviceEntry == null) return res; + Map schemaMap = deviceEntry.getMeasurementMap(); + for (String measurement : schemaMap.keySet()) { + SchemaEntry schemaEntry = schemaMap.get(measurement); + MeasurementPath measurementPath = + new MeasurementPath( + path, + measurement, + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); + res.add(measurementPath); + } + + return res; + } + + private List getMeasurementPathsWithBatchQuery( + PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { + List res = new LinkedList<>(); + List deviceIDs = getDeviceIdFromInvertedIndex(devicePath); + for (IDeviceID deviceID : deviceIDs) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map schemaMap = deviceEntry.getMeasurementMap(); + if (deviceID instanceof SHA256DeviceID) { + for (String measurement : schemaMap.keySet()) { + SchemaEntry schemaEntry = schemaMap.get(measurement); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + MeasurementPath measurementPath = + new MeasurementPath( + new PartialPath(diskSchemaEntry.seriesKey), + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); + res.add(measurementPath); + } + } else { + for (String measurement : schemaMap.keySet()) { + SchemaEntry schemaEntry = schemaMap.get(measurement); + MeasurementPath measurementPath = + new MeasurementPath( + deviceID.toStringID(), + measurement, + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); + res.add(measurementPath); + } + } + } + return res; + } + + // [iotdb/select] [iotdb/select last] [iotdb/select count()] [iotdb/select ...groupby level] + @Override + public Pair, Integer> getMeasurementPathsWithAlias( + PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) + throws MetadataException { + List res = getMeasurementPaths(pathPattern, isPrefixMatch); + Pair, Integer> result = new Pair<>(res, 0); + return result; + } + + @Override + public List fetchSchema( + PartialPath pathPattern, Map templateMap) throws MetadataException { + return null; + } + + // show 时间序列 + @Override // [iotdb/show timeseries] + public Pair, Integer> showTimeseries( + ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { + List res = new ArrayList<>(); + Pair, Integer> result = new Pair<>(res, 0); + String path = plan.getPath().getFullPath(); + if (!path.endsWith(TAIL)) { + Map tags = pathToTags(path); + path = tagsToPath(tags); + DeviceEntry deviceEntry = idTable.getDeviceEntry(path); + if (deviceEntry != null) { + Map measurementMap = deviceEntry.getMeasurementMap(); + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + res.add( + new ShowTimeSeriesResult( + path + "." + m, + "null", + storageGroupFullPath, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>())); + } + } + return result; + } + List deviceIDs = getDeviceIdFromInvertedIndex(plan.getPath()); + for (IDeviceID deviceID : deviceIDs) { + getTimeSeriesResultOfDeviceFromIDTable(res, deviceID); + } + return result; + } + + private List getDeviceIdFromInvertedIndex(PartialPath devicePath) + throws MetadataException { + String path = devicePath.getFullPath(); + if (path.endsWith(TAIL)) { + path = path.substring(0, path.length() - TAIL.length()); + devicePath = new PartialPath(path); + } + if (devicePath.getFullPath().length() <= storageGroupFullPath.length()) { + return deviceIDS; + } else { + List res = new LinkedList<>(); + List ids = getDeviceIDsByInvertedIndex(devicePath); + if (ids.size() > 0) { + for (int id : ids) { + res.add(deviceIDS.get(id)); + } + } + return res; + } + } + + private void getTimeSeriesResultOfDeviceFromIDTable( + List res, IDeviceID deviceID) { + Map measurementMap = + idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap(); + if (deviceID instanceof SHA256DeviceID) { + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + res.add( + new ShowTimeSeriesResult( + diskSchemaEntry.seriesKey, + "null", + storageGroupFullPath, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>())); + } + } else { + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + res.add( + new ShowTimeSeriesResult( + deviceID.toStringID() + "." + m, + "null", + storageGroupFullPath, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>())); + } + } + } + + private List getDeviceIDsByInvertedIndex(PartialPath path) { + Map tags = pathToTags(path.getFullPath()); + List idsCollection = new ArrayList<>(tags.keySet().size()); + for (String tagKey : tags.keySet()) { + if (!tagInvertedIndex.containsKey(tagKey) + || !tagInvertedIndex.get(tagKey).containsKey(tags.get(tagKey))) { + return new ArrayList<>(); + } + List ids = tagInvertedIndex.get(tagKey).get(tags.get(tagKey)); + idsCollection.add(new ArrayList(ids)); + } + if (idsCollection.size() == 0) return new ArrayList<>(); + List ids = idsCollection.get(0); + for (int i = 1; i < idsCollection.size(); i++) { + List list = idsCollection.get(i); + ids.retainAll(list); + } + return ids; + } + + @Override + public List getAllMeasurementByDevicePath(PartialPath devicePath) + throws PathNotExistException { + throw new UnsupportedOperationException(""); + } + + @Override + public IMNode getDeviceNode(PartialPath path) throws MetadataException { + DeviceEntry deviceEntry = idTable.getDeviceEntry(path.getFullPath()); + if (deviceEntry == null) throw new PathNotExistException(path.getFullPath()); + return new EntityMNode(storageGroupMNode, path.getFullPath()); + } + + @Override + public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void upsertTagsAndAttributes( + String alias, + Map tagsMap, + Map attributesMap, + PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void addAttributes(Map attributesMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void addTags(Map tagsMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void dropTagsOrAttributes(Set keySet, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + // insert data + @Override // [iotdb/insert ] + public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) + throws MetadataException, IOException { + PartialPath devicePath = plan.getDevicePath(); + Map tags = pathToTags(devicePath.getFullPath()); + devicePath = new PartialPath(tagsToPath(tags)); + plan.setDevicePath(devicePath); + String[] measurementList = plan.getMeasurements(); + IMeasurementMNode[] measurementMNodes = plan.getMeasurementMNodes(); + checkAlignedAndAutoCreateSeries(plan); + IMNode deviceMNode = getDeviceNode(devicePath); + IMeasurementMNode measurementMNode; + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + Map schemaMap = deviceEntry.getMeasurementMap(); + for (int i = 0; i < measurementList.length; i++) { + SchemaEntry schemaEntry = schemaMap.get(measurementList[i]); + // measurementMNode = + // new MeasurementMNode( + // deviceMNode, + // measurementList[i], + // new MeasurementSchema( + // measurementList[i], + // schemaEntry.getTSDataType(), + // schemaEntry.getTSEncoding(), + // schemaEntry.getCompressionType()), + // null); + measurementMNode = new InsertMeasurementMNode(measurementList[i], schemaEntry, null); + // check type is match + try { + SchemaRegionUtils.checkDataTypeMatch(plan, i, schemaEntry.getTSDataType()); + } catch (DataTypeMismatchException mismatchException) { + if (!config.isEnablePartialInsert()) { + throw mismatchException; + } else { + // mark failed measurement + plan.markFailedMeasurementInsertion(i, mismatchException); + continue; + } + } + measurementMNodes[i] = measurementMNode; + } + plan.setDeviceID(deviceEntry.getDeviceID()); + plan.setDevicePath(new PartialPath(deviceEntry.getDeviceID().toStringID(), false)); + return deviceMNode; + } + + private SchemaEntry getSchemaEntry(String devicePath, String measurementName) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); + if (deviceEntry == null) return null; + return deviceEntry.getSchemaEntry(measurementName); + } + + @Override + public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( + PartialPath devicePath, + String[] measurements, + Function getDataType, + boolean aligned) + throws MetadataException { + List measurementSchemaInfoList = new ArrayList<>(measurements.length); + for (int i = 0; i < measurements.length; i++) { + SchemaEntry schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + if (schemaEntry == null) { + if (config.isAutoCreateSchemaEnabled()) { + if (aligned) { + internalAlignedCreateTimeseries( + devicePath, + Collections.singletonList(measurements[i]), + Collections.singletonList(getDataType.apply(i))); + + } else { + internalCreateTimeseries(devicePath.concatNode(measurements[i]), getDataType.apply(i)); + } + } + schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + } + measurementSchemaInfoList.add( + new MeasurementSchemaInfo( + measurements[i], + new MeasurementSchema( + measurements[i], + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType()), + null)); + } + return new DeviceSchemaInfo(devicePath, aligned, measurementSchemaInfoList); + } + + private void checkAlignedAndAutoCreateSeries(InsertPlan plan) throws MetadataException { + String[] measurementList = plan.getMeasurements(); + try { + if (plan.isAligned()) { + internalAlignedCreateTimeseries( + plan.getDevicePath(), + Arrays.asList(measurementList), + Arrays.asList(plan.getDataTypes())); + } else { + internalCreateTimeseries( + plan.getDevicePath().concatNode(measurementList[0]), plan.getDataTypes()[0]); + } + } catch (MetadataException e) { + if (!(e instanceof PathAlreadyExistException)) { + throw e; + } + } + } + + private void internalCreateTimeseries(PartialPath path, TSDataType dataType) + throws MetadataException { + createTimeseries( + path, + dataType, + getDefaultEncoding(dataType), + TSFileDescriptor.getInstance().getConfig().getCompressor(), + Collections.emptyMap()); + } + + private void internalAlignedCreateTimeseries( + PartialPath prefixPath, List measurements, List dataTypes) + throws MetadataException { + List encodings = new ArrayList<>(); + List compressors = new ArrayList<>(); + for (TSDataType dataType : dataTypes) { + encodings.add(getDefaultEncoding(dataType)); + compressors.add(TSFileDescriptor.getInstance().getConfig().getCompressor()); + } + createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); + } + + @Override + public Set getPathsSetTemplate(String templateName) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getPathsUsingTemplate(String templateName) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public boolean isTemplateAppendable(Template template, List measurements) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template) + throws MetadataException {} + + @Override + public List getPathsUsingTemplate(int templateId) throws MetadataException { + return null; + } + + @Override + public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException { + throw new UnsupportedOperationException(""); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java index ae5e7b97b5716..9974b982c11ae 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.metadata.path.MeasurementPath; +import org.apache.iotdb.db.metadata.schemaregion.SchemaEngineMode; import org.apache.iotdb.db.mpp.common.schematree.node.SchemaEntityNode; import org.apache.iotdb.db.mpp.common.schematree.node.SchemaInternalNode; import org.apache.iotdb.db.mpp.common.schematree.node.SchemaMeasurementNode; @@ -78,13 +79,49 @@ public Pair, Integer> searchMeasurementPaths( @Override public Pair, Integer> searchMeasurementPaths(PartialPath pathPattern) { - SchemaTreeMeasurementVisitor visitor = - new SchemaTreeMeasurementVisitor( - root, - pathPattern, - IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, - 0, - false); + SchemaTreeMeasurementVisitor visitor; + switch (SchemaEngineMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { + case Memory: + case Schema_File: + visitor = + new SchemaTreeMeasurementVisitor( + root, + pathPattern, + IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, + 0, + false); + break; + case Tag: + if (pathPattern.getFullPath().contains(".**")) { + String measurement = pathPattern.getMeasurement(); + visitor = + new SchemaTreeMeasurementVisitor( + root, + ALL_MATCH_PATTERN.concatNode(measurement), + IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, + 0, + false); + } else { + visitor = + new SchemaTreeMeasurementVisitor( + root, + pathPattern, + IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, + 0, + false); + } + break; + default: + visitor = + new SchemaTreeMeasurementVisitor( + root, + ALL_MATCH_PATTERN, + IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, + 0, + false); + break; + } return new Pair<>(visitor.getAllResult(), visitor.getNextOffset()); } @@ -107,7 +144,20 @@ public List getMatchedDevices(PartialPath pathPattern, boolean @Override public List getMatchedDevices(PartialPath pathPattern) { - SchemaTreeDeviceVisitor visitor = new SchemaTreeDeviceVisitor(root, pathPattern, false); + SchemaTreeDeviceVisitor visitor; + switch (SchemaEngineMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { + case Memory: + case Schema_File: + visitor = new SchemaTreeDeviceVisitor(root, pathPattern, false); + break; + case Tag: + visitor = new SchemaTreeDeviceVisitor(root, ALL_MATCH_PATTERN, false); + break; + default: + visitor = new SchemaTreeDeviceVisitor(root, ALL_MATCH_PATTERN, false); + break; + } return visitor.getAllResult(); } From 6e0a46793fcfe93449303cffb046adb642581ca2 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Sun, 18 Sep 2022 16:55:04 +0800 Subject: [PATCH 03/30] spotless apply --- .../iotdb/db/protocol/influxdb/handler/TagQueryHandler.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java index 0f48caa5409f3..5344bba68c90d 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.db.protocol.influxdb.handler; -import org.apache.iotdb.commons.auth.AuthException; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunction; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManagerFactory; From debe656b9129e974bd4405f9465ee71288261ebe Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 7 Sep 2022 12:50:09 +0800 Subject: [PATCH 04/30] lsm --- lsm/pom.xml | 30 +++ .../org/apache/iotdb/lsm/context/Context.java | 101 +++++++++ .../apache/iotdb/lsm/context/ContextType.java | 27 +++ .../iotdb/lsm/context/DeleteContext.java | 43 ++++ .../iotdb/lsm/context/FlushContext.java | 43 ++++ .../iotdb/lsm/context/InsertContext.java | 50 +++++ .../iotdb/lsm/context/QueryContext.java | 43 ++++ .../org/apache/iotdb/lsm/example/Main.java | 209 ++++++++++++++++++ .../apache/iotdb/lsm/example/MemChunk.java | 44 ++++ .../apache/iotdb/lsm/example/MemGroup.java | 45 ++++ .../apache/iotdb/lsm/example/MemTable.java | 45 ++++ .../iotdb/lsm/example/MemTableManager.java | 75 +++++++ .../lsm/levelProcess/BasicLevelProcess.java | 52 +++++ .../lsm/levelProcess/DeleteLevelProcess.java | 31 +++ .../lsm/levelProcess/FlushLevelProcess.java | 30 +++ .../lsm/levelProcess/InsertLevelProcess.java | 31 +++ .../iotdb/lsm/levelProcess/LevelProcess.java | 29 +++ .../lsm/levelProcess/QueryLevelProcess.java | 31 +++ .../iotdb/lsm/manager/BasicLsmManager.java | 46 ++++ .../apache/iotdb/lsm/manager/LsmManager.java | 30 +++ .../iotdb/lsm/strategy/AccessStrategy.java | 35 +++ .../iotdb/lsm/strategy/BFSAccessStrategy.java | 56 +++++ .../lsm/strategy/PostOrderAccessStrategy.java | 47 ++++ .../lsm/strategy/PreOrderAccessStrategy.java | 44 ++++ .../lsm/strategy/RBFSAccessStrategy.java | 74 +++++++ 25 files changed, 1291 insertions(+) create mode 100644 lsm/pom.xml create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/context/ContextType.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java diff --git a/lsm/pom.xml b/lsm/pom.xml new file mode 100644 index 0000000000000..1bbf055742d06 --- /dev/null +++ b/lsm/pom.xml @@ -0,0 +1,30 @@ + + + + + iotdb-parent + org.apache.iotdb + 0.14.0-SNAPSHOT + + 4.0.0 + lsm + + 8 + 8 + + diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java new file mode 100644 index 0000000000000..381586e7c8294 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import org.apache.iotdb.lsm.strategy.AccessStrategy; +import org.apache.iotdb.lsm.strategy.PreOrderAccessStrategy; + +public class Context { + + // 类型 + ContextType type; + + // 访问策略 + AccessStrategy accessStrategy; + + // 所处的树深度 + int level; + + // 多少个线程处理该节点的子节点 + int threadNums; + + // 返回值 + Object result; + + public Context() { + accessStrategy = new PreOrderAccessStrategy(); + type = ContextType.NONE; + level = 0; + threadNums = 1; + } + + public Context( + ContextType type, + AccessStrategy accessStrategy, + int level, + boolean sync, + int threadNums, + Object result) { + this.type = type; + this.accessStrategy = accessStrategy; + this.level = level; + this.threadNums = threadNums; + this.result = result; + } + + public void setLevel(int level) { + this.level = level; + } + + public int getLevel() { + return level; + } + + public ContextType getType() { + return type; + } + + public int getThreadNums() { + return threadNums; + } + + public void setType(ContextType type) { + this.type = type; + } + + public void setThreadNums(int threadNums) { + this.threadNums = threadNums; + } + + public Object getResult() { + return result; + } + + public void setResult(Object result) { + this.result = result; + } + + public AccessStrategy getAccessStrategy() { + return accessStrategy; + } + + public void setAccessStrategy(AccessStrategy accessStrategy) { + this.accessStrategy = accessStrategy; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/ContextType.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/ContextType.java new file mode 100644 index 0000000000000..3ff2e3509afb1 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/ContextType.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +public enum ContextType { + NONE, + INSERT, + QUERY, + DELETE, + FLUSH; +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java new file mode 100644 index 0000000000000..e00db284e7bce --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class DeleteContext extends Context { + + List keys; + + public DeleteContext(Object... ks) { + super(); + keys = new ArrayList<>(); + keys.addAll(Arrays.asList(ks)); + type = ContextType.DELETE; + } + + public Object getKey() { + return keys.get(level); + } + + public int size() { + return keys.size(); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java new file mode 100644 index 0000000000000..ceacd2f636e5f --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import org.apache.iotdb.lsm.strategy.BFSAccessStrategy; +import org.apache.iotdb.lsm.strategy.RBFSAccessStrategy; + +public class FlushContext extends Context { + + // 最小的已经flush的层级 + int minimumFlushedLevel; + + public FlushContext() { + super(); + type = ContextType.FLUSH; + accessStrategy = new RBFSAccessStrategy(); + minimumFlushedLevel = Integer.MAX_VALUE; + } + + public int getMinimumFlushedLevel() { + return minimumFlushedLevel; + } + + public void setMinimumFlushedLevel(int minimumFlushedLevel) { + this.minimumFlushedLevel = minimumFlushedLevel; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java new file mode 100644 index 0000000000000..a34c9d5814ce8 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import org.apache.iotdb.lsm.strategy.PreOrderAccessStrategy; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class InsertContext extends Context { + + List keys; + + Object value; + + public InsertContext(Object value, Object... keys) { + super(); + this.value = value; + this.keys = new ArrayList<>(); + this.keys.addAll(Arrays.asList(keys)); + level = 0; + type = ContextType.INSERT; + accessStrategy = new PreOrderAccessStrategy(); + } + + public Object getKey() { + return keys.get(level); + } + + public Object getValue() { + return value; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java new file mode 100644 index 0000000000000..9b8d440106b48 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class QueryContext extends Context { + + List keys; + + public QueryContext(Object... ks) { + super(); + keys = new ArrayList<>(); + keys.addAll(Arrays.asList(ks)); + type = ContextType.QUERY; + } + + public Object getKey() { + return keys.get(level); + } + + public int size() { + return keys.size(); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java new file mode 100644 index 0000000000000..015963e8af134 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.example; + +import org.apache.iotdb.lsm.context.FlushContext; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.levelProcess.FlushLevelProcess; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; +import org.apache.iotdb.lsm.manager.BasicLsmManager; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Main { + public static void main(String[] args) { + MemTableManager memTableManager = new MemTableManager(); + System.out.println("-------------insert--------------"); + insertionExample(memTableManager); + System.out.println("-------------flush--------------"); + flushExample(memTableManager); + } + + public static void insertionExample(MemTableManager memTableManager) { + + BasicLsmManager baseLsmManager = + new BasicLsmManager().manager(memTableManager); + baseLsmManager + .nextLevel( + new InsertLevelProcess() { + @Override + public List getChildren(MemTableManager memNode, InsertContext context) { + Integer deviceID = (Integer) context.getValue(); + int maxDeviceID = memNode.getMaxDeviceID(); + List children = new ArrayList<>(); + if (deviceID / 65536 == maxDeviceID / 65536) { + children.add(memNode.getWorking()); + } else { + children.add(memNode.getImmutables().get(deviceID / 65536)); + } + return children; + } + + @Override + public void insert(MemTableManager memNode, InsertContext context) { + Integer deviceID = (Integer) context.getValue(); + int maxDeviceID = memNode.getMaxDeviceID(); + if (deviceID / 65536 == maxDeviceID / 65536) { + if (memNode.getWorking() == null) { + memNode.setWorking(new MemTable()); + } + } else if (deviceID > maxDeviceID) { + memNode + .getImmutables() + .put(memNode.getMaxDeviceID() / 65536, memNode.getWorking()); + memNode.setWorking(new MemTable()); + } + if (deviceID > maxDeviceID) { + memNode.setMaxDeviceID(deviceID); + } + } + }) + .nextLevel( + new InsertLevelProcess() { + @Override + public List getChildren(MemTable memNode, InsertContext context) { + String key = (String) context.getKey(); + List children = new ArrayList<>(); + children.add(memNode.getMap().get(key)); + return children; + } + + @Override + public void insert(MemTable memNode, InsertContext context) { + String key = (String) context.getKey(); + Map map = memNode.getMap(); + if (map.containsKey(key)) return; + map.put(key, new MemGroup()); + } + }) + .nextLevel( + new InsertLevelProcess() { + @Override + public List getChildren(MemGroup memNode, InsertContext context) { + String key = (String) context.getKey(); + List children = new ArrayList<>(); + children.add(memNode.getMap().get(key)); + return children; + } + + @Override + public void insert(MemGroup memNode, InsertContext context) { + String key = (String) context.getKey(); + Map map = memNode.getMap(); + if (map.containsKey(key)) return; + map.put(key, new MemChunk()); + } + }) + .nextLevel( + new InsertLevelProcess() { + @Override + public List getChildren(MemChunk memNode, InsertContext context) { + return null; + } + + @Override + public void insert(MemChunk memNode, InsertContext context) { + Integer deviceID = (Integer) context.getValue(); + List deviceIDs = memNode.getDeviceIDS(); + deviceIDs.add(deviceID); + } + }); + + baseLsmManager.process(new InsertContext(1, null, "a", "b")); + baseLsmManager.process(new InsertContext(2, null, "a", "d")); + baseLsmManager.process(new InsertContext(3, null, "a", "e")); + baseLsmManager.process(new InsertContext(4, null, "a", "b")); + baseLsmManager.process(new InsertContext(5, null, "a1", "b")); + baseLsmManager.process(new InsertContext(6, null, "a2", "b")); + baseLsmManager.process(new InsertContext(65535, null, "a", "b")); + baseLsmManager.process(new InsertContext(65536, null, "a", "b")); + baseLsmManager.process(new InsertContext(2, null, "a", "d")); + baseLsmManager.process(new InsertContext(3, null, "a", "e")); + baseLsmManager.process(new InsertContext(4, null, "a", "b")); + baseLsmManager.process(new InsertContext(5, null, "a1", "b")); + baseLsmManager.process(new InsertContext(6, null, "a2", "b")); + System.out.println(memTableManager); + } + + public static void flushExample(MemTableManager memTableManager) { + BasicLsmManager flushManager = + new BasicLsmManager().manager(memTableManager); + + flushManager + .nextLevel( + new FlushLevelProcess() { + @Override + public void flush(MemTableManager memNode, FlushContext context) { + System.out.println(memNode); + } + + @Override + public List getChildren(MemTableManager memNode, FlushContext context) { + List memTables = new ArrayList<>(); + memTables.addAll(memNode.getImmutables().values()); + if (memNode.getWorking() != null) memTables.add(memNode.getWorking()); + return memTables; + } + }) + .nextLevel( + new FlushLevelProcess() { + @Override + public void flush(MemTable memNode, FlushContext context) { + System.out.println(memNode); + } + + @Override + public List getChildren(MemTable memNode, FlushContext context) { + List memGroups = new ArrayList<>(); + memGroups.addAll(memNode.getMap().values()); + return memGroups; + } + }) + .nextLevel( + new FlushLevelProcess() { + @Override + public void flush(MemGroup memNode, FlushContext context) { + System.out.println(memNode); + } + + @Override + public List getChildren(MemGroup memNode, FlushContext context) { + List memChunk = new ArrayList<>(); + memChunk.addAll(memNode.getMap().values()); + return memChunk; + } + }) + .nextLevel( + new FlushLevelProcess() { + @Override + public void flush(MemChunk memNode, FlushContext context) { + System.out.println(memNode); + } + + @Override + public List getChildren(MemChunk memNode, FlushContext context) { + return new ArrayList<>(); + } + }); + + flushManager.process(new FlushContext()); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java new file mode 100644 index 0000000000000..c6e97fd52d9d2 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.example; + +import java.util.ArrayList; +import java.util.List; + +// device list +public class MemChunk { + List deviceIDS; + + public MemChunk() { + deviceIDS = new ArrayList<>(); + } + + public List getDeviceIDS() { + return deviceIDS; + } + + public void setDeviceIDS(List deviceIDS) { + this.deviceIDS = deviceIDS; + } + + @Override + public String toString() { + return "MemChunk{" + deviceIDS.toString() + '}'; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java new file mode 100644 index 0000000000000..69f61d5d451fa --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.example; + +import java.util.HashMap; +import java.util.Map; + +// tagvalue -> memChunk +public class MemGroup { + + Map map; + + public MemGroup() { + map = new HashMap<>(); + } + + public Map getMap() { + return map; + } + + public void setMap(Map map) { + this.map = map; + } + + @Override + public String toString() { + return "MemGroup{" + map.toString() + '}'; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java new file mode 100644 index 0000000000000..7cc82f19ecab7 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.example; + +import java.util.HashMap; +import java.util.Map; + +// tagkey - > MemGroup +public class MemTable { + + private Map map; + + public MemTable() { + map = new HashMap<>(); + } + + public Map getMap() { + return map; + } + + public void setMap(Map map) { + this.map = map; + } + + @Override + public String toString() { + return "MemTable{" + map.toString() + '}'; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java new file mode 100644 index 0000000000000..d0f50071747c1 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.example; + +import java.util.HashMap; +import java.util.Map; + +// 管理working memtable , immutable memtables,框架用户自定义 +public class MemTableManager { + + // 可写的memtable + private MemTable working; + + // 只读的memtables + private Map immutables; + + // 记录已插入的最大的deviceid + private int maxDeviceID; + + public MemTableManager() { + working = new MemTable(); + immutables = new HashMap<>(); + maxDeviceID = 0; + } + + public MemTable getWorking() { + return working; + } + + public void setWorking(MemTable working) { + this.working = working; + } + + public Map getImmutables() { + return immutables; + } + + public void setImmutables(Map immutables) { + this.immutables = immutables; + } + + public int getMaxDeviceID() { + return maxDeviceID; + } + + public void setMaxDeviceID(int maxDeviceID) { + this.maxDeviceID = maxDeviceID; + } + + @Override + public String toString() { + return "MemTableManager{" + + "working=" + + working.toString() + + ", immutables=" + + immutables.toString() + + '}'; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java new file mode 100644 index 0000000000000..9a6359eab1714 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.Context; + +import java.util.List; + +public abstract class BasicLevelProcess implements LevelProcess { + LevelProcess next; + + public abstract void handle(I memNode, C context); + + public abstract List getChildren(I memNode, C context); + + @Override + public LevelProcess nextLevel(LevelProcess next) { + this.next = next; + return next; + } + + @Override + public void process(I memNode, C context) { + int currentLevel = context.getLevel(); + context.getAccessStrategy().execute(this, memNode, context); + context.setLevel(currentLevel); + } + + public boolean hasNext() { + return next != null; + } + + public LevelProcess getNext() { + return next; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java new file mode 100644 index 0000000000000..f4a4c97cd0f5c --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.DeleteContext; + +public abstract class DeleteLevelProcess extends BasicLevelProcess { + + public abstract void delete(I memNode, DeleteContext context); + + @Override + public void handle(I memNode, DeleteContext context) { + delete(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java new file mode 100644 index 0000000000000..00363c7683b75 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.FlushContext; + +public abstract class FlushLevelProcess extends BasicLevelProcess { + + public abstract void flush(I memNode, FlushContext context); + + public void handle(I memNode, FlushContext context) { + flush(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java new file mode 100644 index 0000000000000..91f7e24c1f250 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.InsertContext; + +public abstract class InsertLevelProcess extends BasicLevelProcess { + + public abstract void insert(I memNode, InsertContext context); + + @Override + public void handle(I memNode, InsertContext context) { + insert(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java new file mode 100644 index 0000000000000..aa04c2bf172b6 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.Context; + +public interface LevelProcess { + LevelProcess nextLevel(LevelProcess next); + + void process(I memNode, C context); + + void handle(I memNode, C context); +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java new file mode 100644 index 0000000000000..5b3fc0e9ebb32 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.QueryContext; + +public abstract class QueryLevelProcess extends BasicLevelProcess { + + public abstract void query(I memNode, QueryContext context); + + @Override + public void handle(I memNode, QueryContext context) { + query(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java new file mode 100644 index 0000000000000..3d33dd1712864 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.manager; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.levelProcess.LevelProcess; + +public class BasicLsmManager implements LsmManager { + + T root; + + LevelProcess levelProcess; + + @Override + public BasicLsmManager manager(T memNode) { + root = memNode; + return this; + } + + @Override + public void process(C context) { + levelProcess.process(root, context); + } + + @Override + public LevelProcess nextLevel(LevelProcess levelProcess) { + this.levelProcess = levelProcess; + return levelProcess; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java new file mode 100644 index 0000000000000..3d60af6f69f8e --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.manager; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.levelProcess.LevelProcess; + +public interface LsmManager { + LsmManager manager(T memNode); + + void process(C context); + + LevelProcess nextLevel(LevelProcess next); +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java new file mode 100644 index 0000000000000..d7780a5019877 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +// 表示内存节点访问策略（先序，后序） +public interface AccessStrategy{ + + /** + * + * @param levelProcess 保存当前节点和子节点的处理方法 + * @param memNode 当前待处理的节点 + * @param context 上下文信息 + */ + void execute( + BasicLevelProcess levelProcess, I memNode, C context); +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java new file mode 100644 index 0000000000000..cf794b7097e61 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; + +public class BFSAccessStrategy implements AccessStrategy { + + Queue sameLevelMemNodes; + + @Override + public void execute( + BasicLevelProcess levelProcess, I memNode, C context) { + List children = new ArrayList<>(); + int currentLevel = context.getLevel(); + // 第一个使用bfs策略的节点 + if (sameLevelMemNodes == null) { + sameLevelMemNodes = new LinkedList<>(); + levelProcess.handle(memNode, context); + children = levelProcess.getChildren(memNode, context); + } else { + while (!sameLevelMemNodes.isEmpty()) { + I node = (I) sameLevelMemNodes.poll(); + levelProcess.handle(node, context); + children.addAll(levelProcess.getChildren(node, context)); + } + } + sameLevelMemNodes.addAll(children); + context.setLevel(currentLevel + 1); + if (levelProcess.hasNext() && !sameLevelMemNodes.isEmpty()) { + levelProcess.getNext().process(null, context); + } + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java new file mode 100644 index 0000000000000..3079041763701 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +import java.util.List; + +public class PostOrderAccessStrategy implements AccessStrategy { + + @Override + public void execute( + BasicLevelProcess levelProcess, I memNode, C context) { + int currentLevel = context.getLevel(); + AccessStrategy accessStrategy = context.getAccessStrategy(); + List children = levelProcess.getChildren(memNode, context); + // 处理子节点 + if (levelProcess.hasNext()) { + context.setLevel(currentLevel + 1); + for (O child : children) { + levelProcess.getNext().process(child, context); + } + } + + context.setLevel(currentLevel); + context.setAccessStrategy(accessStrategy); + // 处理该节点 + levelProcess.handle(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java new file mode 100644 index 0000000000000..11f8be0844744 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +import java.util.List; + +public class PreOrderAccessStrategy implements AccessStrategy { + + @Override + public void execute( + BasicLevelProcess levelProcess, I memNode, C context) { + int currentLevel = context.getLevel(); + // 处理该节点 + levelProcess.handle(memNode, context); + List children = levelProcess.getChildren(memNode, context); + + // 处理子节点 + if (levelProcess.hasNext()) { + context.setLevel(currentLevel + 1); + for (O child : children) { + levelProcess.getNext().process(child, context); + } + } + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java new file mode 100644 index 0000000000000..e6b315eb145af --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.FlushContext; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +import java.util.List; + +public class RBFSAccessStrategy implements AccessStrategy{ + @Override + public void execute(BasicLevelProcess levelProcess, I memNode, C context) { + FlushContext flushContext = (FlushContext) context; + int currentLevel = context.getLevel(); + // 如果当前节点是最深的一层节点 + if(!levelProcess.hasNext()){ + flushContext.setMinimumFlushedLevel(currentLevel); + } + // 如果是根节点 + if(currentLevel == 0){ + while(flushContext.getMinimumFlushedLevel() != currentLevel){ + List children = levelProcess.getChildren(memNode, context); + for(O child : children){ + // 处理子节点 + flushContext.setLevel(currentLevel+1); + levelProcess.getNext().process(child,context); + flushContext.setLevel(currentLevel); + } + // 每次处理完-1 + flushContext.setMinimumFlushedLevel(flushContext.getMinimumFlushedLevel() - 1); + } + // 处理root节点 + levelProcess.handle(memNode, context); + return; + } + + // 后序遍历，处理level == minimumFlushedLevel的节点 + + // 已经处理过，直接return + if(currentLevel > flushContext.getMinimumFlushedLevel()) return; + + + // 处理子节点 + if(currentLevel == flushContext.getMinimumFlushedLevel()){ + levelProcess.handle(memNode, context); + return; + } + List children = levelProcess.getChildren(memNode,context); + for(O child : children){ + flushContext.setLevel(currentLevel+1); + levelProcess.getNext().process(child,context); + flushContext.setLevel(currentLevel); + } + + } + +} From 26e9c0f0c537363ec8002d68beed46942d16575a Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 7 Sep 2022 14:14:14 +0800 Subject: [PATCH 05/30] parent pom add lsm --- lsm/pom.xml | 4 ++- .../iotdb/lsm/context/FlushContext.java | 1 - .../iotdb/lsm/strategy/AccessStrategy.java | 5 ++- .../lsm/strategy/RBFSAccessStrategy.java | 32 +++++++++---------- pom.xml | 1 + 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/lsm/pom.xml b/lsm/pom.xml index 1bbf055742d06..d6ff4cf992ad9 100644 --- a/lsm/pom.xml +++ b/lsm/pom.xml @@ -20,9 +20,11 @@ iotdb-parent org.apache.iotdb 0.14.0-SNAPSHOT + ../pom.xml 4.0.0 - lsm + iotdb-lsm + IoTDB lsm 8 8 diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java index ceacd2f636e5f..cb8c938175811 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.lsm.context; -import org.apache.iotdb.lsm.strategy.BFSAccessStrategy; import org.apache.iotdb.lsm.strategy.RBFSAccessStrategy; public class FlushContext extends Context { diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java index d7780a5019877..9ff3f0f0a03e3 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java @@ -22,11 +22,10 @@ import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; // 表示内存节点访问策略（先序，后序） -public interface AccessStrategy{ +public interface AccessStrategy { /** - * - * @param levelProcess 保存当前节点和子节点的处理方法 + * @param levelProcess 保存当前节点和子节点的处理方法 * @param memNode 当前待处理的节点 * @param context 上下文信息 */ diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java index e6b315eb145af..d9f4fa19004c2 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java @@ -24,23 +24,24 @@ import java.util.List; -public class RBFSAccessStrategy implements AccessStrategy{ +public class RBFSAccessStrategy implements AccessStrategy { @Override - public void execute(BasicLevelProcess levelProcess, I memNode, C context) { + public void execute( + BasicLevelProcess levelProcess, I memNode, C context) { FlushContext flushContext = (FlushContext) context; int currentLevel = context.getLevel(); // 如果当前节点是最深的一层节点 - if(!levelProcess.hasNext()){ + if (!levelProcess.hasNext()) { flushContext.setMinimumFlushedLevel(currentLevel); } // 如果是根节点 - if(currentLevel == 0){ - while(flushContext.getMinimumFlushedLevel() != currentLevel){ + if (currentLevel == 0) { + while (flushContext.getMinimumFlushedLevel() != currentLevel) { List children = levelProcess.getChildren(memNode, context); - for(O child : children){ + for (O child : children) { // 处理子节点 - flushContext.setLevel(currentLevel+1); - levelProcess.getNext().process(child,context); + flushContext.setLevel(currentLevel + 1); + levelProcess.getNext().process(child, context); flushContext.setLevel(currentLevel); } // 每次处理完-1 @@ -54,21 +55,18 @@ public void execute(BasicLevelProcess levelPr // 后序遍历，处理level == minimumFlushedLevel的节点 // 已经处理过，直接return - if(currentLevel > flushContext.getMinimumFlushedLevel()) return; - + if (currentLevel > flushContext.getMinimumFlushedLevel()) return; // 处理子节点 - if(currentLevel == flushContext.getMinimumFlushedLevel()){ + if (currentLevel == flushContext.getMinimumFlushedLevel()) { levelProcess.handle(memNode, context); return; } - List children = levelProcess.getChildren(memNode,context); - for(O child : children){ - flushContext.setLevel(currentLevel+1); - levelProcess.getNext().process(child,context); + List children = levelProcess.getChildren(memNode, context); + for (O child : children) { + flushContext.setLevel(currentLevel + 1); + levelProcess.getNext().process(child, context); flushContext.setLevel(currentLevel); } - } - } diff --git a/pom.xml b/pom.xml index a7f5012ec1189..a6c5d56c99384 100644 --- a/pom.xml +++ b/pom.xml @@ -121,6 +121,7 @@ trigger-api rewrite-tsfile-tool external-api + lsm From d12e25f096f4e8bc4c5e0cbd665419af1c3fb3f4 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 7 Sep 2022 18:05:02 +0800 Subject: [PATCH 06/30] lsm preOrder level fix --- lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java | 8 ++++---- .../apache/iotdb/lsm/levelProcess/BasicLevelProcess.java | 2 -- .../apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java | 1 + 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java index 015963e8af134..21796877734da 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java @@ -152,7 +152,7 @@ public static void flushExample(MemTableManager memTableManager) { new FlushLevelProcess() { @Override public void flush(MemTableManager memNode, FlushContext context) { - System.out.println(memNode); + System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); } @Override @@ -167,7 +167,7 @@ public List getChildren(MemTableManager memNode, FlushContext context) new FlushLevelProcess() { @Override public void flush(MemTable memNode, FlushContext context) { - System.out.println(memNode); + System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); } @Override @@ -181,7 +181,7 @@ public List getChildren(MemTable memNode, FlushContext context) { new FlushLevelProcess() { @Override public void flush(MemGroup memNode, FlushContext context) { - System.out.println(memNode); + System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); } @Override @@ -195,7 +195,7 @@ public List getChildren(MemGroup memNode, FlushContext context) { new FlushLevelProcess() { @Override public void flush(MemChunk memNode, FlushContext context) { - System.out.println(memNode); + System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); } @Override diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java index 9a6359eab1714..f4ced68ec5a08 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java @@ -37,9 +37,7 @@ public LevelProcess nextLevel(LevelProcess next) { @Override public void process(I memNode, C context) { - int currentLevel = context.getLevel(); context.getAccessStrategy().execute(this, memNode, context); - context.setLevel(currentLevel); } public boolean hasNext() { diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java index 11f8be0844744..2c0b1b7b03a6c 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java @@ -40,5 +40,6 @@ public void execute( levelProcess.getNext().process(child, context); } } + context.setLevel(currentLevel); } } From 6461111f6af878f57c25bb1ba29a114832f0a344 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 7 Sep 2022 21:32:39 +0800 Subject: [PATCH 07/30] lsm preOrder level fix --- .../java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java index d9f4fa19004c2..1505361947793 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java @@ -52,8 +52,6 @@ public void execute( return; } - // 后序遍历，处理level == minimumFlushedLevel的节点 - // 已经处理过，直接return if (currentLevel > flushContext.getMinimumFlushedLevel()) return; From 77f3dbf1860b5061b3e66ac7e20e43ddf1be9e4b Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Thu, 8 Sep 2022 19:29:37 +0800 Subject: [PATCH 08/30] lsm add levelUpperBound --- .../org/apache/iotdb/lsm/context/Context.java | 26 +++++++++---------- .../iotdb/lsm/context/FlushContext.java | 15 ----------- .../org/apache/iotdb/lsm/example/Main.java | 8 +++--- .../lsm/strategy/RBFSAccessStrategy.java | 23 +++++++--------- 4 files changed, 26 insertions(+), 46 deletions(-) diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java index 381586e7c8294..305b31747f6c3 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java @@ -35,6 +35,9 @@ public class Context { // 多少个线程处理该节点的子节点 int threadNums; + // 上界,大于该值的层级不会被处理 + int levelUpperBound; + // 返回值 Object result; @@ -43,20 +46,7 @@ public Context() { type = ContextType.NONE; level = 0; threadNums = 1; - } - - public Context( - ContextType type, - AccessStrategy accessStrategy, - int level, - boolean sync, - int threadNums, - Object result) { - this.type = type; - this.accessStrategy = accessStrategy; - this.level = level; - this.threadNums = threadNums; - this.result = result; + levelUpperBound = Integer.MAX_VALUE; } public void setLevel(int level) { @@ -98,4 +88,12 @@ public AccessStrategy getAccessStrategy() { public void setAccessStrategy(AccessStrategy accessStrategy) { this.accessStrategy = accessStrategy; } + + public int getLevelUpperBound() { + return levelUpperBound; + } + + public void setLevelUpperBound(int levelUpperBound) { + this.levelUpperBound = levelUpperBound; + } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java index cb8c938175811..2fd266396c8f8 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java @@ -21,22 +21,7 @@ import org.apache.iotdb.lsm.strategy.RBFSAccessStrategy; public class FlushContext extends Context { - - // 最小的已经flush的层级 - int minimumFlushedLevel; - public FlushContext() { - super(); - type = ContextType.FLUSH; accessStrategy = new RBFSAccessStrategy(); - minimumFlushedLevel = Integer.MAX_VALUE; - } - - public int getMinimumFlushedLevel() { - return minimumFlushedLevel; - } - - public void setMinimumFlushedLevel(int minimumFlushedLevel) { - this.minimumFlushedLevel = minimumFlushedLevel; } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java index 21796877734da..bea5c80e15873 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java @@ -152,7 +152,7 @@ public static void flushExample(MemTableManager memTableManager) { new FlushLevelProcess() { @Override public void flush(MemTableManager memNode, FlushContext context) { - System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); } @Override @@ -167,7 +167,7 @@ public List getChildren(MemTableManager memNode, FlushContext context) new FlushLevelProcess() { @Override public void flush(MemTable memNode, FlushContext context) { - System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); } @Override @@ -181,7 +181,7 @@ public List getChildren(MemTable memNode, FlushContext context) { new FlushLevelProcess() { @Override public void flush(MemGroup memNode, FlushContext context) { - System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); } @Override @@ -195,7 +195,7 @@ public List getChildren(MemGroup memNode, FlushContext context) { new FlushLevelProcess() { @Override public void flush(MemChunk memNode, FlushContext context) { - System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); } @Override diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java index 1505361947793..62d62295786fc 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java @@ -19,7 +19,6 @@ package org.apache.iotdb.lsm.strategy; import org.apache.iotdb.lsm.context.Context; -import org.apache.iotdb.lsm.context.FlushContext; import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; import java.util.List; @@ -28,24 +27,22 @@ public class RBFSAccessStrategy implements AccessStrategy { @Override public void execute( BasicLevelProcess levelProcess, I memNode, C context) { - FlushContext flushContext = (FlushContext) context; int currentLevel = context.getLevel(); - // 如果当前节点是最深的一层节点 - if (!levelProcess.hasNext()) { - flushContext.setMinimumFlushedLevel(currentLevel); + if (Integer.MAX_VALUE == context.getLevelUpperBound() && !levelProcess.hasNext()) { + context.setLevelUpperBound(context.getLevel()); } // 如果是根节点 if (currentLevel == 0) { - while (flushContext.getMinimumFlushedLevel() != currentLevel) { + while (context.getLevelUpperBound() != currentLevel) { List children = levelProcess.getChildren(memNode, context); for (O child : children) { // 处理子节点 - flushContext.setLevel(currentLevel + 1); + context.setLevel(currentLevel + 1); levelProcess.getNext().process(child, context); - flushContext.setLevel(currentLevel); + context.setLevel(currentLevel); } // 每次处理完-1 - flushContext.setMinimumFlushedLevel(flushContext.getMinimumFlushedLevel() - 1); + context.setLevelUpperBound(context.getLevelUpperBound() - 1); } // 处理root节点 levelProcess.handle(memNode, context); @@ -53,18 +50,18 @@ public void execute( } // 已经处理过，直接return - if (currentLevel > flushContext.getMinimumFlushedLevel()) return; + if (currentLevel > context.getLevelUpperBound()) return; // 处理子节点 - if (currentLevel == flushContext.getMinimumFlushedLevel()) { + if (currentLevel == context.getLevelUpperBound()) { levelProcess.handle(memNode, context); return; } List children = levelProcess.getChildren(memNode, context); for (O child : children) { - flushContext.setLevel(currentLevel + 1); + context.setLevel(currentLevel + 1); levelProcess.getNext().process(child, context); - flushContext.setLevel(currentLevel); + context.setLevel(currentLevel); } } } From 3e9b59e0b4b830f6575c4d637b1f1ebec8032978 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Fri, 9 Sep 2022 19:50:47 +0800 Subject: [PATCH 09/30] add TagInvertedIndex --- .../tagIndex/ITagInvertedIndex.java | 31 +++++++++++++++++++ .../tagIndex/TagInvertedIndex.java | 23 ++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java create mode 100644 lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java diff --git a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java new file mode 100644 index 0000000000000..a6798ab0096c3 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; + +import java.io.File; +import java.util.List; +import java.util.Map; + +public interface ITagInvertedIndex { + void addTag(String tagKey,String tagValue,int id); + void addTags(Map tags,int id); + void removeTag(String tagKey,String tagValue,int id); + void removeTags(Map tags,int id); + List getMatchedIDs(); +} diff --git a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java new file mode 100644 index 0000000000000..cbfbb9ee58983 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; + +public class TagInvertedIndex { + +} From d99d7aa5e96397d3adbc9d575c94d9f28493f094 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Fri, 9 Sep 2022 19:52:08 +0800 Subject: [PATCH 10/30] add TagInvertedIndex --- .../tagIndex/ITagInvertedIndex.java | 15 +++++++----- .../tagIndex/TagInvertedIndex.java | 23 ------------------- 2 files changed, 9 insertions(+), 29 deletions(-) delete mode 100644 lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java diff --git a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java index a6798ab0096c3..3934a6b680d8a 100644 --- a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java +++ b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java @@ -18,14 +18,17 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; -import java.io.File; import java.util.List; import java.util.Map; public interface ITagInvertedIndex { - void addTag(String tagKey,String tagValue,int id); - void addTags(Map tags,int id); - void removeTag(String tagKey,String tagValue,int id); - void removeTags(Map tags,int id); - List getMatchedIDs(); + void addTag(String tagKey, String tagValue, int id); + + void addTags(Map tags, int id); + + void removeTag(String tagKey, String tagValue, int id); + + void removeTags(Map tags, int id); + + List getMatchedIDs(Map tags); } diff --git a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java deleted file mode 100644 index cbfbb9ee58983..0000000000000 --- a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; - -public class TagInvertedIndex { - -} From 75547c800b1e448bf39489d97ab9758bb3bd15a1 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Tue, 13 Sep 2022 15:30:30 +0800 Subject: [PATCH 11/30] impl TagInvertedIndex --- lsm/pom.xml | 4 - .../iotdb/lsm/context/DeleteContext.java | 12 +- .../iotdb/lsm/context/QueryContext.java | 3 + pom.xml | 1 + schema-engine-tag/pom.xml | 51 ++++++ .../tagIndex/ITagInvertedIndex.java | 4 - .../tagIndex/TagInvertedIndex.java | 153 ++++++++++++++++++ .../tagIndex/deletion/DeletionManager.java | 41 +++++ .../tagIndex/deletion/MemChunkDeletion.java | 38 +++++ .../deletion/MemChunkGroupDeletion.java | 47 ++++++ .../tagIndex/deletion/MemTableDeletion.java | 56 +++++++ .../tagIndex/insertion/InsertionManager.java | 42 +++++ .../insertion/MemChunkGroupInsertion.java | 44 +++++ .../tagIndex/insertion/MemChunkInsertion.java | 38 +++++ .../tagIndex/insertion/MemTableInsertion.java | 48 ++++++ .../tagIndex/memtable/MemChunk.java | 52 ++++++ .../tagIndex/memtable/MemChunkGroup.java | 53 ++++++ .../tagIndex/memtable/MemTable.java | 84 ++++++++++ .../tagIndex/query/MemChunkGroupQuery.java | 41 +++++ .../tagIndex/query/MemChunkQuery.java | 42 +++++ .../tagIndex/query/MemTableQuery.java | 54 +++++++ .../tagIndex/query/QueryManager.java | 41 +++++ .../tagIndex/TagInvertedIndexTest.java | 148 +++++++++++++++++ .../org/apache/iotdb/db/conf/IoTDBConfig.java | 10 ++ 24 files changed, 1098 insertions(+), 9 deletions(-) create mode 100644 schema-engine-tag/pom.xml rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java (90%) create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java create mode 100644 schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java diff --git a/lsm/pom.xml b/lsm/pom.xml index d6ff4cf992ad9..081c60b0abc73 100644 --- a/lsm/pom.xml +++ b/lsm/pom.xml @@ -25,8 +25,4 @@ 4.0.0 iotdb-lsm IoTDB lsm - - 8 - 8 - diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java index e00db284e7bce..46d78b9ed5131 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java @@ -18,6 +18,8 @@ */ package org.apache.iotdb.lsm.context; +import org.apache.iotdb.lsm.strategy.PostOrderAccessStrategy; + import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -26,17 +28,25 @@ public class DeleteContext extends Context { List keys; - public DeleteContext(Object... ks) { + Object value; + + public DeleteContext(Object value, Object... ks) { super(); + this.value = value; keys = new ArrayList<>(); keys.addAll(Arrays.asList(ks)); type = ContextType.DELETE; + accessStrategy = new PostOrderAccessStrategy(); } public Object getKey() { return keys.get(level); } + public Object getValue() { + return value; + } + public int size() { return keys.size(); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java index 9b8d440106b48..7221b643f7fac 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java @@ -18,6 +18,8 @@ */ package org.apache.iotdb.lsm.context; +import org.apache.iotdb.lsm.strategy.PostOrderAccessStrategy; + import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -31,6 +33,7 @@ public QueryContext(Object... ks) { keys = new ArrayList<>(); keys.addAll(Arrays.asList(ks)); type = ContextType.QUERY; + accessStrategy = new PostOrderAccessStrategy(); } public Object getKey() { diff --git a/pom.xml b/pom.xml index a6c5d56c99384..6bb952ace414d 100644 --- a/pom.xml +++ b/pom.xml @@ -122,6 +122,7 @@ rewrite-tsfile-tool external-api lsm + schema-engine-tag diff --git a/schema-engine-tag/pom.xml b/schema-engine-tag/pom.xml new file mode 100644 index 0000000000000..e44c73f76f055 --- /dev/null +++ b/schema-engine-tag/pom.xml @@ -0,0 +1,51 @@ + + + + + iotdb-parent + org.apache.iotdb + 0.14.0-SNAPSHOT + ../pom.xml + + 4.0.0 + schema-engine-tag + schema-engine-tag + + + + org.roaringbitmap + RoaringBitmap + 0.9.32 + + + + org.apache.iotdb + iotdb-lsm + ${project.version} + provided + + + org.apache.iotdb + iotdb-server + ${project.version} + + + + \ No newline at end of file diff --git a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java similarity index 90% rename from lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java rename to schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java index 3934a6b680d8a..cf5e1a19c0ea7 100644 --- a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java @@ -22,12 +22,8 @@ import java.util.Map; public interface ITagInvertedIndex { - void addTag(String tagKey, String tagValue, int id); - void addTags(Map tags, int id); - void removeTag(String tagKey, String tagValue, int id); - void removeTags(Map tags, int id); List getMatchedIDs(Map tags); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java new file mode 100644 index 0000000000000..89c3260ed0e55 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; + +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.DeletionManager; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.InsertionManager; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.QueryManager; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.QueryContext; + +import org.roaringbitmap.RoaringBitmap; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class TagInvertedIndex implements ITagInvertedIndex { + + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + private int numOfDeviceIdsInMemTable; + + private MemTable workingMemTable; + + private MemTable unsequenceMemTable; + + private Map immutableMemTables; + + private int maxDeviceID; + + public TagInvertedIndex() { + workingMemTable = new MemTable(MemTable.WORKING); + unsequenceMemTable = new MemTable(MemTable.UNSEQUENCE); + immutableMemTables = new HashMap<>(); + numOfDeviceIdsInMemTable = config.getNumOfDeviceIdsInMemTable(); + maxDeviceID = 0; + } + + @Override + public synchronized void addTags(Map tags, int id) { + MemTable memTable = null; + // 出现乱序 + if (id < maxDeviceID) { + memTable = unsequenceMemTable; + } else { + if (!inWorkingMemTable(id)) { + workingMemTable.setStatus(MemTable.IMMUTABLE); + immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); + workingMemTable = new MemTable(MemTable.WORKING); + } + memTable = workingMemTable; + maxDeviceID = id; + } + for (Map.Entry tag : tags.entrySet()) { + addTag(memTable, tag.getKey(), tag.getValue(), id); + } + } + + @Override + public synchronized void removeTags(Map tags, int id) { + List memTables = new ArrayList<>(); + // 出现乱序 + if (inWorkingMemTable(id)) { + memTables.add(workingMemTable); + } else { + memTables.add(unsequenceMemTable); + memTables.add(immutableMemTables.get(id / numOfDeviceIdsInMemTable)); + } + for (Map.Entry tag : tags.entrySet()) { + removeTag(memTables, tag.getKey(), tag.getValue(), id); + } + } + + @Override + public synchronized List getMatchedIDs(Map tags) { + List memTables = new ArrayList<>(); + memTables.add(workingMemTable); + memTables.add(unsequenceMemTable); + memTables.addAll(immutableMemTables.values()); + RoaringBitmap roaringBitmap = new RoaringBitmap(); + int i = 0; + for (Map.Entry tag : tags.entrySet()) { + RoaringBitmap rb = getMatchedIDs(memTables, tag.getKey(), tag.getValue()); + if (i == 0) roaringBitmap = rb; + else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb); + i++; + } + return Arrays.stream(roaringBitmap.toArray()).boxed().collect(Collectors.toList()); + } + + @Override + public String toString() { + return "TagInvertedIndex{" + + "numOfDeviceIdsInMemTable=" + + numOfDeviceIdsInMemTable + + ", workingMemTable=" + + workingMemTable + + ", unsequenceMemTable=" + + unsequenceMemTable + + ", immutableMemTables=" + + immutableMemTables + + ", maxDeviceID=" + + maxDeviceID + + '}'; + } + + private synchronized boolean inWorkingMemTable(int id) { + return id / numOfDeviceIdsInMemTable == maxDeviceID / numOfDeviceIdsInMemTable; + } + + private void addTag(MemTable memTable, String tagKey, String tagValue, int id) { + InsertContext insertContext = new InsertContext(id, tagKey, tagValue); + InsertionManager.getInstance().manager(memTable).process(insertContext); + } + + private void removeTag(List memTables, String tagKey, String tagValue, int id) { + DeleteContext deleteContext = new DeleteContext(id, tagKey, tagValue); + for (MemTable memTable : memTables) { + DeletionManager.getInstance().manager(memTable).process(deleteContext); + } + } + + private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue) { + QueryContext queryContext = new QueryContext(tagKey, tagValue); + for (MemTable memTable : memTables) { + QueryManager.getInstance().manager(memTable).process(queryContext); + } + return (RoaringBitmap) queryContext.getResult(); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java new file mode 100644 index 0000000000000..4125a1097db18 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.manager.BasicLsmManager; + +public class DeletionManager extends BasicLsmManager { + private DeletionManager() { + this.nextLevel(new MemTableDeletion()) + .nextLevel(new MemChunkGroupDeletion()) + .nextLevel(new MemChunkDeletion()); + } + + public static DeletionManager getInstance() { + return DeletionManagerHolder.INSTANCE; + } + + private static class DeletionManagerHolder { + private static final DeletionManager INSTANCE = new DeletionManager(); + + private DeletionManagerHolder() {} + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java new file mode 100644 index 0000000000000..c95a617765b66 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess; + +import java.util.List; + +public class MemChunkDeletion extends DeleteLevelProcess { + @Override + public List getChildren(MemChunk memNode, DeleteContext context) { + return null; + } + + @Override + public void delete(MemChunk memNode, DeleteContext context) { + Integer deviceID = (Integer) context.getValue(); + memNode.remove(deviceID); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java new file mode 100644 index 0000000000000..651553fd86032 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess; + +import java.util.ArrayList; +import java.util.List; + +public class MemChunkGroupDeletion extends DeleteLevelProcess { + @Override + public List getChildren(MemChunkGroup memNode, DeleteContext context) { + List memChunks = new ArrayList<>(); + String tagValue = (String) context.getKey(); + MemChunk child = memNode.get(tagValue); + if (child != null) memChunks.add(child); + return memChunks; + } + + @Override + public void delete(MemChunkGroup memNode, DeleteContext context) { + String tagValue = (String) context.getKey(); + MemChunk child = memNode.get(tagValue); + if (child == null || child.isEmpty()) { + memNode.remove(tagValue); + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java new file mode 100644 index 0000000000000..0a1e5b3d05715 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +public class MemTableDeletion extends DeleteLevelProcess { + @Override + public List getChildren(MemTable memNode, DeleteContext context) { + if (memNode.isImmutable()) return new ArrayList<>(); + List memChunkGroups = new ArrayList<>(); + String tagKey = (String) context.getKey(); + MemChunkGroup child = memNode.get(tagKey); + if (child != null) memChunkGroups.add(child); + return memChunkGroups; + } + + @Override + public void delete(MemTable memNode, DeleteContext context) { + if (memNode.isImmutable()) { + Set deletionList = memNode.getDeletionList(); + if (!deletionList.contains(context.getValue())) { + deletionList.add((Integer) context.getValue()); + } + return; + } + String tagKey = (String) context.getKey(); + MemChunkGroup child = memNode.get(tagKey); + if (child == null || child.isEmpty()) { + memNode.remove(tagKey); + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java new file mode 100644 index 0000000000000..3e6d71493bd30 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.manager.BasicLsmManager; + +public class InsertionManager extends BasicLsmManager { + + private InsertionManager() { + this.nextLevel(new MemTableInsertion()) + .nextLevel(new MemChunkGroupInsertion()) + .nextLevel(new MemChunkInsertion()); + } + + public static InsertionManager getInstance() { + return InsertionManagerHolder.INSTANCE; + } + + private static class InsertionManagerHolder { + private static final InsertionManager INSTANCE = new InsertionManager(); + + private InsertionManagerHolder() {} + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java new file mode 100644 index 0000000000000..843084223f4ea --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; + +import java.util.ArrayList; +import java.util.List; + +public class MemChunkGroupInsertion extends InsertLevelProcess { + @Override + public List getChildren(MemChunkGroup memNode, InsertContext context) { + List memChunks = new ArrayList<>(); + String tagValue = (String) context.getKey(); + MemChunk child = memNode.get(tagValue); + if (child != null) memChunks.add(child); + return memChunks; + } + + @Override + public void insert(MemChunkGroup memNode, InsertContext context) { + String tagValue = (String) context.getKey(); + memNode.put(tagValue); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java new file mode 100644 index 0000000000000..5dec23dcb7d1b --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; + +import java.util.List; + +public class MemChunkInsertion extends InsertLevelProcess { + @Override + public List getChildren(MemChunk memNode, InsertContext context) { + return null; + } + + @Override + public void insert(MemChunk memNode, InsertContext context) { + Integer deviceID = (Integer) context.getValue(); + memNode.put(deviceID); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java new file mode 100644 index 0000000000000..ef88f0591bc03 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; + +import java.util.ArrayList; +import java.util.List; + +// memtable的insert操作 +public class MemTableInsertion extends InsertLevelProcess { + + @Override + public List getChildren(MemTable memNode, InsertContext context) { + if (memNode.isImmutable()) return new ArrayList<>(); + List memChunkGroups = new ArrayList<>(); + String tagKey = (String) context.getKey(); + MemChunkGroup child = memNode.get(tagKey); + if (child != null) memChunkGroups.add(child); + return memChunkGroups; + } + + @Override + public void insert(MemTable memNode, InsertContext context) { + if (memNode.isImmutable()) return; + String tagKey = (String) context.getKey(); + memNode.put(tagKey); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java new file mode 100644 index 0000000000000..b46a770ca97e3 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable; + +import org.roaringbitmap.RoaringBitmap; + +// 管理设备id集合 +public class MemChunk { + private RoaringBitmap roaringBitmap; + + public MemChunk() { + roaringBitmap = new RoaringBitmap(); + } + + public boolean isEmpty() { + if (roaringBitmap == null) return true; + return roaringBitmap.isEmpty(); + } + + @Override + public String toString() { + return roaringBitmap.toString(); + } + + public void put(int id) { + roaringBitmap.add(id); + } + + public void remove(int id) { + roaringBitmap.remove(id); + } + + public RoaringBitmap getRoaringBitmap() { + return this.roaringBitmap; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java new file mode 100644 index 0000000000000..f6765030183a5 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable; + +import java.util.HashMap; +import java.util.Map; + +public class MemChunkGroup { + private Map memChunkMap; + + public MemChunkGroup() { + memChunkMap = new HashMap<>(); + } + + public void put(String tagValue) { + if (!memChunkMap.containsKey(tagValue)) { + memChunkMap.put(tagValue, new MemChunk()); + } + } + + @Override + public String toString() { + return memChunkMap.toString(); + } + + public MemChunk get(String tagValue) { + return memChunkMap.get(tagValue); + } + + public void remove(String tagValue) { + memChunkMap.remove(tagValue); + } + + public boolean isEmpty() { + return memChunkMap.isEmpty(); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java new file mode 100644 index 0000000000000..af47581ac9c3d --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class MemTable { + public static final String WORKING = "working"; + + public static final String IMMUTABLE = "immutable"; + + public static final String UNSEQUENCE = "unsequence"; + + private Map memChunkGroupMap; + + private String status; + + private Set deletionList; + + public MemTable(String status) { + memChunkGroupMap = new HashMap<>(); + this.status = status; + deletionList = new HashSet<>(); + } + + public void put(String tagKey) { + if (this.status.equals(IMMUTABLE)) return; + if (!memChunkGroupMap.containsKey(tagKey)) { + memChunkGroupMap.put(tagKey, new MemChunkGroup()); + } + } + + @Override + public String toString() { + return "MemTable{" + + "memChunkGroupMap=" + + memChunkGroupMap + + ", status='" + + status + + '\'' + + ", deletionList=" + + deletionList + + '}'; + } + + public MemChunkGroup get(String tagKey) { + return memChunkGroupMap.get(tagKey); + } + + public void remove(String tagKey) { + memChunkGroupMap.remove(tagKey); + } + + public boolean isImmutable() { + return status.equals(IMMUTABLE); + } + + public Set getDeletionList() { + return deletionList; + } + + public void setStatus(String status) { + this.status = status; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java new file mode 100644 index 0000000000000..bf9656e82205d --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess; + +import java.util.ArrayList; +import java.util.List; + +public class MemChunkGroupQuery extends QueryLevelProcess { + @Override + public List getChildren(MemChunkGroup memNode, QueryContext context) { + List memChunks = new ArrayList<>(); + String tagValue = (String) context.getKey(); + MemChunk child = memNode.get(tagValue); + if (child != null) memChunks.add(child); + return memChunks; + } + + @Override + public void query(MemChunkGroup memNode, QueryContext context) {} +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java new file mode 100644 index 0000000000000..d67de88f62bb4 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess; + +import org.roaringbitmap.RoaringBitmap; + +import java.util.List; + +public class MemChunkQuery extends QueryLevelProcess { + @Override + public List getChildren(MemChunk memNode, QueryContext context) { + return null; + } + + @Override + public void query(MemChunk memNode, QueryContext context) { + RoaringBitmap roaringBitmap = (RoaringBitmap) context.getResult(); + if (roaringBitmap == null) roaringBitmap = new RoaringBitmap(); + RoaringBitmap now = RoaringBitmap.or(roaringBitmap, memNode.getRoaringBitmap()); + context.setResult(now); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java new file mode 100644 index 0000000000000..b1acbf845b14e --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess; + +import org.roaringbitmap.RoaringBitmap; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +public class MemTableQuery extends QueryLevelProcess { + + @Override + public List getChildren(MemTable memNode, QueryContext context) { + List memChunkGroups = new ArrayList<>(); + String tagKey = (String) context.getKey(); + MemChunkGroup child = memNode.get(tagKey); + if (child != null) memChunkGroups.add(child); + return memChunkGroups; + } + + @Override + public void query(MemTable memNode, QueryContext context) { + // 如果是immutable，则需要在查询结果中删除deletionList中的id + if (memNode.isImmutable()) { + RoaringBitmap roaringBitmap = (RoaringBitmap) context.getResult(); + Set deletionList = memNode.getDeletionList(); + for (Integer id : deletionList) { + roaringBitmap.remove(id); + } + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java new file mode 100644 index 0000000000000..8987c49801f36 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.manager.BasicLsmManager; + +public class QueryManager extends BasicLsmManager { + private QueryManager() { + this.nextLevel(new MemTableQuery()) + .nextLevel(new MemChunkGroupQuery()) + .nextLevel(new MemChunkQuery()); + } + + public static QueryManager getInstance() { + return QueryManagerHolder.INSTANCE; + } + + private static class QueryManagerHolder { + private static final QueryManager INSTANCE = new QueryManager(); + + private QueryManagerHolder() {} + } +} diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java new file mode 100644 index 0000000000000..4a17213c77ba8 --- /dev/null +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.tsfile.utils.Pair; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +public class TagInvertedIndexTest { + private String[][] record = + new String[][] { + {"tag1=q", "tag2=a", "1"}, + {"tag1=q", "tag2=s", "2"}, + {"tag1=q", "tag2=a", "tag3=z", "3"}, + {"tag1=q", "tag2=s", "5"}, + {"tag1=w", "tag2=d", "6"}, + {"tag1=q", "tag2=d", "tag3=e", "7"}, + {"tag1=q", "tag3=v", "4"}, + {"tag1=r", "tag2=d", "9"}, + {"tag1=t", "tag2=f", "10"}, + {"tag1=t", "tag2=g", "8"}, + {"tag1=t", "tag2=h", "11"}, + {"tag1=y", "tag2=j", "13"}, + {"tag1=u", "tag2=k", "14"}, + {"tag1=q", "tag2=a", "tag3=l", "12"}, + {"tag1=q", "tag2=a", "tag3=x", "15"}, + {"tag1=q", "tag2=b", "tag3=x", "18"}, + {"tag1=y", "tag2=a", "tag4=z", "17"}, + {"tag1=q", "tag2=a", "tag4=z", "16"}, + }; + + private int numOfDeviceIdsInMemTable; + + private TagInvertedIndex tagInvertedIndex; + + @Before + public void setUp() throws Exception { + numOfDeviceIdsInMemTable = + IoTDBDescriptor.getInstance().getConfig().getNumOfDeviceIdsInMemTable(); + IoTDBDescriptor.getInstance().getConfig().setNumOfDeviceIdsInMemTable(3); + tagInvertedIndex = new TagInvertedIndex(); + } + + @After + public void tearDown() throws Exception { + IoTDBDescriptor.getInstance().getConfig().setNumOfDeviceIdsInMemTable(numOfDeviceIdsInMemTable); + tagInvertedIndex = null; + } + + public void addTags() { + List, Integer>> records = generateTags(); + for (Pair, Integer> pair : records) { + tagInvertedIndex.addTags(pair.left, pair.right); + } + } + + public void removeTags() { + Pair, Integer> tags = generateTag(record[0]); + tagInvertedIndex.removeTags(tags.left, tags.right); + tags = generateTag(record[1]); + tagInvertedIndex.removeTags(tags.left, tags.right); + tags = generateTag(record[6]); + tagInvertedIndex.removeTags(tags.left, tags.right); + tags = generateTag(record[13]); + tagInvertedIndex.removeTags(tags.left, tags.right); + } + + @Test + public void getMatchedIDs() { + addTags(); + System.out.println("------------addTags------------"); + System.out.println(tagInvertedIndex); + Map tags1 = new HashMap<>(); + tags1.put("tag1", "q"); + + Map tags2 = new HashMap<>(); + tags2.put("tag1", "q"); + tags2.put("tag2", "a"); + + List ids = tagInvertedIndex.getMatchedIDs(tags1); + List verify = Arrays.asList(1, 2, 3, 4, 5, 7, 12, 15, 16, 18); + assertEquals(verify, ids); + + ids = tagInvertedIndex.getMatchedIDs(tags2); + verify = Arrays.asList(1, 3, 12, 15, 16); + assertEquals(verify, ids); + + removeTags(); + System.out.println("------------removeTags------------"); + System.out.println(tagInvertedIndex); + ids = tagInvertedIndex.getMatchedIDs(tags1); + verify = Arrays.asList(3, 5, 7, 15, 16, 18); + assertEquals(verify, ids); + + ids = tagInvertedIndex.getMatchedIDs(tags2); + verify = Arrays.asList(3, 15, 16); + assertEquals(verify, ids); + + System.out.println("------------query------------"); + System.out.println(tagInvertedIndex); + } + + private List, Integer>> generateTags() { + List, Integer>> pairs = new ArrayList<>(); + for (String[] strings : record) { + pairs.add(generateTag(strings)); + } + return pairs; + } + + private Pair, Integer> generateTag(String[] strings) { + Map tags = new HashMap<>(); + int i = 0; + for (; i < strings.length - 1; i++) { + String[] str = strings[i].split("="); + tags.put(str[0], str[1]); + } + Pair, Integer> pair = new Pair<>(tags, Integer.valueOf(strings[i])); + return pair; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index c277e6570eb34..29511df14b596 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -857,6 +857,8 @@ public class IoTDBConfig { */ private boolean enableIDTableLogFile = false; + private int numOfDeviceIdsInMemTable = 65536; + /** whether to use persistent schema mode */ private String schemaEngineMode = "Memory"; @@ -2834,6 +2836,14 @@ public void setEnableIDTableLogFile(boolean enableIDTableLogFile) { this.enableIDTableLogFile = enableIDTableLogFile; } + public int getNumOfDeviceIdsInMemTable() { + return numOfDeviceIdsInMemTable; + } + + public void setNumOfDeviceIdsInMemTable(int numOfDeviceIdsInMemTable) { + this.numOfDeviceIdsInMemTable = numOfDeviceIdsInMemTable; + } + public String getSchemaEngineMode() { return schemaEngineMode; } From 954a040a744419b112e0fbb54da8b49ddfd1d061 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Tue, 13 Sep 2022 18:58:32 +0800 Subject: [PATCH 12/30] impl TagInvertedIndex --- schema-engine-tag/pom.xml | 11 +- .../tagSchemaRegion/MockTagSchemaRegion.java | 325 ++++++++++++++++++ .../tagSchemaRegion/TagSchemaRegion.java | 21 ++ .../tagIndex/TagInvertedIndexTest.java | 40 +-- 4 files changed, 368 insertions(+), 29 deletions(-) create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java diff --git a/schema-engine-tag/pom.xml b/schema-engine-tag/pom.xml index e44c73f76f055..c5b8f565525a0 100644 --- a/schema-engine-tag/pom.xml +++ b/schema-engine-tag/pom.xml @@ -15,31 +15,25 @@ See the License for the specific language governing permissions and limitations under the License. --> - + iotdb-parent org.apache.iotdb 0.14.0-SNAPSHOT - ../pom.xml 4.0.0 schema-engine-tag schema-engine-tag - org.roaringbitmap RoaringBitmap 0.9.32 - org.apache.iotdb iotdb-lsm ${project.version} - provided org.apache.iotdb @@ -47,5 +41,4 @@ ${project.version} - - \ No newline at end of file + diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java new file mode 100644 index 0000000000000..184ce789b99cd --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java @@ -0,0 +1,325 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion; + +import org.apache.iotdb.common.rpc.thrift.TSchemaNode; +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.metadata.LocalSchemaProcessor; +import org.apache.iotdb.db.metadata.mnode.IMNode; +import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; +import org.apache.iotdb.db.metadata.path.MeasurementPath; +import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.metadata.template.Template; +import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.qp.physical.crud.InsertPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan; +import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan; +import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan; +import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.dataset.ShowDevicesResult; +import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.Pair; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +public class MockTagSchemaRegion implements ISchemaRegion { + + @Override + public void init() throws MetadataException {} + + @Override + public void clear() {} + + @Override + public void forceMlog() {} + + @Override + public SchemaRegionId getSchemaRegionId() { + return null; + } + + @Override + public String getStorageGroupFullPath() { + return null; + } + + @Override + public void deleteSchemaRegion() throws MetadataException {} + + @Override + public boolean createSnapshot(File snapshotDir) { + return false; + } + + @Override + public void loadSnapshot(File latestSnapshotRootDir) {} + + @Override + public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException {} + + @Override + public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException {} + + @Override + public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException {} + + @Override + public boolean isPathExist(PartialPath path) throws MetadataException { + return false; + } + + @Override + public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, Map templateMap, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains) + throws MetadataException { + return 0; + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { + return null; + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, + int level, + boolean isPrefixMatch, + String key, + String value, + boolean isContains) + throws MetadataException { + return null; + } + + @Override + public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public List getNodesListInGivenLevel( + PartialPath pathPattern, + int nodeLevel, + boolean isPrefixMatch, + LocalSchemaProcessor.StorageGroupFilter filter) + throws MetadataException { + return null; + } + + @Override + public Set getChildNodePathInNextLevel(PartialPath pathPattern) + throws MetadataException { + return null; + } + + @Override + public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException { + return null; + } + + @Override + public Set getBelongedDevices(PartialPath timeseries) throws MetadataException { + return null; + } + + @Override + public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan) + throws MetadataException { + return null; + } + + @Override + public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public Pair, Integer> getMeasurementPathsWithAlias( + PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public List fetchSchema( + PartialPath pathPattern, Map templateMap) throws MetadataException { + return null; + } + + @Override + public Pair, Integer> showTimeseries( + ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { + return null; + } + + @Override + public List getAllMeasurementByDevicePath(PartialPath devicePath) + throws PathNotExistException { + return null; + } + + @Override + public IMNode getDeviceNode(PartialPath path) throws MetadataException { + return null; + } + + @Override + public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException { + return null; + } + + @Override + public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException {} + + @Override + public void upsertTagsAndAttributes( + String alias, + Map tagsMap, + Map attributesMap, + PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void addAttributes(Map attributesMap, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void addTags(Map tagsMap, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void dropTagsOrAttributes(Set keySet, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) + throws MetadataException, IOException { + return null; + } + + @Override + public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( + PartialPath devicePath, + String[] measurements, + Function getDataType, + boolean aligned) + throws MetadataException { + return null; + } + + @Override + public Set getPathsSetTemplate(String templateName) throws MetadataException { + return null; + } + + @Override + public Set getPathsUsingTemplate(String templateName) throws MetadataException { + return null; + } + + @Override + public boolean isTemplateAppendable(Template template, List measurements) + throws MetadataException { + return false; + } + + @Override + public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException {} + + @Override + public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException {} + + @Override + public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException {} + + @Override + public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template) + throws MetadataException {} + + @Override + public List getPathsUsingTemplate(int templateId) throws MetadataException { + return null; + } + + @Override + public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException { + return null; + } + + @Override + public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException {} +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java new file mode 100644 index 0000000000000..2f04d7c095a73 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion; + +public class TagSchemaRegion {} diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java index 4a17213c77ba8..a927c2ebbce79 100644 --- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java @@ -36,24 +36,24 @@ public class TagInvertedIndexTest { private String[][] record = new String[][] { - {"tag1=q", "tag2=a", "1"}, - {"tag1=q", "tag2=s", "2"}, - {"tag1=q", "tag2=a", "tag3=z", "3"}, - {"tag1=q", "tag2=s", "5"}, - {"tag1=w", "tag2=d", "6"}, - {"tag1=q", "tag2=d", "tag3=e", "7"}, - {"tag1=q", "tag3=v", "4"}, - {"tag1=r", "tag2=d", "9"}, - {"tag1=t", "tag2=f", "10"}, - {"tag1=t", "tag2=g", "8"}, - {"tag1=t", "tag2=h", "11"}, - {"tag1=y", "tag2=j", "13"}, - {"tag1=u", "tag2=k", "14"}, - {"tag1=q", "tag2=a", "tag3=l", "12"}, - {"tag1=q", "tag2=a", "tag3=x", "15"}, - {"tag1=q", "tag2=b", "tag3=x", "18"}, - {"tag1=y", "tag2=a", "tag4=z", "17"}, - {"tag1=q", "tag2=a", "tag4=z", "16"}, + {"tag1=q", "tag2=a", "1"}, + {"tag1=q", "tag2=s", "2"}, + {"tag1=q", "tag2=a", "tag3=z", "3"}, + {"tag1=q", "tag3=v", "4"}, + {"tag1=q", "tag2=s", "5"}, + {"tag1=w", "tag2=d", "6"}, + {"tag1=q", "tag2=d", "tag3=e", "7"}, + {"tag1=t", "tag2=g", "8"}, + {"tag1=r", "tag2=d", "9"}, + {"tag1=t", "tag2=f", "10"}, + {"tag1=t", "tag2=h", "11"}, + {"tag1=q", "tag2=a", "tag3=l", "12"}, + {"tag1=y", "tag2=j", "13"}, + {"tag1=u", "tag2=k", "14"}, + {"tag1=q", "tag2=a", "tag3=x", "15"}, + {"tag1=q", "tag2=a", "tag4=z", "16"}, + {"tag1=y", "tag2=a", "tag4=z", "17"}, + {"tag1=q", "tag2=b", "tag3=x", "18"}, }; private int numOfDeviceIdsInMemTable; @@ -86,9 +86,9 @@ public void removeTags() { tagInvertedIndex.removeTags(tags.left, tags.right); tags = generateTag(record[1]); tagInvertedIndex.removeTags(tags.left, tags.right); - tags = generateTag(record[6]); + tags = generateTag(record[3]); tagInvertedIndex.removeTags(tags.left, tags.right); - tags = generateTag(record[13]); + tags = generateTag(record[11]); tagInvertedIndex.removeTags(tags.left, tags.right); } From e619f3f659874ce2fc496054eab1f7cea1f8e433 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 14 Sep 2022 16:40:41 +0800 Subject: [PATCH 13/30] impl wal --- .../iotdb/lsm/context/DeleteContext.java | 9 ++ .../iotdb/lsm/context/InsertContext.java | 9 ++ .../org/apache/iotdb/lsm/example/Main.java | 38 +++--- .../iotdb/lsm/manager/BasicLsmManager.java | 16 +-- .../apache/iotdb/lsm/manager/LsmManager.java | 3 +- .../org/apache/iotdb/lsm/wal/IWALReader.java | 31 +++++ .../org/apache/iotdb/lsm/wal/IWALWriter.java | 30 +++++ .../org/apache/iotdb/lsm/wal/WALReader.java | 92 ++++++++++++++ .../org/apache/iotdb/lsm/wal/WALRecord.java | 38 ++++++ .../org/apache/iotdb/lsm/wal/WALWriter.java | 98 +++++++++++++++ .../tagIndex/TagInvertedIndex.java | 105 ++++++++++------ .../tagIndex/deletion/DeletionManager.java | 23 ++-- .../tagIndex/insertion/InsertionManager.java | 26 ++-- .../tagIndex/memtable/MemTable.java | 2 - .../tagIndex/query/QueryManager.java | 17 +-- .../tagIndex/recover/RecoverManager.java | 33 +++++ .../tagIndex/wal/WALEntry.java | 95 ++++++++++++++ .../tagIndex/wal/WALManager.java | 117 ++++++++++++++++++ .../tagIndex/TagInvertedIndexTest.java | 53 +++++--- 19 files changed, 717 insertions(+), 118 deletions(-) create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java index 46d78b9ed5131..2b99196b0c3aa 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java @@ -39,10 +39,19 @@ public DeleteContext(Object value, Object... ks) { accessStrategy = new PostOrderAccessStrategy(); } + public DeleteContext(Object value,List keys){ + this.value = value; + this.keys = keys; + } + public Object getKey() { return keys.get(level); } + public List getKeys() { + return keys; + } + public Object getValue() { return value; } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java index a34c9d5814ce8..e8d00da4979d7 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java @@ -40,10 +40,19 @@ public InsertContext(Object value, Object... keys) { accessStrategy = new PreOrderAccessStrategy(); } + public InsertContext(Object value,List keys){ + this.value = value; + this.keys = keys; + } + public Object getKey() { return keys.get(level); } + public List getKeys() { + return keys; + } + public Object getValue() { return value; } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java index bea5c80e15873..7dbe793ac5fcd 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java @@ -29,7 +29,7 @@ import java.util.Map; public class Main { - public static void main(String[] args) { + public static void main(String[] args) throws Exception { MemTableManager memTableManager = new MemTableManager(); System.out.println("-------------insert--------------"); insertionExample(memTableManager); @@ -37,10 +37,10 @@ public static void main(String[] args) { flushExample(memTableManager); } - public static void insertionExample(MemTableManager memTableManager) { + public static void insertionExample(MemTableManager memTableManager) throws Exception { BasicLsmManager baseLsmManager = - new BasicLsmManager().manager(memTableManager); + new BasicLsmManager(); baseLsmManager .nextLevel( new InsertLevelProcess() { @@ -127,25 +127,25 @@ public void insert(MemChunk memNode, InsertContext context) { } }); - baseLsmManager.process(new InsertContext(1, null, "a", "b")); - baseLsmManager.process(new InsertContext(2, null, "a", "d")); - baseLsmManager.process(new InsertContext(3, null, "a", "e")); - baseLsmManager.process(new InsertContext(4, null, "a", "b")); - baseLsmManager.process(new InsertContext(5, null, "a1", "b")); - baseLsmManager.process(new InsertContext(6, null, "a2", "b")); - baseLsmManager.process(new InsertContext(65535, null, "a", "b")); - baseLsmManager.process(new InsertContext(65536, null, "a", "b")); - baseLsmManager.process(new InsertContext(2, null, "a", "d")); - baseLsmManager.process(new InsertContext(3, null, "a", "e")); - baseLsmManager.process(new InsertContext(4, null, "a", "b")); - baseLsmManager.process(new InsertContext(5, null, "a1", "b")); - baseLsmManager.process(new InsertContext(6, null, "a2", "b")); + baseLsmManager.process(memTableManager, new InsertContext(1, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertContext(2, null, "a", "d")); + baseLsmManager.process(memTableManager, new InsertContext(3, null, "a", "e")); + baseLsmManager.process(memTableManager, new InsertContext(4, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertContext(5, null, "a1", "b")); + baseLsmManager.process(memTableManager, new InsertContext(6, null, "a2", "b")); + baseLsmManager.process(memTableManager, new InsertContext(65535, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertContext(65536, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertContext(2, null, "a", "d")); + baseLsmManager.process(memTableManager, new InsertContext(3, null, "a", "e")); + baseLsmManager.process(memTableManager, new InsertContext(4, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertContext(5, null, "a1", "b")); + baseLsmManager.process(memTableManager, new InsertContext(6, null, "a2", "b")); System.out.println(memTableManager); } - public static void flushExample(MemTableManager memTableManager) { + public static void flushExample(MemTableManager memTableManager) throws Exception { BasicLsmManager flushManager = - new BasicLsmManager().manager(memTableManager); + new BasicLsmManager(); flushManager .nextLevel( @@ -204,6 +204,6 @@ public List getChildren(MemChunk memNode, FlushContext context) { } }); - flushManager.process(new FlushContext()); + flushManager.process(memTableManager, new FlushContext()); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java index 3d33dd1712864..a5b91439662aa 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java @@ -21,21 +21,21 @@ import org.apache.iotdb.lsm.context.Context; import org.apache.iotdb.lsm.levelProcess.LevelProcess; -public class BasicLsmManager implements LsmManager { +import java.io.IOException; - T root; +public class BasicLsmManager implements LsmManager { LevelProcess levelProcess; - @Override - public BasicLsmManager manager(T memNode) { - root = memNode; - return this; - } + public void preProcess(T root, C context) throws Exception {} + + public void postProcess(T root, C context) throws Exception{} @Override - public void process(C context) { + public void process(T root, C context) throws Exception{ + preProcess(root, context); levelProcess.process(root, context); + postProcess(root, context); } @Override diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java index 3d60af6f69f8e..2940ad60b1751 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java @@ -22,9 +22,8 @@ import org.apache.iotdb.lsm.levelProcess.LevelProcess; public interface LsmManager { - LsmManager manager(T memNode); - void process(C context); + void process(T memNode, C context) throws Exception; LevelProcess nextLevel(LevelProcess next); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java new file mode 100644 index 0000000000000..6d52ef1577fcd --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import java.io.FileNotFoundException; +import java.io.IOException; + +public interface IWALReader { + + void close() throws IOException; + + boolean hasNext() throws FileNotFoundException; + + WALRecord next() throws FileNotFoundException; +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java new file mode 100644 index 0000000000000..6d3fecb21e2eb --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import java.io.IOException; + +public interface IWALWriter { + + void write(WALRecord walRecord) throws IOException; + + void force() throws IOException; + + void close() throws IOException; +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java new file mode 100644 index 0000000000000..a3fef56b0860d --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.NoSuchElementException; + +public class WALReader implements IWALReader { + private static final Logger logger = LoggerFactory.getLogger(WALReader.class); + private final File logFile; + private final WALRecord prototype; + private final DataInputStream logStream; + private WALRecord nextRecord; + private boolean fileCorrupted = false; + + public WALReader(File logFile, WALRecord prototype) throws IOException { + this.logFile = logFile; + this.logStream = + new DataInputStream(new BufferedInputStream(Files.newInputStream(logFile.toPath()))); + this.prototype = prototype; + } + + @Override + public void close() throws IOException { + logStream.close(); + } + + @Override + public boolean hasNext() { + if (nextRecord != null) { + return true; + } + try { + if (fileCorrupted) { + return false; + } + int logSize = logStream.readInt(); + if (logSize <= 0) { + return false; + } + nextRecord = prototype.clone(); + nextRecord.deserialize(logStream); + } catch (EOFException e) { + logger.info(""); + return false; + } catch (IOException e) { + logger.warn(""); + fileCorrupted = true; + return false; + } + return true; + } + + @Override + public WALRecord next() { + if (nextRecord == null) { + throw new NoSuchElementException(); + } + WALRecord walRecord = nextRecord; + nextRecord = null; + return walRecord; + } + + @Override + public String toString() { + return "WALReader{" + "logFile=" + logFile + '}'; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java new file mode 100644 index 0000000000000..a44b8f9d243c5 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public abstract class WALRecord implements Cloneable { + public abstract void serialize(ByteBuffer buffer); + + public abstract void deserialize(DataInputStream stream) throws IOException; + + @Override + public WALRecord clone() { + try { + return (WALRecord) super.clone(); + } catch (CloneNotSupportedException e) { + throw new AssertionError(); + } + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java new file mode 100644 index 0000000000000..dccd8a92233db --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.FileChannel; + +public class WALWriter implements IWALWriter { + private static final Logger logger = LoggerFactory.getLogger(WALWriter.class); + private File logFile; + private FileOutputStream fileOutputStream; + private FileChannel channel; + private final ByteBuffer lengthBuffer = ByteBuffer.allocate(4); + private final ByteBuffer walBuffer = ByteBuffer.allocate(10000); + private final boolean forceEachWrite; + + public WALWriter(File logFile, boolean forceEachWrite) throws FileNotFoundException { + this.logFile = logFile; + this.forceEachWrite = forceEachWrite; + fileOutputStream = new FileOutputStream(logFile, true); + channel = fileOutputStream.getChannel(); + } + + @Override + public void write(WALRecord walRecord) throws IOException { + if (channel == null) { + fileOutputStream = new FileOutputStream(logFile, true); + channel = fileOutputStream.getChannel(); + } + walBuffer.clear(); + walRecord.serialize(walBuffer); + walBuffer.flip(); + int logSize = walBuffer.limit(); + lengthBuffer.clear(); + lengthBuffer.putInt(logSize); + lengthBuffer.flip(); + + try { + channel.write(lengthBuffer); + channel.write(walBuffer); + + if (this.forceEachWrite) { + channel.force(true); + } + } catch (ClosedChannelException ignored) { + logger.warn("someone interrupt current thread, so no need to do write for io safety"); + } + } + + @Override + public void force() throws IOException { + if (channel != null && channel.isOpen()) { + channel.force(true); + } + } + + @Override + public void close() throws IOException { + if (channel != null) { + if (channel.isOpen()) { + channel.force(true); + } + fileOutputStream.close(); + fileOutputStream = null; + channel.close(); + channel = null; + } + } + + @Override + public String toString() { + return "WALLogWriter{" + "logFile=" + logFile + '}'; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java index 89c3260ed0e55..71c2ee923106a 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -18,18 +18,25 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.DeletionManager; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.InsertionManager; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.QueryManager; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.recover.RecoverManager; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; import org.apache.iotdb.lsm.context.DeleteContext; import org.apache.iotdb.lsm.context.InsertContext; import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.wal.WALWriter; import org.roaringbitmap.RoaringBitmap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -38,44 +45,61 @@ import java.util.stream.Collectors; public class TagInvertedIndex implements ITagInvertedIndex { + private static final Logger logger = LoggerFactory.getLogger(TagInvertedIndex.class); private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private int numOfDeviceIdsInMemTable; + private final InsertionManager insertionManager; - private MemTable workingMemTable; + private final DeletionManager deletionManager; + + private final QueryManager queryManager; + + private final WALManager walManager; + + private final RecoverManager recoverManager; - private MemTable unsequenceMemTable; + private final int numOfDeviceIdsInMemTable; - private Map immutableMemTables; + private final Map immutableMemTables; + + private MemTable workingMemTable; private int maxDeviceID; - public TagInvertedIndex() { + public TagInvertedIndex(String schemaDirPath) throws IOException { + walManager = new WALManager(schemaDirPath); + insertionManager = new InsertionManager(walManager); + deletionManager = new DeletionManager(walManager); + recoverManager = new RecoverManager(walManager); + queryManager = new QueryManager(); workingMemTable = new MemTable(MemTable.WORKING); - unsequenceMemTable = new MemTable(MemTable.UNSEQUENCE); immutableMemTables = new HashMap<>(); numOfDeviceIdsInMemTable = config.getNumOfDeviceIdsInMemTable(); maxDeviceID = 0; + recover(); + } + + public synchronized void recover(){ + recoverManager.recover(this); } @Override public synchronized void addTags(Map tags, int id) { MemTable memTable = null; - // 出现乱序 - if (id < maxDeviceID) { - memTable = unsequenceMemTable; - } else { - if (!inWorkingMemTable(id)) { - workingMemTable.setStatus(MemTable.IMMUTABLE); - immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); - workingMemTable = new MemTable(MemTable.WORKING); - } - memTable = workingMemTable; - maxDeviceID = id; + if (!inWorkingMemTable(id)) { + workingMemTable.setStatus(MemTable.IMMUTABLE); + immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); + workingMemTable = new MemTable(MemTable.WORKING); } - for (Map.Entry tag : tags.entrySet()) { - addTag(memTable, tag.getKey(), tag.getValue(), id); + memTable = workingMemTable; + maxDeviceID = id; + try { + for (Map.Entry tag : tags.entrySet()) { + addTag(memTable, tag.getKey(), tag.getValue(), id); + } + }catch (Exception e){ + logger.error(e.getMessage()); } } @@ -86,11 +110,14 @@ public synchronized void removeTags(Map tags, int id) { if (inWorkingMemTable(id)) { memTables.add(workingMemTable); } else { - memTables.add(unsequenceMemTable); memTables.add(immutableMemTables.get(id / numOfDeviceIdsInMemTable)); } - for (Map.Entry tag : tags.entrySet()) { - removeTag(memTables, tag.getKey(), tag.getValue(), id); + try { + for (Map.Entry tag : tags.entrySet()) { + removeTag(memTables, tag.getKey(), tag.getValue(), id); + } + }catch (Exception e){ + logger.error(e.getMessage()); } } @@ -98,15 +125,18 @@ public synchronized void removeTags(Map tags, int id) { public synchronized List getMatchedIDs(Map tags) { List memTables = new ArrayList<>(); memTables.add(workingMemTable); - memTables.add(unsequenceMemTable); memTables.addAll(immutableMemTables.values()); RoaringBitmap roaringBitmap = new RoaringBitmap(); int i = 0; - for (Map.Entry tag : tags.entrySet()) { - RoaringBitmap rb = getMatchedIDs(memTables, tag.getKey(), tag.getValue()); - if (i == 0) roaringBitmap = rb; - else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb); - i++; + try { + for (Map.Entry tag : tags.entrySet()) { + RoaringBitmap rb = getMatchedIDs(memTables, tag.getKey(), tag.getValue()); + if (i == 0) roaringBitmap = rb; + else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb); + i++; + } + }catch (Exception e){ + logger.error(e.getMessage()); } return Arrays.stream(roaringBitmap.toArray()).boxed().collect(Collectors.toList()); } @@ -118,8 +148,6 @@ public String toString() { + numOfDeviceIdsInMemTable + ", workingMemTable=" + workingMemTable - + ", unsequenceMemTable=" - + unsequenceMemTable + ", immutableMemTables=" + immutableMemTables + ", maxDeviceID=" @@ -131,23 +159,28 @@ private synchronized boolean inWorkingMemTable(int id) { return id / numOfDeviceIdsInMemTable == maxDeviceID / numOfDeviceIdsInMemTable; } - private void addTag(MemTable memTable, String tagKey, String tagValue, int id) { + private void addTag(MemTable memTable, String tagKey, String tagValue, int id) throws Exception { InsertContext insertContext = new InsertContext(id, tagKey, tagValue); - InsertionManager.getInstance().manager(memTable).process(insertContext); + insertionManager.process(memTable, insertContext); } - private void removeTag(List memTables, String tagKey, String tagValue, int id) { + private void removeTag(List memTables, String tagKey, String tagValue, int id) throws Exception { DeleteContext deleteContext = new DeleteContext(id, tagKey, tagValue); for (MemTable memTable : memTables) { - DeletionManager.getInstance().manager(memTable).process(deleteContext); + deletionManager.process(memTable, deleteContext); } } - private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue) { + private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue) throws Exception { QueryContext queryContext = new QueryContext(tagKey, tagValue); for (MemTable memTable : memTables) { - QueryManager.getInstance().manager(memTable).process(queryContext); + queryManager.process(memTable, queryContext); } return (RoaringBitmap) queryContext.getResult(); } + + @TestOnly + public void clear() throws IOException { + walManager.close(); + } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java index 4125a1097db18..0d8e62a769f67 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java @@ -19,23 +19,24 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; import org.apache.iotdb.lsm.context.DeleteContext; import org.apache.iotdb.lsm.manager.BasicLsmManager; +import java.io.IOException; + public class DeletionManager extends BasicLsmManager { - private DeletionManager() { - this.nextLevel(new MemTableDeletion()) - .nextLevel(new MemChunkGroupDeletion()) - .nextLevel(new MemChunkDeletion()); - } - public static DeletionManager getInstance() { - return DeletionManagerHolder.INSTANCE; - } + WALManager walManager; - private static class DeletionManagerHolder { - private static final DeletionManager INSTANCE = new DeletionManager(); + public DeletionManager(WALManager walManager) { + this.walManager = walManager; + initLevelProcess(); + } - private DeletionManagerHolder() {} + private void initLevelProcess() { + this.nextLevel(new MemTableDeletion()) + .nextLevel(new MemChunkGroupDeletion()) + .nextLevel(new MemChunkDeletion()); } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java index 3e6d71493bd30..4b4dd29ee8c62 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java @@ -19,24 +19,30 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALEntry; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; import org.apache.iotdb.lsm.context.InsertContext; import org.apache.iotdb.lsm.manager.BasicLsmManager; +import java.io.IOException; + public class InsertionManager extends BasicLsmManager { - private InsertionManager() { - this.nextLevel(new MemTableInsertion()) - .nextLevel(new MemChunkGroupInsertion()) - .nextLevel(new MemChunkInsertion()); - } + private WALManager walManager; - public static InsertionManager getInstance() { - return InsertionManagerHolder.INSTANCE; + public InsertionManager(WALManager walManager){ + this.walManager = walManager; + initLevelProcess(); } - private static class InsertionManagerHolder { - private static final InsertionManager INSTANCE = new InsertionManager(); + @Override + public void preProcess(MemTable root, InsertContext context) throws IOException { + walManager.write(context); + } - private InsertionManagerHolder() {} + private void initLevelProcess() { + this.nextLevel(new MemTableInsertion()) + .nextLevel(new MemChunkGroupInsertion()) + .nextLevel(new MemChunkInsertion()); } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java index af47581ac9c3d..f0730e162df1c 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java @@ -28,8 +28,6 @@ public class MemTable { public static final String IMMUTABLE = "immutable"; - public static final String UNSEQUENCE = "unsequence"; - private Map memChunkGroupMap; private String status; diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java index 8987c49801f36..4a8e3fdbcb0eb 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java @@ -23,19 +23,14 @@ import org.apache.iotdb.lsm.manager.BasicLsmManager; public class QueryManager extends BasicLsmManager { - private QueryManager() { - this.nextLevel(new MemTableQuery()) - .nextLevel(new MemChunkGroupQuery()) - .nextLevel(new MemChunkQuery()); - } - public static QueryManager getInstance() { - return QueryManagerHolder.INSTANCE; + public QueryManager() { + initLevelProcess(); } - private static class QueryManagerHolder { - private static final QueryManager INSTANCE = new QueryManager(); - - private QueryManagerHolder() {} + private void initLevelProcess() { + this.nextLevel(new MemTableQuery()) + .nextLevel(new MemChunkGroupQuery()) + .nextLevel(new MemChunkQuery()); } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java new file mode 100644 index 0000000000000..2c3d38bc06278 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.recover; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.TagInvertedIndex; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; + +public class RecoverManager { + + WALManager walManager; + + public RecoverManager(WALManager walManager) { + this.walManager = walManager; + } + + public void recover(TagInvertedIndex tagInvertedIndex) {} +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java new file mode 100644 index 0000000000000..e70ad62100b10 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal; + +import org.apache.iotdb.lsm.wal.WALRecord; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class WALEntry extends WALRecord { + private int type; + + private List keys; + + private int deviceID; + + public WALEntry() {} + + public WALEntry(int type, List keys, int deviceID) { + this.type = type; + this.keys = keys; + this.deviceID = deviceID; + } + + @Override + public void serialize(ByteBuffer buffer) { + ReadWriteIOUtils.write(type, buffer); + ReadWriteIOUtils.write(deviceID, buffer); + ReadWriteIOUtils.write(keys.size(), buffer); + for (String key : keys) { + ReadWriteIOUtils.write(key, buffer); + } + } + + @Override + public void deserialize(DataInputStream stream) throws IOException { + this.type = stream.readInt(); + this.deviceID = stream.readInt(); + int length = stream.readInt(); + this.keys = new ArrayList<>(); + for (int i = 0; i < length; i++) { + String key = ReadWriteIOUtils.readString(stream); + keys.add(key); + } + } + + public int getType() { + return type; + } + + public void setType(int type) { + this.type = type; + } + + public List getKeys() { + return keys; + } + + public void setKeys(List keys) { + this.keys = keys; + } + + public int getDeviceID() { + return deviceID; + } + + public void setDeviceID(int deviceID) { + this.deviceID = deviceID; + } + + @Override + public String toString() { + return "WALEntry{" + "type=" + type + ", keys=" + keys + ", deviceID=" + deviceID + '}'; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java new file mode 100644 index 0000000000000..88bdf68dd3143 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.wal.WALReader; +import org.apache.iotdb.lsm.wal.WALWriter; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class WALManager { + private static final String WAL_FILE_NAME = "tagInvertedIndex.log"; + private static final int INSERT = 1; + private static final int DELETE = 2; + private final String schemaDirPath; + private File walFile; + private WALWriter walWriter; + private WALReader walReader; + + public WALManager(String schemaDirPath) throws IOException { + this.schemaDirPath = schemaDirPath; + initFile(schemaDirPath); + walWriter = new WALWriter(walFile, false); + walReader = new WALReader(walFile, new WALEntry()); + } + + private void initFile(String schemaDirPath) throws IOException { + File schemaDir = new File(schemaDirPath); + schemaDir.mkdirs(); + walFile = new File(this.schemaDirPath, WAL_FILE_NAME); + if (!walFile.exists()) { + walFile.createNewFile(); + } + } + + public synchronized void write(Context context) throws IOException { + switch (context.getType()) { + case INSERT: + process((InsertContext) context); + break; + case DELETE: + process((DeleteContext) context); + break; + default: + break; + } + } + + public synchronized Context read() { + if (walReader.hasNext()) { + WALEntry walEntry = (WALEntry) walReader.next(); + if (walEntry.getType() == INSERT) { + return generateInsertContext(walEntry); + } + if (walEntry.getType() == DELETE) { + return generateDeleteContext(walEntry); + } + } + return new Context(); + } + + private InsertContext generateInsertContext(WALEntry walEntry) { + InsertContext insertContext = new InsertContext(walEntry.getDeviceID(), walEntry.getKeys()); + return insertContext; + } + + private DeleteContext generateDeleteContext(WALEntry walEntry) { + DeleteContext deleteContext = new DeleteContext(walEntry.getDeviceID(), walEntry.getKeys()); + return deleteContext; + } + + private void process(InsertContext insertContext) throws IOException { + List objects = insertContext.getKeys(); + List keys = new ArrayList<>(); + for (Object o : objects) { + keys.add((String) o); + } + WALEntry walEntry = new WALEntry(INSERT, keys, (Integer) insertContext.getValue()); + walWriter.write(walEntry); + } + + private void process(DeleteContext deleteContext) throws IOException { + List objects = deleteContext.getKeys(); + List keys = new ArrayList<>(); + for (Object o : objects) { + keys.add((String) o); + } + WALEntry walEntry = new WALEntry(DELETE, keys, (Integer) deleteContext.getValue()); + walWriter.write(walEntry); + } + + public void close() throws IOException { + walWriter.close(); + walReader.close(); + } +} diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java index a927c2ebbce79..eba4c1067d3f3 100644 --- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; +import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.tsfile.utils.Pair; @@ -25,6 +26,7 @@ import org.junit.Before; import org.junit.Test; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -36,42 +38,55 @@ public class TagInvertedIndexTest { private String[][] record = new String[][] { - {"tag1=q", "tag2=a", "1"}, - {"tag1=q", "tag2=s", "2"}, - {"tag1=q", "tag2=a", "tag3=z", "3"}, - {"tag1=q", "tag3=v", "4"}, - {"tag1=q", "tag2=s", "5"}, - {"tag1=w", "tag2=d", "6"}, - {"tag1=q", "tag2=d", "tag3=e", "7"}, - {"tag1=t", "tag2=g", "8"}, - {"tag1=r", "tag2=d", "9"}, - {"tag1=t", "tag2=f", "10"}, - {"tag1=t", "tag2=h", "11"}, - {"tag1=q", "tag2=a", "tag3=l", "12"}, - {"tag1=y", "tag2=j", "13"}, - {"tag1=u", "tag2=k", "14"}, - {"tag1=q", "tag2=a", "tag3=x", "15"}, - {"tag1=q", "tag2=a", "tag4=z", "16"}, - {"tag1=y", "tag2=a", "tag4=z", "17"}, - {"tag1=q", "tag2=b", "tag3=x", "18"}, + {"tag1=q", "tag2=a", "1"}, + {"tag1=q", "tag2=s", "2"}, + {"tag1=q", "tag2=a", "tag3=z", "3"}, + {"tag1=q", "tag3=v", "4"}, + {"tag1=q", "tag2=s", "5"}, + {"tag1=w", "tag2=d", "6"}, + {"tag1=q", "tag2=d", "tag3=e", "7"}, + {"tag1=t", "tag2=g", "8"}, + {"tag1=r", "tag2=d", "9"}, + {"tag1=t", "tag2=f", "10"}, + {"tag1=t", "tag2=h", "11"}, + {"tag1=q", "tag2=a", "tag3=l", "12"}, + {"tag1=y", "tag2=j", "13"}, + {"tag1=u", "tag2=k", "14"}, + {"tag1=q", "tag2=a", "tag3=x", "15"}, + {"tag1=q", "tag2=a", "tag4=z", "16"}, + {"tag1=y", "tag2=a", "tag4=z", "17"}, + {"tag1=q", "tag2=b", "tag3=x", "18"}, }; private int numOfDeviceIdsInMemTable; private TagInvertedIndex tagInvertedIndex; + private String storageGroupDirPath; + + private String schemaRegionDirPath; + + private String storageGroupFullPath = "root/testTagIndex"; + + private String schemaDir; + @Before public void setUp() throws Exception { numOfDeviceIdsInMemTable = IoTDBDescriptor.getInstance().getConfig().getNumOfDeviceIdsInMemTable(); IoTDBDescriptor.getInstance().getConfig().setNumOfDeviceIdsInMemTable(3); - tagInvertedIndex = new TagInvertedIndex(); + schemaDir = IoTDBDescriptor.getInstance().getConfig().getSchemaDir(); + storageGroupDirPath = schemaDir + File.separator + storageGroupFullPath; + schemaRegionDirPath = storageGroupDirPath + File.separator + 0; + tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath); } @After public void tearDown() throws Exception { IoTDBDescriptor.getInstance().getConfig().setNumOfDeviceIdsInMemTable(numOfDeviceIdsInMemTable); + tagInvertedIndex.clear(); tagInvertedIndex = null; + FileUtils.deleteDirectoryAndEmptyParent(new File(schemaDir)); } public void addTags() { From 18a7b740d8f2b8343433202d9ae859146c8ba87f Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 14 Sep 2022 20:13:35 +0800 Subject: [PATCH 14/30] impl RecoverManager --- .../org/apache/iotdb/lsm/context/Context.java | 12 +++++ .../iotdb/lsm/context/DeleteContext.java | 13 ++++- .../iotdb/lsm/context/InsertContext.java | 18 +++++-- .../iotdb/lsm/manager/BasicLsmManager.java | 6 +-- .../tagIndex/ITagInvertedIndex.java | 8 +++ .../tagIndex/TagInvertedIndex.java | 52 +++++++++++++++---- .../tagIndex/deletion/DeletionManager.java | 9 +++- .../tagIndex/insertion/InsertionManager.java | 7 +-- .../tagIndex/recover/RecoverManager.java | 19 ++++++- .../tagIndex/wal/WALManager.java | 13 ++++- .../tagIndex/TagInvertedIndexTest.java | 30 ++++++++--- 11 files changed, 153 insertions(+), 34 deletions(-) diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java index 305b31747f6c3..3f369b3017734 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java @@ -41,12 +41,16 @@ public class Context { // 返回值 Object result; + // 是否正在recover + boolean recover; + public Context() { accessStrategy = new PreOrderAccessStrategy(); type = ContextType.NONE; level = 0; threadNums = 1; levelUpperBound = Integer.MAX_VALUE; + recover = false; } public void setLevel(int level) { @@ -96,4 +100,12 @@ public int getLevelUpperBound() { public void setLevelUpperBound(int levelUpperBound) { this.levelUpperBound = levelUpperBound; } + + public boolean isRecover() { + return recover; + } + + public void setRecover(boolean recover) { + this.recover = recover; + } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java index 2b99196b0c3aa..9b4b30ac41a75 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java @@ -30,6 +30,12 @@ public class DeleteContext extends Context { Object value; + public DeleteContext() { + super(); + type = ContextType.DELETE; + accessStrategy = new PostOrderAccessStrategy(); + } + public DeleteContext(Object value, Object... ks) { super(); this.value = value; @@ -39,11 +45,14 @@ public DeleteContext(Object value, Object... ks) { accessStrategy = new PostOrderAccessStrategy(); } - public DeleteContext(Object value,List keys){ - this.value = value; + public void setKeys(List keys) { this.keys = keys; } + public void setValue(Object value) { + this.value = value; + } + public Object getKey() { return keys.get(level); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java index e8d00da4979d7..b09d1540cbc47 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java @@ -30,23 +30,31 @@ public class InsertContext extends Context { Object value; + public InsertContext() { + super(); + type = ContextType.INSERT; + accessStrategy = new PreOrderAccessStrategy(); + } + public InsertContext(Object value, Object... keys) { super(); this.value = value; this.keys = new ArrayList<>(); this.keys.addAll(Arrays.asList(keys)); - level = 0; type = ContextType.INSERT; accessStrategy = new PreOrderAccessStrategy(); } - public InsertContext(Object value,List keys){ - this.value = value; + public Object getKey() { + return keys.get(level); + } + + public void setKeys(List keys) { this.keys = keys; } - public Object getKey() { - return keys.get(level); + public void setValue(Object value) { + this.value = value; } public List getKeys() { diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java index a5b91439662aa..1855f04c0fe48 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java @@ -21,18 +21,16 @@ import org.apache.iotdb.lsm.context.Context; import org.apache.iotdb.lsm.levelProcess.LevelProcess; -import java.io.IOException; - public class BasicLsmManager implements LsmManager { LevelProcess levelProcess; public void preProcess(T root, C context) throws Exception {} - public void postProcess(T root, C context) throws Exception{} + public void postProcess(T root, C context) throws Exception {} @Override - public void process(T root, C context) throws Exception{ + public void process(T root, C context) throws Exception { preProcess(root, context); levelProcess.process(root, context); postProcess(root, context); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java index cf5e1a19c0ea7..d5269bbce987e 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java @@ -18,12 +18,20 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.InsertContext; + import java.util.List; import java.util.Map; public interface ITagInvertedIndex { + + void addTags(InsertContext context); + void addTags(Map tags, int id); + void removeTags(DeleteContext context); + void removeTags(Map tags, int id); List getMatchedIDs(Map tags); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java index 71c2ee923106a..b76ed104d85fa 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -31,7 +31,6 @@ import org.apache.iotdb.lsm.context.InsertContext; import org.apache.iotdb.lsm.context.QueryContext; -import org.apache.iotdb.lsm.wal.WALWriter; import org.roaringbitmap.RoaringBitmap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,25 +79,57 @@ public TagInvertedIndex(String schemaDirPath) throws IOException { recover(); } - public synchronized void recover(){ + public synchronized void recover() { recoverManager.recover(this); } + @Override + public synchronized void addTags(InsertContext context) { + int id = (int) context.getValue(); + if (!inWorkingMemTable(id)) { + workingMemTable.setStatus(MemTable.IMMUTABLE); + immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); + workingMemTable = new MemTable(MemTable.WORKING); + } + MemTable memTable = workingMemTable; + maxDeviceID = id; + try { + insertionManager.process(memTable, context); + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + @Override public synchronized void addTags(Map tags, int id) { - MemTable memTable = null; if (!inWorkingMemTable(id)) { workingMemTable.setStatus(MemTable.IMMUTABLE); immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); workingMemTable = new MemTable(MemTable.WORKING); } - memTable = workingMemTable; + MemTable memTable = workingMemTable; maxDeviceID = id; try { for (Map.Entry tag : tags.entrySet()) { addTag(memTable, tag.getKey(), tag.getValue(), id); } - }catch (Exception e){ + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + + @Override + public void removeTags(DeleteContext context) { + int id = (int) context.getValue(); + MemTable memTable = null; + if (inWorkingMemTable(id)) { + memTable = workingMemTable; + } else { + memTable = immutableMemTables.get(id / numOfDeviceIdsInMemTable); + } + try { + deletionManager.process(memTable, context); + } catch (Exception e) { logger.error(e.getMessage()); } } @@ -106,7 +137,6 @@ public synchronized void addTags(Map tags, int id) { @Override public synchronized void removeTags(Map tags, int id) { List memTables = new ArrayList<>(); - // 出现乱序 if (inWorkingMemTable(id)) { memTables.add(workingMemTable); } else { @@ -116,7 +146,7 @@ public synchronized void removeTags(Map tags, int id) { for (Map.Entry tag : tags.entrySet()) { removeTag(memTables, tag.getKey(), tag.getValue(), id); } - }catch (Exception e){ + } catch (Exception e) { logger.error(e.getMessage()); } } @@ -135,7 +165,7 @@ public synchronized List getMatchedIDs(Map tags) { else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb); i++; } - }catch (Exception e){ + } catch (Exception e) { logger.error(e.getMessage()); } return Arrays.stream(roaringBitmap.toArray()).boxed().collect(Collectors.toList()); @@ -164,14 +194,16 @@ private void addTag(MemTable memTable, String tagKey, String tagValue, int id) t insertionManager.process(memTable, insertContext); } - private void removeTag(List memTables, String tagKey, String tagValue, int id) throws Exception { + private void removeTag(List memTables, String tagKey, String tagValue, int id) + throws Exception { DeleteContext deleteContext = new DeleteContext(id, tagKey, tagValue); for (MemTable memTable : memTables) { deletionManager.process(memTable, deleteContext); } } - private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue) throws Exception { + private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue) + throws Exception { QueryContext queryContext = new QueryContext(tagKey, tagValue); for (MemTable memTable : memTables) { queryManager.process(memTable, queryContext); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java index 0d8e62a769f67..f31b6461b2ca3 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java @@ -23,8 +23,6 @@ import org.apache.iotdb.lsm.context.DeleteContext; import org.apache.iotdb.lsm.manager.BasicLsmManager; -import java.io.IOException; - public class DeletionManager extends BasicLsmManager { WALManager walManager; @@ -34,6 +32,13 @@ public DeletionManager(WALManager walManager) { initLevelProcess(); } + @Override + public void preProcess(MemTable root, DeleteContext context) throws Exception { + if (!context.isRecover()) { + walManager.write(context); + } + } + private void initLevelProcess() { this.nextLevel(new MemTableDeletion()) .nextLevel(new MemChunkGroupDeletion()) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java index 4b4dd29ee8c62..0e203a3d646ba 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; -import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALEntry; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; import org.apache.iotdb.lsm.context.InsertContext; import org.apache.iotdb.lsm.manager.BasicLsmManager; @@ -30,14 +29,16 @@ public class InsertionManager extends BasicLsmManager { private WALManager walManager; - public InsertionManager(WALManager walManager){ + public InsertionManager(WALManager walManager) { this.walManager = walManager; initLevelProcess(); } @Override public void preProcess(MemTable root, InsertContext context) throws IOException { - walManager.write(context); + if (!context.isRecover()) { + walManager.write(context); + } } private void initLevelProcess() { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java index 2c3d38bc06278..98b4ee5745059 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java @@ -20,6 +20,9 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.TagInvertedIndex; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.InsertContext; public class RecoverManager { @@ -29,5 +32,19 @@ public RecoverManager(WALManager walManager) { this.walManager = walManager; } - public void recover(TagInvertedIndex tagInvertedIndex) {} + public void recover(TagInvertedIndex tagInvertedIndex) { + while (true) { + Context context = walManager.read(); + switch (context.getType()) { + case INSERT: + tagInvertedIndex.addTags((InsertContext) context); + break; + case DELETE: + tagInvertedIndex.removeTags((DeleteContext) context); + break; + default: + return; + } + } + } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java index 88bdf68dd3143..b0ac296f6c15f 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java @@ -67,6 +67,7 @@ public synchronized void write(Context context) throws IOException { } } + // 用于recover public synchronized Context read() { if (walReader.hasNext()) { WALEntry walEntry = (WALEntry) walReader.next(); @@ -81,12 +82,22 @@ public synchronized Context read() { } private InsertContext generateInsertContext(WALEntry walEntry) { - InsertContext insertContext = new InsertContext(walEntry.getDeviceID(), walEntry.getKeys()); + InsertContext insertContext = new InsertContext(); + List objects = new ArrayList<>(); + objects.addAll(walEntry.getKeys()); + insertContext.setKeys(objects); + insertContext.setValue(walEntry.getDeviceID()); + insertContext.setRecover(true); return insertContext; } private DeleteContext generateDeleteContext(WALEntry walEntry) { DeleteContext deleteContext = new DeleteContext(walEntry.getDeviceID(), walEntry.getKeys()); + List objects = new ArrayList<>(); + objects.addAll(walEntry.getKeys()); + deleteContext.setKeys(objects); + deleteContext.setValue(walEntry.getDeviceID()); + deleteContext.setRecover(true); return deleteContext; } diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java index eba4c1067d3f3..f193862a68592 100644 --- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java @@ -27,6 +27,7 @@ import org.junit.Test; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -110,8 +111,6 @@ public void removeTags() { @Test public void getMatchedIDs() { addTags(); - System.out.println("------------addTags------------"); - System.out.println(tagInvertedIndex); Map tags1 = new HashMap<>(); tags1.put("tag1", "q"); @@ -128,8 +127,7 @@ public void getMatchedIDs() { assertEquals(verify, ids); removeTags(); - System.out.println("------------removeTags------------"); - System.out.println(tagInvertedIndex); + ids = tagInvertedIndex.getMatchedIDs(tags1); verify = Arrays.asList(3, 5, 7, 15, 16, 18); assertEquals(verify, ids); @@ -137,9 +135,29 @@ public void getMatchedIDs() { ids = tagInvertedIndex.getMatchedIDs(tags2); verify = Arrays.asList(3, 15, 16); assertEquals(verify, ids); + } + + @Test + public void testRecover() throws IOException { + Map tags1 = new HashMap<>(); + tags1.put("tag1", "q"); - System.out.println("------------query------------"); - System.out.println(tagInvertedIndex); + Map tags2 = new HashMap<>(); + tags2.put("tag1", "q"); + tags2.put("tag2", "a"); + addTags(); + removeTags(); + + tagInvertedIndex.clear(); + tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath); + + List ids = tagInvertedIndex.getMatchedIDs(tags1); + List verify = Arrays.asList(3, 5, 7, 15, 16, 18); + assertEquals(verify, ids); + + ids = tagInvertedIndex.getMatchedIDs(tags2); + verify = Arrays.asList(3, 15, 16); + assertEquals(verify, ids); } private List, Integer>> generateTags() { From 085f26813216ee2a981c072bb62e0a454b9d1fd6 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 14 Sep 2022 20:14:37 +0800 Subject: [PATCH 15/30] impl RecoverManager --- .../db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java index b76ed104d85fa..9ae8b0ddb2dce 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -185,7 +185,7 @@ public String toString() { + '}'; } - private synchronized boolean inWorkingMemTable(int id) { + private boolean inWorkingMemTable(int id) { return id / numOfDeviceIdsInMemTable == maxDeviceID / numOfDeviceIdsInMemTable; } From 2d3b38b4b5902a59e35abf4035b219aeb129d3d8 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 14 Sep 2022 20:15:47 +0800 Subject: [PATCH 16/30] impl RecoverManager --- .../tagSchemaRegion/tagIndex/recover/RecoverManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java index 98b4ee5745059..906fafb25503b 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.recover; -import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.TagInvertedIndex; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.ITagInvertedIndex; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; import org.apache.iotdb.lsm.context.Context; import org.apache.iotdb.lsm.context.DeleteContext; @@ -32,7 +32,7 @@ public RecoverManager(WALManager walManager) { this.walManager = walManager; } - public void recover(TagInvertedIndex tagInvertedIndex) { + public void recover(ITagInvertedIndex tagInvertedIndex) { while (true) { Context context = walManager.read(); switch (context.getType()) { From 263f9fb2404aa2c99c0ce180cb79611910d8b754 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 14 Sep 2022 20:20:05 +0800 Subject: [PATCH 17/30] impl RecoverManager --- .../tagSchemaRegion/tagIndex/deletion/DeletionManager.java | 2 +- .../tagSchemaRegion/tagIndex/recover/RecoverManager.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java index f31b6461b2ca3..211eed0bbe944 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java @@ -25,7 +25,7 @@ public class DeletionManager extends BasicLsmManager { - WALManager walManager; + private WALManager walManager; public DeletionManager(WALManager walManager) { this.walManager = walManager; diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java index 906fafb25503b..ec9fd74729565 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java @@ -26,7 +26,7 @@ public class RecoverManager { - WALManager walManager; + private WALManager walManager; public RecoverManager(WALManager walManager) { this.walManager = walManager; From d45cc0fd9f9792a5ba8c0e65b00325faa11e25aa Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 14 Sep 2022 20:36:38 +0800 Subject: [PATCH 18/30] impl RecoverManager --- lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java | 1 + .../db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java index a44b8f9d243c5..4410b6f8a2007 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java @@ -23,6 +23,7 @@ import java.nio.ByteBuffer; public abstract class WALRecord implements Cloneable { + public abstract void serialize(ByteBuffer buffer); public abstract void deserialize(DataInputStream stream) throws IOException; diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java index e70ad62100b10..6cf24548492a2 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java @@ -34,9 +34,10 @@ public class WALEntry extends WALRecord { private int deviceID; - public WALEntry() {} + public WALEntry() {super();} public WALEntry(int type, List keys, int deviceID) { + super(); this.type = type; this.keys = keys; this.deviceID = deviceID; From 31df51d12e3eb9816d303cf6648a8b869ed8fd91 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Mon, 19 Sep 2022 23:28:42 +0800 Subject: [PATCH 19/30] spotless apply --- schema-engine-tag/pom.xml | 1 + .../tagSchemaRegion/MockTagSchemaRegion.java | 325 ------------------ .../tagIndex/wal/WALEntry.java | 4 +- 3 files changed, 4 insertions(+), 326 deletions(-) delete mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java diff --git a/schema-engine-tag/pom.xml b/schema-engine-tag/pom.xml index c5b8f565525a0..731ead4ad0c6a 100644 --- a/schema-engine-tag/pom.xml +++ b/schema-engine-tag/pom.xml @@ -39,6 +39,7 @@ org.apache.iotdb iotdb-server ${project.version} + provided diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java deleted file mode 100644 index 184ce789b99cd..0000000000000 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.metadata.tagSchemaRegion; - -import org.apache.iotdb.common.rpc.thrift.TSchemaNode; -import org.apache.iotdb.commons.consensus.SchemaRegionId; -import org.apache.iotdb.commons.exception.MetadataException; -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.db.exception.metadata.PathNotExistException; -import org.apache.iotdb.db.metadata.LocalSchemaProcessor; -import org.apache.iotdb.db.metadata.mnode.IMNode; -import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; -import org.apache.iotdb.db.metadata.path.MeasurementPath; -import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; -import org.apache.iotdb.db.metadata.template.Template; -import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; -import org.apache.iotdb.db.qp.physical.crud.InsertPlan; -import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan; -import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan; -import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan; -import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; -import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; -import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan; -import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan; -import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan; -import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan; -import org.apache.iotdb.db.query.context.QueryContext; -import org.apache.iotdb.db.query.dataset.ShowDevicesResult; -import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.utils.Pair; - -import java.io.File; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; - -public class MockTagSchemaRegion implements ISchemaRegion { - - @Override - public void init() throws MetadataException {} - - @Override - public void clear() {} - - @Override - public void forceMlog() {} - - @Override - public SchemaRegionId getSchemaRegionId() { - return null; - } - - @Override - public String getStorageGroupFullPath() { - return null; - } - - @Override - public void deleteSchemaRegion() throws MetadataException {} - - @Override - public boolean createSnapshot(File snapshotDir) { - return false; - } - - @Override - public void loadSnapshot(File latestSnapshotRootDir) {} - - @Override - public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException {} - - @Override - public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException {} - - @Override - public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - return null; - } - - @Override - public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException {} - - @Override - public boolean isPathExist(PartialPath path) throws MetadataException { - return false; - } - - @Override - public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - return 0; - } - - @Override - public int getAllTimeseriesCount( - PartialPath pathPattern, Map templateMap, boolean isPrefixMatch) - throws MetadataException { - return 0; - } - - @Override - public int getAllTimeseriesCount( - PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains) - throws MetadataException { - return 0; - } - - @Override - public Map getMeasurementCountGroupByLevel( - PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { - return null; - } - - @Override - public Map getMeasurementCountGroupByLevel( - PartialPath pathPattern, - int level, - boolean isPrefixMatch, - String key, - String value, - boolean isContains) - throws MetadataException { - return null; - } - - @Override - public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - return 0; - } - - @Override - public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch) - throws MetadataException { - return 0; - } - - @Override - public List getNodesListInGivenLevel( - PartialPath pathPattern, - int nodeLevel, - boolean isPrefixMatch, - LocalSchemaProcessor.StorageGroupFilter filter) - throws MetadataException { - return null; - } - - @Override - public Set getChildNodePathInNextLevel(PartialPath pathPattern) - throws MetadataException { - return null; - } - - @Override - public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException { - return null; - } - - @Override - public Set getBelongedDevices(PartialPath timeseries) throws MetadataException { - return null; - } - - @Override - public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - return null; - } - - @Override - public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan) - throws MetadataException { - return null; - } - - @Override - public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - return null; - } - - @Override - public Pair, Integer> getMeasurementPathsWithAlias( - PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) - throws MetadataException { - return null; - } - - @Override - public List fetchSchema( - PartialPath pathPattern, Map templateMap) throws MetadataException { - return null; - } - - @Override - public Pair, Integer> showTimeseries( - ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { - return null; - } - - @Override - public List getAllMeasurementByDevicePath(PartialPath devicePath) - throws PathNotExistException { - return null; - } - - @Override - public IMNode getDeviceNode(PartialPath path) throws MetadataException { - return null; - } - - @Override - public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException { - return null; - } - - @Override - public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException {} - - @Override - public void upsertTagsAndAttributes( - String alias, - Map tagsMap, - Map attributesMap, - PartialPath fullPath) - throws MetadataException, IOException {} - - @Override - public void addAttributes(Map attributesMap, PartialPath fullPath) - throws MetadataException, IOException {} - - @Override - public void addTags(Map tagsMap, PartialPath fullPath) - throws MetadataException, IOException {} - - @Override - public void dropTagsOrAttributes(Set keySet, PartialPath fullPath) - throws MetadataException, IOException {} - - @Override - public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) - throws MetadataException, IOException {} - - @Override - public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) - throws MetadataException, IOException {} - - @Override - public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) - throws MetadataException, IOException { - return null; - } - - @Override - public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( - PartialPath devicePath, - String[] measurements, - Function getDataType, - boolean aligned) - throws MetadataException { - return null; - } - - @Override - public Set getPathsSetTemplate(String templateName) throws MetadataException { - return null; - } - - @Override - public Set getPathsUsingTemplate(String templateName) throws MetadataException { - return null; - } - - @Override - public boolean isTemplateAppendable(Template template, List measurements) - throws MetadataException { - return false; - } - - @Override - public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException {} - - @Override - public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException {} - - @Override - public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException {} - - @Override - public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template) - throws MetadataException {} - - @Override - public List getPathsUsingTemplate(int templateId) throws MetadataException { - return null; - } - - @Override - public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException { - return null; - } - - @Override - public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException {} -} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java index 6cf24548492a2..6c14934ce1062 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java @@ -34,7 +34,9 @@ public class WALEntry extends WALRecord { private int deviceID; - public WALEntry() {super();} + public WALEntry() { + super(); + } public WALEntry(int type, List keys, int deviceID) { super(); From a5e47dd0589bc7a9366332eebd2bababfc095d8c Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 21 Sep 2022 09:49:25 +0800 Subject: [PATCH 20/30] add TagSchemaDescriptor --- .../org/apache/iotdb/lsm/wal/WALWriter.java | 9 +- schema-engine-tag/pom.xml | 32 ++ .../resources/conf/schema-tag.properties | 33 ++ .../src/assembly/schema-engine-tag.xml | 45 +++ .../tagSchemaRegion/TagSchemaConfig.java | 42 +++ .../tagSchemaRegion/TagSchemaDescriptor.java | 85 +++++ .../tagSchemaRegion/TagSchemaRegion.java | 325 +++++++++++++++++- .../tagIndex/TagInvertedIndex.java | 7 +- .../tagIndex/wal/WALManager.java | 16 +- .../tagIndex/TagInvertedIndexTest.java | 9 +- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 10 - .../schemaregion/TagSchemaRegionLoader.java | 112 ++++++ 12 files changed, 705 insertions(+), 20 deletions(-) create mode 100644 schema-engine-tag/src/assembly/resources/conf/schema-tag.properties create mode 100644 schema-engine-tag/src/assembly/schema-engine-tag.xml create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java index dccd8a92233db..abaf89a354dec 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java @@ -34,15 +34,18 @@ public class WALWriter implements IWALWriter { private File logFile; private FileOutputStream fileOutputStream; private FileChannel channel; - private final ByteBuffer lengthBuffer = ByteBuffer.allocate(4); - private final ByteBuffer walBuffer = ByteBuffer.allocate(10000); + private final ByteBuffer lengthBuffer; + private final ByteBuffer walBuffer; private final boolean forceEachWrite; - public WALWriter(File logFile, boolean forceEachWrite) throws FileNotFoundException { + public WALWriter(File logFile, int walBufferSize, boolean forceEachWrite) + throws FileNotFoundException { this.logFile = logFile; this.forceEachWrite = forceEachWrite; fileOutputStream = new FileOutputStream(logFile, true); channel = fileOutputStream.getChannel(); + lengthBuffer = ByteBuffer.allocate(4); + walBuffer = ByteBuffer.allocate(walBufferSize); } @Override diff --git a/schema-engine-tag/pom.xml b/schema-engine-tag/pom.xml index 731ead4ad0c6a..14e61f754aa21 100644 --- a/schema-engine-tag/pom.xml +++ b/schema-engine-tag/pom.xml @@ -42,4 +42,36 @@ provided + + schema-engine-tag + + + org.apache.maven.plugins + maven-assembly-plugin + ${maven.assembly.version} + + + + schema-engine-tag-assembly + package + + single + + + + src/assembly/schema-engine-tag.xml + + false + + + true + true + + + + + + + + diff --git a/schema-engine-tag/src/assembly/resources/conf/schema-tag.properties b/schema-engine-tag/src/assembly/resources/conf/schema-tag.properties new file mode 100644 index 0000000000000..bfb7df48b8ab4 --- /dev/null +++ b/schema-engine-tag/src/assembly/resources/conf/schema-tag.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +#################### +### tag schema region Configuration +#################### + +# This configuration takes effect only when the schema engine mode is Tag. +# The mode is configured in the 'iotdb-datanode.properties'(schema_engine_mode=Tag). + +# Datatype: int +# The size of wal buffer used to store a wal record.(unit: byte) +# wal_buffer_size = 1024*1024 + +# Datatype: int +# How many device ids a memtable can insert, beyond which the memtable will become immutable +# num_of_deviceIds_in_memTable = 65536 \ No newline at end of file diff --git a/schema-engine-tag/src/assembly/schema-engine-tag.xml b/schema-engine-tag/src/assembly/schema-engine-tag.xml new file mode 100644 index 0000000000000..49647f46a7a14 --- /dev/null +++ b/schema-engine-tag/src/assembly/schema-engine-tag.xml @@ -0,0 +1,45 @@ + + + + schema-engine-tag + + dir + zip + + false + + + /lib/tag-schema-region + + org.apache.iotdb:iotdb-lsm + org.roaringbitmap:RoaringBitmap + org.apache.iotdb:schema-engine-tag + + + + + + src/assembly/resources + ${file.separator} + + + diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java new file mode 100644 index 0000000000000..ad13817596a4f --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion; + +public class TagSchemaConfig { + + private int numOfDeviceIdsInMemTable = 65536; + + private int walBufferSize = 1024 * 1024; + + public int getNumOfDeviceIdsInMemTable() { + return numOfDeviceIdsInMemTable; + } + + public void setNumOfDeviceIdsInMemTable(int numOfDeviceIdsInMemTable) { + this.numOfDeviceIdsInMemTable = numOfDeviceIdsInMemTable; + } + + public int getWalBufferSize() { + return walBufferSize; + } + + public void setWalBufferSize(int walBufferSize) { + this.walBufferSize = walBufferSize; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java new file mode 100644 index 0000000000000..97f7d4752b99c --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion; + +import org.apache.iotdb.commons.conf.IoTDBConstant; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +public class TagSchemaDescriptor { + private static final Logger logger = LoggerFactory.getLogger(TagSchemaDescriptor.class); + + private static final String TAG_SCHEMA_CONFIG_FILE_NAME = "schema-tag.properties"; + + private final TagSchemaConfig conf = new TagSchemaConfig(); + + private TagSchemaDescriptor() { + loadProperties(); + } + + public static TagSchemaDescriptor getInstance() { + return TagSchemaDescriptorHolder.INSTANCE; + } + + private void loadProperties() { + String iotdbHomePath = System.getProperty(IoTDBConstant.IOTDB_HOME, null); + String tagSchemaConfigPath = + iotdbHomePath + + File.separatorChar + + "conf" + + File.separatorChar + + TAG_SCHEMA_CONFIG_FILE_NAME; + try (InputStream in = new BufferedInputStream(new FileInputStream(tagSchemaConfigPath))) { + Properties properties = new Properties(); + properties.load(in); + conf.setWalBufferSize( + Integer.parseInt( + properties.getProperty("wal_buffer_size", String.valueOf(conf.getWalBufferSize())))); + conf.setNumOfDeviceIdsInMemTable( + Integer.parseInt( + properties.getProperty( + "num_of_deviceIds_in_memTable", + String.valueOf(conf.getNumOfDeviceIdsInMemTable())))); + } catch (FileNotFoundException e) { + logger.warn("Fail to find tag schema region config file {}", tagSchemaConfigPath); + } catch (IOException e) { + logger.warn("Cannot load tag schema region config file, use default configuration"); + } + } + + public TagSchemaConfig getTagSchemaConfig() { + return conf; + } + + private static class TagSchemaDescriptorHolder { + + private static final TagSchemaDescriptor INSTANCE = new TagSchemaDescriptor(); + + private TagSchemaDescriptorHolder() {} + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java index 2f04d7c095a73..857575d1efca5 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java @@ -18,4 +18,327 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion; -public class TagSchemaRegion {} +import org.apache.iotdb.common.rpc.thrift.TSchemaNode; +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.metadata.LocalSchemaProcessor; +import org.apache.iotdb.db.metadata.mnode.IMNode; +import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; +import org.apache.iotdb.db.metadata.path.MeasurementPath; +import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.metadata.template.Template; +import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.qp.physical.crud.InsertPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan; +import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan; +import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan; +import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.dataset.ShowDevicesResult; +import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.Pair; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +public class TagSchemaRegion implements ISchemaRegion { + + + @Override + public void init() throws MetadataException {} + + @Override + public void clear() {} + + @Override + public void forceMlog() {} + + @Override + public SchemaRegionId getSchemaRegionId() { + return null; + } + + @Override + public String getStorageGroupFullPath() { + return null; + } + + @Override + public void deleteSchemaRegion() throws MetadataException {} + + @Override + public boolean createSnapshot(File snapshotDir) { + return false; + } + + @Override + public void loadSnapshot(File latestSnapshotRootDir) {} + + @Override + public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException {} + + @Override + public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException {} + + @Override + public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public int constructSchemaBlackList(PathPatternTree patternTree) throws MetadataException { + return 0; + } + + @Override + public void rollbackSchemaBlackList(PathPatternTree patternTree) throws MetadataException {} + + @Override + public List fetchSchemaBlackList(PathPatternTree patternTree) + throws MetadataException { + return null; + } + + @Override + public void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws MetadataException {} + + @Override + public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException {} + + @Override + public boolean isPathExist(PartialPath path) throws MetadataException { + return false; + } + + @Override + public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, Map templateMap, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains) + throws MetadataException { + return 0; + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { + return null; + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, + int level, + boolean isPrefixMatch, + String key, + String value, + boolean isContains) + throws MetadataException { + return null; + } + + @Override + public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public List getNodesListInGivenLevel( + PartialPath pathPattern, + int nodeLevel, + boolean isPrefixMatch, + LocalSchemaProcessor.StorageGroupFilter filter) + throws MetadataException { + return null; + } + + @Override + public Set getChildNodePathInNextLevel(PartialPath pathPattern) + throws MetadataException { + return null; + } + + @Override + public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException { + return null; + } + + @Override + public Set getBelongedDevices(PartialPath timeseries) throws MetadataException { + return null; + } + + @Override + public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan) + throws MetadataException { + return null; + } + + @Override + public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public Pair, Integer> getMeasurementPathsWithAlias( + PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public List fetchSchema( + PartialPath pathPattern, Map templateMap) throws MetadataException { + return null; + } + + @Override + public Pair, Integer> showTimeseries( + ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { + return null; + } + + @Override + public List getAllMeasurementByDevicePath(PartialPath devicePath) + throws PathNotExistException { + return null; + } + + @Override + public IMNode getDeviceNode(PartialPath path) throws MetadataException { + return null; + } + + @Override + public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException { + return null; + } + + @Override + public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException {} + + @Override + public void upsertTagsAndAttributes( + String alias, + Map tagsMap, + Map attributesMap, + PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void addAttributes(Map attributesMap, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void addTags(Map tagsMap, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void dropTagsOrAttributes(Set keySet, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) + throws MetadataException, IOException { + return null; + } + + @Override + public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( + PartialPath devicePath, + String[] measurements, + Function getDataType, + boolean aligned) + throws MetadataException { + return null; + } + + @Override + public Set getPathsSetTemplate(String templateName) throws MetadataException { + return null; + } + + @Override + public Set getPathsUsingTemplate(String templateName) throws MetadataException { + return null; + } + + @Override + public boolean isTemplateAppendable(Template template, List measurements) + throws MetadataException { + return false; + } + + @Override + public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException {} + + @Override + public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException {} + + @Override + public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException {} + + @Override + public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template) + throws MetadataException {} + + @Override + public List getPathsUsingTemplate(int templateId) throws MetadataException { + return null; + } + + @Override + public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException { + return null; + } + + @Override + public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException {} +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java index 9ae8b0ddb2dce..ed62c3d2006d7 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -21,6 +21,8 @@ import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaConfig; +import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.DeletionManager; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.InsertionManager; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; @@ -48,6 +50,9 @@ public class TagInvertedIndex implements ITagInvertedIndex { private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final TagSchemaConfig tagSchemaConfig = + TagSchemaDescriptor.getInstance().getTagSchemaConfig(); + private final InsertionManager insertionManager; private final DeletionManager deletionManager; @@ -74,7 +79,7 @@ public TagInvertedIndex(String schemaDirPath) throws IOException { queryManager = new QueryManager(); workingMemTable = new MemTable(MemTable.WORKING); immutableMemTables = new HashMap<>(); - numOfDeviceIdsInMemTable = config.getNumOfDeviceIdsInMemTable(); + numOfDeviceIdsInMemTable = tagSchemaConfig.getNumOfDeviceIdsInMemTable(); maxDeviceID = 0; recover(); } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java index b0ac296f6c15f..65a74bba89144 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java @@ -18,6 +18,8 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal; +import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaConfig; +import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor; import org.apache.iotdb.lsm.context.Context; import org.apache.iotdb.lsm.context.DeleteContext; import org.apache.iotdb.lsm.context.InsertContext; @@ -30,18 +32,28 @@ import java.util.List; public class WALManager { - private static final String WAL_FILE_NAME = "tagInvertedIndex.log"; + private static final String WAL_FILE_NAME = "tag_inverted_index.log"; + private static final int INSERT = 1; + private static final int DELETE = 2; + + private final TagSchemaConfig tagSchemaConfig = + TagSchemaDescriptor.getInstance().getTagSchemaConfig(); + private final String schemaDirPath; + private File walFile; + private WALWriter walWriter; + private WALReader walReader; public WALManager(String schemaDirPath) throws IOException { this.schemaDirPath = schemaDirPath; initFile(schemaDirPath); - walWriter = new WALWriter(walFile, false); + int walBufferSize = tagSchemaConfig.getWalBufferSize(); + walWriter = new WALWriter(walFile, walBufferSize, false); walReader = new WALReader(walFile, new WALEntry()); } diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java index f193862a68592..e8599f4b985c1 100644 --- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java @@ -20,6 +20,7 @@ import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor; import org.apache.iotdb.tsfile.utils.Pair; import org.junit.After; @@ -74,8 +75,8 @@ public class TagInvertedIndexTest { @Before public void setUp() throws Exception { numOfDeviceIdsInMemTable = - IoTDBDescriptor.getInstance().getConfig().getNumOfDeviceIdsInMemTable(); - IoTDBDescriptor.getInstance().getConfig().setNumOfDeviceIdsInMemTable(3); + TagSchemaDescriptor.getInstance().getTagSchemaConfig().getNumOfDeviceIdsInMemTable(); + TagSchemaDescriptor.getInstance().getTagSchemaConfig().setNumOfDeviceIdsInMemTable(3); schemaDir = IoTDBDescriptor.getInstance().getConfig().getSchemaDir(); storageGroupDirPath = schemaDir + File.separator + storageGroupFullPath; schemaRegionDirPath = storageGroupDirPath + File.separator + 0; @@ -84,7 +85,9 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { - IoTDBDescriptor.getInstance().getConfig().setNumOfDeviceIdsInMemTable(numOfDeviceIdsInMemTable); + TagSchemaDescriptor.getInstance() + .getTagSchemaConfig() + .setNumOfDeviceIdsInMemTable(numOfDeviceIdsInMemTable); tagInvertedIndex.clear(); tagInvertedIndex = null; FileUtils.deleteDirectoryAndEmptyParent(new File(schemaDir)); diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 29511df14b596..c277e6570eb34 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -857,8 +857,6 @@ public class IoTDBConfig { */ private boolean enableIDTableLogFile = false; - private int numOfDeviceIdsInMemTable = 65536; - /** whether to use persistent schema mode */ private String schemaEngineMode = "Memory"; @@ -2836,14 +2834,6 @@ public void setEnableIDTableLogFile(boolean enableIDTableLogFile) { this.enableIDTableLogFile = enableIDTableLogFile; } - public int getNumOfDeviceIdsInMemTable() { - return numOfDeviceIdsInMemTable; - } - - public void setNumOfDeviceIdsInMemTable(int numOfDeviceIdsInMemTable) { - this.numOfDeviceIdsInMemTable = numOfDeviceIdsInMemTable; - } - public String getSchemaEngineMode() { return schemaEngineMode; } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java new file mode 100644 index 0000000000000..46a8056d05b89 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.schemaregion; + +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.LinkedList; +import java.util.List; + +public class TagSchemaRegionLoader { + private static final Logger LOGGER = LoggerFactory.getLogger(TagSchemaRegionLoader.class); + private static URLClassLoader urlClassLoader = null; + private static final String TAG_SCHEMA_REGION_CLASS_NAME = + "org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaRegion"; + private static final String TAG_SCHEMA_CONF_DESCRIPTOR_CLASS_NAME = + "org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor"; + private static final String LIB_PATH = + ".." + File.separator + "lib" + File.separator + "tag-schema-region" + File.separator; + + public TagSchemaRegionLoader() {} + + /** + * Load the jar files for RSchemaRegion and create an instance of it. The jar files should be + * located in "../lib/rschema-region". If jar files cannot be found, the function will return + * null. + * + * @param storageGroup + * @param schemaRegionId + * @param node + * @return + */ + public ISchemaRegion loadRSchemaRegion( + PartialPath storageGroup, SchemaRegionId schemaRegionId, IStorageGroupMNode node) { + ISchemaRegion region = null; + LOGGER.info("Creating instance for schema-engine-rocksdb"); + try { + loadRSchemaRegionJar(); + Class classForTagSchemaRegion = urlClassLoader.loadClass(TAG_SCHEMA_REGION_CLASS_NAME); + Class classForTagSchemaDescriptor = + urlClassLoader.loadClass(TAG_SCHEMA_CONF_DESCRIPTOR_CLASS_NAME); + Constructor constructor = + classForTagSchemaRegion.getConstructor( + PartialPath.class, + SchemaRegionId.class, + IStorageGroupMNode.class, + classForTagSchemaDescriptor); + Object rSchemaLoader = classForTagSchemaDescriptor.getConstructor().newInstance(); + region = + (ISchemaRegion) + constructor.newInstance(storageGroup, schemaRegionId, node, rSchemaLoader); + } catch (ClassNotFoundException + | NoSuchMethodException + | InvocationTargetException + | InstantiationException + | IllegalAccessException + | MalformedURLException + | RuntimeException e) { + LOGGER.error("Cannot initialize RSchemaRegion", e); + return null; + } + return region; + } + + /** + * Load the jar files for rocksdb and RSchemaRegion. The jar files should be located in directory + * "../lib/rschema-region". If the jar files have been loaded, it will do nothing. + */ + private void loadRSchemaRegionJar() throws MalformedURLException { + LOGGER.info("Loading jar for schema-engine-rocksdb"); + if (urlClassLoader == null) { + File[] jars = new File(LIB_PATH).listFiles(); + if (jars == null) { + throw new RuntimeException( + String.format("Cannot get jars from %s", new File(LIB_PATH).getAbsolutePath())); + } + List dependentJars = new LinkedList<>(); + for (File jar : jars) { + if (jar.getName().endsWith(".jar")) { + dependentJars.add(new URL("file:" + jar.getAbsolutePath())); + } + } + urlClassLoader = new URLClassLoader(dependentJars.toArray(new URL[] {})); + } + } +} From bef4fcd6c2c56ce4bc0989c27b849bf84fc581af Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Thu, 22 Sep 2022 14:33:33 +0800 Subject: [PATCH 21/30] add DeviceIDList --- .../org/apache/iotdb/lsm/wal/WALReader.java | 4 +- .../org/apache/iotdb/lsm/wal/WALRecord.java | 2 +- schema-engine-tag/README.md | 37 + .../tagSchemaRegion/MockTagSchemaRegion.java | 928 ++++++++++++++++++ .../tagSchemaRegion/TagSchemaConfig.java | 10 + .../tagSchemaRegion/TagSchemaRegion.java | 705 +++++++++++-- .../AppendOnlyDeviceIDListFileManager.java | 105 ++ .../deviceidlist/DeviceIDList.java | 70 ++ .../deviceidlist/IDeviceIDList.java | 39 + .../tagIndex/TagInvertedIndex.java | 40 +- .../tagIndex/recover/RecoverManager.java | 5 + .../utils/MeasurementPathUtils.java | 62 ++ .../utils/PathTagConverterUtils.java | 49 + .../utils/ShowTimeSeriesResultUtils.java | 54 + .../deviceidlist/DeviceIDListTest.java | 122 +++ .../resources/conf/iotdb-datanode.properties | 15 +- .../idtable/entry/DiskSchemaEntry.java | 4 + .../metadata/schemaregion/SchemaEngine.java | 6 +- .../schemaregion/TagSchemaRegionLoader.java | 37 +- 19 files changed, 2187 insertions(+), 107 deletions(-) create mode 100644 schema-engine-tag/README.md create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java create mode 100644 schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java index a3fef56b0860d..42932301d0dc4 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java @@ -65,10 +65,10 @@ public boolean hasNext() { nextRecord = prototype.clone(); nextRecord.deserialize(logStream); } catch (EOFException e) { - logger.info(""); + logger.info(e.getMessage()); return false; } catch (IOException e) { - logger.warn(""); + logger.warn(e.getMessage()); fileCorrupted = true; return false; } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java index 4410b6f8a2007..4311b5debfd2a 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java @@ -33,7 +33,7 @@ public WALRecord clone() { try { return (WALRecord) super.clone(); } catch (CloneNotSupportedException e) { - throw new AssertionError(); + throw new AssertionError(e.getMessage()); } } } diff --git a/schema-engine-tag/README.md b/schema-engine-tag/README.md new file mode 100644 index 0000000000000..70185b9c79731 --- /dev/null +++ b/schema-engine-tag/README.md @@ -0,0 +1,37 @@ + +`TagSchemaRegion` is an implementation of `SchemaRegion`. + +# How To Use + +Firstly, you should package **schema-engine-rocksdb** by the following command: + +```shell +mvn clean package -pl schema-engine-tag -am -DskipTests +``` + +After that, you can get a **conf** directory and a **lib** directory in +schema-engine-tag/target/schema-engine-tag. Copy the file in the conf directory to the conf directory of server, +and copy the files in the lib directory to the lib directory of server. + +Then, open the **iotdb-datanode.properties** in the conf directory of server, and set the `schema_engine_mode` to +Tag, set the `enable_id_table` to true. Restart the IoTDB, the system will use `TagSchemaRegion` to manage +the metadata. \ No newline at end of file diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java new file mode 100644 index 0000000000000..bec4ba83947d7 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java @@ -0,0 +1,928 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.metadata.tagSchemaRegion; + +import org.apache.iotdb.common.rpc.thrift.TSchemaNode; +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.AlignedTimeseriesException; +import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; +import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.metadata.LocalSchemaProcessor; +import org.apache.iotdb.db.metadata.idtable.IDTable; +import org.apache.iotdb.db.metadata.idtable.IDTableManager; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.InsertMeasurementMNode; +import org.apache.iotdb.db.metadata.idtable.entry.SHA256DeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.db.metadata.mnode.EntityMNode; +import org.apache.iotdb.db.metadata.mnode.IMNode; +import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; +import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; +import org.apache.iotdb.db.metadata.path.MeasurementPath; +import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.metadata.schemaregion.SchemaRegionUtils; +import org.apache.iotdb.db.metadata.template.Template; +import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.mpp.common.schematree.MeasurementSchemaInfo; +import org.apache.iotdb.db.qp.physical.crud.InsertPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan; +import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan; +import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan; +import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.dataset.ShowDevicesResult; +import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; +import org.apache.iotdb.external.api.ISeriesNumerLimiter; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; + +public class MockTagSchemaRegion implements ISchemaRegion { + + protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + private final String TAIL = ".**"; + private final IStorageGroupMNode storageGroupMNode; + private String storageGroupFullPath; + private SchemaRegionId schemaRegionId; + + private Map>> tagInvertedIndex; + + private List deviceIDS; + + private IDTable idTable; + + private final ISeriesNumerLimiter seriesNumerLimiter; + + public MockTagSchemaRegion( + PartialPath storageGroup, + SchemaRegionId schemaRegionId, + IStorageGroupMNode storageGroupMNode, + ISeriesNumerLimiter seriesNumerLimiter) + throws MetadataException { + + storageGroupFullPath = storageGroup.getFullPath(); + this.schemaRegionId = schemaRegionId; + this.storageGroupMNode = storageGroupMNode; + this.deviceIDS = new ArrayList<>(); + this.seriesNumerLimiter = seriesNumerLimiter; + tagInvertedIndex = new ConcurrentHashMap<>(); + idTable = IDTableManager.getInstance().getIDTable(storageGroup); + init(); + } + + @NotNull + private Map pathToTags(String path) { + if (path.length() <= storageGroupFullPath.length()) return new TreeMap<>(); + String devicePath = path.substring(storageGroupFullPath.length() + 1); + String[] tags = devicePath.split("\\."); + Map tagsMap = new TreeMap<>(); + for (int i = 0; i < tags.length; i += 2) { + tagsMap.put(tags[i], tags[i + 1]); + } + return tagsMap; + } + + public String tagsToPath(Map tags) { + StringBuilder stringBuilder = new StringBuilder(storageGroupFullPath); + for (String tagKey : tags.keySet()) { + stringBuilder.append(".").append(tagKey).append(".").append(tags.get(tagKey)); + } + return stringBuilder.toString(); + } + + @Override + public void init() throws MetadataException { + if (!config.isEnableIDTableLogFile() + && config.getDeviceIDTransformationMethod().equals("SHA256")) { + throw new MetadataException( + "enableIDTableLogFile OR deviceIDTransformationMethod==\"Plain\""); + } + } + + @Override + public void clear() { + return; + } + + @Override + public void forceMlog() { + return; + } + + @Override + public SchemaRegionId getSchemaRegionId() { + return schemaRegionId; + } + + @Override + public String getStorageGroupFullPath() { + return storageGroupFullPath; + } + + @Override + public void deleteSchemaRegion() throws MetadataException { + return; + } + + @Override + public boolean createSnapshot(File snapshotDir) { + return false; + } + + @Override + public void loadSnapshot(File latestSnapshotRootDir) { + return; + } + + private void createTagInvertedIndex(PartialPath devicePath) { + IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath); + Map tagsMap = pathToTags(devicePath.getFullPath()); + + deviceIDS.add(deviceID); + + for (String tagkey : tagsMap.keySet()) { + String tagValue = tagsMap.get(tagkey); + Map> tagkeyMap = + tagInvertedIndex.computeIfAbsent(tagkey, key -> new HashMap<>()); + List ids = tagkeyMap.computeIfAbsent(tagValue, key -> new ArrayList<>()); + ids.add(deviceIDS.size() - 1); + } + } + + private void createTimeseries( + PartialPath path, + TSDataType dataType, + TSEncoding encoding, + CompressionType compressor, + Map props) + throws MetadataException { + createTimeseries( + new CreateTimeSeriesPlan(path, dataType, encoding, compressor, props, null, null, null), 0); + } + + private void createAlignedTimeSeries( + PartialPath prefixPath, + List measurements, + List dataTypes, + List encodings, + List compressors) + throws MetadataException { + createAlignedTimeSeries( + new CreateAlignedTimeSeriesPlan( + prefixPath, measurements, dataTypes, encodings, compressors, null, null, null)); + } + + @Override // [iotdb|newIotdb/创建非对齐时间序列] [newIotdb/insert 2自动创建时间序列] + public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException { + PartialPath devicePath = plan.getPath().getDevicePath(); + Map tags = pathToTags(devicePath.getFullPath()); + PartialPath path = new PartialPath(tagsToPath(tags) + "." + plan.getPath().getMeasurement()); + plan.setPath(path); + devicePath = plan.getPath().getDevicePath(); + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + if (deviceEntry != null) { + if (deviceEntry.isAligned()) { + throw new AlignedTimeseriesException( + "Timeseries under this entity is not aligned, please use createTimeseries or change entity.", + devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); + } else if (deviceEntry.getMeasurementMap().containsKey(plan.getPath().getMeasurement())) { + throw new PathAlreadyExistException( + devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); + } + } + idTable.createTimeseries(plan); + if (deviceEntry == null) { + createTagInvertedIndex(devicePath); + } + } + + @Override // [iotdb|newIotdb/对齐时间序列] [newIotdb/insert 2自动创建时间序列] + public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException { + PartialPath devicePath = plan.getPrefixPath(); + Map tags = pathToTags(devicePath.getFullPath()); + PartialPath path = new PartialPath(tagsToPath(tags)); + plan.setPrefixPath(path); + devicePath = plan.getPrefixPath(); + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + if (deviceEntry != null) { + if (!deviceEntry.isAligned()) { + throw new AlignedTimeseriesException( + "Timeseries under this entity is aligned, please use createAlignedTimeseries or change entity.", + devicePath.getFullPath()); + } else { + List measurements = plan.getMeasurements(); + List dataTypes = plan.getDataTypes(); + List encodings = plan.getEncodings(); + List compressors = plan.getCompressors(); + + List tmpMeasurements = new LinkedList<>(); + List tmpDataTypes = new LinkedList<>(); + List tmpEncodings = new LinkedList<>(); + List tmpCompressors = new LinkedList<>(); + for (int i = 0; i < measurements.size(); i++) { + String measurement = measurements.get(i); + if (!deviceEntry.getMeasurementMap().containsKey(measurement)) { + tmpMeasurements.add(measurements.get(i)); + tmpDataTypes.add(dataTypes.get(i)); + tmpEncodings.add(encodings.get(i)); + tmpCompressors.add(compressors.get(i)); + } + } + if (tmpMeasurements.size() == 0) + throw new PathAlreadyExistException(devicePath.getFullPath()); + plan.setMeasurements(tmpMeasurements); + plan.setDataTypes(tmpDataTypes); + plan.setEncodings(tmpEncodings); + plan.setCompressors(tmpCompressors); + } + } + idTable.createAlignedTimeseries(plan); + if (deviceEntry == null) { + createTagInvertedIndex(devicePath); + } + } + + @Override + public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public int constructSchemaBlackList(PathPatternTree patternTree) throws MetadataException { + return 0; + } + + @Override + public void rollbackSchemaBlackList(PathPatternTree patternTree) throws MetadataException {} + + @Override + public List fetchSchemaBlackList(PathPatternTree patternTree) + throws MetadataException { + return null; + } + + @Override + public void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws MetadataException {} + + @Override + public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public boolean isPathExist(PartialPath path) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + int res = 0; + List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); + for (IDeviceID deviceID : deviceIDs) { + res += idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap().keySet().size(); + } + return res; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, Map templateMap, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains) + throws MetadataException { + return 0; + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, + int level, + boolean isPrefixMatch, + String key, + String value, + boolean isContains) + throws MetadataException { + return null; + } + + @Override + public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + if (pathPattern.getFullPath().length() <= storageGroupFullPath.length()) { + return deviceIDS.size(); + } else { + return getDeviceIDsByInvertedIndex(pathPattern).size(); + } + } + + @Override + public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public List getNodesListInGivenLevel( + PartialPath pathPattern, + int nodeLevel, + boolean isPrefixMatch, + LocalSchemaProcessor.StorageGroupFilter filter) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getChildNodePathInNextLevel(PartialPath pathPattern) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getBelongedDevices(PartialPath timeseries) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override // [newIotdb/show timeseries] [newIotdb/count device] [newIotdb/count timeseries] + public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); + Set res = new HashSet<>(); + String devicePath = pathPattern.getFullPath(); + if (!devicePath.endsWith(TAIL) && !devicePath.equals(storageGroupFullPath)) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); + if (deviceEntry != null) { + res.add(pathPattern); + } + return res; + } + for (IDeviceID deviceID : deviceIDs) { + if (deviceID instanceof SHA256DeviceID) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map map = deviceEntry.getMeasurementMap(); + for (String m : map.keySet()) { + SchemaEntry schemaEntry = map.get(m); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + res.add( + new PartialPath( + diskSchemaEntry.seriesKey.substring( + 0, + diskSchemaEntry.seriesKey.length() + - diskSchemaEntry.measurementName.length() + - 1))); + break; + } + } else { + res.add(new PartialPath(deviceID.toStringID())); + } + } + return res; + } + + @Override + public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override // [newIotDB / insert1,3] [newIotDB/select] [newIotdb/select count()] [newIotdb/select + // .. groupby level] + public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + PartialPath devicePath = pathPattern.getDevicePath(); + // 批量查询.路径以".**"结尾,如： + // root.sg.tag1.a.** + // root.sg.tagx.c.tag2.v.** + // 点查询.路径不以".**",直接走IDTable,精确查询 + if (devicePath.getFullPath().endsWith(TAIL)) { + return getMeasurementPathsWithBatchQuery(devicePath, isPrefixMatch); + } else { + return getMeasurementPathsWithPointQuery(devicePath, isPrefixMatch); + } + } + + private List getMeasurementPathsWithPointQuery( + PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { + List res = new LinkedList<>(); + String path = devicePath.getFullPath(); + Map tags = pathToTags(path); + path = tagsToPath(tags); + DeviceEntry deviceEntry = idTable.getDeviceEntry(path); + if (deviceEntry == null) return res; + Map schemaMap = deviceEntry.getMeasurementMap(); + for (String measurement : schemaMap.keySet()) { + SchemaEntry schemaEntry = schemaMap.get(measurement); + MeasurementPath measurementPath = + new MeasurementPath( + path, + measurement, + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); + res.add(measurementPath); + } + + return res; + } + + private List getMeasurementPathsWithBatchQuery( + PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { + List res = new LinkedList<>(); + List deviceIDs = getDeviceIdFromInvertedIndex(devicePath); + for (IDeviceID deviceID : deviceIDs) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map schemaMap = deviceEntry.getMeasurementMap(); + if (deviceID instanceof SHA256DeviceID) { + for (String measurement : schemaMap.keySet()) { + SchemaEntry schemaEntry = schemaMap.get(measurement); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + MeasurementPath measurementPath = + new MeasurementPath( + new PartialPath(diskSchemaEntry.seriesKey), + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); + res.add(measurementPath); + } + } else { + for (String measurement : schemaMap.keySet()) { + SchemaEntry schemaEntry = schemaMap.get(measurement); + MeasurementPath measurementPath = + new MeasurementPath( + deviceID.toStringID(), + measurement, + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); + res.add(measurementPath); + } + } + } + return res; + } + + // [iotdb/select] [iotdb/select last] [iotdb/select count()] [iotdb/select ...groupby level] + @Override + public Pair, Integer> getMeasurementPathsWithAlias( + PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) + throws MetadataException { + List res = getMeasurementPaths(pathPattern, isPrefixMatch); + Pair, Integer> result = new Pair<>(res, 0); + return result; + } + + @Override + public List fetchSchema( + PartialPath pathPattern, Map templateMap) throws MetadataException { + return null; + } + + // show 时间序列 + @Override // [iotdb/show timeseries] + public Pair, Integer> showTimeseries( + ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { + List res = new ArrayList<>(); + Pair, Integer> result = new Pair<>(res, 0); + String path = plan.getPath().getFullPath(); + if (!path.endsWith(TAIL)) { + Map tags = pathToTags(path); + path = tagsToPath(tags); + DeviceEntry deviceEntry = idTable.getDeviceEntry(path); + if (deviceEntry != null) { + Map measurementMap = deviceEntry.getMeasurementMap(); + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + res.add( + new ShowTimeSeriesResult( + path + "." + m, + "null", + storageGroupFullPath, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>())); + } + } + return result; + } + List deviceIDs = getDeviceIdFromInvertedIndex(plan.getPath()); + for (IDeviceID deviceID : deviceIDs) { + getTimeSeriesResultOfDeviceFromIDTable(res, deviceID); + } + return result; + } + + private List getDeviceIdFromInvertedIndex(PartialPath devicePath) + throws MetadataException { + String path = devicePath.getFullPath(); + if (path.endsWith(TAIL)) { + path = path.substring(0, path.length() - TAIL.length()); + devicePath = new PartialPath(path); + } + if (devicePath.getFullPath().length() <= storageGroupFullPath.length()) { + return deviceIDS; + } else { + List res = new LinkedList<>(); + List ids = getDeviceIDsByInvertedIndex(devicePath); + if (ids.size() > 0) { + for (int id : ids) { + res.add(deviceIDS.get(id)); + } + } + return res; + } + } + + private void getTimeSeriesResultOfDeviceFromIDTable( + List res, IDeviceID deviceID) { + Map measurementMap = + idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap(); + if (deviceID instanceof SHA256DeviceID) { + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + res.add( + new ShowTimeSeriesResult( + diskSchemaEntry.seriesKey, + "null", + storageGroupFullPath, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>())); + } + } else { + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + res.add( + new ShowTimeSeriesResult( + deviceID.toStringID() + "." + m, + "null", + storageGroupFullPath, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>())); + } + } + } + + private List getDeviceIDsByInvertedIndex(PartialPath path) { + Map tags = pathToTags(path.getFullPath()); + List idsCollection = new ArrayList<>(tags.keySet().size()); + for (String tagKey : tags.keySet()) { + if (!tagInvertedIndex.containsKey(tagKey) + || !tagInvertedIndex.get(tagKey).containsKey(tags.get(tagKey))) { + return new ArrayList<>(); + } + List ids = tagInvertedIndex.get(tagKey).get(tags.get(tagKey)); + idsCollection.add(new ArrayList(ids)); + } + if (idsCollection.size() == 0) return new ArrayList<>(); + List ids = idsCollection.get(0); + for (int i = 1; i < idsCollection.size(); i++) { + List list = idsCollection.get(i); + ids.retainAll(list); + } + return ids; + } + + @Override + public List getAllMeasurementByDevicePath(PartialPath devicePath) + throws PathNotExistException { + throw new UnsupportedOperationException(""); + } + + @Override + public IMNode getDeviceNode(PartialPath path) throws MetadataException { + DeviceEntry deviceEntry = idTable.getDeviceEntry(path.getFullPath()); + if (deviceEntry == null) throw new PathNotExistException(path.getFullPath()); + return new EntityMNode(storageGroupMNode, path.getFullPath()); + } + + @Override + public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void upsertTagsAndAttributes( + String alias, + Map tagsMap, + Map attributesMap, + PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void addAttributes(Map attributesMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void addTags(Map tagsMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void dropTagsOrAttributes(Set keySet, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + // insert data + @Override // [iotdb/insert ] + public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) + throws MetadataException, IOException { + PartialPath devicePath = plan.getDevicePath(); + Map tags = pathToTags(devicePath.getFullPath()); + devicePath = new PartialPath(tagsToPath(tags)); + plan.setDevicePath(devicePath); + String[] measurementList = plan.getMeasurements(); + IMeasurementMNode[] measurementMNodes = plan.getMeasurementMNodes(); + checkAlignedAndAutoCreateSeries(plan); + IMNode deviceMNode = getDeviceNode(devicePath); + IMeasurementMNode measurementMNode; + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + Map schemaMap = deviceEntry.getMeasurementMap(); + for (int i = 0; i < measurementList.length; i++) { + SchemaEntry schemaEntry = schemaMap.get(measurementList[i]); + // measurementMNode = + // new MeasurementMNode( + // deviceMNode, + // measurementList[i], + // new MeasurementSchema( + // measurementList[i], + // schemaEntry.getTSDataType(), + // schemaEntry.getTSEncoding(), + // schemaEntry.getCompressionType()), + // null); + measurementMNode = new InsertMeasurementMNode(measurementList[i], schemaEntry, null); + // check type is match + try { + SchemaRegionUtils.checkDataTypeMatch(plan, i, schemaEntry.getTSDataType()); + } catch (DataTypeMismatchException mismatchException) { + if (!config.isEnablePartialInsert()) { + throw mismatchException; + } else { + // mark failed measurement + plan.markFailedMeasurementInsertion(i, mismatchException); + continue; + } + } + measurementMNodes[i] = measurementMNode; + } + plan.setDeviceID(deviceEntry.getDeviceID()); + plan.setDevicePath(new PartialPath(deviceEntry.getDeviceID().toStringID(), false)); + return deviceMNode; + } + + private SchemaEntry getSchemaEntry(String devicePath, String measurementName) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); + if (deviceEntry == null) return null; + return deviceEntry.getSchemaEntry(measurementName); + } + + @Override + public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( + PartialPath devicePath, + String[] measurements, + Function getDataType, + boolean aligned) + throws MetadataException { + List measurementSchemaInfoList = new ArrayList<>(measurements.length); + for (int i = 0; i < measurements.length; i++) { + SchemaEntry schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + if (schemaEntry == null) { + if (config.isAutoCreateSchemaEnabled()) { + if (aligned) { + internalAlignedCreateTimeseries( + devicePath, + Collections.singletonList(measurements[i]), + Collections.singletonList(getDataType.apply(i))); + + } else { + internalCreateTimeseries(devicePath.concatNode(measurements[i]), getDataType.apply(i)); + } + } + schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + } + measurementSchemaInfoList.add( + new MeasurementSchemaInfo( + measurements[i], + new MeasurementSchema( + measurements[i], + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType()), + null)); + } + return new DeviceSchemaInfo(devicePath, aligned, measurementSchemaInfoList); + } + + private void checkAlignedAndAutoCreateSeries(InsertPlan plan) throws MetadataException { + String[] measurementList = plan.getMeasurements(); + try { + if (plan.isAligned()) { + internalAlignedCreateTimeseries( + plan.getDevicePath(), + Arrays.asList(measurementList), + Arrays.asList(plan.getDataTypes())); + } else { + internalCreateTimeseries( + plan.getDevicePath().concatNode(measurementList[0]), plan.getDataTypes()[0]); + } + } catch (MetadataException e) { + if (!(e instanceof PathAlreadyExistException)) { + throw e; + } + } + } + + private void internalCreateTimeseries(PartialPath path, TSDataType dataType) + throws MetadataException { + createTimeseries( + path, + dataType, + getDefaultEncoding(dataType), + TSFileDescriptor.getInstance().getConfig().getCompressor(), + Collections.emptyMap()); + } + + private void internalAlignedCreateTimeseries( + PartialPath prefixPath, List measurements, List dataTypes) + throws MetadataException { + List encodings = new ArrayList<>(); + List compressors = new ArrayList<>(); + for (TSDataType dataType : dataTypes) { + encodings.add(getDefaultEncoding(dataType)); + compressors.add(TSFileDescriptor.getInstance().getConfig().getCompressor()); + } + createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); + } + + @Override + public Set getPathsSetTemplate(String templateName) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getPathsUsingTemplate(String templateName) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public boolean isTemplateAppendable(Template template, List measurements) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template) + throws MetadataException {} + + @Override + public List getPathsUsingTemplate(int templateId) throws MetadataException { + return null; + } + + @Override + public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException { + throw new UnsupportedOperationException(""); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java index ad13817596a4f..756f0f6f446d4 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java @@ -39,4 +39,14 @@ public int getWalBufferSize() { public void setWalBufferSize(int walBufferSize) { this.walBufferSize = walBufferSize; } + + @Override + public String toString() { + return "TagSchemaConfig[" + + "numOfDeviceIdsInMemTable=" + + numOfDeviceIdsInMemTable + + ", walBufferSize=" + + walBufferSize + + "]"; + } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java index 857575d1efca5..c32466cd346f0 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java @@ -16,21 +16,49 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.metadata.tagSchemaRegion; import org.apache.iotdb.common.rpc.thrift.TSchemaNode; import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.AlignedTimeseriesException; +import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; +import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException; import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.exception.metadata.SchemaDirCreationFailureException; import org.apache.iotdb.db.metadata.LocalSchemaProcessor; +import org.apache.iotdb.db.metadata.idtable.IDTable; +import org.apache.iotdb.db.metadata.idtable.IDTableManager; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.InsertMeasurementMNode; +import org.apache.iotdb.db.metadata.idtable.entry.SHA256DeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.db.metadata.mnode.EntityMNode; import org.apache.iotdb.db.metadata.mnode.IMNode; import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; +import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; import org.apache.iotdb.db.metadata.path.MeasurementPath; import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.metadata.schemaregion.SchemaRegionUtils; +import org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist.DeviceIDList; +import org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist.IDeviceIDList; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.TagInvertedIndex; +import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.MeasurementPathUtils; +import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.PathTagConverterUtils; +import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.ShowTimeSeriesResultUtils; import org.apache.iotdb.db.metadata.template.Template; import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.mpp.common.schematree.MeasurementSchemaInfo; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan; import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan; @@ -44,110 +72,320 @@ import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.dataset.ShowDevicesResult; import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; +import org.apache.iotdb.external.api.ISeriesNumerLimiter; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; +import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; + public class TagSchemaRegion implements ISchemaRegion { + private static final Logger logger = LoggerFactory.getLogger(TagSchemaRegion.class); + + protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + private final String TAIL = ".**"; + private final IStorageGroupMNode storageGroupMNode; + private final String storageGroupFullPath; + private final SchemaRegionId schemaRegionId; + private final String schemaRegionDirPath; + + private final TagInvertedIndex tagInvertedIndex; + + private final IDeviceIDList deviceIDList; + + private final IDTable idTable; + private final ISeriesNumerLimiter seriesNumerLimiter; + + public TagSchemaRegion( + PartialPath storageGroup, + SchemaRegionId schemaRegionId, + IStorageGroupMNode storageGroupMNode, + ISeriesNumerLimiter seriesNumerLimiter) + throws MetadataException { + storageGroupFullPath = storageGroup.getFullPath(); + this.schemaRegionId = schemaRegionId; + String storageGroupDirPath = config.getSchemaDir() + File.separator + storageGroupFullPath; + schemaRegionDirPath = storageGroupDirPath + File.separator + schemaRegionId.getId(); + this.storageGroupMNode = storageGroupMNode; + this.seriesNumerLimiter = seriesNumerLimiter; + idTable = IDTableManager.getInstance().getIDTable(storageGroup); + tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath); + deviceIDList = new DeviceIDList(schemaRegionDirPath); + init(); + } @Override - public void init() throws MetadataException {} + public void init() throws MetadataException { + if (!config.isEnableIDTableLogFile() + && config.getDeviceIDTransformationMethod().equals("SHA256")) { + throw new MetadataException( + "enableIDTableLogFile OR deviceIDTransformationMethod==\"Plain\""); + } + File schemaRegionFolder = SystemFileFactory.INSTANCE.getFile(schemaRegionDirPath); + if (!schemaRegionFolder.exists()) { + if (schemaRegionFolder.mkdirs()) { + logger.info("create schema region folder {}", schemaRegionDirPath); + } else { + if (!schemaRegionFolder.exists()) { + logger.error("create schema region folder {} failed.", schemaRegionDirPath); + throw new SchemaDirCreationFailureException(schemaRegionDirPath); + } + } + } + + logger.info("initialized successfully: {}", this); + } @Override - public void clear() {} + public void clear() { + try { + tagInvertedIndex.clear(); + deviceIDList.clear(); + } catch (IOException e) { + logger.error("clear tag inverted index failed", e); + } + } @Override - public void forceMlog() {} + public void forceMlog() { + // no need to record mlog + } @Override public SchemaRegionId getSchemaRegionId() { - return null; + return schemaRegionId; } @Override public String getStorageGroupFullPath() { - return null; + return storageGroupFullPath; } @Override - public void deleteSchemaRegion() throws MetadataException {} + public void deleteSchemaRegion() throws MetadataException { + clear(); + SchemaRegionUtils.deleteSchemaRegionFolder(schemaRegionDirPath, logger); + } @Override public boolean createSnapshot(File snapshotDir) { - return false; + // todo implement this + throw new UnsupportedOperationException("Tag mode currently doesn't support snapshot feature."); } @Override - public void loadSnapshot(File latestSnapshotRootDir) {} + public void loadSnapshot(File latestSnapshotRootDir) { + // todo implement this + throw new UnsupportedOperationException("Tag mode currently doesn't support snapshot feature."); + } - @Override - public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException {} + private void createTagInvertedIndex(PartialPath devicePath) { + IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath); + Map tagsMap = + PathTagConverterUtils.pathToTags(storageGroupFullPath, devicePath.getFullPath()); + synchronized (deviceIDList) { + deviceIDList.add(deviceID); + tagInvertedIndex.addTags(tagsMap, deviceIDList.size() - 1); + } + } - @Override - public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException {} + private List getDeviceIDsByInvertedIndex(PartialPath path) { + Map tags = + PathTagConverterUtils.pathToTags(storageGroupFullPath, path.getFullPath()); + return tagInvertedIndex.getMatchedIDs(tags); + } + + private void createTimeseries( + PartialPath path, + TSDataType dataType, + TSEncoding encoding, + CompressionType compressor, + Map props) + throws MetadataException { + createTimeseries( + new CreateTimeSeriesPlan(path, dataType, encoding, compressor, props, null, null, null), 0); + } + + private void createAlignedTimeSeries( + PartialPath prefixPath, + List measurements, + List dataTypes, + List encodings, + List compressors) + throws MetadataException { + createAlignedTimeSeries( + new CreateAlignedTimeSeriesPlan( + prefixPath, measurements, dataTypes, encodings, compressors, null, null, null)); + } + + @Override // [iotdb|newIotdb/创建非对齐时间序列] [newIotdb/insert 2自动创建时间序列] + public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException { + PartialPath devicePath = plan.getPath().getDevicePath(); + PartialPath path = + new PartialPath( + PathTagConverterUtils.pathToTagsSortPath(storageGroupFullPath, devicePath.getFullPath()) + + "." + + plan.getPath().getMeasurement()); + plan.setPath(path); + devicePath = plan.getPath().getDevicePath(); + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + if (deviceEntry != null) { + if (deviceEntry.isAligned()) { + throw new AlignedTimeseriesException( + "Timeseries under this entity is not aligned, please use createTimeseries or change entity.", + devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); + } else if (deviceEntry.getMeasurementMap().containsKey(plan.getPath().getMeasurement())) { + throw new PathAlreadyExistException( + devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); + } + } + idTable.createTimeseries(plan); + // write the device path for the first time + if (deviceEntry == null) { + createTagInvertedIndex(devicePath); + } + } + + @Override // [iotdb|newIotdb/对齐时间序列] [newIotdb/insert 2自动创建时间序列] + public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException { + PartialPath devicePath = plan.getPrefixPath(); + PartialPath path = + new PartialPath( + PathTagConverterUtils.pathToTagsSortPath( + storageGroupFullPath, devicePath.getFullPath())); + plan.setPrefixPath(path); + devicePath = plan.getPrefixPath(); + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + if (deviceEntry != null) { + if (!deviceEntry.isAligned()) { + throw new AlignedTimeseriesException( + "Timeseries under this entity is aligned, please use createAlignedTimeseries or change entity.", + devicePath.getFullPath()); + } else { + filterExistingMeasurements(plan, deviceEntry.getMeasurementMap().keySet()); + if (plan.getMeasurements().size() == 0) + throw new PathAlreadyExistException(devicePath.getFullPath()); + } + } + idTable.createAlignedTimeseries(plan); + // write the device path for the first time + if (deviceEntry == null) { + createTagInvertedIndex(devicePath); + } + } + + private void filterExistingMeasurements( + CreateAlignedTimeSeriesPlan plan, Set measurementSet) { + List measurements = plan.getMeasurements(); + List dataTypes = plan.getDataTypes(); + List encodings = plan.getEncodings(); + List compressors = plan.getCompressors(); + + List tmpMeasurements = new LinkedList<>(); + List tmpDataTypes = new LinkedList<>(); + List tmpEncodings = new LinkedList<>(); + List tmpCompressors = new LinkedList<>(); + for (int i = 0; i < measurements.size(); i++) { + String measurement = measurements.get(i); + if (!measurementSet.contains(measurement)) { + tmpMeasurements.add(measurements.get(i)); + tmpDataTypes.add(dataTypes.get(i)); + tmpEncodings.add(encodings.get(i)); + tmpCompressors.add(compressors.get(i)); + } + } + plan.setMeasurements(tmpMeasurements); + plan.setDataTypes(tmpDataTypes); + plan.setEncodings(tmpEncodings); + plan.setCompressors(tmpCompressors); + } @Override public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch) throws MetadataException { - return null; + throw new UnsupportedOperationException("deleteTimeseries"); } @Override public int constructSchemaBlackList(PathPatternTree patternTree) throws MetadataException { - return 0; + throw new UnsupportedOperationException("constructSchemaBlackList"); } @Override - public void rollbackSchemaBlackList(PathPatternTree patternTree) throws MetadataException {} + public void rollbackSchemaBlackList(PathPatternTree patternTree) throws MetadataException { + throw new UnsupportedOperationException("rollbackSchemaBlackList"); + } @Override public List fetchSchemaBlackList(PathPatternTree patternTree) throws MetadataException { - return null; + throw new UnsupportedOperationException("fetchSchemaBlackList"); } @Override - public void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws MetadataException {} + public void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws MetadataException { + throw new UnsupportedOperationException("deleteTimeseriesInBlackList"); + } @Override - public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException {} + public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException { + throw new UnsupportedOperationException("autoCreateDeviceMNode"); + } @Override public boolean isPathExist(PartialPath path) throws MetadataException { - return false; + throw new UnsupportedOperationException("isPathExist"); } @Override public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch) throws MetadataException { - return 0; + int res = 0; + List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); + for (IDeviceID deviceID : deviceIDs) { + res += idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap().keySet().size(); + } + return res; } @Override public int getAllTimeseriesCount( PartialPath pathPattern, Map templateMap, boolean isPrefixMatch) throws MetadataException { - return 0; + throw new UnsupportedOperationException("getAllTimeseriesCount"); } @Override public int getAllTimeseriesCount( PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains) throws MetadataException { - return 0; + throw new UnsupportedOperationException("getAllTimeseriesCount"); } @Override public Map getMeasurementCountGroupByLevel( PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { - return null; + throw new UnsupportedOperationException("getMeasurementCountGroupByLevel"); } @Override @@ -159,19 +397,25 @@ public Map getMeasurementCountGroupByLevel( String value, boolean isContains) throws MetadataException { - return null; + throw new UnsupportedOperationException("getMeasurementCountGroupByLevel"); } @Override public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch) throws MetadataException { - return 0; + synchronized (deviceIDList) { + if (pathPattern.getFullPath().length() <= storageGroupFullPath.length()) { + return deviceIDList.size(); + } else { + return getDeviceIDsByInvertedIndex(pathPattern).size(); + } + } } @Override public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { - return 0; + throw new UnsupportedOperationException("getNodesCountInGivenLevel"); } @Override @@ -181,80 +425,270 @@ public List getNodesListInGivenLevel( boolean isPrefixMatch, LocalSchemaProcessor.StorageGroupFilter filter) throws MetadataException { - return null; + throw new UnsupportedOperationException("getNodesListInGivenLevel"); } @Override public Set getChildNodePathInNextLevel(PartialPath pathPattern) throws MetadataException { - return null; + throw new UnsupportedOperationException("getChildNodePathInNextLevel"); } @Override public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException { - return null; + throw new UnsupportedOperationException("getChildNodeNameInNextLevel"); } @Override public Set getBelongedDevices(PartialPath timeseries) throws MetadataException { - return null; + throw new UnsupportedOperationException("getBelongedDevices"); } - @Override + @Override // [newIotdb/show timeseries] [newIotdb/count device] [newIotdb/count timeseries] public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) throws MetadataException { - return null; + List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); + Set matchedDevices = new HashSet<>(); + String devicePath = pathPattern.getFullPath(); + // exact query + if (!devicePath.endsWith(TAIL) && !devicePath.equals(storageGroupFullPath)) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); + if (deviceEntry != null) { + matchedDevices.add(pathPattern); + } + return matchedDevices; + } + List devicePaths = getDevicePaths(deviceIDs); + for (String path : devicePaths) { + matchedDevices.add(new PartialPath(path)); + } + return matchedDevices; + } + + private List getDevicePaths(List deviceIDS) { + List devicePaths = new ArrayList<>(); + if (config.getDeviceIDTransformationMethod().equals("SHA256")) { + List schemaEntries = new ArrayList<>(); + for (IDeviceID deviceID : deviceIDS) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map map = deviceEntry.getMeasurementMap(); + // For each device, only one SchemaEntry needs to be obtained + for (Map.Entry entry : map.entrySet()) { + schemaEntries.add(entry.getValue()); + break; + } + } + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + for (DiskSchemaEntry diskSchemaEntry : diskSchemaEntries) { + devicePaths.add(diskSchemaEntry.getDevicePath()); + } + } else { + for (IDeviceID deviceID : deviceIDS) { + devicePaths.add(deviceID.toStringID()); + } + } + return devicePaths; + } + + private List getSchemaEntries(List deviceIDS) { + List schemaEntries = new ArrayList<>(); + for (IDeviceID deviceID : deviceIDS) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map schemaMap = deviceEntry.getMeasurementMap(); + for (Map.Entry entry : schemaMap.entrySet()) { + schemaEntries.add(entry.getValue()); + } + } + return schemaEntries; + } + + private List getMeasurementPaths(List deviceIDS) + throws IllegalPathException { + List measurementPaths = new ArrayList<>(); + if (config.getDeviceIDTransformationMethod().equals("SHA256")) { + List schemaEntries = getSchemaEntries(deviceIDS); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + for (DiskSchemaEntry diskSchemaEntry : diskSchemaEntries) { + MeasurementPath measurementPath = + MeasurementPathUtils.generateMeasurementPath(diskSchemaEntry); + measurementPaths.add(measurementPath); + } + } else { + for (IDeviceID deviceID : deviceIDS) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map schemaMap = deviceEntry.getMeasurementMap(); + for (Map.Entry entry : schemaMap.entrySet()) { + MeasurementPath measurementPath = + MeasurementPathUtils.generateMeasurementPath( + deviceID.toStringID(), entry.getKey(), entry.getValue(), deviceEntry.isAligned()); + measurementPaths.add(measurementPath); + } + } + } + return measurementPaths; } @Override public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan) throws MetadataException { - return null; + throw new UnsupportedOperationException("getMatchedDevices"); } - @Override + @Override // [newIotDB / insert1,3] [newIotDB/select] [newIotdb/select count()] [newIotdb/select + // .. groupby level] public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) throws MetadataException { - return null; - } - + PartialPath devicePath = pathPattern.getDevicePath(); + // 批量查询.路径以".**"结尾,如： + // root.sg.tag1.a.** + // root.sg.tagx.c.tag2.v.** + // 点查询.路径不以".**",直接走IDTable,精确查询 + if (devicePath.getFullPath().endsWith(TAIL)) { + return getMeasurementPathsWithBatchQuery(devicePath, isPrefixMatch); + } else { + return getMeasurementPathsWithPointQuery(devicePath, isPrefixMatch); + } + } + + private List getMeasurementPathsWithPointQuery( + PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { + List measurementPaths = new LinkedList<>(); + String path = + PathTagConverterUtils.pathToTagsSortPath(storageGroupFullPath, devicePath.getFullPath()); + DeviceEntry deviceEntry = idTable.getDeviceEntry(path); + if (deviceEntry == null) return measurementPaths; + Map schemaMap = deviceEntry.getMeasurementMap(); + for (Map.Entry entry : schemaMap.entrySet()) { + MeasurementPath measurementPath = + MeasurementPathUtils.generateMeasurementPath( + path, entry.getKey(), entry.getValue(), deviceEntry.isAligned()); + measurementPaths.add(measurementPath); + } + return measurementPaths; + } + + private List getMeasurementPathsWithBatchQuery( + PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { + List deviceIDs = getDeviceIdFromInvertedIndex(devicePath); + return getMeasurementPaths(deviceIDs); + } + + // [iotdb/select] [iotdb/select last] [iotdb/select count()] [iotdb/select ...groupby level] @Override public Pair, Integer> getMeasurementPathsWithAlias( PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) throws MetadataException { - return null; + List res = getMeasurementPaths(pathPattern, isPrefixMatch); + Pair, Integer> result = new Pair<>(res, 0); + return result; } @Override public List fetchSchema( PartialPath pathPattern, Map templateMap) throws MetadataException { - return null; + throw new UnsupportedOperationException("fetchSchema"); } - @Override + // show 时间序列 + @Override // [iotdb/show timeseries] public Pair, Integer> showTimeseries( ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { - return null; + List ShowTimeSeriesResults = new ArrayList<>(); + Pair, Integer> result = new Pair<>(ShowTimeSeriesResults, 0); + String path = plan.getPath().getFullPath(); + // point query + if (!path.endsWith(TAIL)) { + path = PathTagConverterUtils.pathToTagsSortPath(storageGroupFullPath, path); + DeviceEntry deviceEntry = idTable.getDeviceEntry(path); + if (deviceEntry != null) { + Map measurementMap = deviceEntry.getMeasurementMap(); + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + ShowTimeSeriesResults.add( + ShowTimeSeriesResultUtils.generateShowTimeSeriesResult( + storageGroupFullPath, path, m, schemaEntry)); + } + } + return result; + } + // batch query + List deviceIDs = getDeviceIdFromInvertedIndex(plan.getPath()); + for (IDeviceID deviceID : deviceIDs) { + getTimeSeriesResultOfDeviceFromIDTable(ShowTimeSeriesResults, deviceID); + } + return result; + } + + private void getTimeSeriesResultOfDeviceFromIDTable( + List ShowTimeSeriesResults, IDeviceID deviceID) { + Map measurementMap = + idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap(); + if (deviceID instanceof SHA256DeviceID) { + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + ShowTimeSeriesResults.add( + ShowTimeSeriesResultUtils.generateShowTimeSeriesResult( + storageGroupFullPath, diskSchemaEntry.seriesKey, schemaEntry)); + } + } else { + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + ShowTimeSeriesResults.add( + ShowTimeSeriesResultUtils.generateShowTimeSeriesResult( + storageGroupFullPath, deviceID.toStringID(), m, schemaEntry)); + } + } + } + + private List getDeviceIdFromInvertedIndex(PartialPath devicePath) + throws MetadataException { + String path = devicePath.getFullPath(); + if (path.endsWith(TAIL)) { + path = path.substring(0, path.length() - TAIL.length()); + devicePath = new PartialPath(path); + } + synchronized (deviceIDList) { + if (devicePath.getFullPath().length() <= storageGroupFullPath.length()) { + return deviceIDList.getAllDeviceIDS(); + } else { + List IDS = new LinkedList<>(); + List ids = getDeviceIDsByInvertedIndex(devicePath); + if (ids.size() > 0) { + for (int id : ids) { + IDS.add(deviceIDList.get(id)); + } + } + return IDS; + } + } } @Override public List getAllMeasurementByDevicePath(PartialPath devicePath) throws PathNotExistException { - return null; + throw new UnsupportedOperationException("getAllMeasurementByDevicePath"); } @Override public IMNode getDeviceNode(PartialPath path) throws MetadataException { - return null; + DeviceEntry deviceEntry = idTable.getDeviceEntry(path.getFullPath()); + if (deviceEntry == null) throw new PathNotExistException(path.getFullPath()); + return new EntityMNode(storageGroupMNode, path.getFullPath()); } @Override public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException { - return null; + throw new UnsupportedOperationException("getMeasurementMNode"); } @Override - public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException {} + public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException { + throw new UnsupportedOperationException("changeAlias"); + } @Override public void upsertTagsAndAttributes( @@ -262,32 +696,83 @@ public void upsertTagsAndAttributes( Map tagsMap, Map attributesMap, PartialPath fullPath) - throws MetadataException, IOException {} + throws MetadataException, IOException { + throw new UnsupportedOperationException("upsertTagsAndAttributes"); + } @Override public void addAttributes(Map attributesMap, PartialPath fullPath) - throws MetadataException, IOException {} + throws MetadataException, IOException { + throw new UnsupportedOperationException("addAttributes"); + } @Override public void addTags(Map tagsMap, PartialPath fullPath) - throws MetadataException, IOException {} + throws MetadataException, IOException { + throw new UnsupportedOperationException("addTags"); + } @Override public void dropTagsOrAttributes(Set keySet, PartialPath fullPath) - throws MetadataException, IOException {} + throws MetadataException, IOException { + throw new UnsupportedOperationException("dropTagsOrAttributes"); + } @Override public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) - throws MetadataException, IOException {} + throws MetadataException, IOException { + throw new UnsupportedOperationException("setTagsOrAttributesValue"); + } @Override public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) - throws MetadataException, IOException {} + throws MetadataException, IOException { + throw new UnsupportedOperationException("renameTagOrAttributeKey"); + } - @Override + // insert data + @Override // [iotdb/insert ] public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) throws MetadataException, IOException { - return null; + PartialPath devicePath = plan.getDevicePath(); + devicePath = + new PartialPath( + PathTagConverterUtils.pathToTagsSortPath( + storageGroupFullPath, devicePath.getFullPath())); + plan.setDevicePath(devicePath); + String[] measurementList = plan.getMeasurements(); + IMeasurementMNode[] measurementMNodes = plan.getMeasurementMNodes(); + checkAlignedAndAutoCreateSeries(plan); + IMNode deviceMNode = getDeviceNode(devicePath); + IMeasurementMNode measurementMNode; + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + Map schemaMap = deviceEntry.getMeasurementMap(); + for (int i = 0; i < measurementList.length; i++) { + SchemaEntry schemaEntry = schemaMap.get(measurementList[i]); + measurementMNode = new InsertMeasurementMNode(measurementList[i], schemaEntry, null); + // check type is match + try { + SchemaRegionUtils.checkDataTypeMatch(plan, i, schemaEntry.getTSDataType()); + } catch (DataTypeMismatchException mismatchException) { + if (!config.isEnablePartialInsert()) { + throw mismatchException; + } else { + // mark failed measurement + plan.markFailedMeasurementInsertion(i, mismatchException); + continue; + } + } + measurementMNodes[i] = measurementMNode; + } + plan.setDeviceID(deviceEntry.getDeviceID()); + plan.setDevicePath(new PartialPath(deviceEntry.getDeviceID().toStringID(), false)); + return deviceMNode; + } + + private SchemaEntry getSchemaEntry(String devicePath, String measurementName) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); + if (deviceEntry == null) return null; + return deviceEntry.getSchemaEntry(measurementName); } @Override @@ -297,48 +782,140 @@ public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( Function getDataType, boolean aligned) throws MetadataException { - return null; + List measurementSchemaInfoList = new ArrayList<>(measurements.length); + for (int i = 0; i < measurements.length; i++) { + SchemaEntry schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + if (schemaEntry == null) { + if (config.isAutoCreateSchemaEnabled()) { + if (aligned) { + internalAlignedCreateTimeseries( + devicePath, + Collections.singletonList(measurements[i]), + Collections.singletonList(getDataType.apply(i))); + + } else { + internalCreateTimeseries(devicePath.concatNode(measurements[i]), getDataType.apply(i)); + } + } + schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + } + measurementSchemaInfoList.add( + new MeasurementSchemaInfo( + measurements[i], + new MeasurementSchema( + measurements[i], + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType()), + null)); + } + return new DeviceSchemaInfo(devicePath, aligned, measurementSchemaInfoList); + } + + private void checkAlignedAndAutoCreateSeries(InsertPlan plan) throws MetadataException { + String[] measurementList = plan.getMeasurements(); + try { + if (plan.isAligned()) { + internalAlignedCreateTimeseries( + plan.getDevicePath(), + Arrays.asList(measurementList), + Arrays.asList(plan.getDataTypes())); + } else { + internalCreateTimeseries( + plan.getDevicePath().concatNode(measurementList[0]), plan.getDataTypes()[0]); + } + } catch (MetadataException e) { + if (!(e instanceof PathAlreadyExistException)) { + throw e; + } + } + } + + private void internalCreateTimeseries(PartialPath path, TSDataType dataType) + throws MetadataException { + createTimeseries( + path, + dataType, + getDefaultEncoding(dataType), + TSFileDescriptor.getInstance().getConfig().getCompressor(), + Collections.emptyMap()); + } + + private void internalAlignedCreateTimeseries( + PartialPath prefixPath, List measurements, List dataTypes) + throws MetadataException { + List encodings = new ArrayList<>(); + List compressors = new ArrayList<>(); + for (TSDataType dataType : dataTypes) { + encodings.add(getDefaultEncoding(dataType)); + compressors.add(TSFileDescriptor.getInstance().getConfig().getCompressor()); + } + createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); } @Override public Set getPathsSetTemplate(String templateName) throws MetadataException { - return null; + throw new UnsupportedOperationException("getPathsSetTemplate"); } @Override public Set getPathsUsingTemplate(String templateName) throws MetadataException { - return null; + throw new UnsupportedOperationException("getPathsUsingTemplate"); } @Override public boolean isTemplateAppendable(Template template, List measurements) throws MetadataException { - return false; + throw new UnsupportedOperationException("isTemplateAppendable"); } @Override - public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException {} + public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException("setSchemaTemplate"); + } @Override - public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException {} + public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException("unsetSchemaTemplate"); + } @Override - public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException {} + public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException("setUsingSchemaTemplate"); + } @Override public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template) - throws MetadataException {} + throws MetadataException { + throw new UnsupportedOperationException("activateSchemaTemplate"); + } @Override public List getPathsUsingTemplate(int templateId) throws MetadataException { - return null; + throw new UnsupportedOperationException("getPathsUsingTemplate"); } @Override public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException { - return null; + throw new UnsupportedOperationException("getMNodeForTrigger"); } @Override - public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException {} + public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException { + throw new UnsupportedOperationException("releaseMNodeAfterDropTrigger"); + } + + @Override + public String toString() { + return "TagSchemaRegion{" + + "storageGroupFullPath='" + + storageGroupFullPath + + '\'' + + ", schemaRegionId=" + + schemaRegionId + + ", schemaRegionDirPath='" + + schemaRegionDirPath + + '\'' + + '}'; + } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java new file mode 100644 index 0000000000000..32475d650c560 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist; + +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; + +public class AppendOnlyDeviceIDListFileManager { + + private static final Logger logger = + LoggerFactory.getLogger(AppendOnlyDeviceIDListFileManager.class); + + private static final String DEVICE_ID_LIST_FILE_NAME = "device_id.list"; + + private boolean isRecover; + + private String schemaDirPath; + + private File deviceIDSFile; + + private FileOutputStream outputStream; + + public AppendOnlyDeviceIDListFileManager(String schemaDirPath) { + this.schemaDirPath = schemaDirPath; + isRecover = true; + try { + initFile(); + outputStream = new FileOutputStream(deviceIDSFile, true); + } catch (IOException e) { + logger.error(e.getMessage()); + throw new IllegalArgumentException( + "can't initialize device id list file at " + deviceIDSFile); + } + } + + private void initFile() throws IOException { + File schemaDir = new File(schemaDirPath); + schemaDir.mkdirs(); + deviceIDSFile = new File(schemaDir, DEVICE_ID_LIST_FILE_NAME); + if (!deviceIDSFile.exists()) { + // create new file + deviceIDSFile.createNewFile(); + } + } + + public void serialize(String deviceID) { + try { + if (!isRecover) { + ReadWriteIOUtils.write(deviceID, outputStream); + } + } catch (IOException e) { + logger.error("failed to serialize device id: " + deviceID); + throw new IllegalArgumentException("can't serialize device id of " + deviceID); + } + } + + public void recover(DeviceIDList deviceIDList) { + logger.info("recover device id list using file {}", deviceIDSFile); + try (FileInputStream inputStream = new FileInputStream(deviceIDSFile)) { + while (inputStream.available() > 0) { + String deviceID = ReadWriteIOUtils.readString(inputStream); + deviceIDList.add(DeviceIDFactory.getInstance().getDeviceID(deviceID)); + } + } catch (IOException e) { + logger.info("device id list is incomplete, we will recover as much as we can."); + } + isRecover = false; + } + + @TestOnly + public void close() throws IOException { + try { + outputStream.close(); + } catch (IOException e) { + logger.error("close device id list file failed"); + throw e; + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java new file mode 100644 index 0000000000000..c8793d40fdfa4 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist; + +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class DeviceIDList implements IDeviceIDList { + + private final List deviceIDS; + + private AppendOnlyDeviceIDListFileManager appendOnlyDeviceIDListFileManager; + + public DeviceIDList(String schemaDirPath) { + deviceIDS = new ArrayList<>(); + appendOnlyDeviceIDListFileManager = new AppendOnlyDeviceIDListFileManager(schemaDirPath); + recover(); + } + + public void recover() { + appendOnlyDeviceIDListFileManager.recover(this); + } + + @Override + public void add(IDeviceID deviceID) { + deviceIDS.add(deviceID); + appendOnlyDeviceIDListFileManager.serialize(deviceID.toStringID()); + } + + @Override + public IDeviceID get(int index) { + return deviceIDS.get(index); + } + + @Override + public int size() { + return deviceIDS.size(); + } + + @Override + public List getAllDeviceIDS() { + return new ArrayList<>(deviceIDS); + } + + @TestOnly + public void clear() throws IOException { + appendOnlyDeviceIDListFileManager.close(); + deviceIDS.clear(); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java new file mode 100644 index 0000000000000..1a0d72571a8cd --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist; + +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; + +import java.io.IOException; +import java.util.List; + +public interface IDeviceIDList { + + void add(IDeviceID deviceID); + + IDeviceID get(int index); + + int size(); + + List getAllDeviceIDS(); + + @TestOnly + void clear() throws IOException; +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java index ed62c3d2006d7..3ad63e48b8092 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -53,35 +53,39 @@ public class TagInvertedIndex implements ITagInvertedIndex { private static final TagSchemaConfig tagSchemaConfig = TagSchemaDescriptor.getInstance().getTagSchemaConfig(); - private final InsertionManager insertionManager; + private InsertionManager insertionManager; - private final DeletionManager deletionManager; + private DeletionManager deletionManager; - private final QueryManager queryManager; + private QueryManager queryManager; - private final WALManager walManager; + private WALManager walManager; - private final RecoverManager recoverManager; + private RecoverManager recoverManager; - private final int numOfDeviceIdsInMemTable; + private int numOfDeviceIdsInMemTable; - private final Map immutableMemTables; + private Map immutableMemTables; private MemTable workingMemTable; private int maxDeviceID; - public TagInvertedIndex(String schemaDirPath) throws IOException { - walManager = new WALManager(schemaDirPath); - insertionManager = new InsertionManager(walManager); - deletionManager = new DeletionManager(walManager); - recoverManager = new RecoverManager(walManager); - queryManager = new QueryManager(); - workingMemTable = new MemTable(MemTable.WORKING); - immutableMemTables = new HashMap<>(); - numOfDeviceIdsInMemTable = tagSchemaConfig.getNumOfDeviceIdsInMemTable(); - maxDeviceID = 0; - recover(); + public TagInvertedIndex(String schemaDirPath) { + try { + walManager = new WALManager(schemaDirPath); + insertionManager = new InsertionManager(walManager); + deletionManager = new DeletionManager(walManager); + recoverManager = new RecoverManager(walManager); + queryManager = new QueryManager(); + workingMemTable = new MemTable(MemTable.WORKING); + immutableMemTables = new HashMap<>(); + numOfDeviceIdsInMemTable = tagSchemaConfig.getNumOfDeviceIdsInMemTable(); + maxDeviceID = 0; + recover(); + } catch (IOException e) { + logger.error("create TagInvertedIndex fail", e); + } } public synchronized void recover() { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java index ec9fd74729565..7cc0015626030 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java @@ -24,7 +24,11 @@ import org.apache.iotdb.lsm.context.DeleteContext; import org.apache.iotdb.lsm.context.InsertContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class RecoverManager { + private static final Logger logger = LoggerFactory.getLogger(RecoverManager.class); private WALManager walManager; @@ -33,6 +37,7 @@ public RecoverManager(WALManager walManager) { } public void recover(ITagInvertedIndex tagInvertedIndex) { + logger.info("recover tagInvertedIndex"); while (true) { Context context = walManager.read(); switch (context.getType()) { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java new file mode 100644 index 0000000000000..22ae4c3b58162 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.metadata.tagSchemaRegion.utils; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.db.metadata.path.MeasurementPath; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +public class MeasurementPathUtils { + public static MeasurementPath generateMeasurementPath( + String devicePath, String measurement, SchemaEntry schemaEntry, boolean isAligned) + throws IllegalPathException { + MeasurementPath measurementPath = + new MeasurementPath( + devicePath, + measurement, + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(isAligned); + return measurementPath; + } + + public static MeasurementPath generateMeasurementPath(DiskSchemaEntry diskSchemaEntry) + throws IllegalPathException { + MeasurementPath measurementPath = + new MeasurementPath( + new PartialPath(diskSchemaEntry.seriesKey), + new MeasurementSchema( + diskSchemaEntry.measurementName, + TSDataType.deserialize(diskSchemaEntry.type), + TSEncoding.deserialize(diskSchemaEntry.encoding), + CompressionType.deserialize(diskSchemaEntry.compressor))); + measurementPath.setUnderAlignedEntity(diskSchemaEntry.isAligned); + return measurementPath; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java new file mode 100644 index 0000000000000..bd2c3a4e9ab58 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.metadata.tagSchemaRegion.utils; + +import java.util.Map; +import java.util.TreeMap; + +public class PathTagConverterUtils { + + public static Map pathToTags(String storageGroupFullPath, String path) { + if (path.length() <= storageGroupFullPath.length()) return new TreeMap<>(); + String devicePath = path.substring(storageGroupFullPath.length() + 1); + String[] tags = devicePath.split("\\."); + Map tagsMap = new TreeMap<>(); + for (int i = 0; i < tags.length; i += 2) { + tagsMap.put(tags[i], tags[i + 1]); + } + return tagsMap; + } + + public static String tagsToPath(String storageGroupFullPath, Map tags) { + StringBuilder stringBuilder = new StringBuilder(storageGroupFullPath); + for (String tagKey : tags.keySet()) { + stringBuilder.append(".").append(tagKey).append(".").append(tags.get(tagKey)); + } + return stringBuilder.toString(); + } + + public static String pathToTagsSortPath(String storageGroupFullPath, String path) { + return tagsToPath(storageGroupFullPath, pathToTags(storageGroupFullPath, path)); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java new file mode 100644 index 0000000000000..14f02a7dfd205 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.utils; + +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; + +import java.util.HashMap; + +public class ShowTimeSeriesResultUtils { + public static ShowTimeSeriesResult generateShowTimeSeriesResult( + String sgName, String devicePath, String measurement, SchemaEntry schemaEntry) { + return new ShowTimeSeriesResult( + devicePath + "." + measurement, + null, + sgName, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>()); + } + + public static ShowTimeSeriesResult generateShowTimeSeriesResult( + String sgName, String timeSeriesPath, SchemaEntry schemaEntry) { + return new ShowTimeSeriesResult( + timeSeriesPath, + null, + sgName, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>()); + } +} diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java new file mode 100644 index 0000000000000..eddc31d2f9e08 --- /dev/null +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist; + +import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class DeviceIDListTest { + + private IDeviceIDList deviceIDList; + + private String storageGroupDirPath; + + private String schemaRegionDirPath; + + private String storageGroupFullPath = "root/testDeviceIDList"; + + private boolean isEnableIDTable; + + private String originalDeviceIDTransformationMethod; + + private String schemaDir; + + private String[] devicePaths = + new String[] { + storageGroupFullPath + ".a.b.c.d1", + storageGroupFullPath + ".a.b.c.d2", + storageGroupFullPath + ".a.b.c.d3", + storageGroupFullPath + ".a.b.c.d4", + storageGroupFullPath + ".a.b.d.d1", + storageGroupFullPath + ".a.b.d.d2", + storageGroupFullPath + ".a.b.e.d1", + storageGroupFullPath + ".a.b.f.d1", + storageGroupFullPath + ".a.b.g.d1", + storageGroupFullPath + ".a.b.h.d1", + }; + + @Before + public void setUp() throws Exception { + schemaDir = IoTDBDescriptor.getInstance().getConfig().getSchemaDir(); + isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable(); + originalDeviceIDTransformationMethod = + IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod(); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + storageGroupDirPath = schemaDir + File.separator + storageGroupFullPath; + schemaRegionDirPath = storageGroupDirPath + File.separator + 0; + deviceIDList = new DeviceIDList(schemaRegionDirPath); + } + + @After + public void tearDown() throws Exception { + deviceIDList.clear(); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); + IoTDBDescriptor.getInstance() + .getConfig() + .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); + FileUtils.deleteDirectoryAndEmptyParent(new File(schemaDir)); + } + + @Test + public void testAddandGetDeviceID() { + List deviceIDS = generateTestDeviceIDS(); + for (IDeviceID deviceID : deviceIDS) { + deviceIDList.add(deviceID); + System.out.println(deviceID.toStringID()); + } + for (int i = 0; i < deviceIDS.size(); i++) { + assertEquals(deviceIDS.get(i), deviceIDList.get(i)); + } + } + + @Test + public void testRecover() { + List deviceIDS = generateTestDeviceIDS(); + for (IDeviceID deviceID : deviceIDS) { + deviceIDList.add(deviceID); + System.out.println(deviceID.toStringID()); + } + deviceIDList = new DeviceIDList(schemaRegionDirPath); + for (int i = 0; i < deviceIDS.size(); i++) { + assertEquals(deviceIDS.get(i), deviceIDList.get(i)); + } + } + + private List generateTestDeviceIDS() { + List deviceIDS = new ArrayList<>(); + for (String devicePath : devicePaths) { + deviceIDS.add(DeviceIDFactory.getInstance().getDeviceID(devicePath)); + } + return deviceIDS; + } +} diff --git a/server/src/assembly/resources/conf/iotdb-datanode.properties b/server/src/assembly/resources/conf/iotdb-datanode.properties index 6a275d7bf3e2e..8ad232903353c 100644 --- a/server/src/assembly/resources/conf/iotdb-datanode.properties +++ b/server/src/assembly/resources/conf/iotdb-datanode.properties @@ -1071,10 +1071,23 @@ timestamp_precision=ms # Datatype: float # group_by_fill_cache_size_in_mb=1.0 +#################### +### IDTable Configuration +#################### +# Datatype: boolean +# enable_id_table=false + +# Datatype: boolean +# enable_id_table_log_file=false + +# Choose the device id transformation method. The value could be Plain and SHA256. If the provided value doesn't match any pre-defined value, Plain will be used as default. +# Datatype: string +# device_id_transformation_method=Plain + #################### ### Schema Engine Configuration #################### -# Choose the mode of schema engine. The value could be Memory,Schema_File and Rocksdb_based. If the provided value doesn't match any pre-defined value, Memory mode will be used as default. +# Choose the mode of schema engine. The value could be Memory,Schema_File,Rocksdb_based and Tag. If the provided value doesn't match any pre-defined value, Memory mode will be used as default. # Datatype: string # schema_engine_mode=Memory diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java index e0e21ddbfaa69..f6bd30867cf18 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java @@ -74,6 +74,10 @@ public DiskSchemaEntry( this.isAligned = isAligned; } + public String getDevicePath() { + return seriesKey.substring(0, seriesKey.length() - measurementName.length() - 1); + } + public int serialize(OutputStream outputStream) throws IOException { int byteLen = 0; byteLen += ReadWriteIOUtils.write(deviceID, outputStream); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java index 7b3d243242dbd..c6762738695ca 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java @@ -35,7 +35,6 @@ import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; import org.apache.iotdb.db.metadata.mtree.ConfigMTree; import org.apache.iotdb.db.metadata.rescon.SchemaResourceManager; -import org.apache.iotdb.db.metadata.schemaregion.tagschemaregion.MockTagSchemaRegion; import org.apache.iotdb.db.metadata.visitor.SchemaExecutionVisitor; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; import org.apache.iotdb.external.api.ISeriesNumerLimiter; @@ -326,8 +325,9 @@ private ISchemaRegion createSchemaRegionWithoutExistenceCheck( break; case Tag: schemaRegion = - new MockTagSchemaRegion( - storageGroup, schemaRegionId, storageGroupMNode, seriesNumerLimiter); + new TagSchemaRegionLoader() + .loadTagSchemaRegion( + storageGroup, schemaRegionId, storageGroupMNode, seriesNumerLimiter); break; default: throw new UnsupportedOperationException( diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java index 46a8056d05b89..694ffe7929d61 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.consensus.SchemaRegionId; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; +import org.apache.iotdb.external.api.ISeriesNumerLimiter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,16 +40,14 @@ public class TagSchemaRegionLoader { private static URLClassLoader urlClassLoader = null; private static final String TAG_SCHEMA_REGION_CLASS_NAME = "org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaRegion"; - private static final String TAG_SCHEMA_CONF_DESCRIPTOR_CLASS_NAME = - "org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor"; private static final String LIB_PATH = ".." + File.separator + "lib" + File.separator + "tag-schema-region" + File.separator; public TagSchemaRegionLoader() {} /** - * Load the jar files for RSchemaRegion and create an instance of it. The jar files should be - * located in "../lib/rschema-region". If jar files cannot be found, the function will return + * Load the jar files for TagSchemaRegion and create an instance of it. The jar files should be + * located in "../lib/tag-schema-region". If jar files cannot be found, the function will return * null. * * @param storageGroup @@ -56,25 +55,26 @@ public TagSchemaRegionLoader() {} * @param node * @return */ - public ISchemaRegion loadRSchemaRegion( - PartialPath storageGroup, SchemaRegionId schemaRegionId, IStorageGroupMNode node) { + public ISchemaRegion loadTagSchemaRegion( + PartialPath storageGroup, + SchemaRegionId schemaRegionId, + IStorageGroupMNode node, + ISeriesNumerLimiter seriesNumerLimiter) { ISchemaRegion region = null; - LOGGER.info("Creating instance for schema-engine-rocksdb"); + LOGGER.info("Creating instance for schema-engine-tag"); try { - loadRSchemaRegionJar(); + loadTagSchemaRegionJar(); Class classForTagSchemaRegion = urlClassLoader.loadClass(TAG_SCHEMA_REGION_CLASS_NAME); - Class classForTagSchemaDescriptor = - urlClassLoader.loadClass(TAG_SCHEMA_CONF_DESCRIPTOR_CLASS_NAME); Constructor constructor = classForTagSchemaRegion.getConstructor( PartialPath.class, SchemaRegionId.class, IStorageGroupMNode.class, - classForTagSchemaDescriptor); - Object rSchemaLoader = classForTagSchemaDescriptor.getConstructor().newInstance(); + ISeriesNumerLimiter.class); + region = (ISchemaRegion) - constructor.newInstance(storageGroup, schemaRegionId, node, rSchemaLoader); + constructor.newInstance(storageGroup, schemaRegionId, node, seriesNumerLimiter); } catch (ClassNotFoundException | NoSuchMethodException | InvocationTargetException @@ -82,18 +82,18 @@ public ISchemaRegion loadRSchemaRegion( | IllegalAccessException | MalformedURLException | RuntimeException e) { - LOGGER.error("Cannot initialize RSchemaRegion", e); + LOGGER.error("Cannot initialize TagSchemaRegion", e); return null; } return region; } /** - * Load the jar files for rocksdb and RSchemaRegion. The jar files should be located in directory - * "../lib/rschema-region". If the jar files have been loaded, it will do nothing. + * Load the jar files for TagSchemaRegion. The jar files should be located in directory + * "../lib/tag-schema-region". If the jar files have been loaded, it will do nothing. */ - private void loadRSchemaRegionJar() throws MalformedURLException { - LOGGER.info("Loading jar for schema-engine-rocksdb"); + private void loadTagSchemaRegionJar() throws MalformedURLException { + LOGGER.info("Loading jar for schema-engine-tag"); if (urlClassLoader == null) { File[] jars = new File(LIB_PATH).listFiles(); if (jars == null) { @@ -106,6 +106,7 @@ private void loadRSchemaRegionJar() throws MalformedURLException { dependentJars.add(new URL("file:" + jar.getAbsolutePath())); } } + LOGGER.info("load jars: " + dependentJars); urlClassLoader = new URLClassLoader(dependentJars.toArray(new URL[] {})); } } From 035e7c4094f84da617c8761f0f7ab163db668cc3 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Thu, 22 Sep 2022 17:36:31 +0800 Subject: [PATCH 22/30] add SchemaTreeVisitorFactory --- .../common/schematree/ClusterSchemaTree.java | 68 ++--------- .../visitor/SchemaTreeVisitorFactory.java | 112 ++++++++++++++++++ 2 files changed, 120 insertions(+), 60 deletions(-) create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/visitor/SchemaTreeVisitorFactory.java diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java index 9974b982c11ae..112a9137f5e9b 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java @@ -22,15 +22,14 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.metadata.path.MeasurementPath; -import org.apache.iotdb.db.metadata.schemaregion.SchemaEngineMode; import org.apache.iotdb.db.mpp.common.schematree.node.SchemaEntityNode; import org.apache.iotdb.db.mpp.common.schematree.node.SchemaInternalNode; import org.apache.iotdb.db.mpp.common.schematree.node.SchemaMeasurementNode; import org.apache.iotdb.db.mpp.common.schematree.node.SchemaNode; import org.apache.iotdb.db.mpp.common.schematree.visitor.SchemaTreeDeviceVisitor; import org.apache.iotdb.db.mpp.common.schematree.visitor.SchemaTreeMeasurementVisitor; +import org.apache.iotdb.db.mpp.common.schematree.visitor.SchemaTreeVisitorFactory; import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; @@ -79,49 +78,8 @@ public Pair, Integer> searchMeasurementPaths( @Override public Pair, Integer> searchMeasurementPaths(PartialPath pathPattern) { - SchemaTreeMeasurementVisitor visitor; - switch (SchemaEngineMode.valueOf( - IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { - case Memory: - case Schema_File: - visitor = - new SchemaTreeMeasurementVisitor( - root, - pathPattern, - IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, - 0, - false); - break; - case Tag: - if (pathPattern.getFullPath().contains(".**")) { - String measurement = pathPattern.getMeasurement(); - visitor = - new SchemaTreeMeasurementVisitor( - root, - ALL_MATCH_PATTERN.concatNode(measurement), - IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, - 0, - false); - } else { - visitor = - new SchemaTreeMeasurementVisitor( - root, - pathPattern, - IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, - 0, - false); - } - break; - default: - visitor = - new SchemaTreeMeasurementVisitor( - root, - ALL_MATCH_PATTERN, - IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, - 0, - false); - break; - } + SchemaTreeMeasurementVisitor visitor = + SchemaTreeVisitorFactory.getInstance().getSchemaTreeMeasurementVisitor(root, pathPattern); return new Pair<>(visitor.getAllResult(), visitor.getNextOffset()); } @@ -138,26 +96,16 @@ public List getAllMeasurement() { */ @Override public List getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) { - SchemaTreeDeviceVisitor visitor = new SchemaTreeDeviceVisitor(root, pathPattern, isPrefixMatch); + SchemaTreeDeviceVisitor visitor = + SchemaTreeVisitorFactory.getInstance() + .getSchemaTreeDeviceVisitor(root, pathPattern, isPrefixMatch); return visitor.getAllResult(); } @Override public List getMatchedDevices(PartialPath pathPattern) { - SchemaTreeDeviceVisitor visitor; - switch (SchemaEngineMode.valueOf( - IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { - case Memory: - case Schema_File: - visitor = new SchemaTreeDeviceVisitor(root, pathPattern, false); - break; - case Tag: - visitor = new SchemaTreeDeviceVisitor(root, ALL_MATCH_PATTERN, false); - break; - default: - visitor = new SchemaTreeDeviceVisitor(root, ALL_MATCH_PATTERN, false); - break; - } + SchemaTreeDeviceVisitor visitor = + SchemaTreeVisitorFactory.getInstance().getSchemaTreeDeviceVisitor(root, pathPattern, false); return visitor.getAllResult(); } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/visitor/SchemaTreeVisitorFactory.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/visitor/SchemaTreeVisitorFactory.java new file mode 100644 index 0000000000000..cc669ac6f7170 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/visitor/SchemaTreeVisitorFactory.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.mpp.common.schematree.visitor; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.schemaregion.SchemaEngineMode; +import org.apache.iotdb.db.mpp.common.schematree.node.SchemaNode; + +import static org.apache.iotdb.db.metadata.MetadataConstant.ALL_MATCH_PATTERN; + +/** + * Generate corresponding SchemaTreeVisitor for different types of schema regions, such as Memory, + * Schema_File, Rocksdb_based, and Tag + */ +public class SchemaTreeVisitorFactory { + + private SchemaTreeVisitorFactory() {} + + public static SchemaTreeVisitorFactory getInstance() { + return SchemaTreeVisitorFactoryHolder.INSTANCE; + } + + /** + * generate corresponding SchemaTreeDeviceVisitor for different types of schema regions + * + * @param schemaNode schema node + * @param pathPattern can be a pattern or a full path of timeseries. + * @param isPrefixMatch + * @return schemaTreeDeviceVisitor + */ + public SchemaTreeDeviceVisitor getSchemaTreeDeviceVisitor( + SchemaNode schemaNode, PartialPath pathPattern, boolean isPrefixMatch) { + SchemaTreeDeviceVisitor visitor; + switch (SchemaEngineMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { + case Memory: + case Schema_File: + visitor = new SchemaTreeDeviceVisitor(schemaNode, pathPattern, isPrefixMatch); + break; + default: + visitor = new SchemaTreeDeviceVisitor(schemaNode, ALL_MATCH_PATTERN, isPrefixMatch); + break; + } + return visitor; + } + + /** + * generate corresponding SchemaTreeMeasurementVisitor for different types of schema regions + * + * @param schemaNode schema node + * @param pathPattern can be a pattern or a full path of timeseries. + * @return schemaTreeMeasurementVisitor + */ + public SchemaTreeMeasurementVisitor getSchemaTreeMeasurementVisitor( + SchemaNode schemaNode, PartialPath pathPattern) { + SchemaTreeMeasurementVisitor visitor; + switch (SchemaEngineMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { + case Memory: + case Schema_File: + visitor = generateSchemaTreeMeasurementVisitor(schemaNode, pathPattern); + break; + case Tag: + if (pathPattern.getFullPath().contains(".**")) { + visitor = + generateSchemaTreeMeasurementVisitor( + schemaNode, ALL_MATCH_PATTERN.concatNode(pathPattern.getMeasurement())); + } else { + visitor = generateSchemaTreeMeasurementVisitor(schemaNode, pathPattern); + } + break; + default: + visitor = generateSchemaTreeMeasurementVisitor(schemaNode, ALL_MATCH_PATTERN); + break; + } + return visitor; + } + + private SchemaTreeMeasurementVisitor generateSchemaTreeMeasurementVisitor( + SchemaNode schemaNode, PartialPath pathPattern) { + return new SchemaTreeMeasurementVisitor( + schemaNode, + pathPattern, + IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, + 0, + false); + } + + private static class SchemaTreeVisitorFactoryHolder { + private static final SchemaTreeVisitorFactory INSTANCE = new SchemaTreeVisitorFactory(); + + private SchemaTreeVisitorFactoryHolder() {} + } +} From 8fb85b00d9f8e7b67ef0a6aff07390f21576344c Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Thu, 22 Sep 2022 20:33:05 +0800 Subject: [PATCH 23/30] add SchemaTreeVisitorFactory --- .../tagschemaregion/MockTagSchemaRegion.java | 927 ------------------ 1 file changed, 927 deletions(-) delete mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/tagschemaregion/MockTagSchemaRegion.java diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/tagschemaregion/MockTagSchemaRegion.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/tagschemaregion/MockTagSchemaRegion.java deleted file mode 100644 index e4122375469c5..0000000000000 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/tagschemaregion/MockTagSchemaRegion.java +++ /dev/null @@ -1,927 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.metadata.schemaregion.tagschemaregion; - -import org.apache.iotdb.common.rpc.thrift.TSchemaNode; -import org.apache.iotdb.commons.consensus.SchemaRegionId; -import org.apache.iotdb.commons.exception.MetadataException; -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.path.PathPatternTree; -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.exception.metadata.AlignedTimeseriesException; -import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; -import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException; -import org.apache.iotdb.db.exception.metadata.PathNotExistException; -import org.apache.iotdb.db.metadata.LocalSchemaProcessor; -import org.apache.iotdb.db.metadata.idtable.IDTable; -import org.apache.iotdb.db.metadata.idtable.IDTableManager; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; -import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; -import org.apache.iotdb.db.metadata.idtable.entry.InsertMeasurementMNode; -import org.apache.iotdb.db.metadata.idtable.entry.SHA256DeviceID; -import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; -import org.apache.iotdb.db.metadata.mnode.EntityMNode; -import org.apache.iotdb.db.metadata.mnode.IMNode; -import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; -import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; -import org.apache.iotdb.db.metadata.path.MeasurementPath; -import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; -import org.apache.iotdb.db.metadata.schemaregion.SchemaRegionUtils; -import org.apache.iotdb.db.metadata.template.Template; -import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; -import org.apache.iotdb.db.mpp.common.schematree.MeasurementSchemaInfo; -import org.apache.iotdb.db.qp.physical.crud.InsertPlan; -import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan; -import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan; -import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan; -import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; -import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; -import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan; -import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan; -import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan; -import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan; -import org.apache.iotdb.db.query.context.QueryContext; -import org.apache.iotdb.db.query.dataset.ShowDevicesResult; -import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; -import org.apache.iotdb.external.api.ISeriesNumerLimiter; -import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; -import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; -import org.apache.iotdb.tsfile.utils.Pair; -import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; - -import org.jetbrains.annotations.NotNull; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Function; - -import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; - -public class MockTagSchemaRegion implements ISchemaRegion { - - protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private final String TAIL = ".**"; - private final IStorageGroupMNode storageGroupMNode; - private String storageGroupFullPath; - private SchemaRegionId schemaRegionId; - - private Map>> tagInvertedIndex; - - private List deviceIDS; - - private IDTable idTable; - - private final ISeriesNumerLimiter seriesNumerLimiter; - - public MockTagSchemaRegion( - PartialPath storageGroup, - SchemaRegionId schemaRegionId, - IStorageGroupMNode storageGroupMNode, - ISeriesNumerLimiter seriesNumerLimiter) - throws MetadataException { - - storageGroupFullPath = storageGroup.getFullPath(); - this.schemaRegionId = schemaRegionId; - this.storageGroupMNode = storageGroupMNode; - this.deviceIDS = new ArrayList<>(); - this.seriesNumerLimiter = seriesNumerLimiter; - tagInvertedIndex = new ConcurrentHashMap<>(); - idTable = IDTableManager.getInstance().getIDTable(storageGroup); - init(); - } - - @NotNull - private Map pathToTags(String path) { - if (path.length() <= storageGroupFullPath.length()) return new TreeMap<>(); - String devicePath = path.substring(storageGroupFullPath.length() + 1); - String[] tags = devicePath.split("\\."); - Map tagsMap = new TreeMap<>(); - for (int i = 0; i < tags.length; i += 2) { - tagsMap.put(tags[i], tags[i + 1]); - } - return tagsMap; - } - - public String tagsToPath(Map tags) { - StringBuilder stringBuilder = new StringBuilder(storageGroupFullPath); - for (String tagKey : tags.keySet()) { - stringBuilder.append(".").append(tagKey).append(".").append(tags.get(tagKey)); - } - return stringBuilder.toString(); - } - - @Override - public void init() throws MetadataException { - if (!config.isEnableIDTableLogFile() - && config.getDeviceIDTransformationMethod().equals("SHA256")) { - throw new MetadataException( - "enableIDTableLogFile OR deviceIDTransformationMethod==\"Plain\""); - } - } - - @Override - public void clear() { - return; - } - - @Override - public void forceMlog() { - return; - } - - @Override - public SchemaRegionId getSchemaRegionId() { - return schemaRegionId; - } - - @Override - public String getStorageGroupFullPath() { - return storageGroupFullPath; - } - - @Override - public void deleteSchemaRegion() throws MetadataException { - return; - } - - @Override - public boolean createSnapshot(File snapshotDir) { - return false; - } - - @Override - public void loadSnapshot(File latestSnapshotRootDir) { - return; - } - - private void createTagInvertedIndex(PartialPath devicePath) { - IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath); - Map tagsMap = pathToTags(devicePath.getFullPath()); - - deviceIDS.add(deviceID); - - for (String tagkey : tagsMap.keySet()) { - String tagValue = tagsMap.get(tagkey); - Map> tagkeyMap = - tagInvertedIndex.computeIfAbsent(tagkey, key -> new HashMap<>()); - List ids = tagkeyMap.computeIfAbsent(tagValue, key -> new ArrayList<>()); - ids.add(deviceIDS.size() - 1); - } - } - - private void createTimeseries( - PartialPath path, - TSDataType dataType, - TSEncoding encoding, - CompressionType compressor, - Map props) - throws MetadataException { - createTimeseries( - new CreateTimeSeriesPlan(path, dataType, encoding, compressor, props, null, null, null), 0); - } - - private void createAlignedTimeSeries( - PartialPath prefixPath, - List measurements, - List dataTypes, - List encodings, - List compressors) - throws MetadataException { - createAlignedTimeSeries( - new CreateAlignedTimeSeriesPlan( - prefixPath, measurements, dataTypes, encodings, compressors, null, null, null)); - } - - @Override // [iotdb|newIotdb/创建非对齐时间序列] [newIotdb/insert 2自动创建时间序列] - public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException { - PartialPath devicePath = plan.getPath().getDevicePath(); - Map tags = pathToTags(devicePath.getFullPath()); - PartialPath path = new PartialPath(tagsToPath(tags) + "." + plan.getPath().getMeasurement()); - plan.setPath(path); - devicePath = plan.getPath().getDevicePath(); - DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); - if (deviceEntry != null) { - if (deviceEntry.isAligned()) { - throw new AlignedTimeseriesException( - "Timeseries under this entity is not aligned, please use createTimeseries or change entity.", - devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); - } else if (deviceEntry.getMeasurementMap().containsKey(plan.getPath().getMeasurement())) { - throw new PathAlreadyExistException( - devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); - } - } - idTable.createTimeseries(plan); - if (deviceEntry == null) { - createTagInvertedIndex(devicePath); - } - } - - @Override // [iotdb|newIotdb/对齐时间序列] [newIotdb/insert 2自动创建时间序列] - public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException { - PartialPath devicePath = plan.getPrefixPath(); - Map tags = pathToTags(devicePath.getFullPath()); - PartialPath path = new PartialPath(tagsToPath(tags)); - plan.setPrefixPath(path); - devicePath = plan.getPrefixPath(); - DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); - if (deviceEntry != null) { - if (!deviceEntry.isAligned()) { - throw new AlignedTimeseriesException( - "Timeseries under this entity is aligned, please use createAlignedTimeseries or change entity.", - devicePath.getFullPath()); - } else { - List measurements = plan.getMeasurements(); - List dataTypes = plan.getDataTypes(); - List encodings = plan.getEncodings(); - List compressors = plan.getCompressors(); - - List tmpMeasurements = new LinkedList<>(); - List tmpDataTypes = new LinkedList<>(); - List tmpEncodings = new LinkedList<>(); - List tmpCompressors = new LinkedList<>(); - for (int i = 0; i < measurements.size(); i++) { - String measurement = measurements.get(i); - if (!deviceEntry.getMeasurementMap().containsKey(measurement)) { - tmpMeasurements.add(measurements.get(i)); - tmpDataTypes.add(dataTypes.get(i)); - tmpEncodings.add(encodings.get(i)); - tmpCompressors.add(compressors.get(i)); - } - } - if (tmpMeasurements.size() == 0) - throw new PathAlreadyExistException(devicePath.getFullPath()); - plan.setMeasurements(tmpMeasurements); - plan.setDataTypes(tmpDataTypes); - plan.setEncodings(tmpEncodings); - plan.setCompressors(tmpCompressors); - } - } - idTable.createAlignedTimeseries(plan); - if (deviceEntry == null) { - createTagInvertedIndex(devicePath); - } - } - - @Override - public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public int constructSchemaBlackList(PathPatternTree patternTree) throws MetadataException { - return 0; - } - - @Override - public void rollbackSchemaBlackList(PathPatternTree patternTree) throws MetadataException {} - - @Override - public List fetchSchemaBlackList(PathPatternTree patternTree) - throws MetadataException { - return null; - } - - @Override - public void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws MetadataException {} - - @Override - public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public boolean isPathExist(PartialPath path) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - int res = 0; - List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); - for (IDeviceID deviceID : deviceIDs) { - res += idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap().keySet().size(); - } - return res; - } - - @Override - public int getAllTimeseriesCount( - PartialPath pathPattern, Map templateMap, boolean isPrefixMatch) - throws MetadataException { - return 0; - } - - @Override - public int getAllTimeseriesCount( - PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains) - throws MetadataException { - return 0; - } - - @Override - public Map getMeasurementCountGroupByLevel( - PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public Map getMeasurementCountGroupByLevel( - PartialPath pathPattern, - int level, - boolean isPrefixMatch, - String key, - String value, - boolean isContains) - throws MetadataException { - return null; - } - - @Override - public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - if (pathPattern.getFullPath().length() <= storageGroupFullPath.length()) { - return deviceIDS.size(); - } else { - return getDeviceIDsByInvertedIndex(pathPattern).size(); - } - } - - @Override - public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch) - throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public List getNodesListInGivenLevel( - PartialPath pathPattern, - int nodeLevel, - boolean isPrefixMatch, - LocalSchemaProcessor.StorageGroupFilter filter) - throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public Set getChildNodePathInNextLevel(PartialPath pathPattern) - throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public Set getBelongedDevices(PartialPath timeseries) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override // [newIotdb/show timeseries] [newIotdb/count device] [newIotdb/count timeseries] - public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); - Set res = new HashSet<>(); - String devicePath = pathPattern.getFullPath(); - if (!devicePath.endsWith(TAIL) && !devicePath.equals(storageGroupFullPath)) { - DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); - if (deviceEntry != null) { - res.add(pathPattern); - } - return res; - } - for (IDeviceID deviceID : deviceIDs) { - if (deviceID instanceof SHA256DeviceID) { - DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); - Map map = deviceEntry.getMeasurementMap(); - for (String m : map.keySet()) { - SchemaEntry schemaEntry = map.get(m); - List schemaEntries = new ArrayList<>(); - schemaEntries.add(schemaEntry); - List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); - DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); - res.add( - new PartialPath( - diskSchemaEntry.seriesKey.substring( - 0, - diskSchemaEntry.seriesKey.length() - - diskSchemaEntry.measurementName.length() - - 1))); - break; - } - } else { - res.add(new PartialPath(deviceID.toStringID())); - } - } - return res; - } - - @Override - public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan) - throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override // [newIotDB / insert1,3] [newIotDB/select] [newIotdb/select count()] [newIotdb/select - // .. groupby level] - public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - PartialPath devicePath = pathPattern.getDevicePath(); - // 批量查询.路径以".**"结尾,如： - // root.sg.tag1.a.** - // root.sg.tagx.c.tag2.v.** - // 点查询.路径不以".**",直接走IDTable,精确查询 - if (devicePath.getFullPath().endsWith(TAIL)) { - return getMeasurementPathsWithBatchQuery(devicePath, isPrefixMatch); - } else { - return getMeasurementPathsWithPointQuery(devicePath, isPrefixMatch); - } - } - - private List getMeasurementPathsWithPointQuery( - PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { - List res = new LinkedList<>(); - String path = devicePath.getFullPath(); - Map tags = pathToTags(path); - path = tagsToPath(tags); - DeviceEntry deviceEntry = idTable.getDeviceEntry(path); - if (deviceEntry == null) return res; - Map schemaMap = deviceEntry.getMeasurementMap(); - for (String measurement : schemaMap.keySet()) { - SchemaEntry schemaEntry = schemaMap.get(measurement); - MeasurementPath measurementPath = - new MeasurementPath( - path, - measurement, - new MeasurementSchema( - measurement, - schemaEntry.getTSDataType(), - schemaEntry.getTSEncoding(), - schemaEntry.getCompressionType())); - measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); - res.add(measurementPath); - } - - return res; - } - - private List getMeasurementPathsWithBatchQuery( - PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { - List res = new LinkedList<>(); - List deviceIDs = getDeviceIdFromInvertedIndex(devicePath); - for (IDeviceID deviceID : deviceIDs) { - DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); - Map schemaMap = deviceEntry.getMeasurementMap(); - if (deviceID instanceof SHA256DeviceID) { - for (String measurement : schemaMap.keySet()) { - SchemaEntry schemaEntry = schemaMap.get(measurement); - List schemaEntries = new ArrayList<>(); - schemaEntries.add(schemaEntry); - List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); - DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); - MeasurementPath measurementPath = - new MeasurementPath( - new PartialPath(diskSchemaEntry.seriesKey), - new MeasurementSchema( - measurement, - schemaEntry.getTSDataType(), - schemaEntry.getTSEncoding(), - schemaEntry.getCompressionType())); - measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); - res.add(measurementPath); - } - } else { - for (String measurement : schemaMap.keySet()) { - SchemaEntry schemaEntry = schemaMap.get(measurement); - MeasurementPath measurementPath = - new MeasurementPath( - deviceID.toStringID(), - measurement, - new MeasurementSchema( - measurement, - schemaEntry.getTSDataType(), - schemaEntry.getTSEncoding(), - schemaEntry.getCompressionType())); - measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); - res.add(measurementPath); - } - } - } - return res; - } - - // [iotdb/select] [iotdb/select last] [iotdb/select count()] [iotdb/select ...groupby level] - @Override - public Pair, Integer> getMeasurementPathsWithAlias( - PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) - throws MetadataException { - List res = getMeasurementPaths(pathPattern, isPrefixMatch); - Pair, Integer> result = new Pair<>(res, 0); - return result; - } - - @Override - public List fetchSchema( - PartialPath pathPattern, Map templateMap) throws MetadataException { - return null; - } - - // show 时间序列 - @Override // [iotdb/show timeseries] - public Pair, Integer> showTimeseries( - ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { - List res = new ArrayList<>(); - Pair, Integer> result = new Pair<>(res, 0); - String path = plan.getPath().getFullPath(); - if (!path.endsWith(TAIL)) { - Map tags = pathToTags(path); - path = tagsToPath(tags); - DeviceEntry deviceEntry = idTable.getDeviceEntry(path); - if (deviceEntry != null) { - Map measurementMap = deviceEntry.getMeasurementMap(); - for (String m : measurementMap.keySet()) { - SchemaEntry schemaEntry = measurementMap.get(m); - res.add( - new ShowTimeSeriesResult( - path + "." + m, - "null", - storageGroupFullPath, - schemaEntry.getTSDataType(), - schemaEntry.getTSEncoding(), - schemaEntry.getCompressionType(), - schemaEntry.getLastTime(), - new HashMap<>(), - new HashMap<>())); - } - } - return result; - } - List deviceIDs = getDeviceIdFromInvertedIndex(plan.getPath()); - for (IDeviceID deviceID : deviceIDs) { - getTimeSeriesResultOfDeviceFromIDTable(res, deviceID); - } - return result; - } - - private List getDeviceIdFromInvertedIndex(PartialPath devicePath) - throws MetadataException { - String path = devicePath.getFullPath(); - if (path.endsWith(TAIL)) { - path = path.substring(0, path.length() - TAIL.length()); - devicePath = new PartialPath(path); - } - if (devicePath.getFullPath().length() <= storageGroupFullPath.length()) { - return deviceIDS; - } else { - List res = new LinkedList<>(); - List ids = getDeviceIDsByInvertedIndex(devicePath); - if (ids.size() > 0) { - for (int id : ids) { - res.add(deviceIDS.get(id)); - } - } - return res; - } - } - - private void getTimeSeriesResultOfDeviceFromIDTable( - List res, IDeviceID deviceID) { - Map measurementMap = - idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap(); - if (deviceID instanceof SHA256DeviceID) { - for (String m : measurementMap.keySet()) { - SchemaEntry schemaEntry = measurementMap.get(m); - List schemaEntries = new ArrayList<>(); - schemaEntries.add(schemaEntry); - List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); - DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); - res.add( - new ShowTimeSeriesResult( - diskSchemaEntry.seriesKey, - "null", - storageGroupFullPath, - schemaEntry.getTSDataType(), - schemaEntry.getTSEncoding(), - schemaEntry.getCompressionType(), - schemaEntry.getLastTime(), - new HashMap<>(), - new HashMap<>())); - } - } else { - for (String m : measurementMap.keySet()) { - SchemaEntry schemaEntry = measurementMap.get(m); - res.add( - new ShowTimeSeriesResult( - deviceID.toStringID() + "." + m, - "null", - storageGroupFullPath, - schemaEntry.getTSDataType(), - schemaEntry.getTSEncoding(), - schemaEntry.getCompressionType(), - schemaEntry.getLastTime(), - new HashMap<>(), - new HashMap<>())); - } - } - } - - private List getDeviceIDsByInvertedIndex(PartialPath path) { - Map tags = pathToTags(path.getFullPath()); - List idsCollection = new ArrayList<>(tags.keySet().size()); - for (String tagKey : tags.keySet()) { - if (!tagInvertedIndex.containsKey(tagKey) - || !tagInvertedIndex.get(tagKey).containsKey(tags.get(tagKey))) { - return new ArrayList<>(); - } - List ids = tagInvertedIndex.get(tagKey).get(tags.get(tagKey)); - idsCollection.add(new ArrayList(ids)); - } - if (idsCollection.size() == 0) return new ArrayList<>(); - List ids = idsCollection.get(0); - for (int i = 1; i < idsCollection.size(); i++) { - List list = idsCollection.get(i); - ids.retainAll(list); - } - return ids; - } - - @Override - public List getAllMeasurementByDevicePath(PartialPath devicePath) - throws PathNotExistException { - throw new UnsupportedOperationException(""); - } - - @Override - public IMNode getDeviceNode(PartialPath path) throws MetadataException { - DeviceEntry deviceEntry = idTable.getDeviceEntry(path.getFullPath()); - if (deviceEntry == null) throw new PathNotExistException(path.getFullPath()); - return new EntityMNode(storageGroupMNode, path.getFullPath()); - } - - @Override - public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException { - throw new UnsupportedOperationException(""); - } - - @Override - public void upsertTagsAndAttributes( - String alias, - Map tagsMap, - Map attributesMap, - PartialPath fullPath) - throws MetadataException, IOException { - throw new UnsupportedOperationException(""); - } - - @Override - public void addAttributes(Map attributesMap, PartialPath fullPath) - throws MetadataException, IOException { - throw new UnsupportedOperationException(""); - } - - @Override - public void addTags(Map tagsMap, PartialPath fullPath) - throws MetadataException, IOException { - throw new UnsupportedOperationException(""); - } - - @Override - public void dropTagsOrAttributes(Set keySet, PartialPath fullPath) - throws MetadataException, IOException { - throw new UnsupportedOperationException(""); - } - - @Override - public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) - throws MetadataException, IOException { - throw new UnsupportedOperationException(""); - } - - @Override - public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) - throws MetadataException, IOException { - throw new UnsupportedOperationException(""); - } - - // insert data - @Override // [iotdb/insert ] - public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) - throws MetadataException, IOException { - PartialPath devicePath = plan.getDevicePath(); - Map tags = pathToTags(devicePath.getFullPath()); - devicePath = new PartialPath(tagsToPath(tags)); - plan.setDevicePath(devicePath); - String[] measurementList = plan.getMeasurements(); - IMeasurementMNode[] measurementMNodes = plan.getMeasurementMNodes(); - checkAlignedAndAutoCreateSeries(plan); - IMNode deviceMNode = getDeviceNode(devicePath); - IMeasurementMNode measurementMNode; - DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); - Map schemaMap = deviceEntry.getMeasurementMap(); - for (int i = 0; i < measurementList.length; i++) { - SchemaEntry schemaEntry = schemaMap.get(measurementList[i]); - // measurementMNode = - // new MeasurementMNode( - // deviceMNode, - // measurementList[i], - // new MeasurementSchema( - // measurementList[i], - // schemaEntry.getTSDataType(), - // schemaEntry.getTSEncoding(), - // schemaEntry.getCompressionType()), - // null); - measurementMNode = new InsertMeasurementMNode(measurementList[i], schemaEntry, null); - // check type is match - try { - SchemaRegionUtils.checkDataTypeMatch(plan, i, schemaEntry.getTSDataType()); - } catch (DataTypeMismatchException mismatchException) { - if (!config.isEnablePartialInsert()) { - throw mismatchException; - } else { - // mark failed measurement - plan.markFailedMeasurementInsertion(i, mismatchException); - continue; - } - } - measurementMNodes[i] = measurementMNode; - } - plan.setDeviceID(deviceEntry.getDeviceID()); - plan.setDevicePath(new PartialPath(deviceEntry.getDeviceID().toStringID(), false)); - return deviceMNode; - } - - private SchemaEntry getSchemaEntry(String devicePath, String measurementName) { - DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); - if (deviceEntry == null) return null; - return deviceEntry.getSchemaEntry(measurementName); - } - - @Override - public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( - PartialPath devicePath, - String[] measurements, - Function getDataType, - boolean aligned) - throws MetadataException { - List measurementSchemaInfoList = new ArrayList<>(measurements.length); - for (int i = 0; i < measurements.length; i++) { - SchemaEntry schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); - if (schemaEntry == null) { - if (config.isAutoCreateSchemaEnabled()) { - if (aligned) { - internalAlignedCreateTimeseries( - devicePath, - Collections.singletonList(measurements[i]), - Collections.singletonList(getDataType.apply(i))); - - } else { - internalCreateTimeseries(devicePath.concatNode(measurements[i]), getDataType.apply(i)); - } - } - schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); - } - measurementSchemaInfoList.add( - new MeasurementSchemaInfo( - measurements[i], - new MeasurementSchema( - measurements[i], - schemaEntry.getTSDataType(), - schemaEntry.getTSEncoding(), - schemaEntry.getCompressionType()), - null)); - } - return new DeviceSchemaInfo(devicePath, aligned, measurementSchemaInfoList); - } - - private void checkAlignedAndAutoCreateSeries(InsertPlan plan) throws MetadataException { - String[] measurementList = plan.getMeasurements(); - try { - if (plan.isAligned()) { - internalAlignedCreateTimeseries( - plan.getDevicePath(), - Arrays.asList(measurementList), - Arrays.asList(plan.getDataTypes())); - } else { - internalCreateTimeseries( - plan.getDevicePath().concatNode(measurementList[0]), plan.getDataTypes()[0]); - } - } catch (MetadataException e) { - if (!(e instanceof PathAlreadyExistException)) { - throw e; - } - } - } - - private void internalCreateTimeseries(PartialPath path, TSDataType dataType) - throws MetadataException { - createTimeseries( - path, - dataType, - getDefaultEncoding(dataType), - TSFileDescriptor.getInstance().getConfig().getCompressor(), - Collections.emptyMap()); - } - - private void internalAlignedCreateTimeseries( - PartialPath prefixPath, List measurements, List dataTypes) - throws MetadataException { - List encodings = new ArrayList<>(); - List compressors = new ArrayList<>(); - for (TSDataType dataType : dataTypes) { - encodings.add(getDefaultEncoding(dataType)); - compressors.add(TSFileDescriptor.getInstance().getConfig().getCompressor()); - } - createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); - } - - @Override - public Set getPathsSetTemplate(String templateName) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public Set getPathsUsingTemplate(String templateName) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public boolean isTemplateAppendable(Template template, List measurements) - throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template) - throws MetadataException {} - - @Override - public List getPathsUsingTemplate(int templateId) throws MetadataException { - return null; - } - - @Override - public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException { - throw new UnsupportedOperationException(""); - } -} From 0a5c1d817f6837b92c41ad84164ee68c12e78f00 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Thu, 22 Sep 2022 21:39:16 +0800 Subject: [PATCH 24/30] mod getDeviceSchemaInfoWithAutoCreate --- .../tagSchemaRegion/MockTagSchemaRegion.java | 87 ++++++++++++++++--- .../tagSchemaRegion/TagSchemaRegion.java | 55 ++++++++++-- 2 files changed, 122 insertions(+), 20 deletions(-) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java index bec4ba83947d7..bb1fcce53987b 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java @@ -762,16 +762,6 @@ public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) Map schemaMap = deviceEntry.getMeasurementMap(); for (int i = 0; i < measurementList.length; i++) { SchemaEntry schemaEntry = schemaMap.get(measurementList[i]); - // measurementMNode = - // new MeasurementMNode( - // deviceMNode, - // measurementList[i], - // new MeasurementSchema( - // measurementList[i], - // schemaEntry.getTSDataType(), - // schemaEntry.getTSEncoding(), - // schemaEntry.getCompressionType()), - // null); measurementMNode = new InsertMeasurementMNode(measurementList[i], schemaEntry, null); // check type is match try { @@ -792,14 +782,63 @@ public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) return deviceMNode; } + @Override + public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( + PartialPath devicePath, + String[] measurements, + Function getDataType, + TSEncoding[] encodings, + CompressionType[] compressionTypes, + boolean aligned) + throws MetadataException { + List measurementSchemaInfoList = new ArrayList<>(measurements.length); + for (int i = 0; i < measurements.length; i++) { + SchemaEntry schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + if (schemaEntry == null) { + if (config.isAutoCreateSchemaEnabled()) { + if (aligned) { + TSDataType dataType = getDataType.apply(i); + internalAlignedCreateTimeseries( + devicePath, + Collections.singletonList(measurements[i]), + Collections.singletonList(dataType), + Collections.singletonList( + encodings[i] == null ? getDefaultEncoding(dataType) : encodings[i]), + Collections.singletonList( + compressionTypes[i] == null + ? TSFileDescriptor.getInstance().getConfig().getCompressor() + : compressionTypes[i])); + + } else { + internalCreateTimeseries( + devicePath.concatNode(measurements[i]), + getDataType.apply(i), + encodings[i], + compressionTypes[i]); + } + } + schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + } + measurementSchemaInfoList.add( + new MeasurementSchemaInfo( + measurements[i], + new MeasurementSchema( + measurements[i], + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType()), + null)); + } + return new DeviceSchemaInfo(devicePath, aligned, measurementSchemaInfoList); + } + private SchemaEntry getSchemaEntry(String devicePath, String measurementName) { DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); if (deviceEntry == null) return null; return deviceEntry.getSchemaEntry(measurementName); } - @Override - public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( + private DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( PartialPath devicePath, String[] measurements, Function getDataType, @@ -864,6 +903,19 @@ private void internalCreateTimeseries(PartialPath path, TSDataType dataType) Collections.emptyMap()); } + /** create timeseries ignoring PathAlreadyExistException */ + private void internalCreateTimeseries( + PartialPath path, TSDataType dataType, TSEncoding encoding, CompressionType compressor) + throws MetadataException { + if (encoding == null) { + encoding = getDefaultEncoding(dataType); + } + if (compressor == null) { + compressor = TSFileDescriptor.getInstance().getConfig().getCompressor(); + } + createTimeseries(path, dataType, encoding, compressor, Collections.emptyMap()); + } + private void internalAlignedCreateTimeseries( PartialPath prefixPath, List measurements, List dataTypes) throws MetadataException { @@ -876,6 +928,17 @@ private void internalAlignedCreateTimeseries( createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); } + /** create aligned timeseries ignoring PathAlreadyExistException */ + private void internalAlignedCreateTimeseries( + PartialPath prefixPath, + List measurements, + List dataTypes, + List encodings, + List compressors) + throws MetadataException { + createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); + } + @Override public Set getPathsSetTemplate(String templateName) throws MetadataException { throw new UnsupportedOperationException(""); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java index c32466cd346f0..7ad604a9c218c 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java @@ -769,17 +769,13 @@ public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) return deviceMNode; } - private SchemaEntry getSchemaEntry(String devicePath, String measurementName) { - DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); - if (deviceEntry == null) return null; - return deviceEntry.getSchemaEntry(measurementName); - } - @Override public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( PartialPath devicePath, String[] measurements, Function getDataType, + TSEncoding[] encodings, + CompressionType[] compressionTypes, boolean aligned) throws MetadataException { List measurementSchemaInfoList = new ArrayList<>(measurements.length); @@ -788,13 +784,24 @@ public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( if (schemaEntry == null) { if (config.isAutoCreateSchemaEnabled()) { if (aligned) { + TSDataType dataType = getDataType.apply(i); internalAlignedCreateTimeseries( devicePath, Collections.singletonList(measurements[i]), - Collections.singletonList(getDataType.apply(i))); + Collections.singletonList(dataType), + Collections.singletonList( + encodings[i] == null ? getDefaultEncoding(dataType) : encodings[i]), + Collections.singletonList( + compressionTypes[i] == null + ? TSFileDescriptor.getInstance().getConfig().getCompressor() + : compressionTypes[i])); } else { - internalCreateTimeseries(devicePath.concatNode(measurements[i]), getDataType.apply(i)); + internalCreateTimeseries( + devicePath.concatNode(measurements[i]), + getDataType.apply(i), + encodings[i], + compressionTypes[i]); } } schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); @@ -812,6 +819,12 @@ public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( return new DeviceSchemaInfo(devicePath, aligned, measurementSchemaInfoList); } + private SchemaEntry getSchemaEntry(String devicePath, String measurementName) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); + if (deviceEntry == null) return null; + return deviceEntry.getSchemaEntry(measurementName); + } + private void checkAlignedAndAutoCreateSeries(InsertPlan plan) throws MetadataException { String[] measurementList = plan.getMeasurements(); try { @@ -831,6 +844,7 @@ private void checkAlignedAndAutoCreateSeries(InsertPlan plan) throws MetadataExc } } + /** create timeseries ignoring PathAlreadyExistException */ private void internalCreateTimeseries(PartialPath path, TSDataType dataType) throws MetadataException { createTimeseries( @@ -841,6 +855,20 @@ private void internalCreateTimeseries(PartialPath path, TSDataType dataType) Collections.emptyMap()); } + /** create timeseries ignoring PathAlreadyExistException */ + private void internalCreateTimeseries( + PartialPath path, TSDataType dataType, TSEncoding encoding, CompressionType compressor) + throws MetadataException { + if (encoding == null) { + encoding = getDefaultEncoding(dataType); + } + if (compressor == null) { + compressor = TSFileDescriptor.getInstance().getConfig().getCompressor(); + } + createTimeseries(path, dataType, encoding, compressor, Collections.emptyMap()); + } + + /** create aligned timeseries ignoring PathAlreadyExistException */ private void internalAlignedCreateTimeseries( PartialPath prefixPath, List measurements, List dataTypes) throws MetadataException { @@ -853,6 +881,17 @@ private void internalAlignedCreateTimeseries( createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); } + /** create aligned timeseries ignoring PathAlreadyExistException */ + private void internalAlignedCreateTimeseries( + PartialPath prefixPath, + List measurements, + List dataTypes, + List encodings, + List compressors) + throws MetadataException { + createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); + } + @Override public Set getPathsSetTemplate(String templateName) throws MetadataException { throw new UnsupportedOperationException("getPathsSetTemplate"); From c4b22cfa554786e385e14e639f819fd8222fd733 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Fri, 23 Sep 2022 10:53:09 +0800 Subject: [PATCH 25/30] mod Licensed --- lsm/pom.xml | 28 +++++++++++-------- .../org/apache/iotdb/lsm/context/Context.java | 7 ----- .../iotdb/lsm/example/MemTableManager.java | 4 --- .../iotdb/lsm/strategy/AccessStrategy.java | 6 ---- .../iotdb/lsm/strategy/BFSAccessStrategy.java | 1 - .../lsm/strategy/PostOrderAccessStrategy.java | 2 -- .../lsm/strategy/PreOrderAccessStrategy.java | 2 -- .../lsm/strategy/RBFSAccessStrategy.java | 10 +++---- schema-engine-tag/README.md | 1 + schema-engine-tag/pom.xml | 28 +++++++++++-------- .../tagSchemaRegion/MockTagSchemaRegion.java | 21 ++++---------- .../tagSchemaRegion/TagSchemaRegion.java | 21 ++++---------- .../AppendOnlyDeviceIDListFileManager.java | 1 - .../utils/MeasurementPathUtils.java | 1 - .../utils/PathTagConverterUtils.java | 1 - 15 files changed, 49 insertions(+), 85 deletions(-) diff --git a/lsm/pom.xml b/lsm/pom.xml index 081c60b0abc73..b95e98aea9d2b 100644 --- a/lsm/pom.xml +++ b/lsm/pom.xml @@ -1,19 +1,23 @@ diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java index 3f369b3017734..58aa3ed4e666e 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java @@ -23,25 +23,18 @@ public class Context { - // 类型 ContextType type; - // 访问策略 AccessStrategy accessStrategy; - // 所处的树深度 int level; - // 多少个线程处理该节点的子节点 int threadNums; - // 上界,大于该值的层级不会被处理 int levelUpperBound; - // 返回值 Object result; - // 是否正在recover boolean recover; public Context() { diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java index d0f50071747c1..f6438df45f4f8 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java @@ -21,16 +21,12 @@ import java.util.HashMap; import java.util.Map; -// 管理working memtable , immutable memtables,框架用户自定义 public class MemTableManager { - // 可写的memtable private MemTable working; - // 只读的memtables private Map immutables; - // 记录已插入的最大的deviceid private int maxDeviceID; public MemTableManager() { diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java index 9ff3f0f0a03e3..c7d350754e9d4 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java @@ -21,14 +21,8 @@ import org.apache.iotdb.lsm.context.Context; import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; -// 表示内存节点访问策略（先序，后序） public interface AccessStrategy { - /** - * @param levelProcess 保存当前节点和子节点的处理方法 - * @param memNode 当前待处理的节点 - * @param context 上下文信息 - */ void execute( BasicLevelProcess levelProcess, I memNode, C context); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java index cf794b7097e61..3b32027d61992 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java @@ -35,7 +35,6 @@ public void execute( BasicLevelProcess levelProcess, I memNode, C context) { List children = new ArrayList<>(); int currentLevel = context.getLevel(); - // 第一个使用bfs策略的节点 if (sameLevelMemNodes == null) { sameLevelMemNodes = new LinkedList<>(); levelProcess.handle(memNode, context); diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java index 3079041763701..6f86f8eb999f9 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java @@ -31,7 +31,6 @@ public void execute( int currentLevel = context.getLevel(); AccessStrategy accessStrategy = context.getAccessStrategy(); List children = levelProcess.getChildren(memNode, context); - // 处理子节点 if (levelProcess.hasNext()) { context.setLevel(currentLevel + 1); for (O child : children) { @@ -41,7 +40,6 @@ public void execute( context.setLevel(currentLevel); context.setAccessStrategy(accessStrategy); - // 处理该节点 levelProcess.handle(memNode, context); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java index 2c0b1b7b03a6c..41b7861b6e2fa 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java @@ -29,11 +29,9 @@ public class PreOrderAccessStrategy implements AccessStrategy { public void execute( BasicLevelProcess levelProcess, I memNode, C context) { int currentLevel = context.getLevel(); - // 处理该节点 levelProcess.handle(memNode, context); List children = levelProcess.getChildren(memNode, context); - // 处理子节点 if (levelProcess.hasNext()) { context.setLevel(currentLevel + 1); for (O child : children) { diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java index 62d62295786fc..801a733eff2a0 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java @@ -31,28 +31,26 @@ public void execute( if (Integer.MAX_VALUE == context.getLevelUpperBound() && !levelProcess.hasNext()) { context.setLevelUpperBound(context.getLevel()); } - // 如果是根节点 + if (currentLevel == 0) { while (context.getLevelUpperBound() != currentLevel) { List children = levelProcess.getChildren(memNode, context); for (O child : children) { - // 处理子节点 + context.setLevel(currentLevel + 1); levelProcess.getNext().process(child, context); context.setLevel(currentLevel); } - // 每次处理完-1 + context.setLevelUpperBound(context.getLevelUpperBound() - 1); } - // 处理root节点 + levelProcess.handle(memNode, context); return; } - // 已经处理过，直接return if (currentLevel > context.getLevelUpperBound()) return; - // 处理子节点 if (currentLevel == context.getLevelUpperBound()) { levelProcess.handle(memNode, context); return; diff --git a/schema-engine-tag/README.md b/schema-engine-tag/README.md index 70185b9c79731..713e899d49dc8 100644 --- a/schema-engine-tag/README.md +++ b/schema-engine-tag/README.md @@ -18,6 +18,7 @@ under the License. --> + `TagSchemaRegion` is an implementation of `SchemaRegion`. # How To Use diff --git a/schema-engine-tag/pom.xml b/schema-engine-tag/pom.xml index 14e61f754aa21..184921f098956 100644 --- a/schema-engine-tag/pom.xml +++ b/schema-engine-tag/pom.xml @@ -1,19 +1,23 @@ diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java index bb1fcce53987b..e78091b9460f9 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.iotdb.db.metadata.tagSchemaRegion; import org.apache.iotdb.common.rpc.thrift.TSchemaNode; @@ -226,7 +225,7 @@ private void createAlignedTimeSeries( prefixPath, measurements, dataTypes, encodings, compressors, null, null, null)); } - @Override // [iotdb|newIotdb/创建非对齐时间序列] [newIotdb/insert 2自动创建时间序列] + @Override public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException { PartialPath devicePath = plan.getPath().getDevicePath(); Map tags = pathToTags(devicePath.getFullPath()); @@ -250,7 +249,7 @@ public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws Meta } } - @Override // [iotdb|newIotdb/对齐时间序列] [newIotdb/insert 2自动创建时间序列] + @Override public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException { PartialPath devicePath = plan.getPrefixPath(); Map tags = pathToTags(devicePath.getFullPath()); @@ -414,7 +413,7 @@ public Set getBelongedDevices(PartialPath timeseries) throws Metada throw new UnsupportedOperationException(""); } - @Override // [newIotdb/show timeseries] [newIotdb/count device] [newIotdb/count timeseries] + @Override public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) throws MetadataException { List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); @@ -459,15 +458,10 @@ public Pair, Integer> getMatchedDevices(ShowDevicesPlan throw new UnsupportedOperationException(""); } - @Override // [newIotDB / insert1,3] [newIotDB/select] [newIotdb/select count()] [newIotdb/select - // .. groupby level] + @Override public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) throws MetadataException { PartialPath devicePath = pathPattern.getDevicePath(); - // 批量查询.路径以".**"结尾,如： - // root.sg.tag1.a.** - // root.sg.tagx.c.tag2.v.** - // 点查询.路径不以".**",直接走IDTable,精确查询 if (devicePath.getFullPath().endsWith(TAIL)) { return getMeasurementPathsWithBatchQuery(devicePath, isPrefixMatch); } else { @@ -547,7 +541,6 @@ private List getMeasurementPathsWithBatchQuery( return res; } - // [iotdb/select] [iotdb/select last] [iotdb/select count()] [iotdb/select ...groupby level] @Override public Pair, Integer> getMeasurementPathsWithAlias( PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) @@ -563,8 +556,7 @@ public List fetchSchema( return null; } - // show 时间序列 - @Override // [iotdb/show timeseries] + @Override public Pair, Integer> showTimeseries( ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { List res = new ArrayList<>(); @@ -745,8 +737,7 @@ public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fu throw new UnsupportedOperationException(""); } - // insert data - @Override // [iotdb/insert ] + @Override public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) throws MetadataException, IOException { PartialPath devicePath = plan.getDevicePath(); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java index 7ad604a9c218c..06eed3e5eb51c 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.iotdb.db.metadata.tagSchemaRegion; import org.apache.iotdb.common.rpc.thrift.TSchemaNode; @@ -238,7 +237,7 @@ private void createAlignedTimeSeries( prefixPath, measurements, dataTypes, encodings, compressors, null, null, null)); } - @Override // [iotdb|newIotdb/创建非对齐时间序列] [newIotdb/insert 2自动创建时间序列] + @Override public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException { PartialPath devicePath = plan.getPath().getDevicePath(); PartialPath path = @@ -266,7 +265,7 @@ public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws Meta } } - @Override // [iotdb|newIotdb/对齐时间序列] [newIotdb/insert 2自动创建时间序列] + @Override public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException { PartialPath devicePath = plan.getPrefixPath(); PartialPath path = @@ -444,7 +443,7 @@ public Set getBelongedDevices(PartialPath timeseries) throws Metada throw new UnsupportedOperationException("getBelongedDevices"); } - @Override // [newIotdb/show timeseries] [newIotdb/count device] [newIotdb/count timeseries] + @Override public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) throws MetadataException { List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); @@ -534,15 +533,10 @@ public Pair, Integer> getMatchedDevices(ShowDevicesPlan throw new UnsupportedOperationException("getMatchedDevices"); } - @Override // [newIotDB / insert1,3] [newIotDB/select] [newIotdb/select count()] [newIotdb/select - // .. groupby level] + @Override public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) throws MetadataException { PartialPath devicePath = pathPattern.getDevicePath(); - // 批量查询.路径以".**"结尾,如： - // root.sg.tag1.a.** - // root.sg.tagx.c.tag2.v.** - // 点查询.路径不以".**",直接走IDTable,精确查询 if (devicePath.getFullPath().endsWith(TAIL)) { return getMeasurementPathsWithBatchQuery(devicePath, isPrefixMatch); } else { @@ -573,7 +567,6 @@ private List getMeasurementPathsWithBatchQuery( return getMeasurementPaths(deviceIDs); } - // [iotdb/select] [iotdb/select last] [iotdb/select count()] [iotdb/select ...groupby level] @Override public Pair, Integer> getMeasurementPathsWithAlias( PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) @@ -589,8 +582,7 @@ public List fetchSchema( throw new UnsupportedOperationException("fetchSchema"); } - // show 时间序列 - @Override // [iotdb/show timeseries] + @Override public Pair, Integer> showTimeseries( ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { List ShowTimeSeriesResults = new ArrayList<>(); @@ -730,8 +722,7 @@ public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fu throw new UnsupportedOperationException("renameTagOrAttributeKey"); } - // insert data - @Override // [iotdb/insert ] + @Override public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) throws MetadataException, IOException { PartialPath devicePath = plan.getDevicePath(); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java index 32475d650c560..cdbcaea16ed68 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist; import org.apache.iotdb.commons.utils.TestOnly; diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java index 22ae4c3b58162..5c792000d0b9b 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.iotdb.db.metadata.tagSchemaRegion.utils; import org.apache.iotdb.commons.exception.IllegalPathException; diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java index bd2c3a4e9ab58..59fbc6409ed6f 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.iotdb.db.metadata.tagSchemaRegion.utils; import java.util.Map; From d87c265bf704e32d1f0010ae3e0eac6c2618315a Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Fri, 23 Sep 2022 13:44:47 +0800 Subject: [PATCH 26/30] mod Licensed --- lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java | 3 ++- .../deviceidlist/AppendOnlyDeviceIDListFileManager.java | 1 + .../tagSchemaRegion/deviceidlist/DeviceIDListTest.java | 6 +++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java index 42932301d0dc4..858c2c219dd7d 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java @@ -33,7 +33,7 @@ public class WALReader implements IWALReader { private static final Logger logger = LoggerFactory.getLogger(WALReader.class); private final File logFile; private final WALRecord prototype; - private final DataInputStream logStream; + private DataInputStream logStream; private WALRecord nextRecord; private boolean fileCorrupted = false; @@ -47,6 +47,7 @@ public WALReader(File logFile, WALRecord prototype) throws IOException { @Override public void close() throws IOException { logStream.close(); + logStream = null; } @Override diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java index cdbcaea16ed68..9753cf013dddb 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java @@ -96,6 +96,7 @@ public void recover(DeviceIDList deviceIDList) { public void close() throws IOException { try { outputStream.close(); + outputStream = null; } catch (IOException e) { logger.error("close device id list file failed"); throw e; diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java index eddc31d2f9e08..26daa70890f1d 100644 --- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java @@ -29,6 +29,7 @@ import org.junit.Test; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -100,12 +101,15 @@ public void testAddandGetDeviceID() { } @Test - public void testRecover() { + public void testRecover() throws IOException { List deviceIDS = generateTestDeviceIDS(); for (IDeviceID deviceID : deviceIDS) { deviceIDList.add(deviceID); System.out.println(deviceID.toStringID()); } + + deviceIDList.clear(); + deviceIDList = new DeviceIDList(schemaRegionDirPath); for (int i = 0; i < deviceIDS.size(); i++) { assertEquals(deviceIDS.get(i), deviceIDList.get(i)); From b0913c539efaeaaa7a8ada5767054522c8cb2824 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Fri, 23 Sep 2022 21:23:34 +0800 Subject: [PATCH 27/30] add README.md --- lsm/README.md | 396 ++++++++++++++++++ ...Context.java => DeleteRequestContext.java} | 18 +- ...hContext.java => FlushRequestContext.java} | 11 +- ...Context.java => InsertRequestContext.java} | 18 +- ...yContext.java => QueryRequestContext.java} | 12 +- .../{Context.java => RequestContext.java} | 31 +- .../{ContextType.java => RequestType.java} | 2 +- .../org/apache/iotdb/lsm/example/Main.java | 209 --------- .../apache/iotdb/lsm/example/MemChunk.java | 44 -- .../apache/iotdb/lsm/example/MemGroup.java | 45 -- .../apache/iotdb/lsm/example/MemTable.java | 45 -- .../iotdb/lsm/example/MemTableManager.java | 71 ---- .../lsm/levelProcess/BasicLevelProcess.java | 33 +- .../lsm/levelProcess/DeleteLevelProcess.java | 16 +- .../lsm/levelProcess/FlushLevelProcess.java | 15 +- .../lsm/levelProcess/InsertLevelProcess.java | 16 +- .../iotdb/lsm/levelProcess/LevelProcess.java | 20 +- .../lsm/levelProcess/QueryLevelProcess.java | 15 +- .../iotdb/lsm/manager/BasicLsmManager.java | 33 +- .../apache/iotdb/lsm/manager/LsmManager.java | 17 +- .../iotdb/lsm/strategy/AccessStrategy.java | 12 +- .../iotdb/lsm/strategy/BFSAccessStrategy.java | 15 +- .../lsm/strategy/PostOrderAccessStrategy.java | 15 +- .../lsm/strategy/PreOrderAccessStrategy.java | 15 +- .../lsm/strategy/RBFSAccessStrategy.java | 26 +- .../org/apache/iotdb/lsm/wal/IWALReader.java | 17 + .../org/apache/iotdb/lsm/wal/IWALWriter.java | 17 + .../org/apache/iotdb/lsm/wal/WALReader.java | 6 + .../org/apache/iotdb/lsm/wal/WALRecord.java | 12 + .../org/apache/iotdb/lsm/wal/WALWriter.java | 10 + schema-engine-tag/README.md | 2 +- .../tagSchemaRegion/MockTagSchemaRegion.java | 1 + .../tagSchemaRegion/TagSchemaConfig.java | 3 + .../tagSchemaRegion/TagSchemaDescriptor.java | 12 +- .../tagSchemaRegion/TagSchemaRegion.java | 10 +- .../AppendOnlyDeviceIDListFileManager.java | 17 +- .../deviceidlist/DeviceIDList.java | 4 +- .../deviceidlist/IDeviceIDList.java | 22 + .../tagIndex/ITagInvertedIndex.java | 37 +- .../tagIndex/TagInvertedIndex.java | 61 ++- .../tagIndex/deletion/DeletionManager.java | 16 +- .../tagIndex/deletion/MemChunkDeletion.java | 21 +- .../deletion/MemChunkGroupDeletion.java | 21 +- .../tagIndex/deletion/MemTableDeletion.java | 21 +- .../tagIndex/insertion/InsertionManager.java | 16 +- .../insertion/MemChunkGroupInsertion.java | 21 +- .../tagIndex/insertion/MemChunkInsertion.java | 21 +- .../tagIndex/insertion/MemTableInsertion.java | 21 +- .../tagIndex/memtable/MemChunk.java | 4 +- .../tagIndex/memtable/MemChunkGroup.java | 3 + .../tagIndex/memtable/MemTable.java | 5 + .../tagIndex/query/MemChunkGroupQuery.java | 21 +- .../tagIndex/query/MemChunkQuery.java | 21 +- .../tagIndex/query/MemTableQuery.java | 22 +- .../tagIndex/query/QueryManager.java | 6 +- .../tagIndex/recover/RecoverManager.java | 18 +- .../tagIndex/wal/WALEntry.java | 16 + .../tagIndex/wal/WALManager.java | 71 +++- .../utils/MeasurementPathUtils.java | 19 + .../utils/PathTagConverterUtils.java | 22 + .../utils/ShowTimeSeriesResultUtils.java | 19 + 61 files changed, 1219 insertions(+), 567 deletions(-) create mode 100644 lsm/README.md rename lsm/src/main/java/org/apache/iotdb/lsm/context/{DeleteContext.java => DeleteRequestContext.java} (76%) rename lsm/src/main/java/org/apache/iotdb/lsm/context/{FlushContext.java => FlushRequestContext.java} (74%) rename lsm/src/main/java/org/apache/iotdb/lsm/context/{InsertContext.java => InsertRequestContext.java} (76%) rename lsm/src/main/java/org/apache/iotdb/lsm/context/{QueryContext.java => QueryRequestContext.java} (78%) rename lsm/src/main/java/org/apache/iotdb/lsm/context/{Context.java => RequestContext.java} (77%) rename lsm/src/main/java/org/apache/iotdb/lsm/context/{ContextType.java => RequestType.java} (97%) delete mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java delete mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java delete mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java delete mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java delete mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java diff --git a/lsm/README.md b/lsm/README.md new file mode 100644 index 0000000000000..d1c2ae75975fb --- /dev/null +++ b/lsm/README.md @@ -0,0 +1,396 @@ + + +# LSM Document +
+  _____      ______   ____    ____  
+ |_   _|   .' ____ \ |_   \  /   _| 
+   | |     | (___ \_|  |   \/   |   
+   | |   _  _.____`.   | |\  /| |   
+  _| |__/ || \____) | _| |_\/_| |_  
+ |________| \______.'|_____||_____| > version 0.14.0-SNAPSHOT
+                                    
+
+## Abstract + +The lsm framework has implemented the **memory structure** at present, and users only need to define the memory structure and access method of each level. + + +## Example +Suppose we need to implement an lsm storage engine to store each record similar to , we define the following four layers of memory nodes. +### Memory structure +Implement the memory structure of each layer + +- MemChunk + +The last layer of memory nodes, save the id list +```java +public class MemChunk { + List deviceIDS; + + public MemChunk() { + deviceIDS = new ArrayList<>(); + } + + public List getDeviceIDS() { + return deviceIDS; + } + + public void setDeviceIDS(List deviceIDS) { + this.deviceIDS = deviceIDS; + } + + @Override + public String toString() { + return "MemChunk{" + deviceIDS.toString() + '}'; + } +} +``` + +- MemGroup + +Use a map to manage tagValue->MemChunk +```java +public class MemGroup { + + Map map; + + public MemGroup() { + map = new HashMap<>(); + } + + public Map getMap() { + return map; + } + + public void setMap(Map map) { + this.map = map; + } + + @Override + public String toString() { + return "MemGroup{" + map.toString() + '}'; + } +} +``` + +- MemTable + +Use a map to manage tagKey->MemGroup +```java +public class MemTable { + + private Map map; + + public MemTable() { + map = new HashMap<>(); + } + + public Map getMap() { + return map; + } + + public void setMap(Map map) { + this.map = map; + } + + @Override + public String toString() { + return "MemTable{" + map.toString() + '}'; + } +} +``` + +- MemTableManager + +Manage working memTable, and immutable memTable +```java +public class MemTableManager { + + private MemTable working; + + private Map immutables; + + private int maxDeviceID; + + public MemTableManager() { + working = new MemTable(); + immutables = new HashMap<>(); + maxDeviceID = 0; + } + + public MemTable getWorking() { + return working; + } + + public void setWorking(MemTable working) { + this.working = working; + } + + public Map getImmutables() { + return immutables; + } + + public void setImmutables(Map immutables) { + this.immutables = immutables; + } + + public int getMaxDeviceID() { + return maxDeviceID; + } + + public void setMaxDeviceID(int maxDeviceID) { + this.maxDeviceID = maxDeviceID; + } + + @Override + public String toString() { + return "MemTableManager{" + + "working=" + + working.toString() + + ", immutables=" + + immutables.toString() + + '}'; + } +} +``` + +### Access method +Incoming access to each layer for the framework + +- Insertion and flush example +```java +public class Main { + public static void main(String[] args) throws Exception { + MemTableManager memTableManager = new MemTableManager(); + System.out.println("-------------insert--------------"); + insertionExample(memTableManager); + System.out.println("-------------flush--------------"); + flushExample(memTableManager); + } + + public static void insertionExample(MemTableManager memTableManager) throws Exception { + // Initialize a BasicLsmManager to manage insert operations + BasicLsmManager baseLsmManager = + new BasicLsmManager(); + baseLsmManager + .nextLevel( + // The insert method of the MemTableManager level + new InsertLevelProcess() { + @Override + public List getChildren( + MemTableManager memNode, InsertRequestContext context) { + Integer deviceID = (Integer) context.getValue(); + int maxDeviceID = memNode.getMaxDeviceID(); + List children = new ArrayList<>(); + if (deviceID / 65536 == maxDeviceID / 65536) { + children.add(memNode.getWorking()); + } else { + children.add(memNode.getImmutables().get(deviceID / 65536)); + } + return children; + } + + @Override + public void insert(MemTableManager memNode, InsertRequestContext context) { + Integer deviceID = (Integer) context.getValue(); + int maxDeviceID = memNode.getMaxDeviceID(); + if (deviceID / 65536 == maxDeviceID / 65536) { + if (memNode.getWorking() == null) { + memNode.setWorking(new MemTable()); + } + } else if (deviceID > maxDeviceID) { + memNode + .getImmutables() + .put(memNode.getMaxDeviceID() / 65536, memNode.getWorking()); + memNode.setWorking(new MemTable()); + } + if (deviceID > maxDeviceID) { + memNode.setMaxDeviceID(deviceID); + } + } + }) + .nextLevel( + // The insert method of the MemTable level + new InsertLevelProcess() { + @Override + public List getChildren(MemTable memNode, InsertRequestContext context) { + String key = (String) context.getKey(); + List children = new ArrayList<>(); + children.add(memNode.getMap().get(key)); + return children; + } + + @Override + public void insert(MemTable memNode, InsertRequestContext context) { + String key = (String) context.getKey(); + Map map = memNode.getMap(); + if (map.containsKey(key)) return; + map.put(key, new MemGroup()); + } + }) + .nextLevel( + // The insert method of the MemGroup level + new InsertLevelProcess() { + @Override + public List getChildren(MemGroup memNode, InsertRequestContext context) { + String key = (String) context.getKey(); + List children = new ArrayList<>(); + children.add(memNode.getMap().get(key)); + return children; + } + + @Override + public void insert(MemGroup memNode, InsertRequestContext context) { + String key = (String) context.getKey(); + Map map = memNode.getMap(); + if (map.containsKey(key)) return; + map.put(key, new MemChunk()); + } + }) + .nextLevel( + // The insert method of the MemChunk level + new InsertLevelProcess() { + @Override + public List getChildren(MemChunk memNode, InsertRequestContext context) { + return null; + } + + @Override + public void insert(MemChunk memNode, InsertRequestContext context) { + Integer deviceID = (Integer) context.getValue(); + List deviceIDs = memNode.getDeviceIDS(); + deviceIDs.add(deviceID); + } + }); + + // Insert some records + // The key at the MemTableManager level is null + baseLsmManager.process(memTableManager, new InsertRequestContext(1, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(2, null, "a", "d")); + baseLsmManager.process(memTableManager, new InsertRequestContext(3, null, "a", "e")); + baseLsmManager.process(memTableManager, new InsertRequestContext(4, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(5, null, "a1", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(6, null, "a2", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(65535, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(65536, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(2, null, "a", "d")); + baseLsmManager.process(memTableManager, new InsertRequestContext(3, null, "a", "e")); + baseLsmManager.process(memTableManager, new InsertRequestContext(4, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(5, null, "a1", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(6, null, "a2", "b")); + // process memTableManager + System.out.println(memTableManager); + } + + public static void flushExample(MemTableManager memTableManager) throws Exception { + // Initialize a BasicLsmManager to manage insert operations + BasicLsmManager flushManager = + new BasicLsmManager(); + + flushManager + .nextLevel( + // The insert method of the MemTableManager level + new FlushLevelProcess() { + @Override + public void flush(MemTableManager memNode, FlushRequestContext context) { + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); + } + + @Override + public List getChildren( + MemTableManager memNode, FlushRequestContext context) { + List memTables = new ArrayList<>(); + memTables.addAll(memNode.getImmutables().values()); + if (memNode.getWorking() != null) memTables.add(memNode.getWorking()); + return memTables; + } + }) + .nextLevel( + // The insert method of the MemTable level + new FlushLevelProcess() { + @Override + public void flush(MemTable memNode, FlushRequestContext context) { + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); + } + + @Override + public List getChildren(MemTable memNode, FlushRequestContext context) { + List memGroups = new ArrayList<>(); + memGroups.addAll(memNode.getMap().values()); + return memGroups; + } + }) + .nextLevel( + // The insert method of the MemGroup level + new FlushLevelProcess() { + @Override + public void flush(MemGroup memNode, FlushRequestContext context) { + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); + } + + @Override + public List getChildren(MemGroup memNode, FlushRequestContext context) { + List memChunk = new ArrayList<>(); + memChunk.addAll(memNode.getMap().values()); + return memChunk; + } + }) + .nextLevel( + // The insert method of the MemChunk level + new FlushLevelProcess() { + @Override + public void flush(MemChunk memNode, FlushRequestContext context) { + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); + } + + @Override + public List getChildren(MemChunk memNode, FlushRequestContext context) { + return new ArrayList<>(); + } + }); + + // process memTableManager + flushManager.process(memTableManager, new FlushRequestContext()); + } +} + +``` +- Output + +```txt +-------------insert-------------- +MemTableManager{working=MemTable{{a=MemGroup{{b=MemChunk{[65536]}}}}}, immutables={0=MemTable{{a1=MemGroup{{b=MemChunk{[5, 5]}}}, a=MemGroup{{b=MemChunk{[1, 4, 65535, 4]}, d=MemChunk{[2, 2]}, e=MemChunk{[3, 3]}}}, a2=MemGroup{{b=MemChunk{[6, 6]}}}}}}} +-------------flush-------------- +FLUSH: MemChunk{[5, 5]}-->[level:3] +FLUSH: MemChunk{[1, 4, 65535, 4]}-->[level:3] +FLUSH: MemChunk{[2, 2]}-->[level:3] +FLUSH: MemChunk{[3, 3]}-->[level:3] +FLUSH: MemChunk{[6, 6]}-->[level:3] +FLUSH: MemChunk{[65536]}-->[level:3] +FLUSH: MemGroup{{b=MemChunk{[5, 5]}}}-->[level:2] +FLUSH: MemGroup{{b=MemChunk{[1, 4, 65535, 4]}, d=MemChunk{[2, 2]}, e=MemChunk{[3, 3]}}}-->[level:2] +FLUSH: MemGroup{{b=MemChunk{[6, 6]}}}-->[level:2] +FLUSH: MemGroup{{b=MemChunk{[65536]}}}-->[level:2] +FLUSH: MemTable{{a1=MemGroup{{b=MemChunk{[5, 5]}}}, a=MemGroup{{b=MemChunk{[1, 4, 65535, 4]}, d=MemChunk{[2, 2]}, e=MemChunk{[3, 3]}}}, a2=MemGroup{{b=MemChunk{[6, 6]}}}}}-->[level:1] +FLUSH: MemTable{{a=MemGroup{{b=MemChunk{[65536]}}}}}-->[level:1] +FLUSH: MemTableManager{working=MemTable{{a=MemGroup{{b=MemChunk{[65536]}}}}}, immutables={0=MemTable{{a1=MemGroup{{b=MemChunk{[5, 5]}}}, a=MemGroup{{b=MemChunk{[1, 4, 65535, 4]}, d=MemChunk{[2, 2]}, e=MemChunk{[3, 3]}}}, a2=MemGroup{{b=MemChunk{[6, 6]}}}}}}}-->[level:0] +``` diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java similarity index 76% rename from lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java rename to lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java index 9b4b30ac41a75..548874625265d 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java @@ -24,24 +24,32 @@ import java.util.Arrays; import java.util.List; -public class DeleteContext extends Context { +/** + * represents the context of a deletion request, this class can be extended to implement a custom + * context + */ +public class DeleteRequestContext extends RequestContext { + // save the key of each level List keys; + // value to delete Object value; - public DeleteContext() { + public DeleteRequestContext() { super(); - type = ContextType.DELETE; + type = RequestType.DELETE; + // post-order traversal strategy is used by default accessStrategy = new PostOrderAccessStrategy(); } - public DeleteContext(Object value, Object... ks) { + public DeleteRequestContext(Object value, Object... ks) { super(); this.value = value; keys = new ArrayList<>(); keys.addAll(Arrays.asList(ks)); - type = ContextType.DELETE; + type = RequestType.DELETE; + // post-order traversal strategy is used by default accessStrategy = new PostOrderAccessStrategy(); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java similarity index 74% rename from lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java rename to lsm/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java index 2fd266396c8f8..d2a9cd8bae10d 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java @@ -20,8 +20,15 @@ import org.apache.iotdb.lsm.strategy.RBFSAccessStrategy; -public class FlushContext extends Context { - public FlushContext() { +/** + * represents the context of a flush request, this class can be extended to implement a custom + * context + */ +public class FlushRequestContext extends RequestContext { + public FlushRequestContext() { + super(); + type = RequestType.FLUSH; + // use the reverse breadth-first traversal strategy to access memory nodes accessStrategy = new RBFSAccessStrategy(); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java similarity index 76% rename from lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java rename to lsm/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java index b09d1540cbc47..933f006a58b0e 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java @@ -24,24 +24,32 @@ import java.util.Arrays; import java.util.List; -public class InsertContext extends Context { +/** + * represents the context of a insertion request, this class can be extended to implement a custom + * context + */ +public class InsertRequestContext extends RequestContext { + // save the key of each level List keys; + // value to insert Object value; - public InsertContext() { + public InsertRequestContext() { super(); - type = ContextType.INSERT; + type = RequestType.INSERT; + // preorder traversal strategy is used by default accessStrategy = new PreOrderAccessStrategy(); } - public InsertContext(Object value, Object... keys) { + public InsertRequestContext(Object value, Object... keys) { super(); this.value = value; this.keys = new ArrayList<>(); this.keys.addAll(Arrays.asList(keys)); - type = ContextType.INSERT; + type = RequestType.INSERT; + // preorder traversal strategy is used by default accessStrategy = new PreOrderAccessStrategy(); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java similarity index 78% rename from lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java rename to lsm/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java index 7221b643f7fac..8742ac4e2ace8 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java @@ -24,15 +24,21 @@ import java.util.Arrays; import java.util.List; -public class QueryContext extends Context { +/** + * represents the context of a query request, this class can be extended to implement a custom + * context + */ +public class QueryRequestContext extends RequestContext { + // save the key of each level List keys; - public QueryContext(Object... ks) { + public QueryRequestContext(Object... ks) { super(); keys = new ArrayList<>(); keys.addAll(Arrays.asList(ks)); - type = ContextType.QUERY; + type = RequestType.QUERY; + // post-order traversal strategy is used by default accessStrategy = new PostOrderAccessStrategy(); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java similarity index 77% rename from lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java rename to lsm/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java index 58aa3ed4e666e..4866be0ca1fd9 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java @@ -21,27 +21,32 @@ import org.apache.iotdb.lsm.strategy.AccessStrategy; import org.apache.iotdb.lsm.strategy.PreOrderAccessStrategy; -public class Context { +/** represents the context of a request */ +public class RequestContext { - ContextType type; + // request type + RequestType type; + // memory Structure Access Policy AccessStrategy accessStrategy; + // the tree level of the currently pending memory node int level; - int threadNums; - + // the maximum level of memory nodes that can be processed int levelUpperBound; + // return value after request processing is complete Object result; + // whether the request context is only used for recovery boolean recover; - public Context() { + public RequestContext() { + // preorder traversal strategy is used by default accessStrategy = new PreOrderAccessStrategy(); - type = ContextType.NONE; + type = RequestType.NONE; level = 0; - threadNums = 1; levelUpperBound = Integer.MAX_VALUE; recover = false; } @@ -54,22 +59,14 @@ public int getLevel() { return level; } - public ContextType getType() { + public RequestType getType() { return type; } - public int getThreadNums() { - return threadNums; - } - - public void setType(ContextType type) { + public void setType(RequestType type) { this.type = type; } - public void setThreadNums(int threadNums) { - this.threadNums = threadNums; - } - public Object getResult() { return result; } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/ContextType.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/RequestType.java similarity index 97% rename from lsm/src/main/java/org/apache/iotdb/lsm/context/ContextType.java rename to lsm/src/main/java/org/apache/iotdb/lsm/context/RequestType.java index 3ff2e3509afb1..e7dfac6c2a9e9 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/ContextType.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/RequestType.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.lsm.context; -public enum ContextType { +public enum RequestType { NONE, INSERT, QUERY, diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java deleted file mode 100644 index 7dbe793ac5fcd..0000000000000 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.lsm.example; - -import org.apache.iotdb.lsm.context.FlushContext; -import org.apache.iotdb.lsm.context.InsertContext; -import org.apache.iotdb.lsm.levelProcess.FlushLevelProcess; -import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; -import org.apache.iotdb.lsm.manager.BasicLsmManager; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class Main { - public static void main(String[] args) throws Exception { - MemTableManager memTableManager = new MemTableManager(); - System.out.println("-------------insert--------------"); - insertionExample(memTableManager); - System.out.println("-------------flush--------------"); - flushExample(memTableManager); - } - - public static void insertionExample(MemTableManager memTableManager) throws Exception { - - BasicLsmManager baseLsmManager = - new BasicLsmManager(); - baseLsmManager - .nextLevel( - new InsertLevelProcess() { - @Override - public List getChildren(MemTableManager memNode, InsertContext context) { - Integer deviceID = (Integer) context.getValue(); - int maxDeviceID = memNode.getMaxDeviceID(); - List children = new ArrayList<>(); - if (deviceID / 65536 == maxDeviceID / 65536) { - children.add(memNode.getWorking()); - } else { - children.add(memNode.getImmutables().get(deviceID / 65536)); - } - return children; - } - - @Override - public void insert(MemTableManager memNode, InsertContext context) { - Integer deviceID = (Integer) context.getValue(); - int maxDeviceID = memNode.getMaxDeviceID(); - if (deviceID / 65536 == maxDeviceID / 65536) { - if (memNode.getWorking() == null) { - memNode.setWorking(new MemTable()); - } - } else if (deviceID > maxDeviceID) { - memNode - .getImmutables() - .put(memNode.getMaxDeviceID() / 65536, memNode.getWorking()); - memNode.setWorking(new MemTable()); - } - if (deviceID > maxDeviceID) { - memNode.setMaxDeviceID(deviceID); - } - } - }) - .nextLevel( - new InsertLevelProcess() { - @Override - public List getChildren(MemTable memNode, InsertContext context) { - String key = (String) context.getKey(); - List children = new ArrayList<>(); - children.add(memNode.getMap().get(key)); - return children; - } - - @Override - public void insert(MemTable memNode, InsertContext context) { - String key = (String) context.getKey(); - Map map = memNode.getMap(); - if (map.containsKey(key)) return; - map.put(key, new MemGroup()); - } - }) - .nextLevel( - new InsertLevelProcess() { - @Override - public List getChildren(MemGroup memNode, InsertContext context) { - String key = (String) context.getKey(); - List children = new ArrayList<>(); - children.add(memNode.getMap().get(key)); - return children; - } - - @Override - public void insert(MemGroup memNode, InsertContext context) { - String key = (String) context.getKey(); - Map map = memNode.getMap(); - if (map.containsKey(key)) return; - map.put(key, new MemChunk()); - } - }) - .nextLevel( - new InsertLevelProcess() { - @Override - public List getChildren(MemChunk memNode, InsertContext context) { - return null; - } - - @Override - public void insert(MemChunk memNode, InsertContext context) { - Integer deviceID = (Integer) context.getValue(); - List deviceIDs = memNode.getDeviceIDS(); - deviceIDs.add(deviceID); - } - }); - - baseLsmManager.process(memTableManager, new InsertContext(1, null, "a", "b")); - baseLsmManager.process(memTableManager, new InsertContext(2, null, "a", "d")); - baseLsmManager.process(memTableManager, new InsertContext(3, null, "a", "e")); - baseLsmManager.process(memTableManager, new InsertContext(4, null, "a", "b")); - baseLsmManager.process(memTableManager, new InsertContext(5, null, "a1", "b")); - baseLsmManager.process(memTableManager, new InsertContext(6, null, "a2", "b")); - baseLsmManager.process(memTableManager, new InsertContext(65535, null, "a", "b")); - baseLsmManager.process(memTableManager, new InsertContext(65536, null, "a", "b")); - baseLsmManager.process(memTableManager, new InsertContext(2, null, "a", "d")); - baseLsmManager.process(memTableManager, new InsertContext(3, null, "a", "e")); - baseLsmManager.process(memTableManager, new InsertContext(4, null, "a", "b")); - baseLsmManager.process(memTableManager, new InsertContext(5, null, "a1", "b")); - baseLsmManager.process(memTableManager, new InsertContext(6, null, "a2", "b")); - System.out.println(memTableManager); - } - - public static void flushExample(MemTableManager memTableManager) throws Exception { - BasicLsmManager flushManager = - new BasicLsmManager(); - - flushManager - .nextLevel( - new FlushLevelProcess() { - @Override - public void flush(MemTableManager memNode, FlushContext context) { - System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); - } - - @Override - public List getChildren(MemTableManager memNode, FlushContext context) { - List memTables = new ArrayList<>(); - memTables.addAll(memNode.getImmutables().values()); - if (memNode.getWorking() != null) memTables.add(memNode.getWorking()); - return memTables; - } - }) - .nextLevel( - new FlushLevelProcess() { - @Override - public void flush(MemTable memNode, FlushContext context) { - System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); - } - - @Override - public List getChildren(MemTable memNode, FlushContext context) { - List memGroups = new ArrayList<>(); - memGroups.addAll(memNode.getMap().values()); - return memGroups; - } - }) - .nextLevel( - new FlushLevelProcess() { - @Override - public void flush(MemGroup memNode, FlushContext context) { - System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); - } - - @Override - public List getChildren(MemGroup memNode, FlushContext context) { - List memChunk = new ArrayList<>(); - memChunk.addAll(memNode.getMap().values()); - return memChunk; - } - }) - .nextLevel( - new FlushLevelProcess() { - @Override - public void flush(MemChunk memNode, FlushContext context) { - System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); - } - - @Override - public List getChildren(MemChunk memNode, FlushContext context) { - return new ArrayList<>(); - } - }); - - flushManager.process(memTableManager, new FlushContext()); - } -} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java deleted file mode 100644 index c6e97fd52d9d2..0000000000000 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.lsm.example; - -import java.util.ArrayList; -import java.util.List; - -// device list -public class MemChunk { - List deviceIDS; - - public MemChunk() { - deviceIDS = new ArrayList<>(); - } - - public List getDeviceIDS() { - return deviceIDS; - } - - public void setDeviceIDS(List deviceIDS) { - this.deviceIDS = deviceIDS; - } - - @Override - public String toString() { - return "MemChunk{" + deviceIDS.toString() + '}'; - } -} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java deleted file mode 100644 index 69f61d5d451fa..0000000000000 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.lsm.example; - -import java.util.HashMap; -import java.util.Map; - -// tagvalue -> memChunk -public class MemGroup { - - Map map; - - public MemGroup() { - map = new HashMap<>(); - } - - public Map getMap() { - return map; - } - - public void setMap(Map map) { - this.map = map; - } - - @Override - public String toString() { - return "MemGroup{" + map.toString() + '}'; - } -} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java deleted file mode 100644 index 7cc82f19ecab7..0000000000000 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.lsm.example; - -import java.util.HashMap; -import java.util.Map; - -// tagkey - > MemGroup -public class MemTable { - - private Map map; - - public MemTable() { - map = new HashMap<>(); - } - - public Map getMap() { - return map; - } - - public void setMap(Map map) { - this.map = map; - } - - @Override - public String toString() { - return "MemTable{" + map.toString() + '}'; - } -} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java deleted file mode 100644 index f6438df45f4f8..0000000000000 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.lsm.example; - -import java.util.HashMap; -import java.util.Map; - -public class MemTableManager { - - private MemTable working; - - private Map immutables; - - private int maxDeviceID; - - public MemTableManager() { - working = new MemTable(); - immutables = new HashMap<>(); - maxDeviceID = 0; - } - - public MemTable getWorking() { - return working; - } - - public void setWorking(MemTable working) { - this.working = working; - } - - public Map getImmutables() { - return immutables; - } - - public void setImmutables(Map immutables) { - this.immutables = immutables; - } - - public int getMaxDeviceID() { - return maxDeviceID; - } - - public void setMaxDeviceID(int maxDeviceID) { - this.maxDeviceID = maxDeviceID; - } - - @Override - public String toString() { - return "MemTableManager{" - + "working=" - + working.toString() - + ", immutables=" - + immutables.toString() - + '}'; - } -} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java index f4ced68ec5a08..4e10ba794bdc3 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java @@ -18,23 +18,52 @@ */ package org.apache.iotdb.lsm.levelProcess; -import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.RequestContext; import java.util.List; -public abstract class BasicLevelProcess implements LevelProcess { +/** the processing method corresponding to each layer of memory nodes */ +public abstract class BasicLevelProcess + implements LevelProcess { + + // the next level process LevelProcess next; + /** + * process the current layer memory node + * + * @param memNode memory node + * @param context request context + */ public abstract void handle(I memNode, C context); + /** + * get the memory node that needs to be processed in the next layer + * + * @param memNode memory node + * @param context request context + * @return all next-level memory nodes that need to be processed + */ public abstract List getChildren(I memNode, C context); + /** + * add the LevelProcess of the next layer of memory nodes + * + * @param next LevelProcess of the next layer + * @return the next level process + */ @Override public LevelProcess nextLevel(LevelProcess next) { this.next = next; return next; } + /** + * use this method to process memory nodes at each layer according to the access strategy + * + * @param memNode memory node + * @param context request context + */ @Override public void process(I memNode, C context) { context.getAccessStrategy().execute(this, memNode, context); diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java index f4a4c97cd0f5c..bd71d7fc86792 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java @@ -18,14 +18,22 @@ */ package org.apache.iotdb.lsm.levelProcess; -import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.DeleteRequestContext; -public abstract class DeleteLevelProcess extends BasicLevelProcess { +/** indicates the deletion method of each layer of memory nodes */ +public abstract class DeleteLevelProcess + extends BasicLevelProcess { - public abstract void delete(I memNode, DeleteContext context); + /** + * the deletion method of memory node + * + * @param memNode memory node + * @param context deletion request context + */ + public abstract void delete(I memNode, DeleteRequestContext context); @Override - public void handle(I memNode, DeleteContext context) { + public void handle(I memNode, DeleteRequestContext context) { delete(memNode, context); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java index 00363c7683b75..46d72b4b5b50e 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java @@ -18,13 +18,20 @@ */ package org.apache.iotdb.lsm.levelProcess; -import org.apache.iotdb.lsm.context.FlushContext; +import org.apache.iotdb.lsm.context.FlushRequestContext; -public abstract class FlushLevelProcess extends BasicLevelProcess { +/** indicates the flush method of each layer of memory nodes */ +public abstract class FlushLevelProcess extends BasicLevelProcess { - public abstract void flush(I memNode, FlushContext context); + /** + * the flush method of memory node + * + * @param memNode memory node + * @param context flush request context + */ + public abstract void flush(I memNode, FlushRequestContext context); - public void handle(I memNode, FlushContext context) { + public void handle(I memNode, FlushRequestContext context) { flush(memNode, context); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java index 91f7e24c1f250..cf7d9c69bb772 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java @@ -18,14 +18,22 @@ */ package org.apache.iotdb.lsm.levelProcess; -import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; -public abstract class InsertLevelProcess extends BasicLevelProcess { +/** indicates the insertion method of each layer of memory nodes */ +public abstract class InsertLevelProcess + extends BasicLevelProcess { - public abstract void insert(I memNode, InsertContext context); + /** + * the insertion method of memory node + * + * @param memNode memory node + * @param context insertion request context + */ + public abstract void insert(I memNode, InsertRequestContext context); @Override - public void handle(I memNode, InsertContext context) { + public void handle(I memNode, InsertRequestContext context) { insert(memNode, context); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java index aa04c2bf172b6..763d6eb4f6d32 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java @@ -18,12 +18,24 @@ */ package org.apache.iotdb.lsm.levelProcess; -import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.RequestContext; -public interface LevelProcess { +/** the processing method corresponding to each layer of memory nodes */ +public interface LevelProcess { + + /** + * add the LevelProcess of the next layer of memory nodes + * + * @param next LevelProcess of the next layer + * @return LevelProcess of the next layer + */ LevelProcess nextLevel(LevelProcess next); + /** + * use this method to process memory nodes at each layer according to the access strategy + * + * @param memNode memory node + * @param context request context + */ void process(I memNode, C context); - - void handle(I memNode, C context); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java index 5b3fc0e9ebb32..2fef2f213d058 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java @@ -18,14 +18,21 @@ */ package org.apache.iotdb.lsm.levelProcess; -import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.context.QueryRequestContext; -public abstract class QueryLevelProcess extends BasicLevelProcess { +/** indicates the query method of each layer of memory nodes */ +public abstract class QueryLevelProcess extends BasicLevelProcess { - public abstract void query(I memNode, QueryContext context); + /** + * the query method of memory node + * + * @param memNode memory node + * @param context query request context + */ + public abstract void query(I memNode, QueryRequestContext context); @Override - public void handle(I memNode, QueryContext context) { + public void handle(I memNode, QueryRequestContext context) { query(memNode, context); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java index 1855f04c0fe48..5b8816f7f2f26 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java @@ -18,17 +18,40 @@ */ package org.apache.iotdb.lsm.manager; -import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.RequestContext; import org.apache.iotdb.lsm.levelProcess.LevelProcess; -public class BasicLsmManager implements LsmManager { +/** */ +public class BasicLsmManager implements LsmManager { + // the level process of the first layer of memory nodes LevelProcess levelProcess; + /** + * preprocessing of the root memory node + * + * @param root root memory node + * @param context request context + * @throws Exception + */ public void preProcess(T root, C context) throws Exception {} + /** + * postprocessing of the root memory node + * + * @param root root memory node + * @param context request context + * @throws Exception + */ public void postProcess(T root, C context) throws Exception {} + /** + * processing of the root memory node + * + * @param root root memory node + * @param context request context + * @throws Exception + */ @Override public void process(T root, C context) throws Exception { preProcess(root, context); @@ -36,6 +59,12 @@ public void process(T root, C context) throws Exception { postProcess(root, context); } + /** + * add the LevelProcess of the next layer of memory nodes + * + * @param levelProcess LevelProcess of the next layer + * @return LevelProcess of the next layer + */ @Override public LevelProcess nextLevel(LevelProcess levelProcess) { this.levelProcess = levelProcess; diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java index 2940ad60b1751..ea2355378e5d6 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java @@ -18,12 +18,25 @@ */ package org.apache.iotdb.lsm.manager; -import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.RequestContext; import org.apache.iotdb.lsm.levelProcess.LevelProcess; -public interface LsmManager { +// used to implement lsm manager +public interface LsmManager { + /** + * use this method to process root memory node + * + * @param memNode memory node + * @param context request context + */ void process(T memNode, C context) throws Exception; + /** + * add the LevelProcess of the next layer of memory nodes + * + * @param next LevelProcess of the next layer + * @return LevelProcess of the next layer + */ LevelProcess nextLevel(LevelProcess next); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java index c7d350754e9d4..567f83faa3e3b 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java @@ -18,11 +18,19 @@ */ package org.apache.iotdb.lsm.strategy; -import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.RequestContext; import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; +/** access strategy for memory nodes */ public interface AccessStrategy { - void execute( + /** + * implementation of access strategy + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ + void execute( BasicLevelProcess levelProcess, I memNode, C context); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java index 3b32027d61992..e93bd5ca959e6 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.lsm.strategy; -import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.RequestContext; import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; import java.util.ArrayList; @@ -26,18 +26,29 @@ import java.util.List; import java.util.Queue; +/** breadth-first access strategy implementation class */ public class BFSAccessStrategy implements AccessStrategy { + // same level memory nodes, used to implement BFSAccessStrategy Queue sameLevelMemNodes; + /** + * breadth-first access strategy implementation + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ @Override - public void execute( + public void execute( BasicLevelProcess levelProcess, I memNode, C context) { List children = new ArrayList<>(); int currentLevel = context.getLevel(); if (sameLevelMemNodes == null) { sameLevelMemNodes = new LinkedList<>(); + // process the current memory node levelProcess.handle(memNode, context); + // get all memory nodes to be processed in the next layer children = levelProcess.getChildren(memNode, context); } else { while (!sameLevelMemNodes.isEmpty()) { diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java index 6f86f8eb999f9..5e96f29b1b451 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java @@ -18,28 +18,39 @@ */ package org.apache.iotdb.lsm.strategy; -import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.RequestContext; import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; import java.util.List; +/** post-order traversal access strategy implementation class */ public class PostOrderAccessStrategy implements AccessStrategy { + /** + * post-order traversal access strategy + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ @Override - public void execute( + public void execute( BasicLevelProcess levelProcess, I memNode, C context) { int currentLevel = context.getLevel(); AccessStrategy accessStrategy = context.getAccessStrategy(); + // get all memory nodes to be processed in the next layer List children = levelProcess.getChildren(memNode, context); if (levelProcess.hasNext()) { context.setLevel(currentLevel + 1); for (O child : children) { + // process next level memory node levelProcess.getNext().process(child, context); } } context.setLevel(currentLevel); context.setAccessStrategy(accessStrategy); + // process the current memory node levelProcess.handle(memNode, context); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java index 41b7861b6e2fa..d16780cf59e58 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java @@ -18,23 +18,34 @@ */ package org.apache.iotdb.lsm.strategy; -import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.RequestContext; import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; import java.util.List; +/** pre-order traversal access strategy implementation class */ public class PreOrderAccessStrategy implements AccessStrategy { + /** + * pre-order traversal access strategy + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ @Override - public void execute( + public void execute( BasicLevelProcess levelProcess, I memNode, C context) { int currentLevel = context.getLevel(); + // process the current memory node levelProcess.handle(memNode, context); + // get all memory nodes to be processed in the next layer List children = levelProcess.getChildren(memNode, context); if (levelProcess.hasNext()) { context.setLevel(currentLevel + 1); for (O child : children) { + // process next level memory node levelProcess.getNext().process(child, context); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java index 801a733eff2a0..13b05c51717ae 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java @@ -18,43 +18,63 @@ */ package org.apache.iotdb.lsm.strategy; -import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.RequestContext; import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; import java.util.List; +/** reverse breadth first traversal access strategy implementation class */ public class RBFSAccessStrategy implements AccessStrategy { + + /** + * reverse breadth first traversal access strategy + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ @Override - public void execute( + public void execute( BasicLevelProcess levelProcess, I memNode, C context) { int currentLevel = context.getLevel(); + + // if the upper bound has not been set and there is no next-level processing method, set the + // upper bound to the current level if (Integer.MAX_VALUE == context.getLevelUpperBound() && !levelProcess.hasNext()) { context.setLevelUpperBound(context.getLevel()); } + // if the current memory node is the root if (currentLevel == 0) { + // if all the next level nodes of the root node have not been processed while (context.getLevelUpperBound() != currentLevel) { + // process all pending next-level nodes List children = levelProcess.getChildren(memNode, context); for (O child : children) { - context.setLevel(currentLevel + 1); + // use the processing method of the next layer to process the next layer of nodes levelProcess.getNext().process(child, context); context.setLevel(currentLevel); } + // after each layer is processed, the upper bound is reduced by one context.setLevelUpperBound(context.getLevelUpperBound() - 1); } + // process the current memory node levelProcess.handle(memNode, context); return; } if (currentLevel > context.getLevelUpperBound()) return; + // only process memory nodes with equal level and upper bound if (currentLevel == context.getLevelUpperBound()) { levelProcess.handle(memNode, context); return; } + + // process all pending next-level nodes List children = levelProcess.getChildren(memNode, context); for (O child : children) { context.setLevel(currentLevel + 1); diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java index 6d52ef1577fcd..c1e42f74113b4 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java @@ -21,11 +21,28 @@ import java.io.FileNotFoundException; import java.io.IOException; +/** get records in wal file */ public interface IWALReader { + /** + * close resource + * + * @throws IOException + */ void close() throws IOException; + /** + * determine if there is a next record + * + * @return returns true if there is, else returns false + * @throws FileNotFoundException + */ boolean hasNext() throws FileNotFoundException; + /** + * return the next record + * + * @throws FileNotFoundException + */ WALRecord next() throws FileNotFoundException; } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java index 6d3fecb21e2eb..3a7b4e2cda854 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java @@ -20,11 +20,28 @@ import java.io.IOException; +/** write records to wal file */ public interface IWALWriter { + /** + * write walRecord to wal file + * + * @param walRecord record to be written + * @throws IOException + */ void write(WALRecord walRecord) throws IOException; + /** + * force brush + * + * @throws IOException + */ void force() throws IOException; + /** + * close resource + * + * @throws IOException + */ void close() throws IOException; } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java index 858c2c219dd7d..a5a70b62f1722 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java @@ -29,11 +29,15 @@ import java.nio.file.Files; import java.util.NoSuchElementException; +/** get records in wal file */ public class WALReader implements IWALReader { private static final Logger logger = LoggerFactory.getLogger(WALReader.class); + // wal file private final File logFile; + // wal record prototype, clone on read private final WALRecord prototype; private DataInputStream logStream; + // next wal record private WALRecord nextRecord; private boolean fileCorrupted = false; @@ -63,7 +67,9 @@ public boolean hasNext() { if (logSize <= 0) { return false; } + // first clone the object through the prototype nextRecord = prototype.clone(); + // then perform deserialization and assign a value to the new object nextRecord.deserialize(logStream); } catch (EOFException e) { logger.info(e.getMessage()); diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java index 4311b5debfd2a..0aaf77460aab1 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java @@ -22,10 +22,22 @@ import java.io.IOException; import java.nio.ByteBuffer; +/** represents a wal record, which can be extended to implement more complex wal records */ public abstract class WALRecord implements Cloneable { + /** + * serialize the wal record + * + * @param buffer byte buffer + */ public abstract void serialize(ByteBuffer buffer); + /** + * deserialize via input stream + * + * @param stream data input stream + * @throws IOException + */ public abstract void deserialize(DataInputStream stream) throws IOException; @Override diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java index abaf89a354dec..5ce97779ce129 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java @@ -29,12 +29,16 @@ import java.nio.channels.ClosedChannelException; import java.nio.channels.FileChannel; +/** write records to wal file */ public class WALWriter implements IWALWriter { private static final Logger logger = LoggerFactory.getLogger(WALWriter.class); + // wal file private File logFile; private FileOutputStream fileOutputStream; private FileChannel channel; + // 4-bit buffer private final ByteBuffer lengthBuffer; + // save wal record serialized byte data private final ByteBuffer walBuffer; private final boolean forceEachWrite; @@ -48,6 +52,12 @@ public WALWriter(File logFile, int walBufferSize, boolean forceEachWrite) walBuffer = ByteBuffer.allocate(walBufferSize); } + /** + * write walRecord to wal file + * + * @param walRecord record to be written + * @throws IOException + */ @Override public void write(WALRecord walRecord) throws IOException { if (channel == null) { diff --git a/schema-engine-tag/README.md b/schema-engine-tag/README.md index 713e899d49dc8..1b44b95b05038 100644 --- a/schema-engine-tag/README.md +++ b/schema-engine-tag/README.md @@ -23,7 +23,7 @@ # How To Use -Firstly, you should package **schema-engine-rocksdb** by the following command: +Firstly, you should package **schema-engine-tag** by the following command: ```shell mvn clean package -pl schema-engine-tag -am -DskipTests diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java index e78091b9460f9..d360a84327074 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java @@ -89,6 +89,7 @@ import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; +/** Mock tag schema region, only used for test */ public class MockTagSchemaRegion implements ISchemaRegion { protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java index 756f0f6f446d4..a47b66cfa0b9e 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java @@ -18,10 +18,13 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion; +/** tag schema region config */ public class TagSchemaConfig { + // the maximum number of device ids managed by a working memTable private int numOfDeviceIdsInMemTable = 65536; + // the size of wal buffer used to store a wal record private int walBufferSize = 1024 * 1024; public int getNumOfDeviceIdsInMemTable() { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java index 97f7d4752b99c..881a0e28a722d 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java @@ -31,6 +31,7 @@ import java.io.InputStream; import java.util.Properties; +/** manager tag schema config */ public class TagSchemaDescriptor { private static final Logger logger = LoggerFactory.getLogger(TagSchemaDescriptor.class); @@ -47,9 +48,16 @@ public static TagSchemaDescriptor getInstance() { } private void loadProperties() { - String iotdbHomePath = System.getProperty(IoTDBConstant.IOTDB_HOME, null); + String iotDBHomePath = System.getProperty(IoTDBConstant.IOTDB_HOME, null); + if (iotDBHomePath == null) { + logger.warn( + "Cannot find IOTDB_HOME environment variable when loading " + + "config file {}, use default configuration", + TAG_SCHEMA_CONFIG_FILE_NAME); + return; + } String tagSchemaConfigPath = - iotdbHomePath + iotDBHomePath + File.separatorChar + "conf" + File.separatorChar diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java index 06eed3e5eb51c..93ee6e1540296 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.metadata.AlignedTimeseriesException; @@ -96,21 +97,27 @@ import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; +/** tag schema region */ public class TagSchemaRegion implements ISchemaRegion { private static final Logger logger = LoggerFactory.getLogger(TagSchemaRegion.class); protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + // when a path ends with ".**", it represents batch processing private final String TAIL = ".**"; + private final IStorageGroupMNode storageGroupMNode; private final String storageGroupFullPath; private final SchemaRegionId schemaRegionId; private final String schemaRegionDirPath; + // tag inverted index private final TagInvertedIndex tagInvertedIndex; + // manager device id -> INT32 id private final IDeviceIDList deviceIDList; + // manager timeSeries private final IDTable idTable; private final ISeriesNumerLimiter seriesNumerLimiter; @@ -135,6 +142,7 @@ public TagSchemaRegion( @Override public void init() throws MetadataException { + // must enableIDTableLogFile or deviceIDTransformationMethod=="Plain" if (!config.isEnableIDTableLogFile() && config.getDeviceIDTransformationMethod().equals("SHA256")) { throw new MetadataException( @@ -151,11 +159,11 @@ public void init() throws MetadataException { } } } - logger.info("initialized successfully: {}", this); } @Override + @TestOnly public void clear() { try { tagInvertedIndex.clear(); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java index 9753cf013dddb..6dda97a00aa1b 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java @@ -30,6 +30,7 @@ import java.io.FileOutputStream; import java.io.IOException; +/** for append-only writing of device id list to disk */ public class AppendOnlyDeviceIDListFileManager { private static final Logger logger = @@ -68,17 +69,27 @@ private void initFile() throws IOException { } } - public void serialize(String deviceID) { + /** + * write the device id to file + * + * @param deviceID device id + */ + public void write(String deviceID) { try { if (!isRecover) { ReadWriteIOUtils.write(deviceID, outputStream); } } catch (IOException e) { - logger.error("failed to serialize device id: " + deviceID); - throw new IllegalArgumentException("can't serialize device id of " + deviceID); + logger.error("failed to write device id: " + deviceID); + throw new IllegalArgumentException("can't write device id of " + deviceID); } } + /** + * recover device id list + * + * @param deviceIDList device id list + */ public void recover(DeviceIDList deviceIDList) { logger.info("recover device id list using file {}", deviceIDSFile); try (FileInputStream inputStream = new FileInputStream(deviceIDSFile)) { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java index c8793d40fdfa4..4099248724d71 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java @@ -25,8 +25,10 @@ import java.util.ArrayList; import java.util.List; +/** manage device id -> int32 id */ public class DeviceIDList implements IDeviceIDList { + // use an array list to manage device id -> int id private final List deviceIDS; private AppendOnlyDeviceIDListFileManager appendOnlyDeviceIDListFileManager; @@ -44,7 +46,7 @@ public void recover() { @Override public void add(IDeviceID deviceID) { deviceIDS.add(deviceID); - appendOnlyDeviceIDListFileManager.serialize(deviceID.toStringID()); + appendOnlyDeviceIDListFileManager.write(deviceID.toStringID()); } @Override diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java index 1a0d72571a8cd..3d461a4b7097a 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java @@ -24,14 +24,36 @@ import java.io.IOException; import java.util.List; +/** manage device id -> int32 id */ public interface IDeviceIDList { + /** + * insert a device id + * + * @param deviceID device id + */ void add(IDeviceID deviceID); + /** + * get device id using int32 id + * + * @param index int32 id + * @return device id + */ IDeviceID get(int index); + /** + * returns the number of managed device ids + * + * @return the number of managed device ids + */ int size(); + /** + * get all managed device ids + * + * @return all managed device ids + */ List getAllDeviceIDS(); @TestOnly diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java index d5269bbce987e..18c90ada1550e 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java @@ -18,21 +18,50 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; -import org.apache.iotdb.lsm.context.DeleteContext; -import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.DeleteRequestContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; import java.util.List; import java.util.Map; +/** tag inverted index interface */ public interface ITagInvertedIndex { - void addTags(InsertContext context); + /** + * insert tags and id using insert request context + * + * @param context insert request context + */ + void addTags(InsertRequestContext context); + /** + * insert tags and device id + * + * @param tags tags like: + * @param id INT32 device id + */ void addTags(Map tags, int id); - void removeTags(DeleteContext context); + /** + * delete tags and id using delete request context + * + * @param context delete request context + */ + void removeTags(DeleteRequestContext context); + /** + * delete tags and id using delete request context + * + * @param tags tags like: + * @param id INT32 device id + */ void removeTags(Map tags, int id); + /** + * get all matching device ids + * + * @param tags tags like: + * @return device ids + */ List getMatchedIDs(Map tags); } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java index 3ad63e48b8092..e161b5bf93b7d 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -19,8 +19,6 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaConfig; import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.DeletionManager; @@ -29,9 +27,9 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.QueryManager; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.recover.RecoverManager; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; -import org.apache.iotdb.lsm.context.DeleteContext; -import org.apache.iotdb.lsm.context.InsertContext; -import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.context.DeleteRequestContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; +import org.apache.iotdb.lsm.context.QueryRequestContext; import org.roaringbitmap.RoaringBitmap; import org.slf4j.Logger; @@ -45,10 +43,10 @@ import java.util.Map; import java.util.stream.Collectors; +/** tag reverse index implementation class */ public class TagInvertedIndex implements ITagInvertedIndex { - private static final Logger logger = LoggerFactory.getLogger(TagInvertedIndex.class); - private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final Logger logger = LoggerFactory.getLogger(TagInvertedIndex.class); private static final TagSchemaConfig tagSchemaConfig = TagSchemaDescriptor.getInstance().getTagSchemaConfig(); @@ -63,12 +61,15 @@ public class TagInvertedIndex implements ITagInvertedIndex { private RecoverManager recoverManager; + // the maximum number of device ids managed by a working memTable private int numOfDeviceIdsInMemTable; + // (maxDeviceID / numOfDeviceIdsInMemTable) -> MemTable private Map immutableMemTables; private MemTable workingMemTable; + // the largest device id saved by the current MemTable private int maxDeviceID; public TagInvertedIndex(String schemaDirPath) { @@ -92,9 +93,15 @@ public synchronized void recover() { recoverManager.recover(this); } + /** + * insert tags and id using insert request context + * + * @param context insert request context + */ @Override - public synchronized void addTags(InsertContext context) { + public synchronized void addTags(InsertRequestContext context) { int id = (int) context.getValue(); + // if the device id can not be saved to the current working MemTable if (!inWorkingMemTable(id)) { workingMemTable.setStatus(MemTable.IMMUTABLE); immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); @@ -109,8 +116,15 @@ public synchronized void addTags(InsertContext context) { } } + /** + * insert tags and device id + * + * @param tags tags like: + * @param id INT32 device id + */ @Override public synchronized void addTags(Map tags, int id) { + // if the device id can not be saved to the current working MemTable if (!inWorkingMemTable(id)) { workingMemTable.setStatus(MemTable.IMMUTABLE); immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); @@ -127,8 +141,13 @@ public synchronized void addTags(Map tags, int id) { } } + /** + * delete tags and id using delete request context + * + * @param context delete request context + */ @Override - public void removeTags(DeleteContext context) { + public void removeTags(DeleteRequestContext context) { int id = (int) context.getValue(); MemTable memTable = null; if (inWorkingMemTable(id)) { @@ -143,6 +162,12 @@ public void removeTags(DeleteContext context) { } } + /** + * delete tags and id using delete request context + * + * @param tags tags like: + * @param id INT32 device id + */ @Override public synchronized void removeTags(Map tags, int id) { List memTables = new ArrayList<>(); @@ -160,6 +185,12 @@ public synchronized void removeTags(Map tags, int id) { } } + /** + * get all matching device ids + * + * @param tags tags like: + * @return device ids + */ @Override public synchronized List getMatchedIDs(Map tags) { List memTables = new ArrayList<>(); @@ -194,18 +225,24 @@ public String toString() { + '}'; } + /** + * determine whether the id can be saved to the current MemTable + * + * @param id INT32 device id + * @return return true if it can, otherwise return false + */ private boolean inWorkingMemTable(int id) { return id / numOfDeviceIdsInMemTable == maxDeviceID / numOfDeviceIdsInMemTable; } private void addTag(MemTable memTable, String tagKey, String tagValue, int id) throws Exception { - InsertContext insertContext = new InsertContext(id, tagKey, tagValue); + InsertRequestContext insertContext = new InsertRequestContext(id, tagKey, tagValue); insertionManager.process(memTable, insertContext); } private void removeTag(List memTables, String tagKey, String tagValue, int id) throws Exception { - DeleteContext deleteContext = new DeleteContext(id, tagKey, tagValue); + DeleteRequestContext deleteContext = new DeleteRequestContext(id, tagKey, tagValue); for (MemTable memTable : memTables) { deletionManager.process(memTable, deleteContext); } @@ -213,7 +250,7 @@ private void removeTag(List memTables, String tagKey, String tagValue, private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue) throws Exception { - QueryContext queryContext = new QueryContext(tagKey, tagValue); + QueryRequestContext queryContext = new QueryRequestContext(tagKey, tagValue); for (MemTable memTable : memTables) { queryManager.process(memTable, queryContext); } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java index 211eed0bbe944..51fd8ab031a22 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java @@ -20,11 +20,13 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; -import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.DeleteRequestContext; import org.apache.iotdb.lsm.manager.BasicLsmManager; -public class DeletionManager extends BasicLsmManager { +/** manage deletion to MemTable */ +public class DeletionManager extends BasicLsmManager { + // use wal manager object to write wal file on deletion private WALManager walManager; public DeletionManager(WALManager walManager) { @@ -32,13 +34,21 @@ public DeletionManager(WALManager walManager) { initLevelProcess(); } + /** + * write wal file on deletion + * + * @param root root memory node + * @param context request context + * @throws Exception + */ @Override - public void preProcess(MemTable root, DeleteContext context) throws Exception { + public void preProcess(MemTable root, DeleteRequestContext context) throws Exception { if (!context.isRecover()) { walManager.write(context); } } + /** set the delete operation for each layer of memory nodes */ private void initLevelProcess() { this.nextLevel(new MemTableDeletion()) .nextLevel(new MemChunkGroupDeletion()) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java index c95a617765b66..6b11f648710cb 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java @@ -19,19 +19,34 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; -import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.DeleteRequestContext; import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess; import java.util.List; +/** deletion for MemChunk */ public class MemChunkDeletion extends DeleteLevelProcess { + + /** + * MemChunk is the last layer of memory nodes, no children + * + * @param memNode memory node + * @param context request context + * @return null + */ @Override - public List getChildren(MemChunk memNode, DeleteContext context) { + public List getChildren(MemChunk memNode, DeleteRequestContext context) { return null; } + /** + * the delete method corresponding to the MemChunk node + * + * @param memNode memory node + * @param context deletion request context + */ @Override - public void delete(MemChunk memNode, DeleteContext context) { + public void delete(MemChunk memNode, DeleteRequestContext context) { Integer deviceID = (Integer) context.getValue(); memNode.remove(deviceID); } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java index 651553fd86032..a0f40a069aa93 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java @@ -20,15 +20,24 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; -import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.DeleteRequestContext; import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess; import java.util.ArrayList; import java.util.List; +/** deletion for MemChunkGroup */ public class MemChunkGroupDeletion extends DeleteLevelProcess { + + /** + * get all MemChunks that need to be processed in the current MemChunkGroup + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunks + */ @Override - public List getChildren(MemChunkGroup memNode, DeleteContext context) { + public List getChildren(MemChunkGroup memNode, DeleteRequestContext context) { List memChunks = new ArrayList<>(); String tagValue = (String) context.getKey(); MemChunk child = memNode.get(tagValue); @@ -36,8 +45,14 @@ public List getChildren(MemChunkGroup memNode, DeleteContext context) return memChunks; } + /** + * the delete method corresponding to the MemChunkGroup node + * + * @param memNode memory node + * @param context deletion request context + */ @Override - public void delete(MemChunkGroup memNode, DeleteContext context) { + public void delete(MemChunkGroup memNode, DeleteRequestContext context) { String tagValue = (String) context.getKey(); MemChunk child = memNode.get(tagValue); if (child == null || child.isEmpty()) { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java index 0a1e5b3d05715..4795585b32e57 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java @@ -20,16 +20,25 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; -import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.DeleteRequestContext; import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess; import java.util.ArrayList; import java.util.List; import java.util.Set; +/** deletion for MemTable */ public class MemTableDeletion extends DeleteLevelProcess { + + /** + * get all MemChunkGroups that need to be processed in the current MemTable + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunkGroups + */ @Override - public List getChildren(MemTable memNode, DeleteContext context) { + public List getChildren(MemTable memNode, DeleteRequestContext context) { if (memNode.isImmutable()) return new ArrayList<>(); List memChunkGroups = new ArrayList<>(); String tagKey = (String) context.getKey(); @@ -38,8 +47,14 @@ public List getChildren(MemTable memNode, DeleteContext context) return memChunkGroups; } + /** + * the delete method corresponding to the MemTable node + * + * @param memNode memory node + * @param context deletion request context + */ @Override - public void delete(MemTable memNode, DeleteContext context) { + public void delete(MemTable memNode, DeleteRequestContext context) { if (memNode.isImmutable()) { Set deletionList = memNode.getDeletionList(); if (!deletionList.contains(context.getValue())) { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java index 0e203a3d646ba..583f8944fd147 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java @@ -20,13 +20,15 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; -import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; import org.apache.iotdb.lsm.manager.BasicLsmManager; import java.io.IOException; -public class InsertionManager extends BasicLsmManager { +/** manage insertion to MemTable */ +public class InsertionManager extends BasicLsmManager { + // use wal manager object to write wal file on insertion private WALManager walManager; public InsertionManager(WALManager walManager) { @@ -34,13 +36,21 @@ public InsertionManager(WALManager walManager) { initLevelProcess(); } + /** + * write wal file on insertion + * + * @param root root memory node + * @param context insert request context + * @throws Exception + */ @Override - public void preProcess(MemTable root, InsertContext context) throws IOException { + public void preProcess(MemTable root, InsertRequestContext context) throws IOException { if (!context.isRecover()) { walManager.write(context); } } + /** set the insert operation for each layer of memory nodes */ private void initLevelProcess() { this.nextLevel(new MemTableInsertion()) .nextLevel(new MemChunkGroupInsertion()) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java index 843084223f4ea..5877b53f3d46b 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java @@ -20,15 +20,24 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; -import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; import java.util.ArrayList; import java.util.List; +/** insertion for MemChunkGroup */ public class MemChunkGroupInsertion extends InsertLevelProcess { + + /** + * get all MemChunks that need to be processed in the current MemChunkGroup + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunks + */ @Override - public List getChildren(MemChunkGroup memNode, InsertContext context) { + public List getChildren(MemChunkGroup memNode, InsertRequestContext context) { List memChunks = new ArrayList<>(); String tagValue = (String) context.getKey(); MemChunk child = memNode.get(tagValue); @@ -36,8 +45,14 @@ public List getChildren(MemChunkGroup memNode, InsertContext context) return memChunks; } + /** + * the insert method corresponding to the MemChunkGroup node + * + * @param memNode memory node + * @param context insert request context + */ @Override - public void insert(MemChunkGroup memNode, InsertContext context) { + public void insert(MemChunkGroup memNode, InsertRequestContext context) { String tagValue = (String) context.getKey(); memNode.put(tagValue); } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java index 5dec23dcb7d1b..1cf23bd2e1c04 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java @@ -19,19 +19,34 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; -import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; import java.util.List; +/** insertion for MemChunk */ public class MemChunkInsertion extends InsertLevelProcess { + + /** + * MemChunk is the last layer of memory nodes, no children + * + * @param memNode memory node + * @param context request context + * @return null + */ @Override - public List getChildren(MemChunk memNode, InsertContext context) { + public List getChildren(MemChunk memNode, InsertRequestContext context) { return null; } + /** + * the insert method corresponding to the MemChunk node + * + * @param memNode memory node + * @param context insert request context + */ @Override - public void insert(MemChunk memNode, InsertContext context) { + public void insert(MemChunk memNode, InsertRequestContext context) { Integer deviceID = (Integer) context.getValue(); memNode.put(deviceID); } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java index ef88f0591bc03..0d098e88999ab 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java @@ -20,17 +20,24 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; -import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; import java.util.ArrayList; import java.util.List; -// memtable的insert操作 +/** insertion for MemTable */ public class MemTableInsertion extends InsertLevelProcess { + /** + * get all MemChunkGroups that need to be processed in the current MemTable + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunkGroups + */ @Override - public List getChildren(MemTable memNode, InsertContext context) { + public List getChildren(MemTable memNode, InsertRequestContext context) { if (memNode.isImmutable()) return new ArrayList<>(); List memChunkGroups = new ArrayList<>(); String tagKey = (String) context.getKey(); @@ -39,8 +46,14 @@ public List getChildren(MemTable memNode, InsertContext context) return memChunkGroups; } + /** + * the insert method corresponding to the MemTable node + * + * @param memNode memory node + * @param context insert request context + */ @Override - public void insert(MemTable memNode, InsertContext context) { + public void insert(MemTable memNode, InsertRequestContext context) { if (memNode.isImmutable()) return; String tagKey = (String) context.getKey(); memNode.put(tagKey); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java index b46a770ca97e3..80610c1eaec6f 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java @@ -20,8 +20,10 @@ import org.roaringbitmap.RoaringBitmap; -// 管理设备id集合 +/** used to manage the device id collection */ public class MemChunk { + + // manage the device id collection, see: https://github.com/RoaringBitmap/RoaringBitmap private RoaringBitmap roaringBitmap; public MemChunk() { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java index f6765030183a5..c4dca031639a9 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java @@ -21,7 +21,10 @@ import java.util.HashMap; import java.util.Map; +/** used to manage tagValue -> MemChunk */ public class MemChunkGroup { + + // manage tagValue -> MemChunk private Map memChunkMap; public MemChunkGroup() { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java index f0730e162df1c..5f9cd09c6284b 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java @@ -23,15 +23,20 @@ import java.util.Map; import java.util.Set; +/** used to manage tagKey -> MemChunkGroup */ public class MemTable { + public static final String WORKING = "working"; public static final String IMMUTABLE = "immutable"; + // manage tagKey -> MemChunkGroup private Map memChunkGroupMap; private String status; + // if the memTable is immutable, the data cannot be deleted directly, and the deleted data needs + // to be recorded in the deletionList private Set deletionList; public MemTable(String status) { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java index bf9656e82205d..1ba77a754b540 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java @@ -20,15 +20,24 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; -import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.context.QueryRequestContext; import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess; import java.util.ArrayList; import java.util.List; +/** query for MemChunkGroup */ public class MemChunkGroupQuery extends QueryLevelProcess { + + /** + * get all MemChunks that need to be processed in the current MemChunkGroup + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunks + */ @Override - public List getChildren(MemChunkGroup memNode, QueryContext context) { + public List getChildren(MemChunkGroup memNode, QueryRequestContext context) { List memChunks = new ArrayList<>(); String tagValue = (String) context.getKey(); MemChunk child = memNode.get(tagValue); @@ -36,6 +45,12 @@ public List getChildren(MemChunkGroup memNode, QueryContext context) { return memChunks; } + /** + * the insert method corresponding to the MemChunkGroup node + * + * @param memNode memory node + * @param context query request context + */ @Override - public void query(MemChunkGroup memNode, QueryContext context) {} + public void query(MemChunkGroup memNode, QueryRequestContext context) {} } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java index d67de88f62bb4..d60be13c8431c 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java @@ -19,21 +19,36 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; -import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.context.QueryRequestContext; import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess; import org.roaringbitmap.RoaringBitmap; import java.util.List; +/** query for MemChunk */ public class MemChunkQuery extends QueryLevelProcess { + + /** + * MemChunk is the last layer of memory nodes, no children + * + * @param memNode memory node + * @param context request context + * @return null + */ @Override - public List getChildren(MemChunk memNode, QueryContext context) { + public List getChildren(MemChunk memNode, QueryRequestContext context) { return null; } + /** + * the query method corresponding to the MemChunk node + * + * @param memNode memory node + * @param context query request context + */ @Override - public void query(MemChunk memNode, QueryContext context) { + public void query(MemChunk memNode, QueryRequestContext context) { RoaringBitmap roaringBitmap = (RoaringBitmap) context.getResult(); if (roaringBitmap == null) roaringBitmap = new RoaringBitmap(); RoaringBitmap now = RoaringBitmap.or(roaringBitmap, memNode.getRoaringBitmap()); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java index b1acbf845b14e..e5ece3a13a99f 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java @@ -20,7 +20,7 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; -import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.context.QueryRequestContext; import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess; import org.roaringbitmap.RoaringBitmap; @@ -29,10 +29,18 @@ import java.util.List; import java.util.Set; +/** query for MemTable */ public class MemTableQuery extends QueryLevelProcess { + /** + * get all MemChunkGroups that need to be processed in the current MemTable + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunkGroups + */ @Override - public List getChildren(MemTable memNode, QueryContext context) { + public List getChildren(MemTable memNode, QueryRequestContext context) { List memChunkGroups = new ArrayList<>(); String tagKey = (String) context.getKey(); MemChunkGroup child = memNode.get(tagKey); @@ -40,9 +48,15 @@ public List getChildren(MemTable memNode, QueryContext context) { return memChunkGroups; } + /** + * the query method corresponding to the MemTable node + * + * @param memNode memory node + * @param context query request context + */ @Override - public void query(MemTable memNode, QueryContext context) { - // 如果是immutable，则需要在查询结果中删除deletionList中的id + public void query(MemTable memNode, QueryRequestContext context) { + // if the memTable is immutable, we need to delete the id in deletionList in the query result if (memNode.isImmutable()) { RoaringBitmap roaringBitmap = (RoaringBitmap) context.getResult(); Set deletionList = memNode.getDeletionList(); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java index 4a8e3fdbcb0eb..d3dc3a1d30538 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java @@ -19,15 +19,17 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; -import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.context.QueryRequestContext; import org.apache.iotdb.lsm.manager.BasicLsmManager; -public class QueryManager extends BasicLsmManager { +/** manage insertion to MemTable */ +public class QueryManager extends BasicLsmManager { public QueryManager() { initLevelProcess(); } + /** set the query operation for each layer of memory nodes */ private void initLevelProcess() { this.nextLevel(new MemTableQuery()) .nextLevel(new MemChunkGroupQuery()) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java index 7cc0015626030..5c697e1c77b70 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java @@ -20,13 +20,14 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.ITagInvertedIndex; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; -import org.apache.iotdb.lsm.context.Context; -import org.apache.iotdb.lsm.context.DeleteContext; -import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.DeleteRequestContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; +import org.apache.iotdb.lsm.context.RequestContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** for memory structure recovery */ public class RecoverManager { private static final Logger logger = LoggerFactory.getLogger(RecoverManager.class); @@ -36,16 +37,21 @@ public RecoverManager(WALManager walManager) { this.walManager = walManager; } + /** + * recover tagInvertedIndex + * + * @param tagInvertedIndex tag inverted index + */ public void recover(ITagInvertedIndex tagInvertedIndex) { logger.info("recover tagInvertedIndex"); while (true) { - Context context = walManager.read(); + RequestContext context = walManager.read(); switch (context.getType()) { case INSERT: - tagInvertedIndex.addTags((InsertContext) context); + tagInvertedIndex.addTags((InsertRequestContext) context); break; case DELETE: - tagInvertedIndex.removeTags((DeleteContext) context); + tagInvertedIndex.removeTags((DeleteRequestContext) context); break; default: return; diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java index 6c14934ce1062..5d73ed74ae60c 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java @@ -27,11 +27,16 @@ import java.util.ArrayList; import java.util.List; +/** represents a record in the wal file */ public class WALEntry extends WALRecord { + + // can be insertion(1) or deletion(2) private int type; + // keys at each level private List keys; + // device id private int deviceID; public WALEntry() { @@ -45,6 +50,11 @@ public WALEntry(int type, List keys, int deviceID) { this.deviceID = deviceID; } + /** + * serialize the wal entry + * + * @param buffer byte buffer + */ @Override public void serialize(ByteBuffer buffer) { ReadWriteIOUtils.write(type, buffer); @@ -55,6 +65,12 @@ public void serialize(ByteBuffer buffer) { } } + /** + * deserialize from DataInputStream + * + * @param stream data input stream + * @throws IOException + */ @Override public void deserialize(DataInputStream stream) throws IOException { this.type = stream.readInt(); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java index 65a74bba89144..e5047a0233b6e 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java @@ -18,11 +18,12 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaConfig; import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor; -import org.apache.iotdb.lsm.context.Context; -import org.apache.iotdb.lsm.context.DeleteContext; -import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.DeleteRequestContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; +import org.apache.iotdb.lsm.context.RequestContext; import org.apache.iotdb.lsm.wal.WALReader; import org.apache.iotdb.lsm.wal.WALWriter; @@ -31,7 +32,9 @@ import java.util.ArrayList; import java.util.List; +/** Manage wal entry writes and reads */ public class WALManager { + private static final String WAL_FILE_NAME = "tag_inverted_index.log"; private static final int INSERT = 1; @@ -45,8 +48,10 @@ public class WALManager { private File walFile; + // directly use the wal writer that comes with the lsm framework private WALWriter walWriter; + // directly use the wal reader that comes with the lsm framework private WALReader walReader; public WALManager(String schemaDirPath) throws IOException { @@ -66,21 +71,31 @@ private void initFile(String schemaDirPath) throws IOException { } } - public synchronized void write(Context context) throws IOException { + /** + * handle wal log writes for each request context + * + * @param context request context + * @throws IOException + */ + public synchronized void write(RequestContext context) throws IOException { switch (context.getType()) { case INSERT: - process((InsertContext) context); + process((InsertRequestContext) context); break; case DELETE: - process((DeleteContext) context); + process((DeleteRequestContext) context); break; default: break; } } - // 用于recover - public synchronized Context read() { + /** + * for recover + * + * @return request context + */ + public synchronized RequestContext read() { if (walReader.hasNext()) { WALEntry walEntry = (WALEntry) walReader.next(); if (walEntry.getType() == INSERT) { @@ -90,11 +105,17 @@ public synchronized Context read() { return generateDeleteContext(walEntry); } } - return new Context(); + return new RequestContext(); } - private InsertContext generateInsertContext(WALEntry walEntry) { - InsertContext insertContext = new InsertContext(); + /** + * generate insert context from wal entry + * + * @param walEntry wal entry + * @return insert context + */ + private InsertRequestContext generateInsertContext(WALEntry walEntry) { + InsertRequestContext insertContext = new InsertRequestContext(); List objects = new ArrayList<>(); objects.addAll(walEntry.getKeys()); insertContext.setKeys(objects); @@ -103,8 +124,15 @@ private InsertContext generateInsertContext(WALEntry walEntry) { return insertContext; } - private DeleteContext generateDeleteContext(WALEntry walEntry) { - DeleteContext deleteContext = new DeleteContext(walEntry.getDeviceID(), walEntry.getKeys()); + /** + * generate delete context from wal entry + * + * @param walEntry wal entry + * @return delete context + */ + private DeleteRequestContext generateDeleteContext(WALEntry walEntry) { + DeleteRequestContext deleteContext = + new DeleteRequestContext(walEntry.getDeviceID(), walEntry.getKeys()); List objects = new ArrayList<>(); objects.addAll(walEntry.getKeys()); deleteContext.setKeys(objects); @@ -113,7 +141,13 @@ private DeleteContext generateDeleteContext(WALEntry walEntry) { return deleteContext; } - private void process(InsertContext insertContext) throws IOException { + /** + * handle wal log writes for each insert context + * + * @param insertContext insert context + * @throws IOException + */ + private void process(InsertRequestContext insertContext) throws IOException { List objects = insertContext.getKeys(); List keys = new ArrayList<>(); for (Object o : objects) { @@ -123,7 +157,13 @@ private void process(InsertContext insertContext) throws IOException { walWriter.write(walEntry); } - private void process(DeleteContext deleteContext) throws IOException { + /** + * handle wal log writes for each delete context + * + * @param deleteContext delete context + * @throws IOException + */ + private void process(DeleteRequestContext deleteContext) throws IOException { List objects = deleteContext.getKeys(); List keys = new ArrayList<>(); for (Object o : objects) { @@ -133,6 +173,7 @@ private void process(DeleteContext deleteContext) throws IOException { walWriter.write(walEntry); } + @TestOnly public void close() throws IOException { walWriter.close(); walReader.close(); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java index 5c792000d0b9b..b6f5402a7169f 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java @@ -28,7 +28,19 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +/** process MeasurementPath */ public class MeasurementPathUtils { + + /** + * generate MeasurementPath + * + * @param devicePath device path + * @param measurement measurement + * @param schemaEntry schema entry + * @param isAligned is aligned + * @return MeasurementPath + * @throws IllegalPathException + */ public static MeasurementPath generateMeasurementPath( String devicePath, String measurement, SchemaEntry schemaEntry, boolean isAligned) throws IllegalPathException { @@ -45,6 +57,13 @@ public static MeasurementPath generateMeasurementPath( return measurementPath; } + /** + * generate MeasurementPath + * + * @param diskSchemaEntry disk schema entry + * @return MeasurementPath + * @throws IllegalPathException + */ public static MeasurementPath generateMeasurementPath(DiskSchemaEntry diskSchemaEntry) throws IllegalPathException { MeasurementPath measurementPath = diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java index 59fbc6409ed6f..59208fc418b67 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java @@ -21,8 +21,16 @@ import java.util.Map; import java.util.TreeMap; +/** path and tag converter */ public class PathTagConverterUtils { + /** + * convert the path of the tree model to the tags of the tag model + * + * @param storageGroupFullPath storage group full path + * @param path path of the tree model + * @return tags of the tag model + */ public static Map pathToTags(String storageGroupFullPath, String path) { if (path.length() <= storageGroupFullPath.length()) return new TreeMap<>(); String devicePath = path.substring(storageGroupFullPath.length() + 1); @@ -34,6 +42,13 @@ public static Map pathToTags(String storageGroupFullPath, String return tagsMap; } + /** + * convert the tags of the tag model to the path of the tree model + * + * @param storageGroupFullPath storage group full path + * @param tags tags of the tag model + * @return path of the tree model + */ public static String tagsToPath(String storageGroupFullPath, Map tags) { StringBuilder stringBuilder = new StringBuilder(storageGroupFullPath); for (String tagKey : tags.keySet()) { @@ -42,6 +57,13 @@ public static String tagsToPath(String storageGroupFullPath, Map return stringBuilder.toString(); } + /** + * generate unique path for paths with the same semantics + * + * @param storageGroupFullPath storage group full path + * @param path path of the tree model + * @return unique path of the tree model + */ public static String pathToTagsSortPath(String storageGroupFullPath, String path) { return tagsToPath(storageGroupFullPath, pathToTags(storageGroupFullPath, path)); } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java index 14f02a7dfd205..18d4c22662194 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java @@ -23,7 +23,18 @@ import java.util.HashMap; +/** process show timeSeries result */ public class ShowTimeSeriesResultUtils { + + /** + * generate show timeSeries result + * + * @param sgName storage group name + * @param devicePath device path + * @param measurement measurement + * @param schemaEntry schema entry + * @return ShowTimeSeriesResult + */ public static ShowTimeSeriesResult generateShowTimeSeriesResult( String sgName, String devicePath, String measurement, SchemaEntry schemaEntry) { return new ShowTimeSeriesResult( @@ -38,6 +49,14 @@ public static ShowTimeSeriesResult generateShowTimeSeriesResult( new HashMap<>()); } + /** + * generate show timeSeries result + * + * @param sgName storage group name + * @param timeSeriesPath timeSeries path + * @param schemaEntry schema entry + * @return ShowTimeSeriesResult + */ public static ShowTimeSeriesResult generateShowTimeSeriesResult( String sgName, String timeSeriesPath, SchemaEntry schemaEntry) { return new ShowTimeSeriesResult( From 957d33137290996d7f146eaacb10f703a542276f Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Fri, 23 Sep 2022 23:04:05 +0800 Subject: [PATCH 28/30] add README.md --- schema-engine-tag/README.md | 156 +++++++++++++++++++++++++++++++++++- 1 file changed, 153 insertions(+), 3 deletions(-) diff --git a/schema-engine-tag/README.md b/schema-engine-tag/README.md index 1b44b95b05038..0afdc21b0ae5c 100644 --- a/schema-engine-tag/README.md +++ b/schema-engine-tag/README.md @@ -19,7 +19,16 @@ --> -`TagSchemaRegion` is an implementation of `SchemaRegion`. +
+  _____             _____      _                           ______           _             
+|_   _|           /  ___|    | |                          | ___ \         (_)            
+  | | __ _  __ _  \ `--.  ___| |__   ___ _ __ ___   __ _  | |_/ /___  __ _ _  ___  _ __  
+  | |/ _` |/ _` |  `--. \/ __| '_ \ / _ \ '_ ` _ \ / _` | |    // _ \/ _` | |/ _ \| '_ \ 
+  | | (_| | (_| | /\__/ / (__| | | |  __/ | | | | | (_| | | |\ \  __/ (_| | | (_) | | | |
+  \_/\__,_|\__, | \____/ \___|_| |_|\___|_| |_| |_|\__,_| \_| \_\___|\__, |_|\___/|_| |_|
+            __/ |                                                     __/ |              
+           |___/                                                     |___/ > version 0.14.0-SNAPSHOT
+
# How To Use @@ -34,5 +43,146 @@ schema-engine-tag/target/schema-engine-tag. Copy the file in the conf directory and copy the files in the lib directory to the lib directory of server. Then, open the **iotdb-datanode.properties** in the conf directory of server, and set the `schema_engine_mode` to -Tag, set the `enable_id_table` to true. Restart the IoTDB, the system will use `TagSchemaRegion` to manage -the metadata. \ No newline at end of file +**Tag**, set the `enable_id_table` to **true**. Restart the IoTDB, the system will use `TagSchemaRegion` to manage +the metadata. + +## Use Cli + +IoTDB offers different ways to interact with server, here we introduce the basic steps of using Cli tool to insert and query data. +The command line cli is interactive, so you should see the welcome logo and statements if everything is ready: +```txt +--------------------- +Starting IoTDB Cli +--------------------- + _____ _________ ______ ______ +|_ _| | _ _ ||_ _ `.|_ _ \ + | | .--.|_/ | | \_| | | `. \ | |_) | + | | / .'`\ \ | | | | | | | __'. + _| |_| \__. | _| |_ _| |_.' /_| |__) | +|_____|'.__.' |_____| |______.'|_______/ version 0.14.0-SNAPSHOT + + +IoTDB> login successfully +``` +### create timeseries + +- create timeseries + +```txt +IoTDB> create timeseries root.ln.tag1.a.tag2.b.status with datatype=BOOLEAN,encoding=PLAIN +Msg: The statement is executed successfully. +``` +- create aligned timeseries + +```txt +IoTDB> CREATE ALIGNED TIMESERIES root.ln.tag1.a.tag2.c(latitude FLOAT encoding=PLAIN compressor=SNAPPY, longitude FLOAT encoding=PLAIN compressor=SNAPPY) + +Msg: The statement is executed successfully. +``` + +### show timeserie + +- point query + +enter a full path + +```txt +IoTDB> show timeseries root.ln.tag2.c.tag1.a ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| timeseries|alias|storage group|dataType|encoding|compression|tags|attributes| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| root.ln.tag1.a.tag2.c.latitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| +|root.ln.tag1.a.tag2.c.longitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +``` + +- batch query + +paths ending in ".**" indicate batch query + +```txt +IoTDB> show timeseries root.ln.tag1.a.** ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| timeseries|alias|storage group|dataType|encoding|compression|tags|attributes| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| root.ln.tag1.a.tag2.b.status| null| root.ln| BOOLEAN| PLAIN| SNAPPY|null| null| +| root.ln.tag1.a.tag2.c.latitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| +|root.ln.tag1.a.tag2.c.longitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ + +IoTDB> show timeseries root.ln.tag2.c.** ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| timeseries|alias|storage group|dataType|encoding|compression|tags|attributes| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| root.ln.tag1.a.tag2.c.latitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| +|root.ln.tag1.a.tag2.c.longitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ + +IoTDB> show timeseries root.ln.tag2.b.** ++----------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| timeseries|alias|storage group|dataType|encoding|compression|tags|attributes| ++----------------------------+-----+-------------+--------+--------+-----------+----+----------+ +|root.ln.tag1.a.tag2.b.status| null| root.ln| BOOLEAN| PLAIN| SNAPPY|null| null| ++----------------------------+-----+-------------+--------+--------+-----------+----+----------+ +``` + +### insert + +- insert a single column of data + +```txt +IoTDB> insert into root.ln.tag2.d(timestamp,status) values(1,true) +Msg: The statement is executed successfully. +IoTDB> insert into root.ln.tag2.d(timestamp,status) values(2,false) +Msg: The statement is executed successfully. +IoTDB> insert into root.ln.tag2.d(timestamp,status) values(3,true) +Msg: The statement is executed successfully. +IoTDB> insert into root.ln.tag1.a.tag2.d(timestamp,status) values(1,true) +Msg: The statement is executed successfully. +``` + +- insert alignment data + +```txt +IoTDB> insert into root.sg1.tag1.a(time, s1, s2) aligned values(2, 2, 2), (3, 3, 3) +Msg: The statement is executed successfully. +``` + +### select + +- point query + +```txt +IoTDB> select * from root.sg1.tag1.a ++-----------------------------+------------------+------------------+ +| Time|root.sg1.tag1.a.s1|root.sg1.tag1.a.s2| ++-----------------------------+------------------+------------------+ +|1970-01-01T08:00:00.002+08:00| 2.0| 2.0| +|1970-01-01T08:00:00.003+08:00| 3.0| 3.0| ++-----------------------------+------------------+------------------+ +``` + +- align by device + +```txt +IoTDB> select * from root.sg1.tag1.a align by device ++-----------------------------+---------------+---+---+ +| Time| Device| s1| s2| ++-----------------------------+---------------+---+---+ +|1970-01-01T08:00:00.002+08:00|root.sg1.tag1.a|2.0|2.0| +|1970-01-01T08:00:00.003+08:00|root.sg1.tag1.a|3.0|3.0| ++-----------------------------+---------------+---+---+ +``` + +- batch query + +```txt +IoTDB> select status from root.ln.tag2.d.** where time < 2017-11-01T00:08:00.000 ++-----------------------------+----------------------------+---------------------+ +| Time|root.ln.tag1.a.tag2.d.status|root.ln.tag2.d.status| ++-----------------------------+----------------------------+---------------------+ +|1970-01-01T08:00:00.001+08:00| true| true| +|1970-01-01T08:00:00.002+08:00| null| false| +|1970-01-01T08:00:00.003+08:00| null| true| ++-----------------------------+----------------------------+---------------------+ +``` \ No newline at end of file From a011ad990371b51c307ba341824e43897ed015b3 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Fri, 23 Sep 2022 23:07:29 +0800 Subject: [PATCH 29/30] add README.md --- schema-engine-tag/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/schema-engine-tag/README.md b/schema-engine-tag/README.md index 0afdc21b0ae5c..a00fe79b8d49e 100644 --- a/schema-engine-tag/README.md +++ b/schema-engine-tag/README.md @@ -19,6 +19,8 @@ --> +# Tag Schema Region +`TagSchemaRegion` is an implementation of `SchemaRegion` 
   _____             _____      _                           ______           _             
 |_   _|           /  ___|    | |                          | ___ \         (_)            

From 84b493d81e60f95fbd556a4aaf7d753d9a5d4157 Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Fri, 23 Sep 2022 23:10:33 +0800
Subject: [PATCH 30/30] add README.md

---
 schema-engine-tag/README.md | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/schema-engine-tag/README.md b/schema-engine-tag/README.md
index a00fe79b8d49e..e4d70f9e1502e 100644
--- a/schema-engine-tag/README.md
+++ b/schema-engine-tag/README.md
@@ -52,7 +52,7 @@ the metadata.
 
 IoTDB offers different ways to interact with server, here we introduce the basic steps of using Cli tool to insert and query data.
 The command line cli is interactive, so you should see the welcome logo and statements if everything is ready:
-```txt
+```sql
 ---------------------
 Starting IoTDB Cli
 ---------------------
@@ -70,13 +70,13 @@ IoTDB> login successfully
 
 - create timeseries
 
-```txt
+```sql
 IoTDB> create timeseries root.ln.tag1.a.tag2.b.status with datatype=BOOLEAN,encoding=PLAIN
 Msg: The statement is executed successfully.
 ```
 - create aligned timeseries
 
-```txt
+```sql
 IoTDB> CREATE ALIGNED TIMESERIES root.ln.tag1.a.tag2.c(latitude FLOAT encoding=PLAIN compressor=SNAPPY, longitude FLOAT  encoding=PLAIN compressor=SNAPPY)
 
 Msg: The statement is executed successfully.
@@ -88,7 +88,7 @@ Msg: The statement is executed successfully.
 
 enter a full path
 
-```txt
+```sql
 IoTDB> show timeseries root.ln.tag2.c.tag1.a
 +-------------------------------+-----+-------------+--------+--------+-----------+----+----------+
 |                     timeseries|alias|storage group|dataType|encoding|compression|tags|attributes|
@@ -102,7 +102,7 @@ IoTDB> show timeseries root.ln.tag2.c.tag1.a
 
 paths ending in ".**" indicate batch query
 
-```txt
+```sql
 IoTDB> show timeseries root.ln.tag1.a.**
 +-------------------------------+-----+-------------+--------+--------+-----------+----+----------+
 |                     timeseries|alias|storage group|dataType|encoding|compression|tags|attributes|
@@ -132,7 +132,7 @@ IoTDB> show timeseries root.ln.tag2.b.**
 
 - insert a single column of data
 
-```txt
+```sql
 IoTDB> insert into root.ln.tag2.d(timestamp,status) values(1,true)
 Msg: The statement is executed successfully.
 IoTDB> insert into root.ln.tag2.d(timestamp,status) values(2,false)
@@ -145,7 +145,7 @@ Msg: The statement is executed successfully.
 
 - insert alignment data
 
-```txt
+```sql
 IoTDB> insert into root.sg1.tag1.a(time, s1, s2) aligned values(2, 2, 2), (3, 3, 3)
 Msg: The statement is executed successfully.
 ```
@@ -154,7 +154,7 @@ Msg: The statement is executed successfully.
 
 - point query
 
-```txt
+```sql
 IoTDB> select * from root.sg1.tag1.a
 +-----------------------------+------------------+------------------+
 |                         Time|root.sg1.tag1.a.s1|root.sg1.tag1.a.s2|
@@ -166,7 +166,7 @@ IoTDB> select * from root.sg1.tag1.a
 
 - align by device
 
-```txt
+```sql
 IoTDB> select * from root.sg1.tag1.a align by device
 +-----------------------------+---------------+---+---+
 |                         Time|         Device| s1| s2|
@@ -178,7 +178,7 @@ IoTDB> select * from root.sg1.tag1.a align by device
 
 - batch query
 
-```txt
+```sql
 IoTDB> select status from root.ln.tag2.d.** where time < 2017-11-01T00:08:00.000
 +-----------------------------+----------------------------+---------------------+
 |                         Time|root.ln.tag1.a.tag2.d.status|root.ln.tag2.d.status|