diff --git a/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java b/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java index e74adc9a7..0058f20e9 100644 --- a/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java +++ b/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java @@ -24,6 +24,9 @@ import java.util.Map; import javax.xml.bind.*; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlType; +import javax.xml.namespace.QName; import org.apache.lens.api.jaxb.LensJAXBContext; @@ -31,6 +34,11 @@ public abstract class ToXMLString { protected static final Map, JAXBContext> JAXB_CONTEXTS = new HashMap<>(); public static String toString(Object o) { + if (!(o instanceof JAXBElement) && o.getClass().getAnnotation(XmlRootElement.class) == null + && o.getClass().getAnnotation(XmlType.class)!= null) { + o = new JAXBElement(new QName("uri:lens:cube:0.1", o.getClass().getAnnotation(XmlType.class).name()), + o.getClass(), null, o); + } try { StringWriter stringWriter = new StringWriter(); Class cl = null; diff --git a/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java b/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java index 14fc4aa19..8858b9510 100644 --- a/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java +++ b/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java @@ -24,6 +24,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.Reader; import javax.xml.XMLConstants; import javax.xml.bind.*; @@ -114,17 +115,26 @@ public Unmarshaller getUnmarshaller() { return UNMARSHALLER; } + public static T unmarshall(File file) throws JAXBException, IOException { + return ((JAXBElement) UNMARSHALLER.unmarshal(file)).getValue(); + } + public static T unmarshall(InputStream inputStream) throws JAXBException, IOException { + return ((JAXBElement) UNMARSHALLER.unmarshal(inputStream)).getValue(); + } + public static T unmarshall(Reader reader) throws JAXBException, IOException { + return ((JAXBElement) UNMARSHALLER.unmarshal(reader)).getValue(); + } public static T unmarshallFromFile(String filename) throws JAXBException, IOException { File file = new File(filename); if (file.exists()) { - return ((JAXBElement) UNMARSHALLER.unmarshal(file)).getValue(); + return unmarshall(file); } else { // load from classpath InputStream stream = Thread.currentThread().getContextClassLoader().getResourceAsStream(filename); if (stream == null) { throw new IOException("File not found:" + filename); } - return ((JAXBElement) UNMARSHALLER.unmarshal(stream)).getValue(); + return unmarshall(stream); } } } diff --git a/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java new file mode 100644 index 000000000..76cb8b90e --- /dev/null +++ b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.api.metastore; + +import java.io.File; +import java.io.FilenameFilter; +import java.util.Map; +import java.util.function.BiConsumer; + +import com.google.common.collect.Maps; + +/* + * Created on 07/03/17. + */ +public class SchemaTraverser implements Runnable { + final File parent; + final Map> types = Maps.newLinkedHashMap(); + private final SchemaEntityProcessor action; + { + types.put("storages", XStorage.class); + types.put("cubes/base", XBaseCube.class); + types.put("cubes/derived", XDerivedCube.class); + types.put("dimensions", XDimension.class); + types.put("facts", XFactTable.class); + types.put("dimtables", XDimensionTable.class); + types.put("dimensiontables", XDimensionTable.class); + types.put("dimensiontables", XDimensionTable.class); + types.put("segmentations", XSegmentation.class); + } + private static final FilenameFilter XML_FILTER = (dir, name) -> name.endsWith(".xml"); + + public interface SchemaEntityProcessor extends BiConsumer> { + } + + public SchemaTraverser(File parent, SchemaEntityProcessor action) { + this.parent = parent; + this.action = action; + } + + @Override + public void run() { + for (Map.Entry> entry : types.entrySet()) { + File f = new File(parent, entry.getKey()); + if (f.exists()) { + assert f.isDirectory(); + File[] files = f.listFiles(XML_FILTER); + if (files != null) { + for (File entityFile : files) { + action.accept(entityFile.getAbsoluteFile(), entry.getValue()); + } + } + } + } + } +} diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd index 060eb43c6..1d8a6241a 100644 --- a/lens-api/src/main/resources/cube-0.1.xsd +++ b/lens-api/src/main/resources/cube-0.1.xsd @@ -390,7 +390,7 @@ - + @@ -707,7 +707,7 @@ - + diff --git a/lens-api/src/main/resources/lens-errors.conf b/lens-api/src/main/resources/lens-errors.conf index 29e24cf3c..94505efcb 100644 --- a/lens-api/src/main/resources/lens-errors.conf +++ b/lens-api/src/main/resources/lens-errors.conf @@ -326,6 +326,19 @@ lensCubeErrorsForQuery = [ errorMsg = "Could not find queried table or chain: %s" } + { + errorCode = 3034 + httpStatusCode = ${BAD_REQUEST} + errorMsg = "%s does not have any facts that can cover the requested time range : %s and queried measure set : %s" + } + + { + errorCode = 3035 + httpStatusCode = ${BAD_REQUEST} + errorMsg = "%s does not have any facts that can cover the queried measure set : %s" + } + + ] lensCubeErrorsForMetastore = [ diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java index feabf9c22..befe4e661 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java @@ -18,11 +18,21 @@ */ package org.apache.lens.cli.commands; -import java.io.*; +import java.io.File; +import java.io.FilenameFilter; import java.util.List; +import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.lens.api.metastore.SchemaTraverser; +import org.apache.lens.api.metastore.XBaseCube; +import org.apache.lens.api.metastore.XDerivedCube; +import org.apache.lens.api.metastore.XDimension; +import org.apache.lens.api.metastore.XDimensionTable; +import org.apache.lens.api.metastore.XFactTable; +import org.apache.lens.api.metastore.XSegmentation; +import org.apache.lens.api.metastore.XStorage; import org.apache.lens.cli.commands.annotations.UserDocumentation; import org.springframework.beans.factory.annotation.Autowired; @@ -35,6 +45,7 @@ import org.springframework.util.Assert; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; @Component @UserDocumentation(title = "Creating schema with one command", @@ -84,15 +95,52 @@ public class LensSchemaCommands implements CommandMarker { logger.setLevel(Level.FINE); } - private static final FilenameFilter XML_FILTER = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.endsWith(".xml"); - } - }; + private static final FilenameFilter XML_FILTER = (dir, name) -> name.endsWith(".xml"); + private static final Map, String> CREATE_COMMAND_MAP = Maps.newHashMap(); + private static final Map, String> UPDATE_COMMAND_MAP = Maps.newHashMap(); + @Autowired private JLineShellComponent shell; + static { + CREATE_COMMAND_MAP.put(XStorage.class, "create storage --path %s"); + UPDATE_COMMAND_MAP.put(XStorage.class, "update storage --name %s --path %s"); + CREATE_COMMAND_MAP.put(XDimension.class, "create dimension --path %s"); + UPDATE_COMMAND_MAP.put(XDimension.class, "update dimension --name %s --path %s"); + CREATE_COMMAND_MAP.put(XBaseCube.class, "create cube --path %s"); + UPDATE_COMMAND_MAP.put(XBaseCube.class, "update cube --name %s --path %s"); + CREATE_COMMAND_MAP.put(XDerivedCube.class, "create cube --path %s"); + UPDATE_COMMAND_MAP.put(XDerivedCube.class, "update cube --name %s --path %s"); + CREATE_COMMAND_MAP.put(XDimensionTable.class, "create dimtable --path %s"); + UPDATE_COMMAND_MAP.put(XDimensionTable.class, "update dimtable --dimtable_name %s --path %s"); + CREATE_COMMAND_MAP.put(XDimensionTable.class, "create dimtable --path %s"); + UPDATE_COMMAND_MAP.put(XDimensionTable.class, "update dimtable --dimtable_name %s --path %s"); + CREATE_COMMAND_MAP.put(XFactTable.class, "create fact --path %s"); + UPDATE_COMMAND_MAP.put(XFactTable.class, "update fact --fact_name %s --path %s"); + CREATE_COMMAND_MAP.put(XSegmentation.class, "create segmentation --path %s"); + UPDATE_COMMAND_MAP.put(XSegmentation.class, "update segmentation --name %s --path %s"); + } + + private final SchemaTraverser.SchemaEntityProcessor processor = (entityFile, type) -> { + String entityName = entityFile.getName().substring(0, entityFile.getName().length() - 4); + String entityPath = entityFile.getAbsolutePath(); + String createCommand = String.format(CREATE_COMMAND_MAP.get(type), entityPath); + String entityType = createCommand.substring(8, createCommand.indexOf(" ", 9)); + logger.fine(createCommand); + if (shell.executeScriptLine(createCommand)) { + logger.info("Created " + entityType + " " + entityName); + } else { + logger.warning("Create failed, trying update"); + String updateCommand = String.format(UPDATE_COMMAND_MAP.get(type), entityName, entityPath); + logger.fine(updateCommand); + if (shell.executeScriptLine(updateCommand)) { + logger.info("Updated " + entityType + " " + entityName); + } else { + logger.severe("Couldn't create or update " + entityType + " " + entityName); + } + } + }; + @CliCommand(value = {"schema", "create schema"}, help = "Parses the specified resource file and executes commands for " + "creation/updation of schema\nExpected structure is " + STRUCTURE) @@ -108,55 +156,10 @@ public void script( // ignore result. it can fail if database already exists shell.executeCommand("create database " + database); if (shell.executeScriptLine("use " + database)) { - createOrUpdate(new File(schemaDirectory, "storages"), "storage", - "create storage --path %s", "update storage --name %s --path %s"); - createOrUpdate(new File(schemaDirectory, "dimensions"), "dimension", - "create dimension --path %s", "update dimension --name %s --path %s"); - createOrUpdate(new File(new File(schemaDirectory, "cubes"), "base"), "base cube", - "create cube --path %s", "update cube --name %s --path %s"); - createOrUpdate(new File(new File(schemaDirectory, "cubes"), "derived"), "derived cube", - "create cube --path %s", "update cube --name %s --path %s"); - createOrUpdate(new File(schemaDirectory, "dimensiontables"), "dimension table", - "create dimtable --path %s", "update dimtable --dimtable_name %s --path %s"); - createOrUpdate(new File(schemaDirectory, "dimtables"), "dimension table", - "create dimtable --path %s", "update dimtable --dimtable_name %s --path %s"); - createOrUpdate(new File(schemaDirectory, "facts"), "fact", - "create fact --path %s", "update fact --fact_name %s --path %s"); - createOrUpdate(new File(schemaDirectory, "segmentations"), "fact", - "create segmentation --path %s", "update segmentation --name %s --path %s"); + SchemaTraverser schemaTraverser = new SchemaTraverser(schemaDirectory, processor); + schemaTraverser.run(); } else { throw new IllegalStateException("Switching to database " + database + " failed"); } } - - public List createOrUpdate(File parent, String entityType, String createSyntax, String updateSyntax) { - List failedFiles = Lists.newArrayList(); - // Create/update entities - if (parent.exists()) { - Assert.isTrue(parent.isDirectory(), parent.toString() + " must be a directory"); - for (File entityFile : parent.listFiles(XML_FILTER)) { - String entityName = entityFile.getName().substring(0, entityFile.getName().length() - 4); - String entityPath = entityFile.getAbsolutePath(); - String createCommand = String.format(createSyntax, entityPath); - logger.fine(createCommand); - if (shell.executeScriptLine(createCommand)) { - logger.info("Created " + entityType + " " + entityName); - } else { - logger.warning("Create failed, trying update"); - String updateCommand = String.format(updateSyntax, entityName, entityPath); - logger.fine(updateCommand); - if (shell.executeScriptLine(updateCommand)) { - logger.info("Updated " + entityType + " " + entityName); - } else { - logger.severe("Couldn't create or update " + entityType + " " + entityName); - failedFiles.add(entityFile); - } - } - } - } - if (!failedFiles.isEmpty()) { - logger.severe("Failed for " + entityType + ": " + failedFiles); - } - return failedFiles; - } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java index 571b4813c..d98c4c549 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java @@ -57,6 +57,8 @@ public enum LensCubeErrorCode { STORAGE_UNION_DISABLED(3031, 1500), COULD_NOT_PARSE_EXPRESSION(3032, 1500), QUERIED_TABLE_NOT_FOUND(3033, 0), + NO_UNION_CANDIDATE_AVAILABLE(3034, 1501), + NO_JOIN_CANDIDATE_AVAILABLE(3035, 1502), // Error codes greater than 3100 are errors while doing a metastore operation. ERROR_IN_ENTITY_DEFINITION(3101, 100), TIMELINE_ABSENT(3102, 100), diff --git a/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateFactAvailableException.java b/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateFactAvailableException.java index b2568ff79..21dda1685 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateFactAvailableException.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateFactAvailableException.java @@ -18,18 +18,27 @@ */ package org.apache.lens.cube.error; -import org.apache.lens.cube.metadata.CubeFactTable; +import org.apache.lens.cube.parse.CubeQueryContext; import org.apache.lens.cube.parse.PruneCauses; +import org.apache.lens.cube.parse.StorageCandidate; import org.apache.lens.server.api.error.LensException; +import lombok.Getter; + +/** + * Note: This class is mainly meant for test cases to assert the detailed reasons (stored in + * {@link #briefAndDetailedError} leading to "No Candidate was found" + */ public class NoCandidateFactAvailableException extends LensException { - private final PruneCauses briefAndDetailedError; + @Getter + private final PruneCauses briefAndDetailedError; - public NoCandidateFactAvailableException(PruneCauses briefAndDetailedError) { - super(LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo(), briefAndDetailedError.getBriefCause()); - this.briefAndDetailedError = briefAndDetailedError; + public NoCandidateFactAvailableException(CubeQueryContext cubeql) { + super(LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo(), + cubeql.getStoragePruningMsgs().getBriefCause()); + this.briefAndDetailedError = cubeql.getStoragePruningMsgs(); } public PruneCauses.BriefAndDetailedError getJsonMessage() { diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index 896a7a1a3..e00122db0 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -111,21 +111,16 @@ private static void addCubeNames(String factName, Map props, Str private Map> getUpdatePeriodMap(String factName, Map props) { Map> ret = new HashMap<>(); - for (Map.Entry entry : storageUpdatePeriods.entrySet()) { - String storage = (String) entry.getKey(); - for (UpdatePeriod period : (Set) entry.getValue()) { + for (Map.Entry> entry : storageUpdatePeriods.entrySet()) { + String storage = entry.getKey(); + for (UpdatePeriod period : entry.getValue()) { String storagePrefixKey = MetastoreUtil .getUpdatePeriodStoragePrefixKey(factName.trim(), storage, period.getName()); String storageTableNamePrefix = props.get(storagePrefixKey); if (storageTableNamePrefix == null) { storageTableNamePrefix = storage; } - Map mapOfUpdatePeriods = ret.get(storage); - if (mapOfUpdatePeriods == null) { - mapOfUpdatePeriods = new HashMap<>(); - ret.put(storage, mapOfUpdatePeriods); - } - mapOfUpdatePeriods.put(period, storageTableNamePrefix); + ret.computeIfAbsent(storage, k -> new HashMap<>()).put(period, storageTableNamePrefix); } } return ret; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 087c203f1..b5c4c89b3 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -20,12 +20,25 @@ package org.apache.lens.cube.metadata; import static org.apache.lens.cube.metadata.DateUtil.resolveDate; +import static org.apache.lens.cube.metadata.JAXBUtils.getStorageTableDescFromHiveTable; +import static org.apache.lens.cube.metadata.JAXBUtils.segmentationFromXSegmentation; import static org.apache.lens.cube.metadata.MetastoreUtil.*; import java.text.ParseException; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import org.apache.lens.api.metastore.XCube; +import org.apache.lens.api.metastore.XDerivedCube; +import org.apache.lens.api.metastore.XDimension; +import org.apache.lens.api.metastore.XDimensionTable; +import org.apache.lens.api.metastore.XFactTable; +import org.apache.lens.api.metastore.XSegmentation; +import org.apache.lens.api.metastore.XStorage; +import org.apache.lens.api.metastore.XStorageTableElement; +import org.apache.lens.api.metastore.XUpdatePeriod; +import org.apache.lens.api.metastore.XUpdatePeriodTableDescriptor; +import org.apache.lens.api.metastore.XUpdatePeriods; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.Storage.LatestInfo; import org.apache.lens.cube.metadata.Storage.LatestPartColumnInfo; @@ -50,6 +63,10 @@ import org.apache.hadoop.util.ReflectionUtils; import org.apache.thrift.TException; +import org.jvnet.jaxb2_commons.lang.Equals; +import org.jvnet.jaxb2_commons.lang.HashCode; +import org.jvnet.jaxb2_commons.lang.ToString; + import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -302,6 +319,62 @@ public void createCubeFactTable(String cubeName, String factName, List void createEntity(T entity) throws LensException { + if (entity instanceof XStorage) { + createStorage((XStorage) entity); + } else if (entity instanceof XCube) { + createCube((XCube)entity); + } else if (entity instanceof XDimension) { + createDimension((XDimension) entity); + } else if (entity instanceof XFactTable) { + createCubeFactTable((XFactTable) entity); + } else if (entity instanceof XDimensionTable) { + createCubeDimensionTable((XDimensionTable) entity); + } else if (entity instanceof XSegmentation) { + createSegmentation((XSegmentation) entity); + } else { + throw new LensException("Unable to create entity " + entity + " as it's unrecognizable: "+ entity.getClass()); + } + } + + public void updateEntity(String name, T entity) + throws LensException, HiveException { + if (entity instanceof XStorage) { + alterStorage((XStorage) entity); + } else if (entity instanceof XCube) { + alterCube((XCube)entity); + } else if (entity instanceof XDimension) { + alterDimension((XDimension) entity); + } else if (entity instanceof XFactTable) { + alterCubeFactTable((XFactTable) entity); + } else if (entity instanceof XDimensionTable) { + alterCubeDimensionTable((XDimensionTable) entity); + } else if (entity instanceof XSegmentation) { + alterSegmentation((XSegmentation) entity); + } else { + throw new LensException("Unable to alter entity " + entity + " as it's unrecognizable: " + entity.getClass()); + } + } + + + public static Map addFactColStartTimePropertyToFactProperties(XFactTable fact) { + Map props = new HashMap(); + props.putAll(JAXBUtils.mapFromXProperties(fact.getProperties())); + props.putAll(JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns())); + return props; + } + public void createCubeFactTable(XFactTable fact) throws LensException { + createCubeFactTable(fact.getCubeName(), + fact.getName(), + JAXBUtils.fieldSchemaListFromColumns(fact.getColumns()), + JAXBUtils.getFactUpdatePeriodsFromStorageTables(fact.getStorageTables()), + fact.getWeight(), + addFactColStartTimePropertyToFactProperties(fact), + JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()), + JAXBUtils.storageTablePrefixMapOfStorage(fact.getStorageTables())); + } + + /** * In-memory storage of {@link PartitionTimeline} objects for each valid * storagetable-updateperiod-partitioncolumn tuple. also simultaneously stored in metastore table of the @@ -349,7 +422,7 @@ public boolean noPartitionsExist(String fact, String storage, String partCol) public TreeMap> get(String fact, String storage) throws HiveException, LensException { // SUSPEND CHECKSTYLE CHECK DoubleCheckedLockingCheck - // Unique key for the timeline cache, based on storageName and fact. + // Unique key for the timeline cache, based on storage and fact. String timeLineKey = (Storage.getPrefix(storage)+ fact).toLowerCase(); synchronized (this) { if (get(timeLineKey) == null) { @@ -478,17 +551,11 @@ public void addForBatchAddition(String timeLineKey, String storageTableName, Upd */ public PartitionTimeline ensureEntry(String timeLineKey, String storagTableName, UpdatePeriod updatePeriod, String partitionColumn) { - if (get(timeLineKey) == null) { - put(timeLineKey, new TreeMap>()); - } - if (get(timeLineKey).get(updatePeriod) == null) { - get(timeLineKey).put(updatePeriod, new CaseInsensitiveStringHashMap()); - } - if (get(timeLineKey).get(updatePeriod).get(partitionColumn) == null) { - get(timeLineKey).get(updatePeriod).put(partitionColumn, PartitionTimelineFactory.get( - CubeMetastoreClient.this, storagTableName, updatePeriod, partitionColumn)); - } - return get(timeLineKey).get(updatePeriod).get(partitionColumn); + return this + .computeIfAbsent(timeLineKey, s -> new TreeMap<>()) + .computeIfAbsent(updatePeriod, k -> new CaseInsensitiveStringHashMap<>()) + .computeIfAbsent(partitionColumn, c -> PartitionTimelineFactory.get( + CubeMetastoreClient.this, storagTableName, updatePeriod, c)); } /** check partition existence in the appropriate timeline if it exists */ @@ -625,12 +692,22 @@ private Table createCubeHiveTable(AbstractCubeTable table) throws LensException } } + public void createStorage(XStorage storage) throws LensException { + createStorage(JAXBUtils.storageFromXStorage(storage)); + } + public void createStorage(Storage storage) throws LensException { createCubeHiveTable(storage); // do a get to update cache getStorage(storage.getName()); } + public void createCube(XCube cube) throws LensException { + Cube parent = cube instanceof XDerivedCube ? (Cube) getCube( + ((XDerivedCube) cube).getParent()) : null; + createCube(JAXBUtils.hiveCubeFromXCube(cube, parent)); + } + /** * Create cube in metastore defined by {@link Cube} or {@link DerivedCube} object * @@ -720,6 +797,9 @@ public void createDimension(String name, Set attributes, Map columns = JAXBUtils.fieldSchemaListFromColumns(xDimTable.getColumns()); + Map updatePeriodMap = + JAXBUtils.dumpPeriodsFromStorageTables(xDimTable.getStorageTables()); + + Map properties = JAXBUtils.mapFromXProperties(xDimTable.getProperties()); + Map storageDesc = JAXBUtils.tableDescPrefixMapFromXStorageTables( + xDimTable.getStorageTables()); + log.info("# Columns: " + columns); + createCubeDimensionTable(xDimTable.getDimensionName(), xDimTable.getTableName(), columns, xDimTable.getWeight(), + updatePeriodMap, properties, storageDesc); + } /** * Create a cube dimension table * @@ -852,6 +944,14 @@ public void createCubeTable(AbstractCubeTable cubeTable, Map addPartitions(String factOrDimTable, String storageName, } } - private Date getStorageTableStartDate(String storageTable, String factTableName) + public Date getStorageTableStartDate(String storageTable, String factTableName) throws LensException { List startDates = getStorageTimes(storageTable, MetastoreUtil.getStoragetableStartTimesKey()); startDates.add(getFactTable(factTableName).getStartTime()); return Collections.max(startDates); } - private Date getStorageTableEndDate(String storageTable, String factTableName) + public Date getStorageTableEndDate(String storageTable, String factTableName) throws LensException { List endDates = getStorageTimes(storageTable, MetastoreUtil.getStoragetableEndTimesKey()); endDates.add(getFactTable(factTableName).getEndTime()); @@ -1624,6 +1724,48 @@ boolean isDimension(Table tbl) { return CubeTableType.DIMENSION.name().equals(tableType); } + public XFactTable getXFactTable(String tableName) throws LensException { + return getXFactTable(getFactTable(tableName)); + } + public XFactTable getXFactTable(CubeFactTable cft) throws LensException { + + XFactTable factTable = JAXBUtils.factTableFromCubeFactTable(cft); + Map> storageMap = cft.getStoragePrefixUpdatePeriodMap(); + for (String storageName : cft.getStorages()) { + Set updatePeriods = cft.getUpdatePeriods().get(storageName); + // This map tells if there are different tables for different update period. + Map updatePeriodToTableMap = storageMap.get(storageName); + Set tableNames = new HashSet<>(); + for (UpdatePeriod updatePeriod : updatePeriods) { + tableNames.add(updatePeriodToTableMap.get(updatePeriod)); + } + if (tableNames.size() <= 1) { + XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( + getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), storageName))); + tblElement.setStorageName(storageName); + for (UpdatePeriod p : updatePeriods) { + tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); + } + factTable.getStorageTables().getStorageTable().add(tblElement); + } else { + // Multiple storage tables. + XStorageTableElement tblElement = new XStorageTableElement(); + tblElement.setStorageName(storageName); + XUpdatePeriods xUpdatePeriods = new XUpdatePeriods(); + tblElement.setUpdatePeriods(xUpdatePeriods); + for (Map.Entry entry : updatePeriodToTableMap.entrySet()) { + XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor(); + updatePeriodTableDescriptor.setTableDesc(getStorageTableDescFromHiveTable( + this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), + (String) entry.getValue())))); + updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod)entry.getKey()).name())); + xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor); + } + factTable.getStorageTables().getStorageTable().add(tblElement); + } + } + return factTable; + } /** * Get {@link CubeFactTable} object corresponding to the name * @@ -1640,6 +1782,25 @@ public Segmentation getSegmentationTable(String tableName) throws HiveException, return new Segmentation(getTableWithTypeFailFast(tableName, CubeTableType.SEGMENTATION)); } + public XDimensionTable getXDimensionTable(String dimTable) throws LensException { + return getXDimensionTable(getDimensionTable(dimTable)); + } + public XDimensionTable getXDimensionTable(CubeDimensionTable dimTable) throws LensException { + XDimensionTable dt = JAXBUtils.dimTableFromCubeDimTable(dimTable); + if (!dimTable.getStorages().isEmpty()) { + for (String storageName : dimTable.getStorages()) { + XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( + this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTable.getName(), storageName))); + tblElement.setStorageName(storageName); + UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); + if (p != null) { + tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); + } + dt.getStorageTables().getStorageTable().add(tblElement); + } + } + return dt; + } /** * Get {@link CubeDimensionTable} object corresponding to the name * @@ -2101,16 +2262,36 @@ public List getAllDimensionTables(Dimension dim) throws Lens return dimTables; } - public boolean partColExists(String tableName, String partCol) throws LensException { - Table tbl = getTable(tableName); - for (FieldSchema f : tbl.getPartCols()) { - if (f.getName().equalsIgnoreCase(partCol)) { - return true; + public boolean partColExists(String fact, String storage, String partCol) throws LensException { + for (String storageTable : getStorageTables(fact, storage)) { + for (FieldSchema f : getTable(storageTable).getPartCols()) { + if (f.getName().equalsIgnoreCase(partCol)) { + return true; + } } } return false; } + /** + * Returns storage table names for a storage. + * Note: If each update period in the storage has a different storage table, this method will return N Storage Tables + * where N is the number of update periods in the storage (LENS-1386) + * + * @param fact + * @param storage + * @return + * @throws LensException + */ + public Set getStorageTables(String fact, String storage) throws LensException { + Set uniqueStorageTables = new HashSet<>(); + for (UpdatePeriod updatePeriod : getFactTable(fact).getUpdatePeriods().get(storage)) { + uniqueStorageTables.add(getStorageTableName(fact, storage, updatePeriod)); + } + return uniqueStorageTables; + } + + /** * * @param table table name @@ -2150,6 +2331,11 @@ public void alterHiveTable(String table, Table hiveTable) throws HiveException, } } + public void alterCube(XCube cube) throws HiveException, LensException { + Cube parent = cube instanceof XDerivedCube ? (Cube) getCube( + ((XDerivedCube) cube).getParent()) : null; + alterCube(cube.getName(), JAXBUtils.hiveCubeFromXCube(cube, parent)); + } /** * Alter cube specified by the name to new definition * @@ -2168,10 +2354,13 @@ public void alterCube(String cubeName, CubeInterface cube) throws HiveException, /** * Alter dimension specified by the dimension name to new definition * - * @param dimName The cube name to be altered * @param newDim The new dimension definition * @throws HiveException */ + public void alterDimension(XDimension newDim) throws HiveException, LensException { + alterDimension(newDim.getName(), JAXBUtils.dimensionFromXDimension(newDim)); + } + public void alterDimension(String dimName, Dimension newDim) throws HiveException, LensException { Table tbl = getTableWithTypeFailFast(dimName, CubeTableType.DIMENSION); alterCubeTable(dimName, tbl, newDim); @@ -2183,10 +2372,12 @@ public void alterDimension(String dimName, Dimension newDim) throws HiveExceptio /** * Alter storage specified by the name to new definition * - * @param storageName The storage name to be altered * @param storage The new storage definition * @throws LensException */ + public void alterStorage(XStorage storage) throws LensException, HiveException { + alterStorage(storage.getName(), JAXBUtils.storageFromXStorage(storage)); + } public void alterStorage(String storageName, Storage storage) throws LensException, HiveException { Table storageTbl = getTableWithTypeFailFast(storageName, CubeTableType.STORAGE); alterCubeTable(storageName, storageTbl, storage); @@ -2339,7 +2530,11 @@ public void dropDimensionTable(String dimTblName, boolean cascade) throws LensEx dropHiveTable(dimTblName); allDimTables.remove(dimTblName.trim().toLowerCase()); } - + public void alterCubeFactTable(XFactTable fact) throws LensException, HiveException { + alterCubeFactTable(fact.getName(), JAXBUtils.cubeFactFromFactTable(fact), + JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()), + JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns())); + } /** * Alter a cubefact with new definition and alter underlying storage tables as well. * @@ -2367,6 +2562,9 @@ public void alterCubeFactTable(String factTableName, CubeFactTable cubeFactTable updateFactCache(factTableName); } + public void alterSegmentation(XSegmentation cubeSeg) throws LensException, HiveException { + alterSegmentation(cubeSeg.getName(), segmentationFromXSegmentation(cubeSeg)); + } public void alterSegmentation(String segName, Segmentation seg) throws HiveException, LensException { getTableWithTypeFailFast(segName, CubeTableType.SEGMENTATION); @@ -2394,7 +2592,11 @@ private void updateDimCache(String dimTblName) throws LensException { allDimTables.put(dimTblName.trim().toLowerCase(), getDimensionTable(refreshTable(dimTblName))); } } - + public void alterCubeDimensionTable(XDimensionTable dimensionTable) throws LensException, HiveException { + alterCubeDimensionTable(dimensionTable.getTableName(), + JAXBUtils.cubeDimTableFromDimTable(dimensionTable), + JAXBUtils.tableDescPrefixMapFromXStorageTables(dimensionTable.getStorageTables())); + } /** * Alter dimension table with new dimension definition and underlying storage tables as well * diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java index 771708132..99ad2331d 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java @@ -30,6 +30,7 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Stream; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.server.api.error.LensException; @@ -305,11 +306,11 @@ static CoveringInfo getCoveringInfo(Date from, Date to, UpdatePeriod interval) { switch (interval) { case SECONDLY: case CONTINUOUS: - return getMilliSecondCoveringInfo(from, to, 1000); + return getMilliSecondCoveringInfo(from, to, 1000, interval); case MINUTELY: case HOURLY: case DAILY: - return getMilliSecondCoveringInfo(from, to, interval.weight()); + return getMilliSecondCoveringInfo(from, to, interval.weight(), interval); case WEEKLY: return getWeeklyCoveringInfo(from, to); case MONTHLY: @@ -323,18 +324,26 @@ static CoveringInfo getCoveringInfo(Date from, Date to, UpdatePeriod interval) { } } - private static CoveringInfo getMilliSecondCoveringInfo(Date from, Date to, long millisInInterval) { + private static CoveringInfo getMilliSecondCoveringInfo(Date from, Date to, long millisInInterval, + UpdatePeriod interval) { long diff = to.getTime() - from.getTime(); - return new CoveringInfo((int) (diff / millisInInterval), diff % millisInInterval == 0); + return new CoveringInfo((int) (diff / millisInInterval), + Stream.of(from, to).allMatch(a->interval.truncate(a).equals(a))); + // start date and end date should lie on boundaries. } + /** + * Whether the range [from,to) is coverable by intervals + * @param from from time + * @param to to time + * @param intervals intervals to check + * @return true if any of the intervals can completely cover the range + */ static boolean isCoverableBy(Date from, Date to, Set intervals) { - for (UpdatePeriod period : intervals) { - if (getCoveringInfo(from, to, period).isCoverable()) { - return true; - } - } - return false; + return intervals.stream().anyMatch(period->isCoverableBy(from, to, period)); + } + private static boolean isCoverableBy(Date from, Date to, UpdatePeriod period) { + return getCoveringInfo(from, to, period).isCoverable(); } public static int getTimeDiff(Date fromDate, Date toDate, UpdatePeriod updatePeriod) { diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java index 1694b8092..ed940cca3 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java @@ -65,6 +65,16 @@ public FactPartition(String partCol, Date partSpec, UpdatePeriod period, FactPar } } + /** + * Partition should not be used a indicative of the class itself. + * New Fact partition created includes more final partitions with that creation. + * @return + */ + + public FactPartition withoutContaining() { + return new FactPartition(this.getPartCol(), this.getPartSpec(), this.getPeriod(), null, this + .getPartFormat(), this.getStorageTables()); + } public FactPartition(String partCol, TimePartition timePartition) { this(partCol, timePartition, null, null); } diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java similarity index 99% rename from lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java rename to lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java index 7d54c7b2c..e1e3d167e 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.lens.server.metastore; +package org.apache.lens.cube.metadata; import java.lang.reflect.Constructor; import java.text.ParseException; @@ -28,7 +28,6 @@ import javax.xml.datatype.XMLGregorianCalendar; import org.apache.lens.api.metastore.*; -import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.metadata.ExprColumn.ExprSpec; import org.apache.lens.cube.metadata.ReferencedDimAttribute.ChainRefCol; import org.apache.lens.server.api.error.LensException; @@ -743,7 +742,6 @@ public static XFactTable factTableFromCubeFactTable(CubeFactTable cFact) { fact.setColumns(new XColumns()); fact.setProperties(new XProperties()); fact.setStorageTables(new XStorageTables()); - fact.getProperties().getProperty().addAll(xPropertiesFromMap(cFact.getProperties())); fact.getColumns().getColumn().addAll(columnsFromFieldSchemaList(cFact.getColumns())); fact.setWeight(cFact.weight()); diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java index 4e350c8ae..599027fbc 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java @@ -595,5 +595,4 @@ public static String getUpdatePeriodStoragePrefixKey(String factTableName, Strin return MetastoreUtil.getFactKeyPrefix(factTableName) + "." + storageName + "." + updatePeriod; } - } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java index bf6cc5c3e..82868949e 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java @@ -22,7 +22,7 @@ import java.util.Calendar; import java.util.Date; -import java.util.TreeSet; +import java.util.Set; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.server.api.error.LensException; @@ -48,10 +48,28 @@ public class TimeRange { private ASTNode parent; private int childIndex; - public boolean isCoverableBy(TreeSet updatePeriods) { + public boolean isCoverableBy(Set updatePeriods) { return DateUtil.isCoverableBy(fromDate, toDate, updatePeriods); } + /** + * Truncate time range using the update period. + * The lower value of the truncated time range is the smallest date value equal to or larger than original + * time range's lower value which lies at the update period's boundary. Similarly for higher value. + * @param updatePeriod Update period to truncate time range with + * @return truncated time range + * @throws LensException If the truncated time range is invalid. + */ + public TimeRange truncate(UpdatePeriod updatePeriod) throws LensException { + TimeRange timeRange = new TimeRangeBuilder().partitionColumn(partitionColumn) + .fromDate(updatePeriod.getCeilDate(fromDate)).toDate(updatePeriod.getFloorDate(toDate)).build(); + timeRange.validate(); + return timeRange; + } + + public long milliseconds() { + return toDate.getTime() - fromDate.getTime(); + } public static class TimeRangeBuilder { private final TimeRange range; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java index 8681e9013..10f98a1d4 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java @@ -19,6 +19,8 @@ package org.apache.lens.cube.parse; +import static java.util.stream.Collectors.toMap; + import java.util.*; import org.apache.lens.cube.metadata.FactPartition; @@ -33,14 +35,13 @@ * Collapses the time range filters using IN operators */ public class AbridgedTimeRangeWriter implements TimeRangeWriter { - //TODO: minimize use of String, use StringBuilders /** * Return IN clause for the partitions selected in the cube query * - * @param cubeQueryContext - * @param tableName - * @param parts + * @param cubeQueryContext cube query context + * @param tableName table name + * @param parts partitions * @return * @throws LensException */ @@ -80,7 +81,7 @@ private String getClause(CubeQueryContext cubeQueryContext, for (FactPartition factPartition : parts) { String filter = TimeRangeUtils.getTimeRangePartitionFilter(factPartition, cubeQueryContext, tableName); if (filter.contains("AND")) { - allTimeRangeFilters.add(new StringBuilder("(").append(filter).append(")").toString()); + allTimeRangeFilters.add("(" + filter + ")"); } else { extractColumnAndCondition(filter, partFilterMap); } @@ -89,7 +90,7 @@ private String getClause(CubeQueryContext cubeQueryContext, List inClauses = new ArrayList(partFilterMap.size()); for (String column : partFilterMap.keySet()) { String clause = - new StringBuilder("(").append(StringUtils.join(partFilterMap.get(column), ",")).append(")").toString(); + "(" + StringUtils.join(partFilterMap.get(column), ",") + ")"; inClauses.add(column + " IN " + clause); } @@ -120,29 +121,17 @@ private String getClause(CubeQueryContext cubeQueryContext, private Map, Set> groupPartitions(Collection parts) { Map> partitionSetMap = new HashMap>(); for (FactPartition part : parts) { - FactPartition key = part.getContainingPart(); - FactPartition part2 = new FactPartition(part.getPartCol(), part.getPartSpec(), part.getPeriod(), null, part - .getPartFormat(), part.getStorageTables()); - if (partitionSetMap.get(key) == null) { - partitionSetMap.put(key, Sets.newTreeSet()); - } - partitionSetMap.get(key).add(part2); + partitionSetMap.computeIfAbsent(part.getContainingPart(), k -> Sets.newTreeSet()).add(part.withoutContaining()); } Map, Set> setSetOppositeMap = Maps.newHashMap(); for (Map.Entry> entry : partitionSetMap.entrySet()) { - if (setSetOppositeMap.get(entry.getValue()) == null) { - setSetOppositeMap.put(entry.getValue(), Sets.newTreeSet()); - } + setSetOppositeMap.computeIfAbsent(entry.getValue(), k -> Sets.newTreeSet()); if (entry.getKey() != null) { setSetOppositeMap.get(entry.getValue()).add(entry.getKey()); } } - - Map, Set> setSetMap = Maps.newHashMap(); - for (Map.Entry, Set> entry : setSetOppositeMap.entrySet()) { - setSetMap.put(entry.getValue(), entry.getKey()); - } - return setSetMap; + // inverse again + return setSetOppositeMap.entrySet().stream().collect(toMap(Map.Entry::getValue, Map.Entry::getKey)); } // This takes the output of filter generated by TimeRangeUtils.getTimeRangePartitionFilter @@ -156,13 +145,6 @@ private void extractColumnAndCondition(String token, Map> p String column = subTokens[0].trim(); String filterValue = subTokens[1].trim(); - List filterValues = partFilterMap.get(column); - - if (filterValues == null) { - filterValues = new ArrayList(); - partFilterMap.put(column, filterValues); - } - - filterValues.add(filterValue); + partFilterMap.computeIfAbsent(column, k -> new ArrayList<>()).add(filterValue); } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java index 96581007a..30b1a904d 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java @@ -27,7 +27,6 @@ import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.CubeMeasure; import org.apache.lens.cube.metadata.ExprColumn; -import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; import org.apache.lens.cube.parse.ExpressionResolver.ExprSpecContext; import org.apache.lens.server.api.error.LensException; @@ -71,21 +70,23 @@ public void rewriteContext(CubeQueryContext cubeql) throws LensException { || hasMeasuresNotInDefaultAggregates(cubeql, cubeql.getHavingAST(), null, aggregateResolverDisabled) || hasMeasures(cubeql, cubeql.getWhereAST()) || hasMeasures(cubeql, cubeql.getGroupByAST()) || hasMeasures(cubeql, cubeql.getOrderByAST())) { - Iterator factItr = cubeql.getCandidateFacts().iterator(); - while (factItr.hasNext()) { - CandidateFact candidate = factItr.next(); - if (candidate.fact.isAggregated()) { - cubeql.addFactPruningMsgs(candidate.fact, - CandidateTablePruneCause.missingDefaultAggregate()); - factItr.remove(); + Iterator candItr = cubeql.getCandidates().iterator(); + while (candItr.hasNext()) { + Candidate candidate = candItr.next(); + if (candidate instanceof StorageCandidate) { + StorageCandidate sc = (StorageCandidate) candidate; + if (sc.getFact().isAggregated()) { + cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.missingDefaultAggregate()); + candItr.remove(); + } + } else { + throw new LensException("Not a storage candidate!!"); } } nonDefaultAggregates = true; log.info("Query has non default aggregates, no aggregate resolution will be done"); } - cubeql.pruneCandidateFactSet(CandidateTablePruneCode.MISSING_DEFAULT_AGGREGATE); - if (nonDefaultAggregates || aggregateResolverDisabled) { return; } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java index c8b81298a..bd77498a0 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java @@ -92,7 +92,7 @@ public String getTimeRangeWhereClause(CubeQueryContext cubeQueryContext, String } String partCol = start.getPartCol(); - if (cubeQueryContext != null && !cubeQueryContext.shouldReplaceTimeDimWithPart()) { + if (!cubeQueryContext.shouldReplaceTimeDimWithPart()) { partCol = cubeQueryContext.getTimeDimOfPartitionColumn(partCol); } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java new file mode 100644 index 000000000..f241cb311 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + +import java.util.Collection; +import java.util.Date; +import java.util.Set; + +import org.apache.lens.cube.metadata.FactPartition; +import org.apache.lens.cube.metadata.TimeRange; +import org.apache.lens.server.api.error.LensException; + +/** + * This interface represents candidates that are involved in different phases of query rewriting. + * At the lowest level, Candidate is represented by a StorageCandidate that has a fact on a storage + * and other joined dimensions (if any) that are required to answer the query or part of the query. + * At a higher level Candidate can also be a Join or a Union Candidate representing join or union + * between other candidates + * + * Different Re-writers will work on applicable candidates to produce a final candidate which will be used + * for generating the re-written query. + */ +public interface Candidate { + + /** + * Returns all the fact columns + * + * @return + */ + Collection getColumns(); + + /** + * Start Time for this candidate (calculated based on schema) + * + * @return + */ + Date getStartTime(); + + /** + * End Time for this candidate (calculated based on schema) + * + * @return + */ + Date getEndTime(); + + /** + * Returns the cost of this candidate + * + * @return + */ + double getCost(); + + /** + * Returns true if this candidate contains the given candidate + * + * @param candidate + * @return + */ + boolean contains(Candidate candidate); + + /** + * Returns child candidates of this candidate if any. + * Note: StorageCandidate will return null + * + * @return + */ + Collection getChildren(); + + /** + * Is time range coverable based on start and end times configured in schema for the composing storage candidates + * and valid update periods. + * + * Note: This method is different from {@link #evaluateCompleteness(TimeRange, TimeRange, boolean)} . + * isTimeRangeCoverable checks the the possibility of covering time range from schema perspective by using valid + * storages/update periods while evaluateCompleteness checks if a time range can be covered based on + * registered partitions. So isTimeRangeCoverable = false implies evaluateCompleteness = false but vice versa is + * not true. + * + * @param timeRange + * @return + * @throws LensException + */ + boolean isTimeRangeCoverable(TimeRange timeRange) throws LensException; + + /** + * Calculates if this candidate can answer the query for given time range based on actual data registered with + * the underlying candidate storages. This method will also update any internal candidate data structures that are + * required for writing the re-written query and to answer {@link #getParticipatingPartitions()}. + * + * @param timeRange : TimeRange to check completeness for. TimeRange consists of start time, end time and the + * partition column + * @param queriedTimeRange : User quried timerange + * @param failOnPartialData : fail fast if the candidate can answer the query only partially + * @return true if this Candidate can answer query for the given time range. + */ + boolean evaluateCompleteness(TimeRange timeRange, TimeRange queriedTimeRange, boolean failOnPartialData) + throws LensException; + + /** + * Returns the set of fact partitions that will participate in this candidate. + * Note: This method can be called only after call to + * {@link #evaluateCompleteness(TimeRange, TimeRange, boolean)} + * + * @return + */ + Set getParticipatingPartitions(); + + /** + * Checks whether an expression is evaluable by a candidate + * 1. For a JoinCandidate, atleast one of the child candidates should be able to answer the expression + * 2. For a UnionCandidate, all child candidates should answer the expression + * + * @param expr :Expression need to be evaluated for Candidate + * @return + */ + boolean isExpressionEvaluable(ExpressionResolver.ExpressionContext expr); + + /** + * Gets the index positions of answerable measure phrases in CubeQueryContext#selectPhrases + * @return + */ + Set getAnswerableMeasurePhraseIndices(); +} diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java new file mode 100644 index 000000000..b22d97292 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java @@ -0,0 +1,315 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + +import static org.apache.lens.cube.parse.CandidateUtil.getColumns; + +import java.util.*; + +import org.apache.lens.cube.error.LensCubeErrorCode; +import org.apache.lens.cube.metadata.TimeRange; +import org.apache.lens.server.api.error.LensException; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class CandidateCoveringSetsResolver implements ContextRewriter { + + @Override + public void rewriteContext(CubeQueryContext cubeql) throws LensException { + + if (!cubeql.hasCubeInQuery()) { + return; //Dimension query + } + + if (cubeql.getCandidates().size() == 0){ + cubeql.throwNoCandidateFactException(); + } + + List qpcList = cubeql.getQueriedPhrases(); + Set queriedMsrs = new HashSet<>(); + for (QueriedPhraseContext qpc : qpcList) { + if (qpc.hasMeasures(cubeql)) { + queriedMsrs.add(qpc); + } + } + + List timeRangeCoveringSet = resolveTimeRangeCoveringFactSet(cubeql, queriedMsrs, qpcList); + if (timeRangeCoveringSet.isEmpty()) { + throw new LensException(LensCubeErrorCode.NO_UNION_CANDIDATE_AVAILABLE.getLensErrorInfo(), + cubeql.getCube().getName(), cubeql.getTimeRanges().toString(), getColumns(queriedMsrs).toString()); + } + log.info("Time covering candidates :{}", timeRangeCoveringSet); + + if (queriedMsrs.isEmpty()) { + cubeql.getCandidates().clear(); + cubeql.getCandidates().addAll(timeRangeCoveringSet); + } else if (!timeRangeCoveringSet.isEmpty()) { + List> measureCoveringSets = resolveJoinCandidates(timeRangeCoveringSet, queriedMsrs, cubeql); + if (measureCoveringSets.isEmpty()) { + throw new LensException(LensCubeErrorCode.NO_JOIN_CANDIDATE_AVAILABLE.getLensErrorInfo(), + cubeql.getCube().getName(), getColumns(queriedMsrs).toString()); + } + updateFinalCandidates(measureCoveringSets, cubeql); + } + + log.info("Final Time and Measure covering candidates :{}", cubeql.getCandidates()); + } + + private Candidate createJoinCandidate(List childCandidates, CubeQueryContext cubeql) { + Candidate cand; + Candidate first = childCandidates.get(0); + Candidate second = childCandidates.get(1); + cand = new JoinCandidate(first, second, cubeql); + for (int i = 2; i < childCandidates.size(); i++) { + cand = new JoinCandidate(cand, childCandidates.get(i), cubeql); + } + return cand; + } + + private void updateFinalCandidates(List> joinCandidates, CubeQueryContext cubeql) { + List finalCandidates = new ArrayList<>(); + + for (List joinCandidate : joinCandidates) { + if (joinCandidate.size() == 1) { + finalCandidates.add(joinCandidate.iterator().next()); + } else { + finalCandidates.add(createJoinCandidate(joinCandidate, cubeql)); + } + } + cubeql.getCandidates().clear(); + cubeql.getCandidates().addAll(finalCandidates); + } + + private boolean isCandidateCoveringTimeRanges(UnionCandidate uc, List ranges) { + for (TimeRange range : ranges) { + if (!CandidateUtil.isTimeRangeCovered(uc.getChildren(), range.getFromDate(), range.getToDate())) { + return false; + } + } + return true; + } + + private void pruneUnionCandidatesNotCoveringAllRanges(List ucs, CubeQueryContext cubeql) { + for (Iterator itr = ucs.iterator(); itr.hasNext();) { + UnionCandidate uc = itr.next(); + if (!isCandidateCoveringTimeRanges(uc, cubeql.getTimeRanges())) { + itr.remove(); + cubeql.addCandidatePruningMsg(uc, CandidateTablePruneCause.storageNotAvailableInRange(cubeql.getTimeRanges())); + } + } + } + + private List resolveTimeRangeCoveringFactSet(CubeQueryContext cubeql, + Set queriedMsrs, List qpcList) throws LensException { + List candidateSet = new ArrayList<>(); + if (!cubeql.getCandidates().isEmpty()) { + // All Candidates + List allCandidates = new ArrayList<>(cubeql.getCandidates()); + // Partially valid candidates + List allCandidatesPartiallyValid = new ArrayList<>(); + for (Candidate cand : allCandidates) { + // Assuming initial list of candidates populated are StorageCandidate + assert (cand instanceof StorageCandidate); + StorageCandidate sc = (StorageCandidate) cand; + if (CandidateUtil.isValidForTimeRanges(sc, cubeql.getTimeRanges())) { + candidateSet.add(CandidateUtil.cloneStorageCandidate(sc)); + } else if (CandidateUtil.isPartiallyValidForTimeRanges(sc, cubeql.getTimeRanges())) { + allCandidatesPartiallyValid.add(CandidateUtil.cloneStorageCandidate(sc)); + } else { + cubeql.addCandidatePruningMsg(sc, CandidateTablePruneCause.storageNotAvailableInRange( + cubeql.getTimeRanges())); + } + + } + // Get all covering fact sets + List unionCoveringSet = + getCombinations(new ArrayList<>(allCandidatesPartiallyValid), cubeql); + // Sort the Collection based on no of elements + unionCoveringSet.sort(new CandidateUtil.ChildrenSizeBasedCandidateComparator()); + // prune non covering sets + pruneUnionCandidatesNotCoveringAllRanges(unionCoveringSet, cubeql); + // prune candidate set which doesn't contain any common measure i + pruneUnionCoveringSetWithoutAnyCommonMeasure(unionCoveringSet, queriedMsrs, cubeql); + // prune redundant covering sets + pruneRedundantUnionCoveringSets(unionCoveringSet); + // pruing done in the previous steps, now create union candidates + candidateSet.addAll(unionCoveringSet); + updateQueriableMeasures(candidateSet, qpcList, cubeql); + } + return candidateSet; + } + + private boolean isMeasureAnswerablebyUnionCandidate(QueriedPhraseContext msr, Candidate uc, + CubeQueryContext cubeql) throws LensException { + // Candidate is a single StorageCandidate + if ((uc instanceof StorageCandidate) && !msr.isEvaluable(cubeql, (StorageCandidate) uc)) { + return false; + } else if ((uc instanceof UnionCandidate)){ + for (Candidate cand : uc.getChildren()) { + if (!msr.isEvaluable(cubeql, (StorageCandidate) cand)) { + return false; + } + } + } + return true; + } + + private void pruneUnionCoveringSetWithoutAnyCommonMeasure(List ucs, + Set queriedMsrs, + CubeQueryContext cubeql) throws LensException { + for (ListIterator itr = ucs.listIterator(); itr.hasNext();) { + boolean toRemove = true; + UnionCandidate uc = itr.next(); + for (QueriedPhraseContext msr : queriedMsrs) { + if (isMeasureAnswerablebyUnionCandidate(msr, uc, cubeql)) { + toRemove = false; + break; + } + } + if (toRemove) { + itr.remove(); + } + } + } + + private void pruneRedundantUnionCoveringSets(List candidates) { + for (int i = 0; i < candidates.size(); i++) { + UnionCandidate current = candidates.get(i); + int j = i + 1; + for (ListIterator itr = candidates.listIterator(j); itr.hasNext();) { + UnionCandidate next = itr.next(); + if (next.getChildren().containsAll(current.getChildren())) { + itr.remove(); + } + } + } + } + + private List getCombinations(final List candidates, CubeQueryContext cubeql) { + List combinations = new LinkedList<>(); + int size = candidates.size(); + int threshold = Double.valueOf(Math.pow(2, size)).intValue() - 1; + + for (int i = 1; i <= threshold; ++i) { + LinkedList individualCombinationList = new LinkedList<>(); + int count = size - 1; + int clonedI = i; + while (count >= 0) { + if ((clonedI & 1) != 0) { + individualCombinationList.addFirst(candidates.get(count)); + } + clonedI = clonedI >>> 1; + --count; + } + combinations.add(new UnionCandidate(individualCombinationList, cubeql)); + } + return combinations; + } + + private List> resolveJoinCandidates(List unionCandidates, + Set msrs, CubeQueryContext cubeql) throws LensException { + List> msrCoveringSets = new ArrayList<>(); + List ucSet = new ArrayList<>(unionCandidates); + // Check if a single set can answer all the measures and exprsWithMeasures + for (Iterator i = ucSet.iterator(); i.hasNext();) { + boolean evaluable = false; + Candidate uc = i.next(); + for (QueriedPhraseContext msr : msrs) { + evaluable = isMeasureAnswerablebyUnionCandidate(msr, uc, cubeql); + if (!evaluable) { + break; + } + } + if (evaluable) { + // single set can answer all the measures as an UnionCandidate + List one = new ArrayList<>(); + one.add(uc); + msrCoveringSets.add(one); + i.remove(); + } + } + // Sets that contain all measures or no measures are removed from iteration. + // find other facts + for (Iterator i = ucSet.iterator(); i.hasNext();) { + Candidate candidate = i.next(); + i.remove(); + // find the remaining measures in other facts + if (i.hasNext()) { + Set remainingMsrs = new HashSet<>(msrs); + Set coveredMsrs = CandidateUtil.coveredMeasures(candidate, msrs, cubeql); + remainingMsrs.removeAll(coveredMsrs); + + List> coveringSets = resolveJoinCandidates(ucSet, remainingMsrs, cubeql); + if (!coveringSets.isEmpty()) { + for (List candSet : coveringSets) { + candSet.add(candidate); + msrCoveringSets.add(candSet); + } + } else { + log.info("Couldnt find any set containing remaining measures:{} {} in {}", remainingMsrs, + ucSet); + } + } + } + log.info("Covering set {} for measures {} with factsPassed {}", msrCoveringSets, msrs, ucSet); + return msrCoveringSets; + } + + private void updateQueriableMeasures(List cands, + List qpcList, CubeQueryContext cubeql) throws LensException { + for (Candidate cand : cands) { + updateStorageCandidateQueriableMeasures(cand, qpcList, cubeql); + } + } + + + private void updateStorageCandidateQueriableMeasures(Candidate unionCandidate, + List qpcList, CubeQueryContext cubeql) throws LensException { + QueriedPhraseContext msrPhrase; + boolean isEvaluable; + for (int index = 0; index < qpcList.size(); index++) { + + if (!qpcList.get(index).hasMeasures(cubeql)) { + //Not a measure phrase. Skip it + continue; + } + + msrPhrase = qpcList.get(index); + if (unionCandidate instanceof StorageCandidate && msrPhrase.isEvaluable(cubeql, + (StorageCandidate) unionCandidate)) { + ((StorageCandidate) unionCandidate).setAnswerableMeasurePhraseIndices(index); + } else if (unionCandidate instanceof UnionCandidate) { + isEvaluable = true; + for (Candidate childCandidate : unionCandidate.getChildren()) { + if (!msrPhrase.isEvaluable(cubeql, (StorageCandidate) childCandidate)) { + isEvaluable = false; + break; + } + } + if (isEvaluable) { + //Set the index for all the children in this case + for (Candidate childCandidate : unionCandidate.getChildren()) { + ((StorageCandidate) childCandidate).setAnswerableMeasurePhraseIndices(index); + } + } + } + } + } +} diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateDim.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateDim.java index 4dcdbcf27..ce734cf33 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateDim.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateDim.java @@ -124,12 +124,7 @@ public Collection getColumns() { } @Override - public Set getStorageTables() { - return Collections.singleton(storageTable); - } - - @Override - public Set getPartsQueried() { + public Set getParticipatingPartitions() { if (StringUtils.isBlank(whereClause)) { return Collections.emptySet(); } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java deleted file mode 100644 index b42262ded..000000000 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java +++ /dev/null @@ -1,367 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.cube.parse; - -import static org.apache.hadoop.hive.ql.parse.HiveParser.*; - -import java.util.*; - -import org.apache.lens.cube.metadata.*; -import org.apache.lens.server.api.error.LensException; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.ql.lib.Node; -import org.apache.hadoop.hive.ql.parse.ASTNode; -import org.apache.hadoop.hive.ql.parse.HiveParser; -import org.apache.hadoop.hive.ql.session.SessionState; - -import org.antlr.runtime.CommonToken; - -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import lombok.Getter; -import lombok.Setter; - -/** - * Holds context of a candidate fact table. - */ -public class CandidateFact implements CandidateTable, QueryAST { - final CubeFactTable fact; - @Getter - private Set storageTables; - @Getter - private int numQueriedParts = 0; - @Getter - private final Set partsQueried = Sets.newHashSet(); - - private CubeInterface baseTable; - @Getter - @Setter - private ASTNode selectAST; - @Getter - @Setter - private ASTNode whereAST; - @Getter - @Setter - private ASTNode groupByAST; - @Getter - @Setter - private ASTNode havingAST; - @Getter - @Setter - private ASTNode joinAST; - @Getter - @Setter - private ASTNode orderByAST; - @Getter - @Setter - private Integer limitValue; - @Getter - private String fromString; - private final List selectIndices = Lists.newArrayList(); - private final List dimFieldIndices = Lists.newArrayList(); - private Collection columns; - @Getter - private final Map storgeWhereClauseMap = new HashMap<>(); - @Getter - private final Map storgeWhereStringMap = new HashMap<>(); - @Getter - private final Map>> rangeToStoragePartMap = new HashMap<>(); - @Getter - private final Map> rangeToStorageWhereMap = new HashMap<>(); - @Getter - @Setter - private Map> dataCompletenessMap; - - CandidateFact(CubeFactTable fact, CubeInterface cube) { - this.fact = fact; - this.baseTable = cube; - } - - @Override - public String toString() { - return fact.toString(); - } - - public Collection getColumns() { - if (columns == null) { - columns = fact.getValidColumns(); - if (columns == null) { - columns = fact.getAllFieldNames(); - } - } - return columns; - } - - public boolean isValidForTimeRange(TimeRange timeRange) { - return (!timeRange.getFromDate().before(fact.getStartTime())) && (!timeRange.getToDate().after(fact.getEndTime())); - } - - public void addToHaving(ASTNode ast) { - if (getHavingAST() == null) { - setHavingAST(new ASTNode(new CommonToken(TOK_HAVING, "TOK_HAVING"))); - getHavingAST().addChild(ast); - return; - } - ASTNode existingHavingAST = (ASTNode) getHavingAST().getChild(0); - ASTNode newHavingAST = new ASTNode(new CommonToken(KW_AND, "AND")); - newHavingAST.addChild(existingHavingAST); - newHavingAST.addChild(ast); - getHavingAST().setChild(0, newHavingAST); - } - - public String addAndGetAliasFromSelect(ASTNode ast, AliasDecider aliasDecider) { - for (Node n : getSelectAST().getChildren()) { - ASTNode astNode = (ASTNode) n; - if (HQLParser.equalsAST(ast, (ASTNode) astNode.getChild(0))) { - if (astNode.getChildCount() > 1) { - return astNode.getChild(1).getText(); - } - String alias = aliasDecider.decideAlias(astNode); - astNode.addChild(new ASTNode(new CommonToken(Identifier, alias))); - return alias; - } - } - // Not found, have to add to select - String alias = aliasDecider.decideAlias(ast); - ASTNode selectExprNode = new ASTNode(new CommonToken(TOK_SELEXPR)); - selectExprNode.addChild(ast); - selectExprNode.addChild(new ASTNode(new CommonToken(Identifier, alias))); - getSelectAST().addChild(selectExprNode); - return alias; - } - - void incrementPartsQueried(int incr) { - numQueriedParts += incr; - } - - // copy ASTs from CubeQueryContext - public void copyASTs(CubeQueryContext cubeql) throws LensException { - setSelectAST(MetastoreUtil.copyAST(cubeql.getSelectAST())); - setWhereAST(MetastoreUtil.copyAST(cubeql.getWhereAST())); - if (cubeql.getJoinAST() != null) { - setJoinAST(MetastoreUtil.copyAST(cubeql.getJoinAST())); - } - if (cubeql.getGroupByAST() != null) { - setGroupByAST(MetastoreUtil.copyAST(cubeql.getGroupByAST())); - } - } - - - public ASTNode getStorageWhereClause(String storageTable) { - return storgeWhereClauseMap.get(storageTable); - } - public String getStorageWhereString(String storageTable) { - return storgeWhereStringMap.get(storageTable); - } - - public boolean isExpressionAnswerable(ASTNode node, CubeQueryContext context) throws LensException { - return getColumns().containsAll(HQLParser.getColsInExpr(context.getAliasForTableName(context.getCube()), node)); - } - - /** - * Update the ASTs to include only the fields queried from this fact, in all the expressions - * - * @param cubeql - * @throws LensException - */ - public void updateASTs(CubeQueryContext cubeql) throws LensException { - // update select AST with selected fields - int currentChild = 0; - for (int i = 0; i < cubeql.getSelectAST().getChildCount(); i++) { - ASTNode selectExpr = (ASTNode) this.selectAST.getChild(currentChild); - Set exprCols = HQLParser.getColsInExpr(cubeql.getAliasForTableName(cubeql.getCube()), selectExpr); - if (getColumns().containsAll(exprCols)) { - selectIndices.add(i); - if (exprCols.isEmpty() // no direct fact columns - // does not have measure names - || (!containsAny(cubeql.getCube().getMeasureNames(), exprCols))) { - dimFieldIndices.add(i); - } - ASTNode aliasNode = HQLParser.findNodeByPath(selectExpr, Identifier); - String alias = cubeql.getSelectPhrases().get(i).getSelectAlias(); - if (aliasNode != null) { - String queryAlias = aliasNode.getText(); - if (!queryAlias.equals(alias)) { - // replace the alias node - ASTNode newAliasNode = new ASTNode(new CommonToken(HiveParser.Identifier, alias)); - this.selectAST.getChild(currentChild).replaceChildren(selectExpr.getChildCount() - 1, - selectExpr.getChildCount() - 1, newAliasNode); - } - } else { - // add column alias - ASTNode newAliasNode = new ASTNode(new CommonToken(HiveParser.Identifier, alias)); - this.selectAST.getChild(currentChild).addChild(newAliasNode); - } - } else { - this.selectAST.deleteChild(currentChild); - currentChild--; - } - currentChild++; - } - - // don't need to update where ast, since where is only on dim attributes and dim attributes - // are assumed to be common in multi fact queries. - - // push down of having clauses happens just after this call in cubequerycontext - } - - // The source set contains atleast one column in the colSet - static boolean containsAny(Collection srcSet, Collection colSet) { - if (colSet == null || colSet.isEmpty()) { - return true; - } - for (String column : colSet) { - if (srcSet.contains(column)) { - return true; - } - } - return false; - } - - @Override - public String getStorageString(String alias) { - return StringUtils.join(storageTables, ",") + " " + alias; - } - - public void setStorageTables(Set storageTables) { - String database = SessionState.get().getCurrentDatabase(); - // Add database name prefix for non default database - if (StringUtils.isNotBlank(database) && !"default".equalsIgnoreCase(database)) { - Set storageTbls = new TreeSet<>(); - Iterator names = storageTables.iterator(); - while (names.hasNext()) { - storageTbls.add(database + "." + names.next()); - } - this.storageTables = storageTbls; - } else { - this.storageTables = storageTables; - } - } - - @Override - public AbstractCubeTable getBaseTable() { - return (AbstractCubeTable) baseTable; - } - - @Override - public CubeFactTable getTable() { - return fact; - } - - @Override - public String getName() { - return fact.getName(); - } - - @Override - public boolean equals(Object obj) { - if (!super.equals(obj)) { - return false; - } - CandidateFact other = (CandidateFact) obj; - - if (this.getTable() == null) { - if (other.getTable() != null) { - return false; - } - } - return true; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = super.hashCode(); - result = prime * result + ((getTable() == null) ? 0 : getTable().getName().toLowerCase().hashCode()); - return result; - } - - public String getSelectString() { - return HQLParser.getString(selectAST); - } - - public String getWhereString() { - if (whereAST != null) { - return HQLParser.getString(whereAST); - } - return null; - } - - public String getHavingString() { - if (havingAST != null) { - return HQLParser.getString(havingAST); - } - return null; - } - - @Override - public String getOrderByString() { - if (orderByAST != null) { - return HQLParser.getString(orderByAST); - } - return null; - } - - /** - * @return the selectIndices - */ - public List getSelectIndices() { - return selectIndices; - } - - /** - * @return the groupbyIndices - */ - public List getDimFieldIndices() { - return dimFieldIndices; - } - - public String getGroupByString() { - if (groupByAST != null) { - return HQLParser.getString(groupByAST); - } - return null; - } - - public Set getTimePartCols(CubeQueryContext query) throws LensException { - Set cubeTimeDimensions = baseTable.getTimedDimensions(); - Set timePartDimensions = new HashSet(); - String singleStorageTable = storageTables.iterator().next(); - List partitionKeys = null; - partitionKeys = query.getMetastoreClient().getTable(singleStorageTable).getPartitionKeys(); - for (FieldSchema fs : partitionKeys) { - if (cubeTimeDimensions.contains(CubeQueryContext.getTimeDimOfPartitionColumn(baseTable, fs.getName()))) { - timePartDimensions.add(fs.getName()); - } - } - return timePartDimensions; - } - - public void updateFromString(CubeQueryContext query, Set queryDims, - Map dimsToQuery) throws LensException { - fromString = "%s"; // to update the storage alias later - if (query.isAutoJoinResolved()) { - fromString = - query.getAutoJoinCtx().getFromString(fromString, this, queryDims, dimsToQuery, - query, this); - } - } -} diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java index e001ca42d..c909545cc 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java @@ -37,10 +37,10 @@ public interface CandidateTable { String getStorageString(String alias); /** - * Get storage tables corresponding to this candidate + * Get storage table corresponding to this candidate * @return */ - Set getStorageTables(); + String getStorageTable(); /** * Get candidate table @@ -73,5 +73,5 @@ public interface CandidateTable { /** * Get partitions queried */ - Set getPartsQueried(); + Set getParticipatingPartitions(); } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java index bd6e27c2e..1c0d35684 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,12 +18,17 @@ */ package org.apache.lens.cube.parse; +import static java.util.stream.Collectors.toSet; + import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*; +import static com.google.common.collect.Lists.newArrayList; + import java.util.*; import org.apache.lens.cube.metadata.TimeRange; + import org.codehaus.jackson.annotate.JsonWriteNullProperties; import com.google.common.collect.Lists; @@ -43,43 +48,72 @@ public class CandidateTablePruneCause { public enum CandidateTablePruneCode { // other fact set element is removed ELEMENT_IN_SET_PRUNED("Other candidate from measure covering set is pruned"), - FACT_NOT_AVAILABLE_IN_RANGE("No facts available for all of these time ranges: %s") { - @Override - Object[] getFormatPlaceholders(Set causes) { - Set allRanges = Sets.newHashSet(); - for (CandidateTablePruneCause cause : causes) { - allRanges.addAll(cause.getInvalidRanges()); - } - return new Object[]{ - allRanges.toString(), - }; - } - }, // least weight not satisfied MORE_WEIGHT("Picked table had more weight than minimum."), // partial data is enabled, another fact has more data. LESS_DATA("Picked table has less data than the maximum"), // cube table has more partitions MORE_PARTITIONS("Picked table has more partitions than minimum"), + // storage is not supported by execution engine/driver + UNSUPPORTED_STORAGE("Unsupported Storage"), // invalid cube table INVALID("Invalid cube table provided in query"), // expression is not evaluable in the candidate - EXPRESSION_NOT_EVALUABLE("%s expressions not evaluable") { + COLUMN_NOT_FOUND("%s are not %s") { Object[] getFormatPlaceholders(Set causes) { - List columns = new ArrayList(); - for (CandidateTablePruneCause cause : causes) { - columns.addAll(cause.getMissingExpressions()); + if (causes.size() == 1) { + return new String[]{ + "Columns " + causes.iterator().next().getMissingColumns(), + "present in any table", + }; + } else { + return new String[]{ + "Column Sets: " + causes.stream().map(CandidateTablePruneCause::getMissingColumns).collect(toSet()), + "queriable together", + }; } - return new String[]{columns.toString()}; } }, // candidate table tries to get denormalized field from dimension and the // referred dimension is invalid. INVALID_DENORM_TABLE("Referred dimension is invalid in one of the candidate tables"), - // column not valid in cube table - COLUMN_NOT_VALID("Column not valid in cube table"), + + // Moved from Stoarge causes . + //The storage is removed as its not set in property "lens.cube.query.valid.fact..storagetables" + INVALID_STORAGE("Invalid Storage"), + // storage table does not exist. Commented as its not being used anywhere in master. + // STOARGE_TABLE_DOES_NOT_EXIST("Storage table does not exist"), + // storage has no update periods queried. Commented as its not being used anywhere in master. + // MISSING_UPDATE_PERIODS("Storage has no update periods"), + + // storage table has no partitions queried + NO_PARTITIONS("Storage table has no partitions"), + // partition column does not exist + PART_COL_DOES_NOT_EXIST("Partition column does not exist"), + // Range is not supported by this storage table + TIME_RANGE_NOT_ANSWERABLE("Range not answerable"), + STORAGE_NOT_AVAILABLE_IN_RANGE("No storages available for all of these time ranges: %s") { + @Override + Object[] getFormatPlaceholders(Set causes) { + return new Object[]{ + causes.stream().map(CandidateTablePruneCause::getInvalidRanges).flatMap(Collection::stream) + .collect(toSet()).toString(), + }; + } + }, + + EXPRESSION_NOT_EVALUABLE("%s expressions not evaluable") { + Object[] getFormatPlaceholders(Set causes) { + return new String[]{ + causes.stream().map(CandidateTablePruneCause::getMissingExpressions).flatMap(Collection::stream) + .collect(toSet()).toString(), + }; + } + }, + // column not valid in cube table. Commented the below line as it's not being used in master. + //COLUMN_NOT_VALID("Column not valid in cube table"), // column not found in cube table - COLUMN_NOT_FOUND("%s are not %s") { + DENORM_COLUMN_NOT_FOUND("%s are not %s") { Object[] getFormatPlaceholders(Set causes) { if (causes.size() == 1) { return new String[]{ @@ -87,12 +121,8 @@ Object[] getFormatPlaceholders(Set causes) { "present in any table", }; } else { - List> columnSets = new ArrayList>(); - for (CandidateTablePruneCause cause : causes) { - columnSets.add(cause.getMissingColumns()); - } return new String[]{ - "Column Sets: " + columnSets, + "Column Sets: " + causes.stream().map(CandidateTablePruneCause::getMissingColumns).collect(toSet()), "queriable together", }; } @@ -107,61 +137,54 @@ Object[] getFormatPlaceholders(Set causes) { TIMEDIM_NOT_SUPPORTED("Queried data not available for time dimensions: %s") { @Override Object[] getFormatPlaceholders(Set causes) { - Set dims = Sets.newHashSet(); - for(CandidateTablePruneCause cause: causes){ - dims.addAll(cause.getUnsupportedTimeDims()); - } return new Object[]{ - dims.toString(), + causes.stream().map(CandidateTablePruneCause::getUnsupportedTimeDims).flatMap(Collection::stream) + .collect(toSet()).toString(), }; } }, NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE("No fact update periods for given range"), + + // no candidate update periods, update period cause will have why each + // update period is not a candidate + NO_CANDIDATE_UPDATE_PERIODS("Storage update periods are not valid for given time range"), + NO_COLUMN_PART_OF_A_JOIN_PATH("No column part of a join path. Join columns: [%s]") { Object[] getFormatPlaceholders(Set causes) { - List columns = new ArrayList(); - for (CandidateTablePruneCause cause : causes) { - columns.addAll(cause.getJoinColumns()); - } - return new String[]{columns.toString()}; + return new String[]{ + causes.stream().map(CandidateTablePruneCause::getJoinColumns).flatMap(Collection::stream) + .collect(toSet()).toString(), + }; } }, // cube table is an aggregated fact and queried column is not under default // aggregate MISSING_DEFAULT_AGGREGATE("Columns: [%s] are missing default aggregate") { Object[] getFormatPlaceholders(Set causes) { - List columns = new ArrayList(); - for (CandidateTablePruneCause cause : causes) { - columns.addAll(cause.getColumnsMissingDefaultAggregate()); - } - return new String[]{columns.toString()}; + return new String[]{ + causes.stream().map(CandidateTablePruneCause::getColumnsMissingDefaultAggregate).flatMap(Collection::stream) + .collect(toSet()).toString(), + }; } }, // missing partitions for cube table MISSING_PARTITIONS("Missing partitions for the cube table: %s") { Object[] getFormatPlaceholders(Set causes) { - Set> missingPartitions = Sets.newHashSet(); - for (CandidateTablePruneCause cause : causes) { - missingPartitions.add(cause.getMissingPartitions()); - } - return new String[]{missingPartitions.toString()}; + return new String[]{ + causes.stream().map(CandidateTablePruneCause::getMissingPartitions).collect(toSet()).toString(), + }; } }, // incomplete data in the fact INCOMPLETE_PARTITION("Data for the requested metrics is only partially complete. Partially complete metrics are:" + " %s. Please try again later or rerun after removing incomplete metrics") { Object[] getFormatPlaceholders(Set causes) { - Set incompleteMetrics = Sets.newHashSet(); - for (CandidateTablePruneCause cause : causes) { - if (cause.getIncompletePartitions() != null) { - incompleteMetrics.addAll(cause.getIncompletePartitions().keySet()); - } - } - return new String[]{incompleteMetrics.toString()}; + return new String[]{ + causes.stream().map(CandidateTablePruneCause::getIncompletePartitions).collect(toSet()).toString(), + }; } }; - String errorFormat; CandidateTablePruneCode(String format) { @@ -181,67 +204,20 @@ String getBriefError(Set causes) { } } - public enum SkipStorageCode { - // invalid storage table - INVALID, - // storage table does not exist - TABLE_NOT_EXIST, - // storage has no update periods queried - MISSING_UPDATE_PERIODS, - // no candidate update periods, update period cause will have why each - // update period is not a candidate - NO_CANDIDATE_PERIODS, - // storage table has no partitions queried - NO_PARTITIONS, - // partition column does not exist - PART_COL_DOES_NOT_EXIST, - // Range is not supported by this storage table - RANGE_NOT_ANSWERABLE, - // storage is not supported by execution engine - UNSUPPORTED - } - public enum SkipUpdatePeriodCode { // invalid update period INVALID, - // Query max interval is more than update period - QUERY_INTERVAL_BIGGER + //this update period is greater than the Query max interval as provided by user with lens.cube.query.max.interval + UPDATE_PERIOD_BIGGER_THAN_MAX, + TIME_RANGE_NOT_ANSWERABLE_BY_UPDATE_PERIOD } - @JsonWriteNullProperties(false) - @Data - @NoArgsConstructor - public static class SkipStorageCause { - private SkipStorageCode cause; - // update period to skip cause - private Map updatePeriodRejectionCause; - private List nonExistantPartCols; - - public SkipStorageCause(SkipStorageCode cause) { - this.cause = cause; - } - - public static SkipStorageCause partColDoesNotExist(String... partCols) { - SkipStorageCause ret = new SkipStorageCause(SkipStorageCode.PART_COL_DOES_NOT_EXIST); - ret.nonExistantPartCols = new ArrayList(); - for (String s : partCols) { - ret.nonExistantPartCols.add(s); - } - return ret; - } - - public static SkipStorageCause noCandidateUpdatePeriod(Map causes) { - SkipStorageCause ret = new SkipStorageCause(SkipStorageCode.NO_CANDIDATE_PERIODS); - ret.updatePeriodRejectionCause = causes; - return ret; - } - } + // Used for Test cases only. + // storage to skip storage cause for dim table + private Map dimStoragePruningCauses; // cause for cube table private CandidateTablePruneCode cause; - // storage to skip storage cause - private Map storageCauses; - // populated only incase of missing partitions cause private Set missingPartitions; // populated only incase of incomplete partitions cause @@ -249,110 +225,129 @@ public static SkipStorageCause noCandidateUpdatePeriod(Map missingUpdatePeriods; // populated in case of missing columns - private List missingColumns; + private Set missingColumns; // populated in case of expressions not evaluable private List missingExpressions; // populated in case of no column part of a join path - private List joinColumns; + private Collection joinColumns; // the columns that are missing default aggregate. only set in case of MISSING_DEFAULT_AGGREGATE private List columnsMissingDefaultAggregate; // if a time dim is not supported by the fact. Would be set if and only if // the fact is not partitioned by part col of the time dim and time dim is not a dim attribute private Set unsupportedTimeDims; // time covered - private MaxCoveringFactResolver.TimeCovered maxTimeCovered; // ranges in which fact is invalid private List invalidRanges; + private List nonExistantPartCols; + + private Map updatePeriodRejectionCause; + + public CandidateTablePruneCause(CandidateTablePruneCode cause) { this.cause = cause; } // Different static constructors for different causes. - public static CandidateTablePruneCause factNotAvailableInRange(List ranges) { - CandidateTablePruneCause cause = new CandidateTablePruneCause(FACT_NOT_AVAILABLE_IN_RANGE); + static CandidateTablePruneCause storageNotAvailableInRange(List ranges) { + CandidateTablePruneCause cause = new CandidateTablePruneCause(STORAGE_NOT_AVAILABLE_IN_RANGE); cause.invalidRanges = ranges; return cause; } - public static CandidateTablePruneCause timeDimNotSupported(Set unsupportedTimeDims) { + static CandidateTablePruneCause timeDimNotSupported(Set unsupportedTimeDims) { CandidateTablePruneCause cause = new CandidateTablePruneCause(TIMEDIM_NOT_SUPPORTED); cause.unsupportedTimeDims = unsupportedTimeDims; return cause; } - public static CandidateTablePruneCause columnNotFound(Collection... missingColumns) { - List colList = new ArrayList(); - for (Collection missing : missingColumns) { - colList.addAll(missing); - } + static CandidateTablePruneCause columnNotFound(Collection missingColumns) { CandidateTablePruneCause cause = new CandidateTablePruneCause(COLUMN_NOT_FOUND); - cause.setMissingColumns(colList); + cause.setMissingColumns(Sets.newHashSet(missingColumns)); + return cause; + } + static CandidateTablePruneCause denormColumnNotFound(Collection missingColumns) { + CandidateTablePruneCause cause = new CandidateTablePruneCause(DENORM_COLUMN_NOT_FOUND); + cause.setMissingColumns(Sets.newHashSet(missingColumns)); return cause; } - public static CandidateTablePruneCause columnNotFound(String... columns) { - List colList = new ArrayList(); - for (String column : columns) { - colList.add(column); - } - return columnNotFound(colList); + static CandidateTablePruneCause columnNotFound(String... columns) { + return columnNotFound(newArrayList(columns)); } - public static CandidateTablePruneCause expressionNotEvaluable(String... exprs) { - List colList = new ArrayList(); - for (String column : exprs) { - colList.add(column); - } + static CandidateTablePruneCause expressionNotEvaluable(String... exprs) { CandidateTablePruneCause cause = new CandidateTablePruneCause(EXPRESSION_NOT_EVALUABLE); - cause.setMissingExpressions(colList); + cause.setMissingExpressions(newArrayList(exprs)); return cause; } - public static CandidateTablePruneCause missingPartitions(Set nonExistingParts) { + static CandidateTablePruneCause missingPartitions(Set nonExistingParts) { CandidateTablePruneCause cause = new CandidateTablePruneCause(MISSING_PARTITIONS); cause.setMissingPartitions(nonExistingParts); return cause; } - public static CandidateTablePruneCause incompletePartitions(Map> incompleteParts) { + static CandidateTablePruneCause incompletePartitions(Map> incompleteParts) { CandidateTablePruneCause cause = new CandidateTablePruneCause(INCOMPLETE_PARTITION); //incompleteParts may be null when partial data is allowed. cause.setIncompletePartitions(incompleteParts); return cause; } - public static CandidateTablePruneCause lessData(MaxCoveringFactResolver.TimeCovered timeCovered) { - CandidateTablePruneCause cause = new CandidateTablePruneCause(LESS_DATA); - cause.setMaxTimeCovered(timeCovered); - return cause; - } - public static CandidateTablePruneCause noColumnPartOfAJoinPath(final Collection colSet) { CandidateTablePruneCause cause = new CandidateTablePruneCause(NO_COLUMN_PART_OF_A_JOIN_PATH); - cause.setJoinColumns(new ArrayList() { - { - addAll(colSet); - } - }); + cause.setJoinColumns(colSet); + return cause; + } + + static CandidateTablePruneCause missingDefaultAggregate(String... names) { + CandidateTablePruneCause cause = new CandidateTablePruneCause(MISSING_DEFAULT_AGGREGATE); + cause.setColumnsMissingDefaultAggregate(newArrayList(names)); return cause; } - public static CandidateTablePruneCause noCandidateStorages(Map storageCauses) { + /** + * This factroy menthod can be used when a Dim Table is pruned because all its Storages are pruned. + * @param dimStoragePruningCauses + * @return + */ + static CandidateTablePruneCause noCandidateStoragesForDimtable( + Map dimStoragePruningCauses) { CandidateTablePruneCause cause = new CandidateTablePruneCause(NO_CANDIDATE_STORAGES); - cause.setStorageCauses(new HashMap()); - for (Map.Entry entry : storageCauses.entrySet()) { + cause.setDimStoragePruningCauses(new HashMap()); + for (Map.Entry entry : dimStoragePruningCauses.entrySet()) { String key = entry.getKey(); key = key.substring(0, (key.indexOf("_") + key.length() + 1) % (key.length() + 1)); // extract the storage part - cause.getStorageCauses().put(key.toLowerCase(), entry.getValue()); + cause.getDimStoragePruningCauses().put(key.toLowerCase(), entry.getValue()); } return cause; } - public static CandidateTablePruneCause missingDefaultAggregate(String... names) { - CandidateTablePruneCause cause = new CandidateTablePruneCause(MISSING_DEFAULT_AGGREGATE); - cause.setColumnsMissingDefaultAggregate(Lists.newArrayList(names)); + /** + * Queried partition columns are not present in this Storage Candidate + * @param missingPartitionColumns + * @return + */ + public static CandidateTablePruneCause partitionColumnsMissing(final String... missingPartitionColumns) { + return partitionColumnsMissing(Lists.newArrayList(missingPartitionColumns)); + } + public static CandidateTablePruneCause partitionColumnsMissing(final List missingPartitionColumns) { + CandidateTablePruneCause cause = new CandidateTablePruneCause(PART_COL_DOES_NOT_EXIST); + cause.nonExistantPartCols = missingPartitionColumns; + return cause; + } + + /** + * All update periods of this Stoarge Candidate are rejected. + * @param updatePeriodRejectionCause + * @return + */ + static CandidateTablePruneCause updatePeriodsRejected( + final Map updatePeriodRejectionCause) { + CandidateTablePruneCause cause = new CandidateTablePruneCause(NO_CANDIDATE_UPDATE_PERIODS); + cause.updatePeriodRejectionCause = updatePeriodRejectionCause; return cause; } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java index ed37bc587..6d61f1fc1 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java @@ -32,7 +32,6 @@ import org.apache.commons.lang.StringUtils; import com.google.common.collect.Sets; - import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -70,7 +69,8 @@ public void rewriteContext(CubeQueryContext cubeql) throws LensException { if (cubeql.getAutoJoinCtx() != null) { // Before checking for candidate table columns, prune join paths containing non existing columns // in populated candidate tables - cubeql.getAutoJoinCtx().pruneAllPaths(cubeql.getCube(), cubeql.getCandidateFacts(), null); + cubeql.getAutoJoinCtx().pruneAllPaths(cubeql.getCube(), + CandidateUtil.getStorageCandidates(cubeql.getCandidates()), null); cubeql.getAutoJoinCtx().pruneAllPathsForCandidateDims(cubeql.getCandidateDimTables()); cubeql.getAutoJoinCtx().refreshJoinPathColumns(); } @@ -78,7 +78,6 @@ public void rewriteContext(CubeQueryContext cubeql) throws LensException { // check for joined columns and denorm columns on refered tables resolveCandidateFactTablesForJoins(cubeql); resolveCandidateDimTablesForJoinsAndDenorms(cubeql); - cubeql.pruneCandidateFactSet(CandidateTablePruneCode.INVALID_DENORM_TABLE); checkForQueriedColumns = true; } } @@ -91,10 +90,16 @@ private void populateCandidateTables(CubeQueryContext cubeql) throws LensExcepti cubeql.getCube().getName() + " does not have any facts"); } for (CubeFactTable fact : factTables) { - CandidateFact cfact = new CandidateFact(fact, cubeql.getCube()); - cubeql.getCandidateFacts().add(cfact); + if (fact.getUpdatePeriods().isEmpty()) { + log.info("Not considering fact: {} as it has no update periods", fact.getName()); + } else { + for (String s : fact.getStorages()) { + StorageCandidate sc = new StorageCandidate(cubeql.getCube(), fact, s, cubeql); + cubeql.getCandidates().add(sc); + } + } } - log.info("Populated candidate facts: {}", cubeql.getCandidateFacts()); + log.info("Populated storage candidates: {}", cubeql.getCandidates()); } if (cubeql.getDimensions().size() != 0) { @@ -154,10 +159,10 @@ private void removeOptionalDim(CubeQueryContext cubeql, Aliased dim) OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().remove(dim); // remove all the depending candidate table as well for (CandidateTable candidate : optdim.requiredForCandidates) { - if (candidate instanceof CandidateFact) { - log.info("Not considering fact:{} as refered table does not have any valid dimtables", candidate); - cubeql.getCandidateFacts().remove(candidate); - cubeql.addFactPruningMsgs(((CandidateFact) candidate).fact, new CandidateTablePruneCause( + if (candidate instanceof StorageCandidate) { + log.info("Not considering storage candidate:{} as refered table does not have any valid dimtables", candidate); + cubeql.getCandidates().remove(candidate); + cubeql.addStoragePruningMsg(((StorageCandidate) candidate), new CandidateTablePruneCause( CandidateTablePruneCode.INVALID_DENORM_TABLE)); } else { log.info("Not considering dimtable:{} as refered table does not have any valid dimtables", candidate); @@ -172,20 +177,20 @@ private void removeOptionalDim(CubeQueryContext cubeql, Aliased dim) } } - public static boolean isColumnAvailableInRange(final TimeRange range, Date startTime, Date endTime) { + private static boolean isColumnAvailableInRange(final TimeRange range, Date startTime, Date endTime) { return (isColumnAvailableFrom(range.getFromDate(), startTime) && isColumnAvailableTill(range.getToDate(), endTime)); } - public static boolean isColumnAvailableFrom(@NonNull final Date date, Date startTime) { + private static boolean isColumnAvailableFrom(@NonNull final Date date, Date startTime) { return (startTime == null) ? true : date.equals(startTime) || date.after(startTime); } - public static boolean isColumnAvailableTill(@NonNull final Date date, Date endTime) { + private static boolean isColumnAvailableTill(@NonNull final Date date, Date endTime) { return (endTime == null) ? true : date.equals(endTime) || date.before(endTime); } - public static boolean isFactColumnValidForRange(CubeQueryContext cubeql, CandidateTable cfact, String col) { + private static boolean isFactColumnValidForRange(CubeQueryContext cubeql, CandidateTable cfact, String col) { for(TimeRange range : cubeql.getTimeRanges()) { if (!isColumnAvailableInRange(range, getFactColumnStartTime(cfact, col), getFactColumnEndTime(cfact, col))) { return false; @@ -194,14 +199,14 @@ public static boolean isFactColumnValidForRange(CubeQueryContext cubeql, Candida return true; } - public static Date getFactColumnStartTime(CandidateTable table, String factCol) { + private static Date getFactColumnStartTime(CandidateTable table, String factCol) { Date startTime = null; - if (table instanceof CandidateFact) { - for (String key : ((CandidateFact) table).fact.getProperties().keySet()) { + if (table instanceof StorageCandidate) { + for (String key : ((StorageCandidate) table).getFact().getProperties().keySet()) { if (key.contains(MetastoreConstants.FACT_COL_START_TIME_PFX)) { String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_START_TIME_PFX); if (factCol.equals(propCol)) { - startTime = ((CandidateFact) table).fact.getDateFromProperty(key, false, true); + startTime = ((StorageCandidate) table).getFact().getDateFromProperty(key, false, true); } } } @@ -209,14 +214,14 @@ public static Date getFactColumnStartTime(CandidateTable table, String factCol) return startTime; } - public static Date getFactColumnEndTime(CandidateTable table, String factCol) { + private static Date getFactColumnEndTime(CandidateTable table, String factCol) { Date endTime = null; - if (table instanceof CandidateFact) { - for (String key : ((CandidateFact) table).fact.getProperties().keySet()) { + if (table instanceof StorageCandidate) { + for (String key : ((StorageCandidate) table).getFact().getProperties().keySet()) { if (key.contains(MetastoreConstants.FACT_COL_END_TIME_PFX)) { String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_END_TIME_PFX); if (factCol.equals(propCol)) { - endTime = ((CandidateFact) table).fact.getDateFromProperty(key, false, true); + endTime = ((StorageCandidate) table).getFact().getDateFromProperty(key, false, true); } } } @@ -228,7 +233,7 @@ private void resolveCandidateFactTables(CubeQueryContext cubeql) throws LensExce if (cubeql.getCube() != null) { String str = cubeql.getConf().get(CubeQueryConfUtil.getValidFactTablesKey(cubeql.getCube().getName())); List validFactTables = - StringUtils.isBlank(str) ? null : Arrays.asList(StringUtils.split(str.toLowerCase(), ",")); + StringUtils.isBlank(str) ? null : Arrays.asList(StringUtils.split(str.toLowerCase(), ",")); Set queriedMsrs = new HashSet<>(); Set dimExprs = new HashSet<>(); @@ -239,101 +244,79 @@ private void resolveCandidateFactTables(CubeQueryContext cubeql) throws LensExce dimExprs.add(qur); } } - // Remove fact tables based on whether they are valid or not. - for (Iterator i = cubeql.getCandidateFacts().iterator(); i.hasNext();) { - CandidateFact cfact = i.next(); - - if (validFactTables != null) { - if (!validFactTables.contains(cfact.getName().toLowerCase())) { - log.info("Not considering fact table:{} as it is not a valid fact", cfact); - cubeql - .addFactPruningMsgs(cfact.fact, new CandidateTablePruneCause(CandidateTablePruneCode.INVALID)); - i.remove(); - continue; + // Remove storage candidates based on whether they are valid or not. + for (Iterator i = cubeql.getCandidates().iterator(); i.hasNext();) { + Candidate cand = i.next(); + if (cand instanceof StorageCandidate) { + StorageCandidate sc = (StorageCandidate) cand; + if (validFactTables != null) { + if (!validFactTables.contains(sc.getFact().getName().toLowerCase())) { + log.info("Not considering storage candidate:{} as it is not a valid candidate", sc); + cubeql.addStoragePruningMsg(sc, new CandidateTablePruneCause(CandidateTablePruneCode.INVALID)); + i.remove(); + continue; + } } - } - - // update expression evaluability for this fact - for (String expr : cubeql.getQueriedExprs()) { - cubeql.getExprCtx().updateEvaluables(expr, cfact); - } - // go over the columns accessed in the query and find out which tables - // can answer the query - // the candidate facts should have all the dimensions queried and - // atleast - // one measure - boolean toRemove = false; - for (QueriedPhraseContext qur : dimExprs) { - if (!qur.isEvaluable(cubeql, cfact)) { - log.info("Not considering fact table:{} as columns {} are not available", cfact, qur.getColumns()); - cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(qur.getColumns())); - toRemove = true; - break; + // update expression evaluability for this fact + for (String expr : cubeql.getQueriedExprs()) { + cubeql.getExprCtx().updateEvaluables(expr, sc); } - } - // check if the candidate fact has atleast one measure queried - // if expression has measures, they should be considered along with other measures and see if the fact can be - // part of measure covering set - if (!checkForFactColumnExistsAndValidForRange(cfact, queriedMsrs, cubeql)) { - Set columns = getColumns(queriedMsrs); - - log.info("Not considering fact table:{} as columns {} is not available", cfact, columns); - cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(columns)); - toRemove = true; - } - // go over join chains and prune facts that dont have any of the columns in each chain - for (JoinChain chain : cubeql.getJoinchains().values()) { - OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(Aliased.create((Dimension)cubeql.getCubeTbls() - .get(chain.getName()), chain.getName())); - if (!checkForFactColumnExistsAndValidForRange(cfact, chain.getSourceColumns(), cubeql)) { - // check if chain is optional or not - if (optdim == null) { - log.info("Not considering fact table:{} as columns {} are not available", cfact, - chain.getSourceColumns()); - cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(chain.getSourceColumns())); + // go over the columns accessed in the query and find out which tables + // can answer the query + // the candidate facts should have all the dimensions queried and + // atleast + // one measure + boolean toRemove = false; + for (QueriedPhraseContext qur : dimExprs) { + if (!qur.isEvaluable(cubeql, sc)) { + log.info("Not considering storage candidate:{} as columns {} are not available", sc, qur.getColumns()); + cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.columnNotFound( + qur.getColumns())); toRemove = true; break; } } - } - if (toRemove) { - i.remove(); + // check if the candidate fact has atleast one measure queried + // if expression has measures, they should be considered along with other measures and see if the fact can be + // part of measure covering set + if (!checkForFactColumnExistsAndValidForRange(sc, queriedMsrs, cubeql)) { + Set columns = getColumns(queriedMsrs); + log.info("Not considering storage candidate:{} as columns {} is not available", sc, columns); + cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.columnNotFound( + columns)); + toRemove = true; + } + + // go over join chains and prune facts that dont have any of the columns in each chain + for (JoinChain chain : cubeql.getJoinchains().values()) { + OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(Aliased.create((Dimension) cubeql.getCubeTbls() + .get(chain.getName()), chain.getName())); + if (!checkForFactColumnExistsAndValidForRange(sc, chain.getSourceColumns(), cubeql)) { + // check if chain is optional or not + if (optdim == null) { + log.info("Not considering storage candidate:{} as columns {} are not available", sc, + chain.getSourceColumns()); + cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.columnNotFound( + chain.getSourceColumns())); + toRemove = true; + break; + } + } + } + + if (toRemove) { + i.remove(); + } + } else { + throw new LensException("Not a storage candidate!!"); } } - if (cubeql.getCandidateFacts().size() == 0) { + if (cubeql.getCandidates().size() == 0) { throw new LensException(LensCubeErrorCode.NO_FACT_HAS_COLUMN.getLensErrorInfo(), - getColumns(cubeql.getQueriedPhrases()).toString()); - } - Set> cfactset; - if (queriedMsrs.isEmpty()) { - // if no measures are queried, add all facts individually as single covering sets - cfactset = new HashSet<>(); - for (CandidateFact cfact : cubeql.getCandidateFacts()) { - Set one = new LinkedHashSet<>(); - one.add(cfact); - cfactset.add(one); - } - cubeql.getCandidateFactSets().addAll(cfactset); - } else { - // Find out candidate fact table sets which contain all the measures - // queried - - List cfacts = new ArrayList<>(cubeql.getCandidateFacts()); - cfactset = findCoveringSets(cubeql, cfacts, queriedMsrs); - log.info("Measure covering fact sets :{}", cfactset); - String msrString = getColumns(queriedMsrs).toString(); - if (cfactset.isEmpty()) { - throw new LensException(LensCubeErrorCode.NO_FACT_HAS_COLUMN.getLensErrorInfo(), msrString); - } - cubeql.getCandidateFactSets().addAll(cfactset); - cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCause.columnNotFound(getColumns(queriedMsrs))); - - if (cubeql.getCandidateFacts().size() == 0) { - throw new LensException(LensCubeErrorCode.NO_FACT_HAS_COLUMN.getLensErrorInfo(), msrString); - } + getColumns(cubeql.getQueriedPhrases()).toString()); } } } @@ -345,51 +328,6 @@ private static Set getColumns(Collection queriedPh } return cols; } - static Set> findCoveringSets(CubeQueryContext cubeql, List cfactsPassed, - Set msrs) throws LensException { - Set> cfactset = new HashSet<>(); - List cfacts = new ArrayList<>(cfactsPassed); - for (Iterator i = cfacts.iterator(); i.hasNext();) { - CandidateFact cfact = i.next(); - if (!checkForFactColumnExistsAndValidForRange(cfact, msrs, cubeql)) { - // cfact does not contain any of msrs and none of exprsWithMeasures are evaluable. - // ignore the fact - i.remove(); - continue; - } else if (allEvaluable(cfact, msrs, cubeql)) { - // return single set - Set one = new LinkedHashSet<>(); - one.add(cfact); - cfactset.add(one); - i.remove(); - } - } - // facts that contain all measures or no measures are removed from iteration. - // find other facts - for (Iterator i = cfacts.iterator(); i.hasNext();) { - CandidateFact cfact = i.next(); - i.remove(); - // find the remaining measures in other facts - if (i.hasNext()) { - Set remainingMsrs = new HashSet<>(msrs); - Set coveredMsrs = coveredMeasures(cfact, msrs, cubeql); - remainingMsrs.removeAll(coveredMsrs); - - Set> coveringSets = findCoveringSets(cubeql, cfacts, remainingMsrs); - if (!coveringSets.isEmpty()) { - for (Set set : coveringSets) { - set.add(cfact); - cfactset.add(set); - } - } else { - log.info("Couldnt find any set containing remaining measures:{} {} in {}", remainingMsrs, - cfactsPassed); - } - } - } - log.info("Covering set {} for measures {} with factsPassed {}", cfactset, msrs, cfactsPassed); - return cfactset; - } private void resolveCandidateDimTablesForJoinsAndDenorms(CubeQueryContext cubeql) throws LensException { if (cubeql.getAutoJoinCtx() == null) { @@ -484,11 +422,10 @@ private void resolveCandidateFactTablesForJoins(CubeQueryContext cubeql) throws return; } Collection colSet = null; - if (cubeql.getCube() != null && !cubeql.getCandidateFacts().isEmpty()) { - for (Iterator i = cubeql.getCandidateFacts().iterator(); i.hasNext();) { - CandidateFact cfact = i.next(); - CubeFactTable fact = cfact.fact; - + if (cubeql.getCube() != null && !cubeql.getCandidates().isEmpty()) { + for (Iterator i = + CandidateUtil.getStorageCandidates(cubeql.getCandidates()).iterator(); i.hasNext();) { + StorageCandidate sc = i.next(); // for each join path check for columns involved in path for (Map.Entry, Map>> joincolumnsEntry : cubeql .getAutoJoinCtx() @@ -497,19 +434,19 @@ private void resolveCandidateFactTablesForJoins(CubeQueryContext cubeql) throws OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(reachableDim); colSet = joincolumnsEntry.getValue().get(cubeql.getCube()); - if (!checkForFactColumnExistsAndValidForRange(cfact, colSet, cubeql)) { + if (!checkForFactColumnExistsAndValidForRange(sc, colSet, cubeql)) { if (optdim == null || optdim.isRequiredInJoinChain - || (optdim != null && optdim.requiredForCandidates.contains(cfact))) { + || (optdim != null && optdim.requiredForCandidates.contains(sc))) { i.remove(); - log.info("Not considering fact table:{} as it does not have columns in any of the join paths." - + " Join columns:{}", fact, colSet); - cubeql.addFactPruningMsgs(fact, CandidateTablePruneCause.noColumnPartOfAJoinPath(colSet)); + log.info("Not considering storage candidate :{} as it does not have columns in any of the join paths." + + " Join columns:{}", sc, colSet); + cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.noColumnPartOfAJoinPath(colSet)); break; } } } } - if (cubeql.getCandidateFacts().size() == 0) { + if (cubeql.getCandidates().size() == 0) { throw new LensException(LensCubeErrorCode.NO_FACT_HAS_COLUMN.getLensErrorInfo(), colSet == null ? "NULL" : colSet.toString()); } @@ -586,12 +523,16 @@ private void checkForSourceReachabilityForDenormCandidates(CubeQueryContext cube if (removedCandidates.get(dim) != null) { for (CandidateTable candidate : removedCandidates.get(dim)) { if (!candidatesReachableThroughRefs.contains(candidate)) { - if (candidate instanceof CandidateFact) { - if (cubeql.getCandidateFacts().contains(candidate)) { - log.info("Not considering fact:{} as its required optional dims are not reachable", candidate); - cubeql.getCandidateFacts().remove(candidate); - cubeql.addFactPruningMsgs(((CandidateFact) candidate).fact, - CandidateTablePruneCause.columnNotFound(col)); + if (candidate instanceof StorageCandidate) { + if (cubeql.getCandidates().contains(candidate)) { + log.info("Not considering Storage:{} as its required optional dims are not reachable", candidate); + cubeql.getCandidates().remove(candidate); + cubeql.addStoragePruningMsg((StorageCandidate) candidate, + CandidateTablePruneCause.columnNotFound(col)); + Collection prunedCandidates = CandidateUtil. + filterCandidates(cubeql.getCandidates(), (StorageCandidate) candidate); + cubeql.addCandidatePruningMsg(prunedCandidates, + new CandidateTablePruneCause(CandidateTablePruneCode.ELEMENT_IN_SET_PRUNED)); } } else if (cubeql.getCandidateDimTables().containsKey(((CandidateDim) candidate).getBaseTable())) { log.info("Not considering dimtable:{} as its required optional dims are not reachable", candidate); @@ -639,11 +580,11 @@ private void checkForSourceReachabilityForDenormCandidates(CubeQueryContext cube // candidate has other evaluable expressions continue; } - if (candidate instanceof CandidateFact) { - if (cubeql.getCandidateFacts().contains(candidate)) { + if (candidate instanceof StorageCandidate) { + if (cubeql.getCandidates().contains(candidate)) { log.info("Not considering fact:{} as is not reachable through any optional dim", candidate); - cubeql.getCandidateFacts().remove(candidate); - cubeql.addFactPruningMsgs(((CandidateFact) candidate).fact, + cubeql.getCandidates().remove(candidate); + cubeql.addStoragePruningMsg(((StorageCandidate) candidate), CandidateTablePruneCause.expressionNotEvaluable(col.getExprCol())); } } else if (cubeql.getCandidateDimTables().containsKey(((CandidateDim) candidate).getBaseTable())) { @@ -697,7 +638,8 @@ private void resolveCandidateDimTables(CubeQueryContext cubeql) throws LensExcep // check if it available as reference, if not remove the // candidate log.info("Not considering dimtable: {} as column {} is not available", cdim, col); - cubeql.addDimPruningMsgs(dim, cdim.getTable(), CandidateTablePruneCause.columnNotFound(col)); + cubeql.addDimPruningMsgs(dim, cdim.getTable(), CandidateTablePruneCause.columnNotFound( + col)); i.remove(); break; } @@ -716,7 +658,7 @@ private void resolveCandidateDimTables(CubeQueryContext cubeql) throws LensExcep // The candidate table contains atleast one column in the colSet and // column can the queried in the range specified - static boolean checkForFactColumnExistsAndValidForRange(CandidateTable table, Collection colSet, + private static boolean checkForFactColumnExistsAndValidForRange(CandidateTable table, Collection colSet, CubeQueryContext cubeql) { if (colSet == null || colSet.isEmpty()) { return true; @@ -729,40 +671,18 @@ static boolean checkForFactColumnExistsAndValidForRange(CandidateTable table, Co return false; } - static boolean checkForFactColumnExistsAndValidForRange(CandidateFact table, Collection colSet, - CubeQueryContext cubeql) throws LensException { + + private static boolean checkForFactColumnExistsAndValidForRange(StorageCandidate sc, + Collection colSet, + CubeQueryContext cubeql) throws LensException { if (colSet == null || colSet.isEmpty()) { return true; } for (QueriedPhraseContext qur : colSet) { - if (qur.isEvaluable(cubeql, table)) { + if (qur.isEvaluable(cubeql, sc)) { return true; } } return false; } - - static boolean allEvaluable(CandidateFact table, Collection colSet, - CubeQueryContext cubeql) throws LensException { - if (colSet == null || colSet.isEmpty()) { - return true; - } - for (QueriedPhraseContext qur : colSet) { - if (!qur.isEvaluable(cubeql, table)) { - return false; - } - } - return true; - } - - static Set coveredMeasures(CandidateFact table, Collection msrs, - CubeQueryContext cubeql) throws LensException { - Set coveringSet = new HashSet<>(); - for (QueriedPhraseContext msr : msrs) { - if (msr.isEvaluable(cubeql, table)) { - coveringSet.add(msr); - } - } - return coveringSet; - } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java new file mode 100644 index 000000000..b9ff0eff2 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java @@ -0,0 +1,319 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for theJoinCandidate.java + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + +import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier; + +import java.util.*; + +import org.apache.lens.cube.metadata.*; +import org.apache.lens.server.api.error.LensException; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.HiveParser; + +import org.antlr.runtime.CommonToken; + +import com.google.common.collect.BoundType; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; +import com.google.common.collect.TreeRangeSet; + +/** + * Placeholder for Util methods that will be required for {@link Candidate} + */ +public final class CandidateUtil { + + private CandidateUtil() { + // Added due to checkstyle error getting below : + // (design) HideUtilityClassConstructor: Utility classes should not have a public or default constructor. + } + + /** + * Returns true if the Candidate is valid for all the timeranges based on its start and end times. + * @param candidate + * @param timeRanges + * @return + */ + public static boolean isValidForTimeRanges(Candidate candidate, List timeRanges) { + for (TimeRange timeRange : timeRanges) { + if (!(timeRange.getFromDate().after(candidate.getStartTime()) + && timeRange.getToDate().before(candidate.getEndTime()))) { + return false; + } + } + return true; + } + + static boolean isCandidatePartiallyValidForTimeRange(Date candidateStartTime, Date candidateEndTime, + Date timeRangeStart, Date timeRangeEnd) { + Date start = candidateStartTime.after(timeRangeStart) ? candidateStartTime : timeRangeStart; + Date end = candidateEndTime.before(timeRangeEnd) ? candidateEndTime : timeRangeEnd; + if (end.after(start)) { + return true; + } + return false; + } + + + static boolean isPartiallyValidForTimeRange(Candidate cand, TimeRange timeRange) { + return isPartiallyValidForTimeRanges(cand, Arrays.asList(timeRange)); + } + + static boolean isPartiallyValidForTimeRanges(Candidate cand, List timeRanges) { + return timeRanges.stream().anyMatch(timeRange -> + isCandidatePartiallyValidForTimeRange(cand.getStartTime(), cand.getEndTime(), + timeRange.getFromDate(), timeRange.getToDate())); + } + + /** + * Copy Query AST from sourceAst to targetAst + * + * @param sourceAst + * @param targetAst + * @throws LensException + */ + static void copyASTs(QueryAST sourceAst, QueryAST targetAst) throws LensException { + + targetAst.setSelectAST(MetastoreUtil.copyAST(sourceAst.getSelectAST())); + targetAst.setWhereAST(MetastoreUtil.copyAST(sourceAst.getWhereAST())); + if (sourceAst.getJoinAST() != null) { + targetAst.setJoinAST(MetastoreUtil.copyAST(sourceAst.getJoinAST())); + } + if (sourceAst.getGroupByAST() != null) { + targetAst.setGroupByAST(MetastoreUtil.copyAST(sourceAst.getGroupByAST())); + } + if (sourceAst.getHavingAST() != null) { + targetAst.setHavingAST(MetastoreUtil.copyAST(sourceAst.getHavingAST())); + } + if (sourceAst.getOrderByAST() != null) { + targetAst.setOrderByAST(MetastoreUtil.copyAST(sourceAst.getOrderByAST())); + } + + targetAst.setLimitValue(sourceAst.getLimitValue()); + targetAst.setFromString(sourceAst.getFromString()); + targetAst.setWhereString(sourceAst.getWhereString()); + } + + public static Set getStorageCandidates(final Candidate candidate) { + return getStorageCandidates(new HashSet(1) {{ add(candidate); }}); + } + + // this function should only be used for union candidates and never for join candidates. + // future scope of improvement: move the data model to use polymorphism + static Set coveredMeasures(Candidate candSet, Collection msrs, + CubeQueryContext cubeql) throws LensException { + Set coveringSet = new HashSet<>(); + for (QueriedPhraseContext msr : msrs) { + if (candSet.getChildren() == null) { + if (msr.isEvaluable(cubeql, (StorageCandidate) candSet)) { + coveringSet.add(msr); + } + } else { + boolean allCanAnswer = true; + for (Candidate cand : candSet.getChildren()) { + if (!msr.isEvaluable(cubeql, (StorageCandidate) cand)) { + allCanAnswer = false; + break; + } + } + if (allCanAnswer) { + coveringSet.add(msr); + } + } + } + return coveringSet; + } + + /** + * Returns true is the Candidates cover the entire time range. + * @param candidates + * @param startTime + * @param endTime + * @return + */ + public static boolean isTimeRangeCovered(Collection candidates, Date startTime, Date endTime) { + RangeSet set = TreeRangeSet.create(); + for (Candidate candidate : candidates) { + set.add(Range.range(candidate.getStartTime(), BoundType.CLOSED, candidate.getEndTime(), BoundType.OPEN)); + } + return set.encloses(Range.range(startTime, BoundType.CLOSED, endTime, BoundType.OPEN)); + } + + public static Set getColumns(Collection queriedPhraseContexts) { + Set cols = new HashSet<>(); + for (QueriedPhraseContext qur : queriedPhraseContexts) { + cols.addAll(qur.getColumns()); + } + return cols; + } + + /** + * Filters Candidates that contain the filterCandidate + * + * @param candidates + * @param filterCandidate + * @return pruned Candidates + */ + public static Collection filterCandidates(Collection candidates, Candidate filterCandidate) { + List prunedCandidates = new ArrayList<>(); + Iterator itr = candidates.iterator(); + while (itr.hasNext()) { + if (itr.next().contains(filterCandidate)) { + prunedCandidates.add(itr.next()); + itr.remove(); + } + } + return prunedCandidates; + } + + /** + * Gets all the Storage Candidates that participate in the collection of passed candidates + * + * @param candidates + * @return + */ + public static Set getStorageCandidates(Collection candidates) { + Set storageCandidateSet = new HashSet<>(); + getStorageCandidates(candidates, storageCandidateSet); + return storageCandidateSet; + } + + private static void getStorageCandidates(Collection candidates, + Set storageCandidateSet) { + for (Candidate candidate : candidates) { + if (candidate.getChildren() == null) { + //Expecting this to be a StorageCandidate as it has no children. + storageCandidateSet.add((StorageCandidate)candidate); + } else { + getStorageCandidates(candidate.getChildren(), storageCandidateSet); + } + } + } + + public static StorageCandidate cloneStorageCandidate(StorageCandidate sc) throws LensException{ + return new StorageCandidate(sc); + } + + public static boolean factHasColumn(CubeFactTable fact, String column) { + for (FieldSchema factField : fact.getColumns()) { + if (factField.getName().equals(column)) { + return true; + } + } + return false; + } + + public static String getTimeRangeWhereClasue(TimeRangeWriter rangeWriter, + StorageCandidate sc, TimeRange range) throws LensException { + String rangeWhere = rangeWriter.getTimeRangeWhereClause(sc.getCubeql(), + sc.getCubeql().getAliasForTableName(sc.getCube().getName()), + sc.getRangeToPartitions().get(range)); + if (sc.getRangeToExtraWhereFallBack().containsKey(range)) { + rangeWhere = "((" + rangeWhere + ") and (" + sc.getRangeToExtraWhereFallBack().get(range) + "))"; + } + return rangeWhere; + } + + public static class ChildrenSizeBasedCandidateComparator implements Comparator { + @Override + public int compare(Candidate o1, Candidate o2) { + return o1.getChildren().size() - o2.getChildren().size(); + } + } + + private static final String BASE_QUERY_FORMAT = "SELECT %s FROM %s"; + + public static String buildHQLString(String select, String from, String where, + String groupby, String orderby, String having, Integer limit) { + List qstrs = new ArrayList(); + qstrs.add(select); + qstrs.add(from); + if (!StringUtils.isBlank(where)) { + qstrs.add(where); + } + if (!StringUtils.isBlank(groupby)) { + qstrs.add(groupby); + } + if (!StringUtils.isBlank(having)) { + qstrs.add(having); + } + if (!StringUtils.isBlank(orderby)) { + qstrs.add(orderby); + } + if (limit != null) { + qstrs.add(String.valueOf(limit)); + } + + StringBuilder queryFormat = new StringBuilder(); + queryFormat.append(BASE_QUERY_FORMAT); + if (!StringUtils.isBlank(where)) { + queryFormat.append(" WHERE %s"); + } + if (!StringUtils.isBlank(groupby)) { + queryFormat.append(" GROUP BY %s"); + } + if (!StringUtils.isBlank(having)) { + queryFormat.append(" HAVING %s"); + } + if (!StringUtils.isBlank(orderby)) { + queryFormat.append(" ORDER BY %s"); + } + if (limit != null) { + queryFormat.append(" LIMIT %s"); + } + return String.format(queryFormat.toString(), qstrs.toArray(new String[qstrs.size()])); + } + + /** + * + * @param selectAST Outer query selectAST + * @param cubeql Cubequery Context + * + * Update the final alias in the outer select expressions + * 1. Replace queriedAlias with finalAlias if both are not same + * 2. If queriedAlias is missing add finalAlias as alias + */ + public static void updateFinalAlias(ASTNode selectAST, CubeQueryContext cubeql) { + for (int i = 0; i < selectAST.getChildCount(); i++) { + ASTNode selectExpr = (ASTNode) selectAST.getChild(i); + ASTNode aliasNode = HQLParser.findNodeByPath(selectExpr, Identifier); + String finalAlias = cubeql.getSelectPhrases().get(i).getFinalAlias().replaceAll("`", ""); + if (aliasNode != null) { + String queryAlias = aliasNode.getText(); + if (!queryAlias.equals(finalAlias)) { + // replace the alias node + ASTNode newAliasNode = new ASTNode(new CommonToken(HiveParser.Identifier, finalAlias)); + selectAST.getChild(i).replaceChildren(selectExpr.getChildCount() - 1, + selectExpr.getChildCount() - 1, newAliasNode); + } + } else { + // add column alias + ASTNode newAliasNode = new ASTNode(new CommonToken(HiveParser.Identifier, finalAlias)); + selectAST.getChild(i).addChild(newAliasNode); + } + } + } + + + + +} diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CheckTableNames.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CheckTableNames.java index 858626241..df35a429f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CheckTableNames.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CheckTableNames.java @@ -30,7 +30,6 @@ public CheckTableNames(Configuration conf) { @Override public boolean validate(CubeQueryContext ctx) throws LensException { - // TODO return true; } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnLifetimeChecker.java similarity index 52% rename from lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java rename to lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnLifetimeChecker.java index f18ae3679..c3d12a4e8 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnLifetimeChecker.java @@ -28,113 +28,18 @@ import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.metadata.join.JoinPath; import org.apache.lens.cube.parse.join.AutoJoinContext; -import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.error.LensException; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.ql.parse.ASTNode; -import org.apache.hadoop.hive.ql.plan.PlanUtils; - -import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; @Slf4j -public class TimeRangeChecker implements ContextRewriter { - public TimeRangeChecker(Configuration conf) { - } +public class ColumnLifetimeChecker implements ContextRewriter { @Override public void rewriteContext(CubeQueryContext cubeql) throws LensException { if (cubeql.getCube() == null) { return; } doColLifeValidation(cubeql); - doFactRangeValidation(cubeql); - } - private void extractTimeRange(CubeQueryContext cubeql) throws LensException { - // get time range - - // Time range should be direct child of where condition - // TOK_WHERE.TOK_FUNCTION.Identifier Or, it should be right hand child of - // AND condition TOK_WHERE.KW_AND.TOK_FUNCTION.Identifier - if (cubeql.getWhereAST() == null || cubeql.getWhereAST().getChildCount() < 1) { - throw new LensException(LensCubeErrorCode.NO_TIMERANGE_FILTER.getLensErrorInfo()); - } - searchTimeRanges(cubeql.getWhereAST(), cubeql, null, 0); - } - - private void searchTimeRanges(ASTNode root, CubeQueryContext cubeql, ASTNode parent, int childIndex) - throws LensException { - if (root == null) { - return; - } else if (root.getToken().getType() == TOK_FUNCTION) { - ASTNode fname = HQLParser.findNodeByPath(root, Identifier); - if (fname != null && CubeQueryContext.TIME_RANGE_FUNC.equalsIgnoreCase(fname.getText())) { - processTimeRangeFunction(cubeql, root, parent, childIndex); - } - } else { - for (int i = 0; i < root.getChildCount(); i++) { - ASTNode child = (ASTNode) root.getChild(i); - searchTimeRanges(child, cubeql, root, i); - } - } - } - - private String getColumnName(ASTNode node) { - String column = null; - if (node.getToken().getType() == DOT) { - ASTNode colIdent = (ASTNode) node.getChild(1); - column = colIdent.getText().toLowerCase(); - } else if (node.getToken().getType() == TOK_TABLE_OR_COL) { - // Take child ident.totext - ASTNode ident = (ASTNode) node.getChild(0); - column = ident.getText().toLowerCase(); - } - return column; - } - - private void processTimeRangeFunction(CubeQueryContext cubeql, ASTNode timenode, ASTNode parent, int childIndex) - throws LensException { - TimeRange.TimeRangeBuilder builder = TimeRange.getBuilder(); - builder.astNode(timenode); - builder.parent(parent); - builder.childIndex(childIndex); - - String timeDimName = getColumnName((ASTNode) timenode.getChild(1)); - - if (!cubeql.getCube().getTimedDimensions().contains(timeDimName)) { - throw new LensException(LensCubeErrorCode.NOT_A_TIMED_DIMENSION.getLensErrorInfo(), timeDimName); - } - // Replace timeDimName with column which is used for partitioning. Assume - // the same column - // is used as a partition column in all storages of the fact - timeDimName = cubeql.getPartitionColumnOfTimeDim(timeDimName); - builder.partitionColumn(timeDimName); - - String fromDateRaw = PlanUtils.stripQuotes(timenode.getChild(2).getText()); - String toDateRaw = null; - if (timenode.getChildCount() > 3) { - ASTNode toDateNode = (ASTNode) timenode.getChild(3); - if (toDateNode != null) { - toDateRaw = PlanUtils.stripQuotes(timenode.getChild(3).getText()); - } - } - long currentTime = cubeql.getConf().getLong(LensConfConstants.QUERY_CURRENT_TIME_IN_MILLIS, 0); - Date now; - if (currentTime != 0) { - now = new Date(currentTime); - } else { - now = new Date(); - } - builder.fromDate(DateUtil.resolveDate(fromDateRaw, now)); - if (StringUtils.isNotBlank(toDateRaw)) { - builder.toDate(DateUtil.resolveDate(toDateRaw, now)); - } else { - builder.toDate(now); - } - - TimeRange range = builder.build(); - range.validate(); - cubeql.getTimeRanges().add(range); } private void doColLifeValidation(CubeQueryContext cubeql) throws LensException, @@ -204,7 +109,6 @@ private void doColLifeValidation(CubeQueryContext cubeql) throws LensException, } // End column loop } - private void throwException(CubeColumn column) throws ColUnAvailableInTimeRangeException { final Long availabilityStartTime = (column.getStartTimeMillisSinceEpoch().isPresent()) @@ -218,23 +122,4 @@ private void throwException(CubeColumn column) throws ColUnAvailableInTimeRangeE throw new ColUnAvailableInTimeRangeException(col); } - - private void doFactRangeValidation(CubeQueryContext cubeql) { - Iterator iter = cubeql.getCandidateFacts().iterator(); - while (iter.hasNext()) { - CandidateFact cfact = iter.next(); - List invalidTimeRanges = Lists.newArrayList(); - for (TimeRange timeRange : cubeql.getTimeRanges()) { - if (!cfact.isValidForTimeRange(timeRange)) { - invalidTimeRanges.add(timeRange); - } - } - if (!invalidTimeRanges.isEmpty()){ - cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.factNotAvailableInRange(invalidTimeRanges)); - log.info("Not considering {} as it's not available for time ranges: {}", cfact, invalidTimeRanges); - iter.remove(); - } - } - cubeql.pruneCandidateFactSet(CandidateTablePruneCause.CandidateTablePruneCode.FACT_NOT_AVAILABLE_IN_RANGE); - } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java index 4d8910ade..8b47f86ea 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java @@ -290,7 +290,7 @@ private static Optional getNameIfFunc(final ASTNode node) { return Optional.fromNullable(funcName); } - private static void addColumnsForSelectExpr(final TrackQueriedColumns sel, ASTNode node, ASTNode parent, + static void addColumnsForSelectExpr(final TrackQueriedColumns sel, ASTNode node, ASTNode parent, Set cols) { if (node.getToken().getType() == TOK_TABLE_OR_COL && (parent != null && parent.getToken().getType() != DOT)) { // Take child ident.totext diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java index 125b4328d..c63c4c0f2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,24 +19,25 @@ package org.apache.lens.cube.parse; + +import static java.util.stream.Collectors.toSet; + import static org.apache.lens.cube.parse.CubeQueryConfUtil.*; import static org.apache.hadoop.hive.ql.parse.HiveParser.*; import static com.google.common.base.Preconditions.checkArgument; - - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.*; +import java.util.function.Predicate; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.error.NoCandidateDimAvailableException; import org.apache.lens.cube.error.NoCandidateFactAvailableException; import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.metadata.join.TableRelationship; -import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; import org.apache.lens.cube.parse.join.AutoJoinContext; import org.apache.lens.cube.parse.join.JoinClause; import org.apache.lens.cube.parse.join.JoinTree; @@ -47,8 +48,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.*; +import org.apache.hadoop.util.ReflectionUtils; import org.codehaus.jackson.map.ObjectMapper; @@ -60,9 +63,9 @@ @Slf4j public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, TrackDenormContext { - public static final String TIME_RANGE_FUNC = "time_range_in"; + static final String TIME_RANGE_FUNC = "time_range_in"; public static final String NOW = "now"; - public static final String DEFAULT_TABLE = "_default_"; + static final String DEFAULT_TABLE = "_default_"; private final ASTNode ast; @Getter private final QB qb; @@ -102,8 +105,14 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, // Mapping of a qualified column name to its table alias private final Map colToTableAlias = new HashMap<>(); + /** + * This is the set of working Candidates that gets updated during different phases of + * query resolution. Each {@link ContextRewriter} may add/remove/update Candiadtes in + * this working set and from the final set of Candidates single {@link #pickedCandidate} + * is chosen. + */ @Getter - private final Set> candidateFactSets = new HashSet<>(); + private final Set candidates = new HashSet<>(); @Getter // would be added through join chains and de-normalized resolver @@ -119,17 +128,14 @@ void addSelectPhrase(SelectPhraseContext sel) { } boolean isColumnAnAlias(String col) { - for (SelectPhraseContext sel : selectPhrases) { - if (col.equals(sel.getActualAlias())) { - return true; - } - } - return false; + return selectPhrases.stream().map(SelectPhraseContext::getActualAlias).anyMatch(Predicate.isEqual(col)); } void addQueriedPhrase(QueriedPhraseContext qur) { queriedPhrases.add(qur); + qur.setPosition(queriedPhrases.size() -1); } + @Getter private final List selectPhrases = new ArrayList<>(); @@ -138,14 +144,9 @@ void addQueriedPhrase(QueriedPhraseContext qur) { // Join conditions used in all join expressions @Getter - private final Map joinConds = new HashMap(); - - // storage specific - @Getter - protected final Set candidateFacts = new HashSet(); + private final Map joinConds = new HashMap<>(); @Getter - protected final Map> candidateDims = new HashMap>(); - + protected final Map> candidateDims = new HashMap<>(); // query trees @Getter @Setter @@ -178,20 +179,22 @@ void addQueriedPhrase(QueriedPhraseContext qur) { @Setter private DenormalizationResolver.DenormalizationContext deNormCtx; @Getter - private PruneCauses factPruningMsgs = - new PruneCauses(); + private PruneCauses storagePruningMsgs = new PruneCauses<>(); @Getter private Map> dimPruningMsgs = new HashMap>(); + @Setter @Getter private String fromString; + @Getter + private TimeRangeWriter rangeWriter = null; public CubeQueryContext(ASTNode ast, QB qb, Configuration queryConf, HiveConf metastoreConf) throws LensException { this.ast = ast; this.qb = qb; this.conf = queryConf; this.clauseName = getClause(); - this.timeRanges = new ArrayList(); + this.timeRanges = new ArrayList<>(); try { metastoreClient = CubeMetastoreClient.getInstance(metastoreConf); } catch (HiveException e) { @@ -212,16 +215,14 @@ public CubeQueryContext(ASTNode ast, QB qb, Configuration queryConf, HiveConf me if (qb.getParseInfo().getSelForClause(clauseName) != null) { this.selectAST = qb.getParseInfo().getSelForClause(clauseName); } - extractMetaTables(); - } - public boolean hasCubeInQuery() { - return cube != null; + this.rangeWriter = ReflectionUtils.newInstance(conf.getClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, + CubeQueryConfUtil.DEFAULT_TIME_RANGE_WRITER, TimeRangeWriter.class), conf); } - public boolean hasDimensionInQuery() { - return dimensions != null && !dimensions.isEmpty(); + boolean hasCubeInQuery() { + return cube != null; } private void extractMetaTables() throws LensException { @@ -279,10 +280,10 @@ private boolean addJoinChain(String alias, boolean isOptional) throws LensExcept return true; } - return retVal; + return false; } - public boolean addQueriedTable(String alias) throws LensException { + boolean addQueriedTable(String alias) throws LensException { return addQueriedTable(alias, false); } @@ -340,42 +341,29 @@ private boolean addQueriedTable(String alias, String tblName, boolean isOptional return false; } } catch (LensException e) { - //TODO: check if catch can be removed return false; } return true; } - public boolean isAutoJoinResolved() { + boolean isAutoJoinResolved() { return autoJoinCtx != null && autoJoinCtx.isJoinsResolved(); } - public Cube getBaseCube() { - if (cube instanceof Cube) { - return (Cube) cube; - } - return ((DerivedCube) cube).getParent(); + Cube getBaseCube() { + return cube instanceof Cube ? (Cube) cube : ((DerivedCube) cube).getParent(); } - public Set getPartitionColumnsQueried() { - Set partsQueried = Sets.newHashSet(); - for (TimeRange range : getTimeRanges()) { - partsQueried.add(range.getPartitionColumn()); - } - return partsQueried; + Set getPartitionColumnsQueried() { + return getTimeRanges().stream().map(TimeRange::getPartitionColumn).collect(toSet()); } // map of ref column in query to set of Dimension that have the column - which are added as optional dims @Getter private Map>> refColToDim = Maps.newHashMap(); - public void updateRefColDim(String col, Aliased dim) { - Set> refDims = refColToDim.get(col.toLowerCase()); - if (refDims == null) { - refDims = Sets.newHashSet(); - refColToDim.put(col.toLowerCase(), refDims); - } - refDims.add(dim); + private void updateRefColDim(String col, Aliased dim) { + refColToDim.computeIfAbsent(col.toLowerCase(), k -> Sets.newHashSet()).add(dim); } @Data @@ -390,15 +378,8 @@ static class QueriedExprColumn { @Getter private Map>> exprColToDim = Maps.newHashMap(); - public void updateExprColDim(String tblAlias, String col, Aliased dim) { - - QueriedExprColumn qexpr = new QueriedExprColumn(col, tblAlias); - Set> exprDims = exprColToDim.get(qexpr); - if (exprDims == null) { - exprDims = Sets.newHashSet(); - exprColToDim.put(qexpr, exprDims); - } - exprDims.add(dim); + private void updateExprColDim(String tblAlias, String col, Aliased dim) { + exprColToDim.computeIfAbsent(new QueriedExprColumn(col, tblAlias), k -> Sets.newHashSet()).add(dim); } // Holds the context of optional dimension @@ -406,7 +387,7 @@ public void updateExprColDim(String tblAlias, String col, Aliased dim // required by a candidate table to get a denormalized field from reference // or required in a join chain @ToString - public static class OptionalDimCtx { + static class OptionalDimCtx { OptionalDimCtx() { } @@ -415,16 +396,16 @@ public static class OptionalDimCtx { boolean isRequiredInJoinChain = false; } - public void addOptionalJoinDimTable(String alias, boolean isRequired) throws LensException { + void addOptionalJoinDimTable(String alias, boolean isRequired) throws LensException { addOptionalDimTable(alias, null, isRequired, null, false, (String[]) null); } - public void addOptionalExprDimTable(String dimAlias, String queriedExpr, String srcTableAlias, + void addOptionalExprDimTable(String dimAlias, String queriedExpr, String srcTableAlias, CandidateTable candidate, String... cols) throws LensException { addOptionalDimTable(dimAlias, candidate, false, queriedExpr, false, srcTableAlias, cols); } - public void addOptionalDimTable(String alias, CandidateTable candidate, boolean isRequiredInJoin, String cubeCol, + void addOptionalDimTable(String alias, CandidateTable candidate, boolean isRequiredInJoin, String cubeCol, boolean isRef, String... cols) throws LensException { addOptionalDimTable(alias, candidate, isRequiredInJoin, cubeCol, isRef, null, cols); } @@ -437,15 +418,9 @@ private void addOptionalDimTable(String alias, CandidateTable candidate, boolean } Dimension dim = (Dimension) cubeTbls.get(alias); Aliased aliasedDim = Aliased.create(dim, alias); - OptionalDimCtx optDim = optionalDimensionMap.get(aliasedDim); - if (optDim == null) { - optDim = new OptionalDimCtx(); - optionalDimensionMap.put(aliasedDim, optDim); - } + OptionalDimCtx optDim = optionalDimensionMap.computeIfAbsent(aliasedDim, k -> new OptionalDimCtx()); if (cols != null && candidate != null) { - for (String col : cols) { - optDim.colQueried.add(col); - } + optDim.colQueried.addAll(Arrays.asList(cols)); optDim.requiredForCandidates.add(candidate); } if (cubeCol != null) { @@ -480,18 +455,28 @@ public Map> getCandidateDimTables() { return candidateDims; } - public void addFactPruningMsgs(CubeFactTable fact, CandidateTablePruneCause factPruningMsg) { - log.info("Pruning fact {} with cause: {}", fact, factPruningMsg); - factPruningMsgs.addPruningMsg(fact, factPruningMsg); + void addCandidatePruningMsg(Collection candidateCollection, CandidateTablePruneCause pruneCause) { + for (Candidate c : candidateCollection){ + addCandidatePruningMsg(c, pruneCause); + } } - public void addDimPruningMsgs(Dimension dim, CubeDimensionTable dimtable, CandidateTablePruneCause msg) { - PruneCauses dimMsgs = dimPruningMsgs.get(dim); - if (dimMsgs == null) { - dimMsgs = new PruneCauses(); - dimPruningMsgs.put(dim, dimMsgs); + void addCandidatePruningMsg(Candidate cand, CandidateTablePruneCause pruneCause) { + Set scs = CandidateUtil.getStorageCandidates(cand); + for (StorageCandidate sc : scs) { + addStoragePruningMsg(sc, pruneCause); + } + } + + void addStoragePruningMsg(StorageCandidate sc, CandidateTablePruneCause... factPruningMsgs) { + for (CandidateTablePruneCause factPruningMsg: factPruningMsgs) { + log.info("Pruning Storage {} with cause: {}", sc, factPruningMsg); + storagePruningMsgs.addPruningMsg(sc, factPruningMsg); } - dimMsgs.addPruningMsg(dimtable, msg); + } + + public void addDimPruningMsgs(Dimension dim, CubeDimensionTable dimtable, CandidateTablePruneCause msg) { + dimPruningMsgs.computeIfAbsent(dim, k -> new PruneCauses<>()).addPruningMsg(dimtable, msg); } public String getAliasForTableName(Named named) { @@ -518,73 +503,75 @@ public void print() { if (!log.isDebugEnabled()) { return; } - StringBuilder builder = new StringBuilder(); - builder.append("ASTNode:" + ast.dump() + "\n"); - builder.append("QB:"); - builder.append("\n numJoins:" + qb.getNumJoins()); - builder.append("\n numGbys:" + qb.getNumGbys()); - builder.append("\n numSels:" + qb.getNumSels()); - builder.append("\n numSelDis:" + qb.getNumSelDi()); - builder.append("\n aliasToTabs:"); + StringBuilder builder = new StringBuilder() + .append("ASTNode:").append(ast.dump()).append("\n") + .append("QB:") + .append("\n numJoins:").append(qb.getNumJoins()) + .append("\n numGbys:").append(qb.getNumGbys()) + .append("\n numSels:").append(qb.getNumSels()) + .append("\n numSelDis:").append(qb.getNumSelDi()) + .append("\n aliasToTabs:"); Set tabAliases = qb.getTabAliases(); for (String alias : tabAliases) { - builder.append("\n\t" + alias + ":" + qb.getTabNameForAlias(alias)); + builder.append("\n\t").append(alias).append(":").append(qb.getTabNameForAlias(alias)); } builder.append("\n aliases:"); for (String alias : qb.getAliases()) { builder.append(alias); builder.append(", "); } - builder.append("id:" + qb.getId()); - builder.append("isQuery:" + qb.getIsQuery()); - builder.append("\n QBParseInfo"); + builder + .append("id:").append(qb.getId()) + .append("isQuery:").append(qb.getIsQuery()) + .append("\n QBParseInfo"); QBParseInfo parseInfo = qb.getParseInfo(); - builder.append("\n isSubQ: " + parseInfo.getIsSubQ()); - builder.append("\n alias: " + parseInfo.getAlias()); + builder + .append("\n isSubQ: ").append(parseInfo.getIsSubQ()) + .append("\n alias: ").append(parseInfo.getAlias()); if (parseInfo.getJoinExpr() != null) { - builder.append("\n joinExpr: " + parseInfo.getJoinExpr().dump()); + builder.append("\n joinExpr: ").append(parseInfo.getJoinExpr().dump()); } - builder.append("\n hints: " + parseInfo.getHints()); + builder.append("\n hints: ").append(parseInfo.getHints()); builder.append("\n aliasToSrc: "); for (String alias : tabAliases) { - builder.append("\n\t" + alias + ": " + parseInfo.getSrcForAlias(alias).dump()); + builder.append("\n\t").append(alias).append(": ").append(parseInfo.getSrcForAlias(alias).dump()); } TreeSet clauses = new TreeSet(parseInfo.getClauseNames()); for (String clause : clauses) { - builder.append("\n\t" + clause + ": " + parseInfo.getClauseNamesForDest()); + builder.append("\n\t").append(clause).append(": ").append(parseInfo.getClauseNamesForDest()); } String clause = clauses.first(); if (parseInfo.getWhrForClause(clause) != null) { - builder.append("\n whereexpr: " + parseInfo.getWhrForClause(clause).dump()); + builder.append("\n whereexpr: ").append(parseInfo.getWhrForClause(clause).dump()); } if (parseInfo.getGroupByForClause(clause) != null) { - builder.append("\n groupby expr: " + parseInfo.getGroupByForClause(clause).dump()); + builder.append("\n groupby expr: ").append(parseInfo.getGroupByForClause(clause).dump()); } if (parseInfo.getSelForClause(clause) != null) { - builder.append("\n sel expr: " + parseInfo.getSelForClause(clause).dump()); + builder.append("\n sel expr: ").append(parseInfo.getSelForClause(clause).dump()); } if (parseInfo.getHavingForClause(clause) != null) { - builder.append("\n having expr: " + parseInfo.getHavingForClause(clause).dump()); + builder.append("\n having expr: ").append(parseInfo.getHavingForClause(clause).dump()); } if (parseInfo.getDestLimit(clause) != null) { - builder.append("\n limit: " + parseInfo.getDestLimit(clause)); + builder.append("\n limit: ").append(parseInfo.getDestLimit(clause)); } if (parseInfo.getAllExprToColumnAlias() != null && !parseInfo.getAllExprToColumnAlias().isEmpty()) { builder.append("\n exprToColumnAlias:"); for (Map.Entry entry : parseInfo.getAllExprToColumnAlias().entrySet()) { - builder.append("\n\t expr: " + entry.getKey().dump() + " ColumnAlias: " + entry.getValue()); + builder.append("\n\t expr: ").append(entry.getKey().dump()).append(" ColumnAlias: ").append(entry.getValue()); } } if (parseInfo.getAggregationExprsForClause(clause) != null) { builder.append("\n aggregateexprs:"); for (Map.Entry entry : parseInfo.getAggregationExprsForClause(clause).entrySet()) { - builder.append("\n\t key: " + entry.getKey() + " expr: " + entry.getValue().dump()); + builder.append("\n\t key: ").append(entry.getKey()).append(" expr: ").append(entry.getValue().dump()); } } if (parseInfo.getDistinctFuncExprsForClause(clause) != null) { builder.append("\n distinctFuncExprs:"); for (ASTNode entry : parseInfo.getDistinctFuncExprsForClause(clause)) { - builder.append("\n\t expr: " + entry.dump()); + builder.append("\n\t expr: ").append(entry.dump()); } } @@ -595,24 +582,24 @@ public void print() { } if (qb.getParseInfo().getDestForClause(clause) != null) { - builder.append("\n Destination:"); - builder.append("\n\t dest expr:" + qb.getParseInfo().getDestForClause(clause).dump()); + builder.append("\n Destination:") + .append("\n\t dest expr:").append(qb.getParseInfo().getDestForClause(clause).dump()); } log.debug(builder.toString()); } - void printJoinTree(QBJoinTree joinTree, StringBuilder builder) { - builder.append("leftAlias:" + joinTree.getLeftAlias()); + private void printJoinTree(QBJoinTree joinTree, StringBuilder builder) { + builder.append("leftAlias:").append(joinTree.getLeftAlias()); if (joinTree.getLeftAliases() != null) { builder.append("\n leftAliases:"); for (String alias : joinTree.getLeftAliases()) { - builder.append("\n\t " + alias); + builder.append("\n\t ").append(alias); } } if (joinTree.getRightAliases() != null) { builder.append("\n rightAliases:"); for (String alias : joinTree.getRightAliases()) { - builder.append("\n\t " + alias); + builder.append("\n\t ").append(alias); } } if (joinTree.getJoinSrc() != null) { @@ -623,26 +610,28 @@ void printJoinTree(QBJoinTree joinTree, StringBuilder builder) { if (joinTree.getBaseSrc() != null) { builder.append("\n baseSrcs:"); for (String src : joinTree.getBaseSrc()) { - builder.append("\n\t " + src); + builder.append("\n\t ").append(src); } } - builder.append("\n noOuterJoin: " + joinTree.getNoOuterJoin()); - builder.append("\n noSemiJoin: " + joinTree.getNoSemiJoin()); - builder.append("\n mapSideJoin: " + joinTree.isMapSideJoin()); + builder.append("\n noOuterJoin: ").append(joinTree.getNoOuterJoin()); + builder.append("\n noSemiJoin: ").append(joinTree.getNoSemiJoin()); + builder.append("\n mapSideJoin: ").append(joinTree.isMapSideJoin()); if (joinTree.getJoinCond() != null) { builder.append("\n joinConds:"); for (JoinCond cond : joinTree.getJoinCond()) { - builder.append("\n\t left: " + cond.getLeft() + " right: " + cond.getRight() + " type:" + cond.getJoinType() - + " preserved:" + cond.getPreserved()); + builder.append("\n\t left: ").append(cond.getLeft()) + .append(" right: ").append(cond.getRight()) + .append(" type:").append(cond.getJoinType()) + .append(" preserved:").append(cond.getPreserved()); } } } - void updateFromString(CandidateFact fact, Map dimsToQuery) throws LensException { + private void updateFromString(StorageCandidate sc, Map dimsToQuery) throws LensException { fromString = "%s"; // storage string is updated later if (isAutoJoinResolved()) { fromString = - getAutoJoinCtx().getFromString(fromString, fact, dimsToQuery.keySet(), dimsToQuery, this, this); + getAutoJoinCtx().getFromString(fromString, sc, dimsToQuery.keySet(), dimsToQuery, this, this); } } @@ -650,6 +639,11 @@ public String getSelectString() { return HQLParser.getString(selectAST); } + + public void setWhereString(String whereString) { + //NO OP + } + public String getWhereString() { if (whereAST != null) { return HQLParser.getString(whereAST); @@ -675,6 +669,11 @@ public ASTNode getJoinAST() { return qb.getParseInfo().getJoinExpr(); } + @Override + public void setJoinAST(ASTNode node) { + //NO-OP + } + public String getOrderByString() { if (orderByAST != null) { return HQLParser.getString(orderByAST); @@ -690,9 +689,10 @@ public void setLimitValue(Integer value) { qb.getParseInfo().setDestLimit(getClause(), 0, value); } - private String getStorageStringWithAlias(CandidateFact fact, Map dimsToQuery, String alias) { + private String getStorageStringWithAlias(StorageCandidate candidate, Map dimsToQuery, String alias) { if (cubeTbls.get(alias) instanceof CubeInterface) { - return fact.getStorageString(alias); + return candidate.getAliasForTable(alias); } else { return dimsToQuery.get(cubeTbls.get(alias)).getStorageString(alias); } @@ -702,14 +702,14 @@ private String getWhereClauseWithAlias(Map dimsToQuery, return StorageUtil.getWhereClause(dimsToQuery.get(cubeTbls.get(alias)), alias); } - String getQBFromString(CandidateFact fact, Map dimsToQuery) throws LensException { + String getQBFromString(StorageCandidate candidate, Map dimsToQuery) throws LensException { String fromString; if (getJoinAST() == null) { if (cube != null) { if (dimensions.size() > 0) { throw new LensException(LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE.getLensErrorInfo()); } - fromString = fact.getStorageString(getAliasForTableName(cube.getName())); + fromString = candidate.getAliasForTable(getAliasForTableName(cube.getName())); } else { if (dimensions.size() != 1) { throw new LensException(LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE.getLensErrorInfo()); @@ -719,22 +719,23 @@ String getQBFromString(CandidateFact fact, Map dimsToQu } } else { StringBuilder builder = new StringBuilder(); - getQLString(qb.getQbJoinTree(), builder, fact, dimsToQuery); + getQLString(qb.getQbJoinTree(), builder, candidate, dimsToQuery); fromString = builder.toString(); } return fromString; } - private void getQLString(QBJoinTree joinTree, StringBuilder builder, CandidateFact fact, + + private void getQLString(QBJoinTree joinTree, StringBuilder builder, StorageCandidate candidate, Map dimsToQuery) throws LensException { List joiningTables = new ArrayList<>(); if (joinTree.getBaseSrc()[0] == null) { if (joinTree.getJoinSrc() != null) { - getQLString(joinTree.getJoinSrc(), builder, fact, dimsToQuery); + getQLString(joinTree.getJoinSrc(), builder, candidate, dimsToQuery); } } else { // (joinTree.getBaseSrc()[0] != null){ String alias = joinTree.getBaseSrc()[0].toLowerCase(); - builder.append(getStorageStringWithAlias(fact, dimsToQuery, alias)); + builder.append(getStorageStringWithAlias(candidate, dimsToQuery, alias)); joiningTables.add(alias); } if (joinTree.getJoinCond() != null) { @@ -743,11 +744,11 @@ private void getQLString(QBJoinTree joinTree, StringBuilder builder, CandidateFa } if (joinTree.getBaseSrc()[1] == null) { if (joinTree.getJoinSrc() != null) { - getQLString(joinTree.getJoinSrc(), builder, fact, dimsToQuery); + getQLString(joinTree.getJoinSrc(), builder, candidate, dimsToQuery); } } else { // (joinTree.getBaseSrc()[1] != null){ String alias = joinTree.getBaseSrc()[1].toLowerCase(); - builder.append(getStorageStringWithAlias(fact, dimsToQuery, alias)); + builder.append(getStorageStringWithAlias(candidate, dimsToQuery, alias)); joiningTables.add(alias); } @@ -795,7 +796,7 @@ void setNonexistingParts(Map> nonExistingParts) throws LensE } } - public String getNonExistingParts() { + String getNonExistingParts() { return conf.get(NON_EXISTING_PARTITIONS); } @@ -809,24 +810,13 @@ private Map pickCandidateDimsToQuery(Set dim cdim.dimtable); dimsToQuery.put(dim, cdim); } else { - String reason = ""; if (dimPruningMsgs.get(dim) != null && !dimPruningMsgs.get(dim).isEmpty()) { - ByteArrayOutputStream out = null; - try { + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { ObjectMapper mapper = new ObjectMapper(); - out = new ByteArrayOutputStream(); mapper.writeValue(out, dimPruningMsgs.get(dim).getJsonObject()); - reason = out.toString("UTF-8"); + log.info("No candidate dim found because: {}", out.toString("UTF-8")); } catch (Exception e) { throw new LensException("Error writing dim pruning messages", e); - } finally { - if (out != null) { - try { - out.close(); - } catch (IOException e) { - throw new LensException(e); - } - } } } log.error("Query rewrite failed due to NO_CANDIDATE_DIM_AVAILABLE, Cause {}", @@ -838,192 +828,226 @@ private Map pickCandidateDimsToQuery(Set dim return dimsToQuery; } - private Set pickCandidateFactToQuery() throws LensException { - Set facts = null; + private Candidate pickCandidateToQuery() throws LensException { + Candidate cand = null; if (hasCubeInQuery()) { - if (candidateFactSets.size() > 0) { - facts = candidateFactSets.iterator().next(); - log.info("Available candidate facts:{}, picking up {} for querying", candidateFactSets, facts); + if (candidates.size() > 0) { + cand = candidates.iterator().next(); + log.info("Available Candidates:{}, picking up Candidate: {} for querying", candidates, cand); } else { - String reason = ""; - if (!factPruningMsgs.isEmpty()) { - ByteArrayOutputStream out = null; + throwNoCandidateFactException(); + } + } + return cand; + } + + void throwNoCandidateFactException() throws LensException { + String reason = ""; + if (!storagePruningMsgs.isEmpty()) { + ByteArrayOutputStream out = null; + try { + ObjectMapper mapper = new ObjectMapper(); + out = new ByteArrayOutputStream(); + mapper.writeValue(out, storagePruningMsgs.getJsonObject()); + reason = out.toString("UTF-8"); + } catch (Exception e) { + throw new LensException("Error writing fact pruning messages", e); + } finally { + if (out != null) { try { - ObjectMapper mapper = new ObjectMapper(); - out = new ByteArrayOutputStream(); - mapper.writeValue(out, factPruningMsgs.getJsonObject()); - reason = out.toString("UTF-8"); - } catch (Exception e) { - throw new LensException("Error writing fact pruning messages", e); - } finally { - if (out != null) { - try { - out.close(); - } catch (IOException e) { - throw new LensException(e); - } - } + out.close(); + } catch (IOException e) { + throw new LensException(e); } } - log.error("Query rewrite failed due to NO_CANDIDATE_FACT_AVAILABLE, Cause {}", factPruningMsgs.toJsonObject()); - throw new NoCandidateFactAvailableException(factPruningMsgs); } } - return facts; + log.error("Query rewrite failed due to NO_CANDIDATE_FACT_AVAILABLE, Cause {}", storagePruningMsgs.toJsonObject()); + throw new NoCandidateFactAvailableException(this); } private HQLContextInterface hqlContext; + @Getter - private Collection pickedFacts; + private Candidate pickedCandidate; @Getter private Collection pickedDimTables; - private void addRangeClauses(CandidateFact fact) throws LensException { - if (fact != null) { + private void addRangeClauses(StorageCandidate sc) throws LensException { + if (sc != null) { // resolve timerange positions and replace it by corresponding where clause for (TimeRange range : getTimeRanges()) { - for (Map.Entry entry : fact.getRangeToStorageWhereMap().get(range).entrySet()) { - String table = entry.getKey(); - String rangeWhere = entry.getValue(); - if (!StringUtils.isBlank(rangeWhere)) { - ASTNode rangeAST = HQLParser.parseExpr(rangeWhere, conf); - range.getParent().setChild(range.getChildIndex(), rangeAST); - } - fact.getStorgeWhereClauseMap().put(table, HQLParser.parseExpr(getWhereString(), conf)); + String rangeWhere = CandidateUtil.getTimeRangeWhereClasue(rangeWriter, sc, range); + if (!StringUtils.isBlank(rangeWhere)) { + ASTNode updatedRangeAST = HQLParser.parseExpr(rangeWhere, conf); + updateTimeRangeNode(sc.getQueryAst().getWhereAST(), range.getAstNode(), updatedRangeAST); + } + } + } + } + + + /** + * Find the appropriate time range node in the AST and update it with "updatedTimeRange". + * Time Range node looks like this + * time_range_in(dt, '2017', '2018') -> + * TOK_FUNCTION [TOK_FUNCTION] (l5c2p37) { + * time_range_in [Identifier] (l6c1p37)$ + * TOK_TABLE_OR_COL [TOK_TABLE_OR_COL] (l6c2p51) { + * dt [Identifier] (l7c1p51)$ + * } + * '2017' [StringLiteral] (l6c3p55)$ + * '2018' [StringLiteral] (l6c4p63)$ + } + * @param root + * @param timeRangeFuncNode + * @param updatedTimeRange + */ + private void updateTimeRangeNode(ASTNode root, ASTNode timeRangeFuncNode, ASTNode updatedTimeRange) { + ASTNode childNode; + if (root.getChildCount() == 0) { + return; + } + for (Node child : root.getChildren()) { + childNode = (ASTNode) child; + if (childNode.getType() == timeRangeFuncNode.getType() + && childNode.getChildCount() == timeRangeFuncNode.getChildCount() + && childNode.getChild(0).getText().equalsIgnoreCase(timeRangeFuncNode.getChild(0).getText())) { + //Found the "time_range_in" function node. Check the details further as there can be more than one time ranges + if (HQLParser.getString(timeRangeFuncNode).equalsIgnoreCase(HQLParser.getString(childNode))) { + //This is the correct time range node . Replace it with "updatedTimeRange" + childNode.getParent().setChild(childNode.getChildIndex(), updatedTimeRange); + return; } } + updateTimeRangeNode(childNode, timeRangeFuncNode, updatedTimeRange); } } + public String toHQL() throws LensException { - Set cfacts = pickCandidateFactToQuery(); + Candidate cand = pickCandidateToQuery(); Map dimsToQuery = pickCandidateDimsToQuery(dimensions); - log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery); + Collection scSet = new HashSet<>(); + if (cand != null) { + scSet.addAll(CandidateUtil.getStorageCandidates(cand)); + } + + //Expand and get update period specific storage candidates if required. + scSet = expandStorageCandidates(scSet); + + log.info("Candidate: {}, DimsToQuery: {}", cand, dimsToQuery); if (autoJoinCtx != null) { // prune join paths for picked fact and dimensions - autoJoinCtx.pruneAllPaths(cube, cfacts, dimsToQuery); + autoJoinCtx.pruneAllPaths(cube, scSet, dimsToQuery); } - Map> factDimMap = new HashMap<>(); - if (cfacts != null) { - if (cfacts.size() > 1) { - // copy ASTs for each fact - for (CandidateFact cfact : cfacts) { - cfact.copyASTs(this); - factDimMap.put(cfact, new HashSet<>(dimsToQuery.keySet())); - } + Map> factDimMap = new HashMap<>(); + if (cand != null) { + // Set the default queryAST for StorageCandidate and copy child ASTs from cubeql. + // Later in the rewrite flow each Storage candidate will modify them accordingly. + for (StorageCandidate sc : scSet) { + sc.setQueryAst(DefaultQueryAST.fromStorageCandidate(sc, this)); + CandidateUtil.copyASTs(this, sc.getQueryAst()); + factDimMap.put(sc, new HashSet<>(dimsToQuery.keySet())); } - for (CandidateFact fact : cfacts) { - addRangeClauses(fact); + for (StorageCandidate sc : scSet) { + addRangeClauses(sc); } } // pick dimension tables required during expression expansion for the picked fact and dimensions Set exprDimensions = new HashSet<>(); - if (cfacts != null) { - for (CandidateFact cfact : cfacts) { - Set factExprDimTables = exprCtx.rewriteExprCtx(this, cfact, dimsToQuery, - cfacts.size() > 1 ? cfact : this); - exprDimensions.addAll(factExprDimTables); - if (cfacts.size() > 1) { - factDimMap.get(cfact).addAll(factExprDimTables); - } - } - if (cfacts.size() > 1) { - havingAST = MultiFactHQLContext.pushDownHaving(havingAST, this, cfacts); + if (!scSet.isEmpty()) { + for (StorageCandidate sc : scSet) { + Set scExprDimTables = exprCtx.rewriteExprCtx(this, sc, dimsToQuery, sc.getQueryAst()); + exprDimensions.addAll(scExprDimTables); + factDimMap.get(sc).addAll(scExprDimTables); } } else { // dim only query exprDimensions.addAll(exprCtx.rewriteExprCtx(this, null, dimsToQuery, this)); } dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimensions)); - log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery); + log.info("StorageCandidates: {}, DimsToQuery: {}", scSet, dimsToQuery); // pick denorm tables for the picked fact and dimensions Set denormTables = new HashSet<>(); - if (cfacts != null) { - for (CandidateFact cfact : cfacts) { - Set factDenormTables = deNormCtx.rewriteDenormctx(this, cfact, dimsToQuery, cfacts.size() > 1); - denormTables.addAll(factDenormTables); - if (cfacts.size() > 1) { - factDimMap.get(cfact).addAll(factDenormTables); - } + if (!scSet.isEmpty()) { + for (StorageCandidate sc : scSet) { + Set scDenormTables = deNormCtx.rewriteDenormctx(this, sc, dimsToQuery, !scSet.isEmpty()); + denormTables.addAll(scDenormTables); + factDimMap.get(sc).addAll(scDenormTables); } } else { denormTables.addAll(deNormCtx.rewriteDenormctx(this, null, dimsToQuery, false)); } dimsToQuery.putAll(pickCandidateDimsToQuery(denormTables)); - log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery); + log.info("StorageCandidates: {}, DimsToQuery: {}", scSet, dimsToQuery); // Prune join paths once denorm tables are picked if (autoJoinCtx != null) { // prune join paths for picked fact and dimensions - autoJoinCtx.pruneAllPaths(cube, cfacts, dimsToQuery); + autoJoinCtx.pruneAllPaths(cube, scSet, dimsToQuery); } if (autoJoinCtx != null) { // add optional dims from Join resolver Set joiningTables = new HashSet<>(); - if (cfacts != null && cfacts.size() > 1) { - for (CandidateFact cfact : cfacts) { - Set factJoiningTables = autoJoinCtx.pickOptionalTables(cfact, factDimMap.get(cfact), this); - factDimMap.get(cfact).addAll(factJoiningTables); - joiningTables.addAll(factJoiningTables); + if (scSet != null && scSet.size() > 1) { + for (StorageCandidate sc : scSet) { + Set scJoiningTables = autoJoinCtx.pickOptionalTables(sc, factDimMap.get(sc), this); + factDimMap.get(sc).addAll(scJoiningTables); + joiningTables.addAll(scJoiningTables); } } else { joiningTables.addAll(autoJoinCtx.pickOptionalTables(null, dimsToQuery.keySet(), this)); } dimsToQuery.putAll(pickCandidateDimsToQuery(joiningTables)); } - log.info("Picked Fact:{} dimsToQuery: {}", cfacts, dimsToQuery); + log.info("Picked StorageCandidates: {} DimsToQuery: {}", scSet, dimsToQuery); pickedDimTables = dimsToQuery.values(); - pickedFacts = cfacts; - if (cfacts != null) { - if (cfacts.size() > 1) { - // Update ASTs for each fact - for (CandidateFact cfact : cfacts) { - cfact.updateASTs(this); - } - whereAST = MultiFactHQLContext.convertHavingToWhere(havingAST, this, cfacts, new DefaultAliasDecider()); - for (CandidateFact cFact : cfacts) { - cFact.updateFromString(this, factDimMap.get(cFact), dimsToQuery); - } + pickedCandidate = cand; + + //Set From string and time range clause + if (!scSet.isEmpty()) { + for (StorageCandidate sc : scSet) { + sc.updateFromString(this, factDimMap.get(sc), dimsToQuery); } - } - if (cfacts == null || cfacts.size() == 1) { - updateFromString(cfacts == null ? null : cfacts.iterator().next(), dimsToQuery); - } - //update dim filter with fact filter - if (cfacts != null && cfacts.size() > 0) { - for (CandidateFact cfact : cfacts) { - if (!cfact.getStorageTables().isEmpty()) { - for (String qualifiedStorageTable : cfact.getStorageTables()) { - String storageTable = qualifiedStorageTable.substring(qualifiedStorageTable.indexOf(".") + 1); - String where = getWhere(cfact, autoJoinCtx, - cfact.getStorageWhereClause(storageTable), getAliasForTableName(cfact.getBaseTable().getName()), - shouldReplaceDimFilterWithFactFilter(), storageTable, dimsToQuery); - cfact.getStorgeWhereStringMap().put(storageTable, where); - } - } + } else { + updateFromString(null, dimsToQuery); + } + + //update dim filter with fact filter, set where string in sc + if (scSet.size() > 0) { + for (StorageCandidate sc : scSet) { + String qualifiedStorageTable = sc.getStorageTable(); + String storageTable = qualifiedStorageTable.substring(qualifiedStorageTable.indexOf(".") + 1); + String where = getWhere(sc, autoJoinCtx, + sc.getQueryAst().getWhereAST(), getAliasForTableName(sc.getBaseTable().getName()), + shouldReplaceDimFilterWithFactFilter(), storageTable, dimsToQuery); + sc.setWhereString(where); } } - hqlContext = createHQLContext(cfacts, dimsToQuery, factDimMap); - return hqlContext.toHQL(); - } - - private HQLContextInterface createHQLContext(Set facts, Map dimsToQuery, - Map> factDimMap) throws LensException { - if (facts == null || facts.size() == 0) { - return new DimOnlyHQLContext(dimsToQuery, this, this); - } else if (facts.size() == 1 && facts.iterator().next().getStorageTables().size() > 1) { - //create single fact with multiple storage context - return new SingleFactMultiStorageHQLContext(facts.iterator().next(), dimsToQuery, this, this); - } else if (facts.size() == 1 && facts.iterator().next().getStorageTables().size() == 1) { - CandidateFact fact = facts.iterator().next(); - // create single fact context - return new SingleFactSingleStorageHQLContext(fact, null, - dimsToQuery, this, DefaultQueryAST.fromCandidateFact(fact, fact.getStorageTables().iterator().next(), this)); + + if (cand == null) { + hqlContext = new DimOnlyHQLContext(dimsToQuery, this, this); + return hqlContext.toHQL(); + } else if (scSet.size() == 1) { + StorageCandidate sc = (StorageCandidate) scSet.iterator().next(); + sc.updateAnswerableSelectColumns(this); + return getInsertClause() + sc.toHQL(factDimMap.get(sc)); } else { - return new MultiFactHQLContext(facts, dimsToQuery, factDimMap, this); + UnionQueryWriter uqc = new UnionQueryWriter(scSet, this); + return getInsertClause() + uqc.toHQL(factDimMap); + } + } + + private Collection expandStorageCandidates(Collection scSet) + throws LensException { + Collection expandedList = new ArrayList(); + for (StorageCandidate sc : scSet) { + expandedList.addAll(sc.splitAtUpdatePeriodLevelIfReq()); } + return expandedList; } public ASTNode toAST(Context ctx) throws LensException { @@ -1039,18 +1063,18 @@ public ASTNode toAST(Context ctx) throws LensException { return ParseUtils.findRootNonNullToken(tree); } - public Set getColumnsQueriedForTable(String tblName) { + Set getColumnsQueriedForTable(String tblName) { return getColumnsQueried(getAliasForTableName(tblName)); } - public void addColumnsQueriedWithTimeDimCheck(QueriedPhraseContext qur, String alias, String timeDimColumn) { + void addColumnsQueriedWithTimeDimCheck(QueriedPhraseContext qur, String alias, String timeDimColumn) { if (!shouldReplaceTimeDimWithPart()) { qur.addColumnsQueried(alias, timeDimColumn); } } - public boolean isCubeMeasure(String col) { + boolean isCubeMeasure(String col) { if (col == null) { return false; } @@ -1100,6 +1124,7 @@ boolean isCubeMeasure(ASTNode node) { ASTNode colIdent = (ASTNode) node.getChild(1); colname = colIdent.getText(); + assert tabident != null; tabname = tabident.getText(); } @@ -1108,7 +1133,7 @@ boolean isCubeMeasure(ASTNode node) { return isCubeMeasure(msrname); } - public boolean hasAggregates() { + boolean hasAggregates() { if (getExprCtx().hasAggregates()) { return true; } @@ -1120,7 +1145,7 @@ public boolean hasAggregates() { return false; } - public void setJoinCond(QBJoinTree qb, String cond) { + void setJoinCond(QBJoinTree qb, String cond) { joinConds.put(qb, cond); } @@ -1136,22 +1161,22 @@ public AbstractCubeTable getQueriedTable(String alias) { return null; } - public String getInsertClause() { + String getInsertClause() { ASTNode destTree = qb.getParseInfo().getDestForClause(clauseName); if (destTree != null && ((ASTNode) (destTree.getChild(0))).getToken().getType() != TOK_TMP_FILE) { - return "INSERT OVERWRITE" + HQLParser.getString(destTree); + return "INSERT OVERWRITE " + HQLParser.getString(destTree) + " "; } return ""; } - public Set> getOptionalDimensions() { + Set> getOptionalDimensions() { return optionalDimensionMap.keySet(); } /** * @return the hqlContext */ - public HQLContextInterface getHqlContext() { + HQLContextInterface getHqlContext() { return hqlContext; } @@ -1159,15 +1184,15 @@ public boolean shouldReplaceTimeDimWithPart() { return getConf().getBoolean(REPLACE_TIMEDIM_WITH_PART_COL, DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL); } - public boolean shouldReplaceDimFilterWithFactFilter() { + private boolean shouldReplaceDimFilterWithFactFilter() { return getConf().getBoolean(REWRITE_DIM_FILTER_TO_FACT_FILTER, DEFAULT_REWRITE_DIM_FILTER_TO_FACT_FILTER); } - public String getPartitionColumnOfTimeDim(String timeDimName) { + String getPartitionColumnOfTimeDim(String timeDimName) { return getPartitionColumnOfTimeDim(cube, timeDimName); } - public static String getPartitionColumnOfTimeDim(CubeInterface cube, String timeDimName) { + private static String getPartitionColumnOfTimeDim(CubeInterface cube, String timeDimName) { if (cube == null) { return timeDimName; } @@ -1178,11 +1203,11 @@ public static String getPartitionColumnOfTimeDim(CubeInterface cube, String time } } - public String getTimeDimOfPartitionColumn(String partCol) { + String getTimeDimOfPartitionColumn(String partCol) { return getTimeDimOfPartitionColumn(cube, partCol); } - public static String getTimeDimOfPartitionColumn(CubeInterface cube, String partCol) { + private static String getTimeDimOfPartitionColumn(CubeInterface cube, String partCol) { if (cube == null) { return partCol; } @@ -1193,93 +1218,44 @@ public static String getTimeDimOfPartitionColumn(CubeInterface cube, String part } } - public void addQueriedMsrs(Set msrs) { + void addQueriedMsrs(Set msrs) { queriedMsrs.addAll(msrs); } - public void addQueriedExprs(Set exprs) { + void addQueriedExprs(Set exprs) { queriedExprs.addAll(exprs); } - public void addQueriedExprsWithMeasures(Set exprs) { + void addQueriedExprsWithMeasures(Set exprs) { queriedExprsWithMeasures.addAll(exprs); } - /** - * Prune candidate fact sets with respect to available candidate facts. - *

- * Prune a candidate set, if any of the fact is missing. - * - * @param pruneCause - */ - public void pruneCandidateFactSet(CandidateTablePruneCode pruneCause) { - // remove candidate fact sets that have missing facts - for (Iterator> i = candidateFactSets.iterator(); i.hasNext();) { - Set cfacts = i.next(); - if (!candidateFacts.containsAll(cfacts)) { - log.info("Not considering fact table set:{} as they have non candidate tables and facts missing because of {}", - cfacts, pruneCause); - i.remove(); - } - } - // prune candidate facts - pruneCandidateFactWithCandidateSet(CandidateTablePruneCode.ELEMENT_IN_SET_PRUNED); - } - - /** - * Prune candidate fact with respect to available candidate fact sets. - *

- * If candidate fact is not present in any of the candidate fact sets, remove it. - * - * @param pruneCause - */ - public void pruneCandidateFactWithCandidateSet(CandidateTablePruneCode pruneCause) { - // remove candidate facts that are not part of any covering set - pruneCandidateFactWithCandidateSet(new CandidateTablePruneCause(pruneCause)); - } - - public void pruneCandidateFactWithCandidateSet(CandidateTablePruneCause pruneCause) { - // remove candidate facts that are not part of any covering set - Set allCoveringFacts = new HashSet(); - for (Set set : candidateFactSets) { - allCoveringFacts.addAll(set); - } - for (Iterator i = candidateFacts.iterator(); i.hasNext();) { - CandidateFact cfact = i.next(); - if (!allCoveringFacts.contains(cfact)) { - log.info("Not considering fact table:{} as {}", cfact, pruneCause); - addFactPruningMsgs(cfact.fact, pruneCause); - i.remove(); - } - } - } - - public void addQueriedTimeDimensionCols(final String timeDimColName) { + void addQueriedTimeDimensionCols(final String timeDimColName) { checkArgument(StringUtils.isNotBlank(timeDimColName)); this.queriedTimeDimCols.add(timeDimColName); } - public ImmutableSet getQueriedTimeDimCols() { + ImmutableSet getQueriedTimeDimCols() { return ImmutableSet.copyOf(this.queriedTimeDimCols); } - private String getWhere(CandidateFact cfact, AutoJoinContext autoJoinCtx, + private String getWhere(StorageCandidate sc, AutoJoinContext autoJoinCtx, ASTNode node, String cubeAlias, boolean shouldReplaceDimFilter, String storageTable, Map dimToQuery) throws LensException { String whereString; if (autoJoinCtx != null && shouldReplaceDimFilter) { List allfilters = new ArrayList<>(); - getAllFilters(node, cubeAlias, allfilters, autoJoinCtx.getJoinClause(cfact), dimToQuery); + getAllFilters(node, cubeAlias, allfilters, autoJoinCtx.getJoinClause(sc), dimToQuery); whereString = StringUtils.join(allfilters, " and "); } else { - whereString = HQLParser.getString(cfact.getStorageWhereClause(storageTable)); + whereString = HQLParser.getString(sc.getQueryAst().getWhereAST()); } return whereString; } - private List getAllFilters(ASTNode node, String cubeAlias, List allFilters, + private void getAllFilters(ASTNode node, String cubeAlias, List allFilters, JoinClause joinClause, Map dimToQuery) throws LensException { @@ -1304,7 +1280,6 @@ private List getAllFilters(ASTNode node, String cubeAlias, List ASTNode child = (ASTNode) node.getChild(i); getAllFilters(child, cubeAlias, allFilters, joinClause, dimToQuery); } - return allFilters; } private String getFilter(String table, String cubeAlias, ASTNode node, JoinClause joinClause, @@ -1322,7 +1297,6 @@ private String getFilter(String table, String cubeAlias, ASTNode node, JoinClau } private TableRelationship getStarJoin(JoinClause joinClause, String table) { - TableRelationship rel; for (Map.Entry entry : joinClause.getJoinTree().getSubtrees().entrySet()) { if (entry.getValue().getDepthFromRoot() == 1 && table.equals(entry.getValue().getAlias())) { return entry.getKey(); @@ -1334,8 +1308,9 @@ private TableRelationship getStarJoin(JoinClause joinClause, String table) { private String getTableFromFilterAST(ASTNode node) { if (node.getToken().getType() == HiveParser.DOT) { - return HQLParser.findNodeByPath((ASTNode) node, - TOK_TABLE_OR_COL, Identifier).getText(); + ASTNode n = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier); + assert n != null; + return n.getText(); } else { // recurse down for (int i = 0; i < node.getChildCount(); i++) { @@ -1354,7 +1329,8 @@ private String buildFactSubqueryFromDimFilter(TableRelationship tabRelation, AST String cubeAlias) throws LensException { StringBuilder builder = new StringBuilder(); - String storageClause = dimToQuery.get(tabRelation.getToTable()).getWhereClause(); + CandidateDim dim = dimToQuery.get(tabRelation.getToTable()); + String storageClause = dim.getWhereClause(); builder.append(cubeAlias) .append(".") @@ -1363,9 +1339,9 @@ private String buildFactSubqueryFromDimFilter(TableRelationship tabRelation, AST .append("select ") .append(tabRelation.getToColumn()) .append(" from ") - .append(dimToQuery.get(tabRelation.getToTable()).getStorageString(dimAlias)) + .append(dim.getStorageString(dimAlias)) .append(" where ") - .append(HQLParser.getString((ASTNode) dimFilter)); + .append(HQLParser.getString(dimFilter)); if (storageClause != null) { builder.append(" and ") .append(String.format(storageClause, dimAlias)) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java index ea2eb7e6d..6bee3864e 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java @@ -139,52 +139,68 @@ private void setupRewriters() { // Rewrite base trees (groupby, having, orderby, limit) using aliases rewriters.add(new AliasReplacer()); ExpressionResolver exprResolver = new ExpressionResolver(); - DenormalizationResolver denormResolver = new DenormalizationResolver(conf); + DenormalizationResolver denormResolver = new DenormalizationResolver(); CandidateTableResolver candidateTblResolver = new CandidateTableResolver(); StorageTableResolver storageTableResolver = new StorageTableResolver(conf); + + // Phase 1 of exprResolver: Resolve expressions rewriters.add(exprResolver); - // De-normalized columns resolved + // Phase 1 of denormResolver: De-normalized columns resolved rewriters.add(denormResolver); // Resolve time ranges rewriters.add(new TimerangeResolver()); - // Resolve candidate fact tables and dimension tables for columns queried + // Phase 1 of candidateTblResolver: Resolve candidate storages and dimension tables for columns queried rewriters.add(candidateTblResolver); // Resolve aggregations and generate base select tree rewriters.add(new AggregateResolver()); rewriters.add(new GroupbyResolver(conf)); + //validate fields queryability (in case of derived cubes setup) rewriters.add(new FieldValidator()); // Resolve joins and generate base join tree rewriters.add(new JoinResolver()); - // Do col life validation - rewriters.add(new TimeRangeChecker(conf)); - // Resolve candidate fact tables and dimension tables for columns included + // Do col life validation for the time range(s) queried + rewriters.add(new ColumnLifetimeChecker()); + // Phase 1 of storageTableResolver: Validate and prune candidate storages + rewriters.add(storageTableResolver); + // Phase 2 of candidateTblResolver: Resolve candidate storages and dimension tables for columns included // in join and denorm resolvers rewriters.add(candidateTblResolver); + // Find Union and Join combinations over Storage Candidates that can answer the queried time range(s) and all + // queried measures + rewriters.add(new CandidateCoveringSetsResolver()); - // Phase 1: resolve fact tables. - rewriters.add(storageTableResolver); + // If lightest fact first option is enabled for this driver (via lens.cube.query.pick.lightest.fact.first = true), + // run LightestFactResolver and keep only the lighted combination(s) generated by CandidateCoveringSetsResolver if (lightFactFirst) { // Prune candidate tables for which denorm column references do not exist rewriters.add(denormResolver); - // Prune candidate facts without any valid expressions + // Phase 2 of exprResolver:Prune candidate facts without any valid expressions rewriters.add(exprResolver); + // Pick the least cost combination(s) (and prune others) out of a set of combinations produced + // by CandidateCoveringSetsResolver rewriters.add(new LightestFactResolver()); } - // Phase 2: resolve fact table partitions. + + // Phase 2 of storageTableResolver: resolve storage table partitions. rewriters.add(storageTableResolver); + // In case partial data is allowed (via lens.cube.query.fail.if.data.partial = false) and there are many + // combinations with partial data, pick the one that covers the maximum part of time ranges(s) queried rewriters.add(new MaxCoveringFactResolver(conf)); - // Phase 3: resolve dimension tables and partitions. + // Phase 3 of storageTableResolver: resolve dimension tables and partitions. rewriters.add(storageTableResolver); // Prune candidate tables for which denorm column references do not exist + //TODO union: phase 2 of denormResolver needs to be moved before CoveringSetResolver.. check if this makes sense rewriters.add(denormResolver); - // Prune candidate facts without any valid expressions + // Phase 2 of exprResolver : Prune candidate facts without any valid expressions rewriters.add(exprResolver); - // We can have LightestFactResolver before LeastPartitionResolver - that says - // "if two facts have the same least weight, then the fact with least number of time partitions queried will be - // picked". This will be useful, if users did not set fact weights. + if (!lightFactFirst) { + // Pick the least cost combination(s) (and prune others) out of a set of combinations produced + // by CandidateCoveringSetsResolver rewriters.add(new LightestFactResolver()); } + // if two combinations have the same least weight/cost, then the combination with least number of time partitions + // queried will be picked. Rest of the combinations will be pruned rewriters.add(new LeastPartitionResolver()); rewriters.add(new LightestDimensionResolver()); } @@ -228,6 +244,7 @@ private void rewrite(List rewriters, CubeQueryContext ctx) thro */ MethodMetricsContext mgauge = MethodMetricsFactory.createMethodGauge(ctx.getConf(), true, rewriter.getClass().getCanonicalName() + ITER_STR + i); + rewriter.rewriteContext(ctx); mgauge.markSuccess(); i++; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java index fc960555c..e2015c2b6 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java @@ -33,16 +33,12 @@ * Accepts cube query AST and rewrites into storage table query */ public class CubeSemanticAnalyzer extends SemanticAnalyzer { - private final Configuration queryConf; - private final HiveConf hiveConf; - private final List validationRules = new ArrayList(); + private final List validationRules = new ArrayList<>(); @Getter private QB cubeQB; public CubeSemanticAnalyzer(Configuration queryConf, HiveConf hiveConf) throws SemanticException { super(new QueryState(hiveConf)); - this.queryConf = queryConf; - this.hiveConf = hiveConf; setupRules(); } @@ -65,10 +61,6 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { ast.deleteChild(ast.getChildCount() - 1); } } - // analyzing from the ASTNode. - if (!doPhase1(ast, cubeQB, initPhase1Ctx(), null)) { - // if phase1Result false return - return; - } + doPhase1(ast, cubeQB, initPhase1Ctx(), null); } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultAliasDecider.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultAliasDecider.java index 80ceae4f6..c8bf7879d 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultAliasDecider.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultAliasDecider.java @@ -20,8 +20,12 @@ import org.apache.hadoop.hive.ql.parse.ASTNode; +import lombok.Getter; +import lombok.Setter; public class DefaultAliasDecider implements AliasDecider { + @Getter + @Setter int counter = 0; private static final String ALIAS_PREFIX = "alias"; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java index c9993f3b1..29da0a2cf 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java @@ -24,16 +24,17 @@ import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; @Data @AllArgsConstructor +@NoArgsConstructor public class DefaultQueryAST implements QueryAST { private ASTNode selectAST, whereAST, groupByAST, havingAST, joinAST, orderByAST; private Integer limitValue; private String fromString; private String whereString; - public String getSelectString() { return HQLParser.getString(selectAST); } @@ -60,12 +61,12 @@ public String getOrderByString() { return null; } - public static DefaultQueryAST fromCandidateFact(CandidateFact fact, String storageTable, QueryAST ast) throws - LensException { + public static DefaultQueryAST fromStorageCandidate(StorageCandidate sc, QueryAST ast) throws + LensException { return new DefaultQueryAST(ast.getSelectAST(), - null, - ast.getGroupByAST(), ast.getHavingAST(), ast.getJoinAST(), ast.getOrderByAST(), ast.getLimitValue(), - ast.getFromString(), - fact.getStorageWhereString(storageTable.substring(storageTable.indexOf(".") + 1))); + null, + ast.getGroupByAST(), ast.getHavingAST(), ast.getJoinAST(), ast.getOrderByAST(), ast.getLimitValue(), + ast.getFromString(), + sc.getWhereString()); } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java index cb2687853..30fa873c2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,6 +18,8 @@ */ package org.apache.lens.cube.parse; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.denormColumnNotFound; + import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TABLE_OR_COL; @@ -26,12 +28,10 @@ import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.metadata.ReferencedDimAttribute.ChainRefCol; -import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; import org.apache.lens.cube.parse.ExpressionResolver.ExprSpecContext; import org.apache.lens.cube.parse.ExpressionResolver.ExpressionContext; import org.apache.lens.server.api.error.LensException; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; @@ -50,11 +50,8 @@ @Slf4j public class DenormalizationResolver implements ContextRewriter { - public DenormalizationResolver(Configuration conf) { - } - @ToString - public static class ReferencedQueriedColumn { + static class ReferencedQueriedColumn { ReferencedDimAttribute col; AbstractCubeTable srcTable; transient List chainRefCols = new ArrayList<>(); @@ -67,17 +64,12 @@ public static class ReferencedQueriedColumn { } @ToString - public static class PickedReference { + static class PickedReference { @Getter ChainRefCol chainRef; String srcAlias; String pickedFor; - PickedReference(String srcAlias, String pickedFor) { - this.srcAlias = srcAlias; - this.pickedFor = pickedFor; - } - PickedReference(ChainRefCol chainRef, String srcAlias, String pickedFor) { this.srcAlias = srcAlias; this.chainRef = chainRef; @@ -85,7 +77,7 @@ public static class PickedReference { } } - public static class DenormalizationContext { + static class DenormalizationContext { // map of column name to all references @Getter private Map> referencedCols = new HashMap<>(); @@ -101,12 +93,7 @@ public static class DenormalizationContext { private Map> pickedReferences = new HashMap<>(); void addReferencedCol(String col, ReferencedQueriedColumn refer) { - Set refCols = referencedCols.get(col); - if (refCols == null) { - refCols = new HashSet<>(); - referencedCols.put(col, refCols); - } - refCols.add(refer); + referencedCols.computeIfAbsent(col, k -> new HashSet<>()).add(refer); } // When candidate table does not have the field, this method checks @@ -122,12 +109,8 @@ boolean addRefUsage(CubeQueryContext cubeql, CandidateTable table, String col, S // there is no path // to the source table log.info("Adding denormalized column for column:{} for table:{}", col, table); - Set refCols = tableToRefCols.get(table.getName()); - if (refCols == null) { - refCols = new HashSet<>(); - tableToRefCols.put(table.getName(), refCols); - } - refCols.add(refer); + String name = (table instanceof CandidateDim) ? table.getName() : table.getStorageTable(); + tableToRefCols.computeIfAbsent(name, k -> new HashSet<>()).add(refer); // Add to optional tables for (ChainRefCol refCol : refer.col.getChainRefColumns()) { cubeql.addOptionalDimTable(refCol.getChainName(), table, false, refer.col.getName(), true, @@ -141,12 +124,7 @@ boolean addRefUsage(CubeQueryContext cubeql, CandidateTable table, String col, S } private void addPickedReference(String col, PickedReference refer) { - Set refCols = pickedReferences.get(col); - if (refCols == null) { - refCols = new HashSet<>(); - pickedReferences.put(col, refCols); - } - refCols.add(refer); + pickedReferences.computeIfAbsent(col, k -> new HashSet<>()).add(refer); } private PickedReference getPickedReference(String col, String srcAlias) { @@ -161,28 +139,27 @@ private PickedReference getPickedReference(String col, String srcAlias) { return null; } - public Set rewriteDenormctx(CubeQueryContext cubeql, CandidateFact cfact, Map dimsToQuery, boolean replaceFact) throws LensException { + Set rewriteDenormctx(CubeQueryContext cubeql, + StorageCandidate sc, Map dimsToQuery, boolean replaceFact) throws LensException { Set refTbls = new HashSet<>(); - log.info("Doing denorm changes for fact :{}", cfact); + log.info("Doing denorm changes for fact :{}", sc); if (!tableToRefCols.isEmpty()) { // pick referenced columns for fact - if (cfact != null) { - pickColumnsForTable(cubeql, cfact.getName()); + if (sc != null) { + pickColumnsForTable(cubeql, sc.getStorageTable()); } // pick referenced columns for dimensions - if (dimsToQuery != null && !dimsToQuery.isEmpty()) { + if (dimsToQuery != null) { for (CandidateDim cdim : dimsToQuery.values()) { pickColumnsForTable(cubeql, cdim.getName()); } } // Replace picked reference in all the base trees - replaceReferencedColumns(cubeql, cfact, replaceFact); - + replaceReferencedColumns(cubeql, sc, replaceFact); // Add the picked references to dimsToQuery for (PickedReference picked : pickedRefs) { - if (isPickedFor(picked, cfact, dimsToQuery)) { + if (isPickedFor(picked, sc, dimsToQuery)) { refTbls.add((Dimension) cubeql.getCubeTableForAlias(picked.getChainRef().getChainName())); cubeql.addColumnsQueried(picked.getChainRef().getChainName(), picked.getChainRef().getRefColumn()); } @@ -193,21 +170,19 @@ public Set rewriteDenormctx(CubeQueryContext cubeql, CandidateFact cf return refTbls; } - public boolean hasReferences() { + boolean hasReferences() { return !tableToRefCols.isEmpty(); } - public Set rewriteDenormctxInExpression(CubeQueryContext cubeql, CandidateFact cfact, Map rewriteDenormctxInExpression(CubeQueryContext cubeql, StorageCandidate sc, Map dimsToQuery, ASTNode exprAST) throws LensException { Set refTbls = new HashSet<>(); - - log.info("Doing denorm changes for expressions in fact :{}", cfact); if (!tableToRefCols.isEmpty()) { // pick referenced columns for fact - if (cfact != null) { - pickColumnsForTable(cubeql, cfact.getName()); + if (sc != null) { + pickColumnsForTable(cubeql, sc.getStorageTable()); } // pick referenced columns for dimensions - if (dimsToQuery != null && !dimsToQuery.isEmpty()) { + if (dimsToQuery != null) { for (CandidateDim cdim : dimsToQuery.values()) { pickColumnsForTable(cubeql, cdim.getName()); } @@ -217,7 +192,7 @@ public Set rewriteDenormctxInExpression(CubeQueryContext cubeql, Cand // Add the picked references to dimsToQuery for (PickedReference picked : pickedRefs) { - if (isPickedFor(picked, cfact, dimsToQuery)) { + if (isPickedFor(picked, sc, dimsToQuery)) { refTbls.add((Dimension) cubeql.getCubeTableForAlias(picked.getChainRef().getChainName())); cubeql.addColumnsQueried(picked.getChainRef().getChainName(), picked.getChainRef().getRefColumn()); } @@ -228,8 +203,8 @@ public Set rewriteDenormctxInExpression(CubeQueryContext cubeql, Cand return refTbls; } // checks if the reference if picked for facts and dimsToQuery passed - private boolean isPickedFor(PickedReference picked, CandidateFact cfact, Map dimsToQuery) { - if (cfact != null && picked.pickedFor.equalsIgnoreCase(cfact.getName())) { + private boolean isPickedFor(PickedReference picked, StorageCandidate sc, Map dimsToQuery) { + if (sc != null && picked.pickedFor.equalsIgnoreCase(sc.getStorageTable())) { return true; } if (dimsToQuery != null) { @@ -245,15 +220,9 @@ private boolean isPickedFor(PickedReference picked, CandidateFact cfact, Map iter = refered.chainRefCols.iterator(); - while (iter.hasNext()) { - // remove unreachable references - ChainRefCol reference = iter.next(); - if (!cubeql.getAutoJoinCtx().isReachableDim( - (Dimension) cubeql.getCubeTableForAlias(reference.getChainName()), reference.getChainName())) { - iter.remove(); - } - } + // remove unreachable references + refered.chainRefCols.removeIf(reference -> !cubeql.getAutoJoinCtx().isReachableDim( + (Dimension) cubeql.getCubeTableForAlias(reference.getChainName()), reference.getChainName())); if (refered.chainRefCols.isEmpty()) { throw new LensException(LensCubeErrorCode.NO_REF_COL_AVAILABLE.getLensErrorInfo(), refered.col.getName()); } @@ -265,8 +234,7 @@ private void pickColumnsForTable(CubeQueryContext cubeql, String tbl) throws Len } } } - - public void pruneReferences(CubeQueryContext cubeql) { + void pruneReferences(CubeQueryContext cubeql) { for (Set referencedQueriedColumns : referencedCols.values()) { for(Iterator iterator = referencedQueriedColumns.iterator(); iterator.hasNext();) { ReferencedQueriedColumn rqc = iterator.next(); @@ -295,25 +263,24 @@ public void pruneReferences(CubeQueryContext cubeql) { } } } - private void replaceReferencedColumns(CubeQueryContext cubeql, CandidateFact cfact, boolean replaceFact) - throws LensException { + + private void replaceReferencedColumns(CubeQueryContext cubeql, StorageCandidate sc, boolean replaceFact) + throws LensException { QueryAST ast = cubeql; - boolean factRefExists = cfact != null && tableToRefCols.get(cfact.getName()) != null && !tableToRefCols.get(cfact - .getName()).isEmpty(); + boolean factRefExists = sc != null && tableToRefCols.get(sc.getStorageTable()) != null + && !tableToRefCols.get(sc.getStorageTable()).isEmpty(); if (replaceFact && factRefExists) { - ast = cfact; + ast = sc.getQueryAst(); } resolveClause(ast.getSelectAST()); if (factRefExists) { - for (ASTNode storageWhereClauseAST : cfact.getStorgeWhereClauseMap().values()) { - resolveClause(storageWhereClauseAST); - } + resolveClause(sc.getQueryAst().getWhereAST()); } else { resolveClause(ast.getWhereAST()); } resolveClause(ast.getGroupByAST()); resolveClause(ast.getHavingAST()); - resolveClause(cubeql.getOrderByAST()); + resolveClause(ast.getOrderByAST()); } private void resolveClause(ASTNode node) throws LensException { @@ -332,6 +299,7 @@ private void resolveClause(ASTNode node) throws LensException { ASTNode tableNode = (ASTNode) node.getChild(0); ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier); + assert tabident != null; PickedReference refered = getPickedReference(colName, tabident.getText().toLowerCase()); if (refered == null) { return; @@ -352,7 +320,7 @@ private void resolveClause(ASTNode node) throws LensException { } } - public Set getNonReachableReferenceFields(String table) { + Set getNonReachableReferenceFields(String table) { Set nonReachableFields = new HashSet<>(); if (tableToRefCols.containsKey(table)) { for (ReferencedQueriedColumn refcol : tableToRefCols.get(table)) { @@ -369,7 +337,7 @@ public Set getNonReachableReferenceFields(String table) { private void addRefColsQueried(CubeQueryContext cubeql, TrackQueriedColumns tqc, DenormalizationContext denormCtx) { for (Map.Entry> entry : tqc.getTblAliasToColumns().entrySet()) { // skip default alias - if (entry.getKey() == CubeQueryContext.DEFAULT_TABLE) { + if (Objects.equals(entry.getKey(), CubeQueryContext.DEFAULT_TABLE)) { continue; } // skip join chain aliases @@ -408,9 +376,10 @@ private static DenormalizationContext getOrCreateDeNormCtx(TrackDenormContext td public void rewriteContext(CubeQueryContext cubeql) throws LensException { DenormalizationContext denormCtx = cubeql.getDeNormCtx(); if (denormCtx == null) { + DenormalizationContext ctx = getOrCreateDeNormCtx(cubeql); // Adds all the reference dimensions as eligible for denorm fields // add ref columns in cube - addRefColsQueried(cubeql, cubeql, getOrCreateDeNormCtx(cubeql)); + addRefColsQueried(cubeql, cubeql, ctx); // add ref columns from expressions for (Set ecSet : cubeql.getExprCtx().getAllExprsQueried().values()) { for (ExpressionContext ec : ecSet) { @@ -424,21 +393,22 @@ public void rewriteContext(CubeQueryContext cubeql) throws LensException { // In the second iteration of denorm resolver // candidate tables which require denorm fields and the refernces are no // more valid will be pruned - if (cubeql.getCube() != null && !cubeql.getCandidateFacts().isEmpty()) { - for (Iterator i = cubeql.getCandidateFacts().iterator(); i.hasNext();) { - CandidateFact cfact = i.next(); - Set nonReachableFields = denormCtx.getNonReachableReferenceFields(cfact.getName()); + if (cubeql.getCube() != null && !cubeql.getCandidates().isEmpty()) { + for (Iterator i = + CandidateUtil.getStorageCandidates(cubeql.getCandidates()).iterator(); i.hasNext();) { + StorageCandidate candidate = i.next(); + Set nonReachableFields = denormCtx.getNonReachableReferenceFields(candidate.getStorageTable()); if (!nonReachableFields.isEmpty()) { - log.info("Not considering fact table:{} as columns {} are not available", cfact, nonReachableFields); - cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(nonReachableFields)); + log.info("Not considering fact table:{} as columns {} are not available", candidate, nonReachableFields); + cubeql.addCandidatePruningMsg(candidate, denormColumnNotFound(nonReachableFields)); i.remove(); } } - if (cubeql.getCandidateFacts().size() == 0) { + if (cubeql.getCandidates().size() == 0) { throw new LensException(LensCubeErrorCode.NO_FACT_HAS_COLUMN.getLensErrorInfo(), cubeql.getColumnsQueriedForTable(cubeql.getCube().getName()).toString()); } - cubeql.pruneCandidateFactSet(CandidateTablePruneCode.COLUMN_NOT_FOUND); + } if (cubeql.getDimensions() != null && !cubeql.getDimensions().isEmpty()) { for (Dimension dim : cubeql.getDimensions()) { @@ -447,8 +417,7 @@ public void rewriteContext(CubeQueryContext cubeql) throws LensException { Set nonReachableFields = denormCtx.getNonReachableReferenceFields(cdim.getName()); if (!nonReachableFields.isEmpty()) { log.info("Not considering dim table:{} as column {} is not available", cdim, nonReachableFields); - cubeql.addDimPruningMsgs(dim, cdim.dimtable, - CandidateTablePruneCause.columnNotFound(nonReachableFields)); + cubeql.addDimPruningMsgs(dim, cdim.dimtable, denormColumnNotFound(nonReachableFields)); i.remove(); } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index c99fdf1c7..926a4d0e0 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -25,9 +25,6 @@ import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.metadata.ExprColumn.ExprSpec; -import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; -import org.apache.lens.cube.parse.HQLParser.ASTNodeVisitor; -import org.apache.lens.cube.parse.HQLParser.TreeNode; import org.apache.lens.server.api.error.LensException; import org.apache.hadoop.hive.ql.parse.ASTNode; @@ -58,7 +55,7 @@ static class ExpressionContext { private Map> evaluableExpressions = new HashMap<>(); private boolean hasMeasures = false; - public boolean hasMeasures() { + boolean hasMeasures() { return hasMeasures; } @@ -148,11 +145,6 @@ void addDirectlyAvailable(CandidateTable cTable) { } void addEvaluable(CubeQueryContext cubeql, CandidateTable cTable, ExprSpecContext esc) throws LensException { - Set evalSet = evaluableExpressions.get(cTable); - if (evalSet == null) { - evalSet = new LinkedHashSet<>(); - evaluableExpressions.put(cTable, evalSet); - } // add optional dimensions involved in expressions for (String table : esc.getTblAliasToColumns().keySet()) { if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && !srcAlias.equals(table)) { @@ -161,7 +153,7 @@ void addEvaluable(CubeQueryContext cubeql, CandidateTable cTable, ExprSpecContex esc.exprDims.add((Dimension) cubeql.getCubeTableForAlias(table)); } } - evalSet.add(esc); + evaluableExpressions.computeIfAbsent(cTable, k -> new LinkedHashSet<>()).add(esc); } Set getAllASTNodes() { @@ -182,13 +174,8 @@ boolean hasAggregates() { } boolean isEvaluable(CandidateTable cTable) { - if (directlyAvailableIn.contains(cTable)) { - return true; - } - if (evaluableExpressions.get(cTable) == null) { - return false; - } - return !evaluableExpressions.get(cTable).isEmpty(); + return directlyAvailableIn.contains(cTable) + || (evaluableExpressions.get(cTable) != null && !evaluableExpressions.get(cTable).isEmpty()); } } @@ -252,16 +239,16 @@ Date getEndTime() { return null; } - public boolean isValidInTimeRange(final TimeRange range) { + boolean isValidInTimeRange(final TimeRange range) { return isValidFrom(range.getFromDate()) && isValidTill(range.getToDate()); } - public boolean isValidFrom(@NonNull final Date date) { - return (getStartTime() == null) ? true : date.equals(getStartTime()) || date.after(getStartTime()); + boolean isValidFrom(@NonNull final Date date) { + return (getStartTime() == null) || (date.equals(getStartTime()) || date.after(getStartTime())); } - public boolean isValidTill(@NonNull final Date date) { - return (getEndTime() == null) ? true : date.equals(getEndTime()) || date.before(getEndTime()); + boolean isValidTill(@NonNull final Date date) { + return (getEndTime() == null) || (date.equals(getEndTime()) || date.before(getEndTime())); } public String toString() { @@ -304,13 +291,7 @@ static class ExpressionResolverContext { this.cubeql = cubeql; } void addExpressionQueried(ExpressionContext expr) { - String exprCol = expr.getExprCol().getName().toLowerCase(); - Set ecSet = allExprsQueried.get(exprCol); - if (ecSet == null) { - ecSet = new LinkedHashSet(); - allExprsQueried.put(exprCol, ecSet); - } - ecSet.add(expr); + allExprsQueried.computeIfAbsent(expr.getExprCol().getName().toLowerCase(), k -> new LinkedHashSet<>()).add(expr); } boolean isQueriedExpression(String column) { @@ -337,7 +318,7 @@ ExpressionContext getExpressionContext(String expr, String alias) { throw new IllegalArgumentException("no expression available for " + expr + " alias:" + alias); } - public boolean hasMeasures(String expr, CubeInterface cube) { + boolean hasMeasures(String expr, CubeInterface cube) { String alias = cubeql.getAliasForTableName(cube.getName()); ExpressionContext ec = getExpressionContext(expr, alias); boolean hasMeasures = false; @@ -356,7 +337,7 @@ public boolean hasMeasures(String expr, CubeInterface cube) { } //updates all expression specs which are evaluable - public void updateEvaluables(String expr, CandidateTable cTable) + void updateEvaluables(String expr, CandidateTable cTable) throws LensException { String alias = cubeql.getAliasForTableName(cTable.getBaseTable().getName()); ExpressionContext ec = getExpressionContext(expr, alias); @@ -392,19 +373,20 @@ public void updateEvaluables(String expr, CandidateTable cTable) } // checks if expr is evaluable - public boolean isEvaluable(String expr, CandidateTable cTable) { + boolean isEvaluable(String expr, CandidateTable cTable) { ExpressionContext ec = getExpressionContext(expr, cubeql.getAliasForTableName(cTable.getBaseTable().getName())); return ec.isEvaluable(cTable); } - public Set rewriteExprCtx(CubeQueryContext cubeql, CandidateFact cfact, Map dimsToQuery, QueryAST queryAST) throws LensException { + Set rewriteExprCtx(CubeQueryContext cubeql, StorageCandidate sc, + Map dimsToQuery, + QueryAST queryAST) throws LensException { Set exprDims = new HashSet(); - log.info("Picking expressions for fact {} ", cfact); + log.info("Picking expressions for candidate {} ", sc); if (!allExprsQueried.isEmpty()) { // pick expressions for fact - if (cfact != null) { - pickExpressionsForTable(cfact); + if (sc != null) { + pickExpressionsForTable(sc); } // pick expressions for dimensions if (dimsToQuery != null && !dimsToQuery.isEmpty()) { @@ -412,17 +394,17 @@ public Set rewriteExprCtx(CubeQueryContext cubeql, CandidateFact cfac pickExpressionsForTable(cdim); } } - log.info("Picked expressions: {}", pickedExpressions); + log.debug("Picked expressions: {}", pickedExpressions); for (Set peSet : pickedExpressions.values()) { for (PickedExpression pe : peSet) { exprDims.addAll(pe.pickedCtx.exprDims); pe.initRewrittenAST(pe.pickedCtx.deNormCtx.hasReferences()); - exprDims.addAll(pe.pickedCtx.deNormCtx.rewriteDenormctxInExpression(cubeql, cfact, dimsToQuery, + exprDims.addAll(pe.pickedCtx.deNormCtx.rewriteDenormctxInExpression(cubeql, sc, dimsToQuery, pe.getRewrittenAST())); } } // Replace picked expressions in all the base trees - replacePickedExpressions(cfact, queryAST); + replacePickedExpressions(sc, queryAST); } pickedExpressions.clear(); @@ -430,13 +412,11 @@ public Set rewriteExprCtx(CubeQueryContext cubeql, CandidateFact cfac return exprDims; } - private void replacePickedExpressions(CandidateFact cfact, QueryAST queryAST) + private void replacePickedExpressions(StorageCandidate sc, QueryAST queryAST) throws LensException { replaceAST(cubeql, queryAST.getSelectAST()); - if (cfact != null) { - for (ASTNode storageWhereClauseAST : cfact.getStorgeWhereClauseMap().values()) { - replaceAST(cubeql, storageWhereClauseAST); - } + if (sc != null) { + replaceAST(cubeql, sc.getQueryAst().getWhereAST()); } else { replaceAST(cubeql, queryAST.getWhereAST()); } @@ -445,7 +425,7 @@ private void replacePickedExpressions(CandidateFact cfact, QueryAST queryAST) // Having AST is resolved by each fact, so that all facts can expand their expressions. // Having ast is not copied now, it's maintained in cubeql, each fact processes that serially. replaceAST(cubeql, cubeql.getHavingAST()); - replaceAST(cubeql, cubeql.getOrderByAST()); + replaceAST(cubeql, queryAST.getOrderByAST()); } private void replaceAST(final CubeQueryContext cubeql, ASTNode node) throws LensException { @@ -453,27 +433,25 @@ private void replaceAST(final CubeQueryContext cubeql, ASTNode node) throws Lens return; } // Traverse the tree and resolve expression columns - HQLParser.bft(node, new ASTNodeVisitor() { - @Override - public void visit(TreeNode visited) throws LensException { - ASTNode node = visited.getNode(); - int childcount = node.getChildCount(); - for (int i = 0; i < childcount; i++) { - ASTNode current = (ASTNode) node.getChild(i); - if (current.getToken().getType() == DOT) { - // This is for the case where column name is prefixed by table name - // or table alias - // For example 'select fact.id, dim2.id ...' - // Right child is the column name, left child.ident is table name - ASTNode tabident = HQLParser.findNodeByPath(current, TOK_TABLE_OR_COL, Identifier); - ASTNode colIdent = (ASTNode) current.getChild(1); - String column = colIdent.getText().toLowerCase(); - - if (pickedExpressions.containsKey(column)) { - PickedExpression expr = getPickedExpression(column, tabident.getText().toLowerCase()); - if (expr != null) { - node.setChild(i, replaceAlias(expr.getRewrittenAST(), cubeql)); - } + HQLParser.bft(node, visited -> { + ASTNode node1 = visited.getNode(); + int childcount = node1.getChildCount(); + for (int i = 0; i < childcount; i++) { + ASTNode current = (ASTNode) node1.getChild(i); + if (current.getToken().getType() == DOT) { + // This is for the case where column name is prefixed by table name + // or table alias + // For example 'select fact.id, dim2.id ...' + // Right child is the column name, left child.ident is table name + ASTNode tabident = HQLParser.findNodeByPath(current, TOK_TABLE_OR_COL, Identifier); + ASTNode colIdent = (ASTNode) current.getChild(1); + String column = colIdent.getText().toLowerCase(); + + if (pickedExpressions.containsKey(column)) { + assert tabident != null; + PickedExpression expr = getPickedExpression(column, tabident.getText().toLowerCase()); + if (expr != null) { + node1.setChild(i, replaceAlias(expr.getRewrittenAST(), cubeql)); } } } @@ -502,12 +480,8 @@ private void pickExpressionsForTable(CandidateTable cTable) { log.debug("{} is not directly evaluable in {}", ec, cTable); if (ec.evaluableExpressions.get(cTable) != null && !ec.evaluableExpressions.get(cTable).isEmpty()) { // pick first evaluable expression - Set peSet = pickedExpressions.get(ecEntry.getKey()); - if (peSet == null) { - peSet = new HashSet(); - pickedExpressions.put(ecEntry.getKey(), peSet); - } - peSet.add(new PickedExpression(ec.srcAlias, ec.evaluableExpressions.get(cTable).iterator().next())); + pickedExpressions.computeIfAbsent(ecEntry.getKey(), k -> new HashSet<>()) + .add(new PickedExpression(ec.srcAlias, ec.evaluableExpressions.get(cTable).iterator().next())); } } } @@ -616,7 +590,7 @@ public void rewriteContext(CubeQueryContext cubeql) throws LensException { for (Map.Entry> entry : cubeql.getTblAliasToColumns().entrySet()) { String alias = entry.getKey(); // skip default alias - if (alias == CubeQueryContext.DEFAULT_TABLE) { + if (Objects.equals(alias, CubeQueryContext.DEFAULT_TABLE)) { continue; } AbstractCubeTable tbl = cubeql.getCubeTableForAlias(alias); @@ -646,41 +620,39 @@ public void rewriteContext(CubeQueryContext cubeql) throws LensException { // prune invalid expressions cubeql.getExprCtx().pruneExpressions(); // prune candidate facts without any valid expressions - if (cubeql.getCube() != null && !cubeql.getCandidateFacts().isEmpty()) { + if (cubeql.getCube() != null && !cubeql.getCandidates().isEmpty()) { for (Map.Entry> ecEntry : exprCtx.allExprsQueried.entrySet()) { String expr = ecEntry.getKey(); Set ecSet = ecEntry.getValue(); for (ExpressionContext ec : ecSet) { if (ec.getSrcTable().getName().equals(cubeql.getCube().getName())) { if (cubeql.getQueriedExprsWithMeasures().contains(expr)) { - for (Iterator> sItr = cubeql.getCandidateFactSets().iterator(); sItr.hasNext();) { - Set factSet = sItr.next(); - boolean evaluableInSet = false; - for (CandidateFact cfact : factSet) { - if (ec.isEvaluable(cfact)) { - evaluableInSet = true; - } - } - if (!evaluableInSet) { - log.info("Not considering fact table set:{} as {} is not evaluable", factSet, ec.exprCol.getName()); + for (Iterator sItr = cubeql.getCandidates().iterator(); sItr.hasNext();) { + Candidate cand = sItr.next(); + if (!cand.isExpressionEvaluable(ec)) { + log.info("Not considering Candidate :{} as {} is not evaluable", cand, ec.exprCol.getName()); sItr.remove(); + cubeql.addCandidatePruningMsg(cand, + CandidateTablePruneCause.expressionNotEvaluable(ec.exprCol.getName())); } } } else { - for (Iterator i = cubeql.getCandidateFacts().iterator(); i.hasNext();) { - CandidateFact cfact = i.next(); - if (!ec.isEvaluable(cfact)) { - log.info("Not considering fact table:{} as {} is not evaluable", cfact, ec.exprCol.getName()); - cubeql.addFactPruningMsgs(cfact.fact, - CandidateTablePruneCause.expressionNotEvaluable(ec.exprCol.getName())); - i.remove(); + // prune dimension only expressions + Set storageCandidates = CandidateUtil.getStorageCandidates(cubeql.getCandidates()); + for (StorageCandidate sc : storageCandidates) { + if (!sc.isExpressionEvaluable(ec)) { + Collection prunedCandidates = + CandidateUtil.filterCandidates(cubeql.getCandidates(), sc); + log.info("Not considering candidate(s) :{} as expr :{} in storage :{} is not evaluable", + prunedCandidates, ec.exprCol.getName(), sc); + cubeql.addStoragePruningMsg(sc, + CandidateTablePruneCause.expressionNotEvaluable(ec.exprCol.getName())); } } } } } } - cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCode.EXPRESSION_NOT_EVALUABLE); } // prune candidate dims without any valid expressions if (cubeql.getDimensions() != null && !cubeql.getDimensions().isEmpty()) { @@ -707,24 +679,21 @@ public void rewriteContext(CubeQueryContext cubeql) throws LensException { } private static ASTNode replaceAlias(final ASTNode expr, final CubeQueryContext cubeql) throws LensException { - ASTNode finalAST = MetastoreUtil.copyAST(expr); - HQLParser.bft(finalAST, new ASTNodeVisitor() { - @Override - public void visit(TreeNode visited) { - ASTNode node = visited.getNode(); - ASTNode parent = null; - if (visited.getParent() != null) { - parent = visited.getParent().getNode(); - } - - if (node.getToken().getType() == TOK_TABLE_OR_COL && (parent != null && parent.getToken().getType() == DOT)) { - ASTNode current = (ASTNode) node.getChild(0); - if (current.getToken().getType() == Identifier) { - String tableName = current.getToken().getText().toLowerCase(); - String alias = cubeql.getAliasForTableName(tableName); - if (!alias.equalsIgnoreCase(tableName)) { - node.setChild(0, new ASTNode(new CommonToken(HiveParser.Identifier, alias))); - } + final ASTNode finalAST = MetastoreUtil.copyAST(expr); + HQLParser.bft(finalAST, visited -> { + ASTNode node = visited.getNode(); + ASTNode parent = null; + if (visited.getParent() != null) { + parent = visited.getParent().getNode(); + } + + if (node.getToken().getType() == TOK_TABLE_OR_COL && (parent != null && parent.getToken().getType() == DOT)) { + ASTNode current = (ASTNode) node.getChild(0); + if (current.getToken().getType() == Identifier) { + String tableName = current.getToken().getText().toLowerCase(); + String alias = cubeql.getAliasForTableName(tableName); + if (!alias.equalsIgnoreCase(tableName)) { + node.setChild(0, new ASTNode(new CommonToken(HiveParser.Identifier, alias))); } } } @@ -738,33 +707,30 @@ private static void replaceColumnInAST(ASTNode expr, final String toReplace, fin return; } // Traverse the tree and resolve expression columns - HQLParser.bft(expr, new ASTNodeVisitor() { - @Override - public void visit(TreeNode visited) throws LensException { - ASTNode node = visited.getNode(); - int childcount = node.getChildCount(); - for (int i = 0; i < childcount; i++) { - ASTNode current = (ASTNode) node.getChild(i); - if (current.getToken().getType() == TOK_TABLE_OR_COL && (node != null && node.getToken().getType() != DOT)) { - // Take child ident.totext - ASTNode ident = (ASTNode) current.getChild(0); - String column = ident.getText().toLowerCase(); - if (toReplace.equals(column)) { - node.setChild(i, MetastoreUtil.copyAST(columnAST)); - } - } else if (current.getToken().getType() == DOT) { - // This is for the case where column name is prefixed by table name - // or table alias - // For example 'select fact.id, dim2.id ...' - // Right child is the column name, left child.ident is table name - ASTNode tabident = HQLParser.findNodeByPath(current, TOK_TABLE_OR_COL, Identifier); - ASTNode colIdent = (ASTNode) current.getChild(1); - - String column = colIdent.getText().toLowerCase(); - - if (toReplace.equals(column)) { - node.setChild(i, MetastoreUtil.copyAST(columnAST)); - } + HQLParser.bft(expr, visited -> { + ASTNode node = visited.getNode(); + int childcount = node.getChildCount(); + for (int i = 0; i < childcount; i++) { + ASTNode current = (ASTNode) node.getChild(i); + if (current.getToken().getType() == TOK_TABLE_OR_COL && node.getToken().getType() != DOT) { + // Take child ident.totext + ASTNode ident = (ASTNode) current.getChild(0); + String column = ident.getText().toLowerCase(); + if (toReplace.equals(column)) { + node.setChild(i, MetastoreUtil.copyAST(columnAST)); + } + } else if (current.getToken().getType() == DOT) { + // This is for the case where column name is prefixed by table name + // or table alias + // For example 'select fact.id, dim2.id ...' + // Right child is the column name, left child.ident is table name + ASTNode tabident = HQLParser.findNodeByPath(current, TOK_TABLE_OR_COL, Identifier); + ASTNode colIdent = (ASTNode) current.getChild(1); + + String column = colIdent.getText().toLowerCase(); + + if (toReplace.equals(column)) { + node.setChild(i, MetastoreUtil.copyAST(columnAST)); } } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java index 48af0c908..94f9c7d72 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java @@ -104,7 +104,6 @@ public void validateFields(CubeQueryContext cubeql) throws LensException { conflictingFields.addAll(queriedMsrs); throw new FieldsCannotBeQueriedTogetherException(new ConflictingFields(conflictingFields)); } else { - conflictingFields.addAll(queriedMsrs); throw new FieldsCannotBeQueriedTogetherException(new ConflictingFields(conflictingFields)); } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java index 052b87a31..1b30c0b25 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -18,10 +18,11 @@ */ package org.apache.lens.cube.parse; +import static org.apache.lens.cube.parse.ColumnResolver.addColumnsForSelectExpr; + import static org.apache.hadoop.hive.ql.parse.HiveParser.*; -import java.util.ArrayList; -import java.util.List; +import java.util.*; import org.apache.lens.cube.metadata.AbstractBaseTable; import org.apache.lens.server.api.error.LensException; @@ -42,6 +43,7 @@ @Slf4j class GroupbyResolver implements ContextRewriter { + private static final String SELECT_ALIAS_PREFIX = "select_expr"; private final boolean selectPromotionEnabled; private final boolean groupbyPromotionEnabled; @@ -53,8 +55,8 @@ class GroupbyResolver implements ContextRewriter { CubeQueryConfUtil.DEFAULT_ENABLE_GROUP_BY_TO_SELECT); } - private void promoteSelect(CubeQueryContext cubeql, List selectExprs, - List groupByExprs) throws LensException { + private void promoteSelect(CubeQueryContext cubeql, List selectExprs, List groupByExprs) + throws LensException { if (!selectPromotionEnabled) { return; } @@ -79,7 +81,7 @@ private void promoteSelect(CubeQueryContext cubeql, List se groupbyAST.addChild(exprAST); } else { // no group by ast exist, create one - ASTNode newAST = new ASTNode(new CommonToken(TOK_GROUPBY)); + ASTNode newAST = new ASTNode(new CommonToken(TOK_GROUPBY, "TOK_GROUPBY")); newAST.addChild(exprAST); cubeql.setGroupByAST(newAST); } @@ -97,7 +99,6 @@ private boolean isConstantsUsed(ASTNode node) { return node != null && node.getToken() != null && !hasTableOrColumn(node); } - /* * Check if table or column used in node */ @@ -115,8 +116,7 @@ private boolean hasTableOrColumn(ASTNode node) { return false; } - private void promoteGroupby(CubeQueryContext cubeql, List selectExprs, - List groupByExprs) + private void promoteGroupby(CubeQueryContext cubeql, List selectExprs, List groupByExprs) throws LensException { if (!groupbyPromotionEnabled) { return; @@ -131,12 +131,44 @@ private void promoteGroupby(CubeQueryContext cubeql, List s for (String expr : groupByExprs) { if (!contains(selectExprs, expr)) { ASTNode exprAST = HQLParser.parseExpr(expr, cubeql.getConf()); - addChildAtIndex(index, cubeql.getSelectAST(), exprAST); + ASTNode parent = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR, "TOK_SELEXPR")); + parent.addChild(exprAST); + exprAST.setParent(parent); + addChildAtIndex(index, cubeql.getSelectAST(), parent); + updateSelectPhrase(cubeql, index, parent); index++; } } } + private void updateSelectPhrase(CubeQueryContext cubeql, int index, ASTNode selectExpr) { + int exprInd = index; + ASTNode selectExprChild = (ASTNode) selectExpr.getChild(0); + Set cols = new HashSet<>(); + SelectPhraseContext sel = new SelectPhraseContext(selectExpr); + addColumnsForSelectExpr(sel, selectExpr, cubeql.getSelectAST(), cols); + String alias = selectExpr.getChildCount() > 1 ? selectExpr.getChild(1).getText() : null; + String selectAlias; + String selectFinalAlias = null; + if (alias != null) { + selectFinalAlias = alias; + selectAlias = SELECT_ALIAS_PREFIX + exprInd; + } else if (cols.size() == 1 && (selectExprChild.getToken().getType() == TOK_TABLE_OR_COL + || selectExprChild.getToken().getType() == DOT)) { + // select expression is same as the column + selectAlias = cols.iterator().next().toLowerCase(); + } else { + selectAlias = SELECT_ALIAS_PREFIX + exprInd; + selectFinalAlias = HQLParser.getString(selectExprChild); + } + cubeql.addColumnsQueried(sel.getTblAliasToColumns()); + sel.setSelectAlias(selectAlias); + sel.setFinalAlias(!StringUtils.isBlank(selectFinalAlias) ? "`" + selectFinalAlias + "`" : selectAlias); + sel.setActualAlias(alias != null ? alias.toLowerCase() : null); + cubeql.getSelectPhrases().add(exprInd, sel); + //cubeql.addSelectPhrase(sel); + } + private void addChildAtIndex(int index, ASTNode parent, ASTNode child) { // add the last child int count = parent.getChildCount(); @@ -158,7 +190,7 @@ public void rewriteContext(CubeQueryContext cubeql) throws LensException { List selectExprs = getSelectNonAggregateNonMeasureExpressions(cubeql); List groupByExprs = new ArrayList<>(); if (cubeql.getGroupByString() != null) { - String[] gby = getGroupbyExpressions(cubeql.getGroupByAST()).toArray(new String[]{}); + String[] gby = getGroupbyExpressions(cubeql.getGroupByAST()).toArray(new String[] {}); for (String g : gby) { groupByExprs.add(g.trim()); } @@ -228,7 +260,7 @@ boolean hasAggregate(ASTNode node, CubeQueryContext cubeql) { // by the time Groupby resolver is looking for aggregate, all columns should be aliased with correct // alias name. if (cubeql.getCubeTableForAlias(alias) instanceof AbstractBaseTable) { - if (((AbstractBaseTable)cubeql.getCubeTableForAlias(alias)).getExpressionByName(colname) != null) { + if (((AbstractBaseTable) cubeql.getCubeTableForAlias(alias)).getExpressionByName(colname) != null) { return cubeql.getExprCtx().getExpressionContext(colname, alias).hasAggregates(); } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java new file mode 100644 index 000000000..52085ea35 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + +import java.util.*; + +import org.apache.lens.cube.metadata.FactPartition; +import org.apache.lens.cube.metadata.TimeRange; +import org.apache.lens.server.api.error.LensException; + +/** + * Represents a join of two candidates + */ +public class JoinCandidate implements Candidate { + + /** + * Child candidates that will participate in the join + */ + private Candidate childCandidate1; + private Candidate childCandidate2; + private String toStr; + private QueryAST queryAST; + private CubeQueryContext cubeql; + + public JoinCandidate(Candidate childCandidate1, Candidate childCandidate2, CubeQueryContext cubeql) { + this.childCandidate1 = childCandidate1; + this.childCandidate2 = childCandidate2; + this.cubeql = cubeql; + } + + @Override + public Collection getColumns() { + Set columns = new HashSet<>(); + columns.addAll(childCandidate1.getColumns()); + columns.addAll(childCandidate2.getColumns()); + return columns; + } + + @Override + public Date getStartTime() { + return childCandidate1.getStartTime().after(childCandidate2.getStartTime()) + ? childCandidate1.getStartTime() : childCandidate2.getStartTime(); + } + + @Override + public Date getEndTime() { + return childCandidate1.getEndTime().before(childCandidate2.getEndTime()) + ? childCandidate1.getEndTime() : childCandidate2.getEndTime(); + } + + @Override + public double getCost() { + return childCandidate1.getCost() + childCandidate2.getCost(); + } + + @Override + public boolean contains(Candidate candidate) { + if (this.equals(candidate)) { + return true; + } else { + return childCandidate1.contains(candidate) || childCandidate2.contains(candidate); + } + } + + @Override + public Collection getChildren() { + ArrayList joinCandidates = new ArrayList<>(); + joinCandidates.add(childCandidate1); + joinCandidates.add(childCandidate2); + return joinCandidates; + } + + /** + * @param timeRange + * @return + */ + @Override + public boolean evaluateCompleteness(TimeRange timeRange, TimeRange parentTimeRange, boolean failOnPartialData) + throws LensException { + return this.childCandidate1.evaluateCompleteness(timeRange, parentTimeRange, failOnPartialData) + && this.childCandidate2.evaluateCompleteness(timeRange, parentTimeRange, failOnPartialData); + } + + /** + * @return all the partitions from the children + */ + @Override + public Set getParticipatingPartitions() { + Set factPartitionsSet = new HashSet<>(); + factPartitionsSet.addAll(childCandidate1.getParticipatingPartitions()); + factPartitionsSet.addAll(childCandidate2.getParticipatingPartitions()); + return factPartitionsSet; + } + + @Override + public boolean isExpressionEvaluable(ExpressionResolver.ExpressionContext expr) { + return childCandidate1.isExpressionEvaluable(expr) || childCandidate2.isExpressionEvaluable(expr); + } + + @Override + public Set getAnswerableMeasurePhraseIndices() { + Set mesureIndices = new HashSet<>(); + for (Candidate cand : getChildren()) { + mesureIndices.addAll(cand.getAnswerableMeasurePhraseIndices()); + } + return mesureIndices; + } + + @Override + public boolean isTimeRangeCoverable(TimeRange timeRange) throws LensException { + return this.childCandidate1.isTimeRangeCoverable(timeRange) + && this.childCandidate2.isTimeRangeCoverable(timeRange); + } + + @Override + public String toString() { + if (this.toStr == null) { + this.toStr = getToString(); + } + return this.toStr; + } + + private String getToString() { + return "JOIN[" + childCandidate1.toString() + ", " + childCandidate2.toString() + "]"; + } +} diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java index fce16624d..02e3dc7b6 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java @@ -42,7 +42,10 @@ class JoinResolver implements ContextRewriter { private Map tableJoinTypeMap; private AbstractCubeTable target; - private HashMap> dimensionInJoinChain = new HashMap>(); + /** + * Dimension as key and all the participating join chains for this dimension as value. + */ + private HashMap> dimensionToJoinChainsMap = new HashMap>(); @Override public void rewriteContext(CubeQueryContext cubeql) throws LensException { @@ -91,10 +94,10 @@ private void processJoinChains(CubeQueryContext cubeql) throws HiveException, Le dims.add(chain.getDestTable()); for (String dim : dims) { Dimension dimension = cubeql.getMetastoreClient().getDimension(dim); - if (dimensionInJoinChain.get(dimension) == null) { - dimensionInJoinChain.put(dimension, new ArrayList()); + if (dimensionToJoinChainsMap.get(dimension) == null) { + dimensionToJoinChainsMap.put(dimension, new ArrayList()); } - dimensionInJoinChain.get(dimension).add(chain); + dimensionToJoinChainsMap.get(dimension).add(chain); } } } @@ -139,7 +142,7 @@ private void autoResolveJoins(CubeQueryContext cubeql) throws LensException, Hiv Map, List> multipleJoinPaths = new LinkedHashMap<>(); - // populate paths from joinchains + // populate paths from joinchains. For a destination Dimension get all the join paths that lead to it. for (JoinChain chain : cubeql.getJoinchains().values()) { Dimension dimension = cubeql.getMetastoreClient().getDimension(chain.getDestTable()); Aliased aliasedDimension = Aliased.create(dimension, chain.getName()); @@ -149,6 +152,7 @@ private void autoResolveJoins(CubeQueryContext cubeql) throws LensException, Hiv multipleJoinPaths.get(aliasedDimension).addAll( chain.getRelationEdges(cubeql.getMetastoreClient())); } + boolean flattenBridgeTables = cubeql.getConf().getBoolean(CubeQueryConfUtil.ENABLE_FLATTENING_FOR_BRIDGETABLES, CubeQueryConfUtil.DEFAULT_ENABLE_FLATTENING_FOR_BRIDGETABLES); String bridgeTableFieldAggr = cubeql.getConf().get(CubeQueryConfUtil.BRIDGE_TABLE_FIELD_AGGREGATOR, diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java index 0bc7f8243..a9bd164b8 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -18,9 +18,11 @@ */ package org.apache.lens.cube.parse; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; -import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; import org.apache.lens.server.api.error.LensException; import lombok.extern.slf4j.Slf4j; @@ -33,35 +35,36 @@ class LeastPartitionResolver implements ContextRewriter { @Override public void rewriteContext(CubeQueryContext cubeql) throws LensException { - if (cubeql.getCube() != null && !cubeql.getCandidateFactSets().isEmpty()) { - Map, Integer> factPartCount = new HashMap, Integer>(); + if (cubeql.getCube() != null && !cubeql.getCandidates().isEmpty()) { + Map factPartCount = new HashMap<>(); //The number of partitions being calculated is not the actual number of partitions, // they are number of time values now instead of partitions. // This seems fine, as the less number of time values actually represent the rollups on time. And with // MaxCoveringFactResolver facts with less partitions which are not covering the range would be removed. - for (Set facts : cubeql.getCandidateFactSets()) { - factPartCount.put(facts, getPartCount(facts)); + for (Candidate candidate : cubeql.getCandidates()) { + factPartCount.put(candidate, getPartCount(candidate)); } double minPartitions = Collections.min(factPartCount.values()); - for (Iterator> i = cubeql.getCandidateFactSets().iterator(); i.hasNext();) { - Set facts = i.next(); - if (factPartCount.get(facts) > minPartitions) { - log.info("Not considering facts:{} from candidate fact tables as it requires more partitions to be" - + " queried:{} minimum:{}", facts, factPartCount.get(facts), minPartitions); + for (Iterator i = cubeql.getCandidates().iterator(); i.hasNext();) { + Candidate candidate = i.next(); + if (factPartCount.get(candidate) > minPartitions) { + log.info("Not considering Candidate:{} as it requires more partitions to be" + " queried:{} minimum:{}", + candidate, factPartCount.get(candidate), minPartitions); i.remove(); + cubeql.addCandidatePruningMsg(candidate, + new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.MORE_PARTITIONS)); } } - cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCode.MORE_PARTITIONS); } } - private int getPartCount(Set set) { + private int getPartCount(Candidate candidate) { int parts = 0; - for (CandidateFact f : set) { - parts += f.getNumQueriedParts(); + for (StorageCandidate sc : CandidateUtil.getStorageCandidates(candidate)) { + parts += sc.getNumQueriedParts(); } return parts; } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java index 43564010a..dd25f3e2b 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java @@ -34,32 +34,24 @@ public class LightestFactResolver implements ContextRewriter { @Override public void rewriteContext(CubeQueryContext cubeql) throws LensException { - if (cubeql.getCube() != null && !cubeql.getCandidateFactSets().isEmpty()) { - Map, Double> factWeightMap = new HashMap, Double>(); + if (cubeql.getCube() != null && !cubeql.getCandidates().isEmpty()) { + Map factWeightMap = new HashMap(); - for (Set facts : cubeql.getCandidateFactSets()) { - factWeightMap.put(facts, getWeight(facts)); + for (Candidate cand : cubeql.getCandidates()) { + factWeightMap.put(cand, cand.getCost()); } double minWeight = Collections.min(factWeightMap.values()); - for (Iterator> i = cubeql.getCandidateFactSets().iterator(); i.hasNext();) { - Set facts = i.next(); - if (factWeightMap.get(facts) > minWeight) { - log.info("Not considering facts:{} from candidate fact tables as it has more fact weight:{} minimum:{}", - facts, factWeightMap.get(facts), minWeight); + for (Iterator i = cubeql.getCandidates().iterator(); i.hasNext();) { + Candidate cand = i.next(); + if (factWeightMap.get(cand) > minWeight) { + log.info("Not considering candidate:{} from final candidates as it has more fact weight:{} minimum:{}", + cand, factWeightMap.get(cand), minWeight); + cubeql.addCandidatePruningMsg(cand, new CandidateTablePruneCause(CandidateTablePruneCode.MORE_WEIGHT)); i.remove(); } } - cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCode.MORE_WEIGHT); } } - - private Double getWeight(Set set) { - Double weight = 0.0; - for (CandidateFact f : set) { - weight += f.fact.weight(); - } - return weight; - } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java index 2822857e6..34180d1d1 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -21,7 +21,6 @@ import java.util.Collections; import java.util.Iterator; import java.util.Map; -import java.util.Set; import org.apache.lens.cube.metadata.FactPartition; import org.apache.lens.cube.metadata.UpdatePeriod; @@ -31,11 +30,10 @@ import org.apache.hadoop.conf.Configuration; import com.google.common.collect.Maps; - import lombok.extern.slf4j.Slf4j; /** - * Prune candidate fact sets so that the facts except the ones that are covering maximum of range are pruned + * Prune candidates except the ones that are covering maximum of range are pruned */ @Slf4j class MaxCoveringFactResolver implements ContextRewriter { @@ -53,7 +51,7 @@ public void rewriteContext(CubeQueryContext cubeql) { // redundant computation. return; } - if (cubeql.getCube() == null || cubeql.getCandidateFactSets().size() <= 1) { + if (cubeql.getCube() == null || cubeql.getCandidates().size() <= 1) { // nothing to prune. return; } @@ -66,13 +64,13 @@ public void rewriteContext(CubeQueryContext cubeql) { private void resolveByTimeCovered(CubeQueryContext cubeql) { // For each part column, which candidate fact sets are covering how much amount. // Later, we'll maximize coverage for each queried part column. - Map, Long>> partCountsPerPartCol = Maps.newHashMap(); - for (Set facts : cubeql.getCandidateFactSets()) { - for (Map.Entry entry : getTimeCoveredForEachPartCol(facts).entrySet()) { + Map> partCountsPerPartCol = Maps.newHashMap(); + for (Candidate cand : cubeql.getCandidates()) { + for (Map.Entry entry : getTimeCoveredForEachPartCol(cand).entrySet()) { if (!partCountsPerPartCol.containsKey(entry.getKey())) { - partCountsPerPartCol.put(entry.getKey(), Maps., Long>newHashMap()); + partCountsPerPartCol.put(entry.getKey(), Maps.newHashMap()); } - partCountsPerPartCol.get(entry.getKey()).put(facts, entry.getValue()); + partCountsPerPartCol.get(entry.getKey()).put(cand, entry.getValue()); } } // for each queried partition, prune fact sets that are covering less range than max @@ -80,29 +78,30 @@ private void resolveByTimeCovered(CubeQueryContext cubeql) { if (partCountsPerPartCol.get(partColQueried) != null) { long maxTimeCovered = Collections.max(partCountsPerPartCol.get(partColQueried).values()); TimeCovered timeCovered = new TimeCovered(maxTimeCovered); - Iterator> iter = cubeql.getCandidateFactSets().iterator(); + Iterator iter = cubeql.getCandidates().iterator(); while (iter.hasNext()) { - Set facts = iter.next(); - Long timeCoveredLong = partCountsPerPartCol.get(partColQueried).get(facts); + Candidate candidate = iter.next(); + Long timeCoveredLong = partCountsPerPartCol.get(partColQueried).get(candidate); if (timeCoveredLong == null) { timeCoveredLong = 0L; } if (timeCoveredLong < maxTimeCovered) { - log.info("Not considering facts:{} from candidate fact tables as it covers less time than the max" - + " for partition column: {} which is: {}", facts, partColQueried, timeCovered); + log.info("Not considering Candidate:{} from Candidate set as it covers less time than the max" + + " for partition column: {} which is: {}", candidate, partColQueried, timeCovered); iter.remove(); + cubeql.addCandidatePruningMsg(candidate, + new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.LESS_DATA)); } } } } - cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCause.lessData(null)); } private void resolveByDataCompleteness(CubeQueryContext cubeql) { // From the list of candidate fact sets, we calculate the maxDataCompletenessFactor. float maxDataCompletenessFactor = 0f; - for (Set facts : cubeql.getCandidateFactSets()) { - float dataCompletenessFactor = computeDataCompletenessFactor(facts); + for (Candidate cand : cubeql.getCandidates()) { + float dataCompletenessFactor = computeDataCompletenessFactor(cand); if (dataCompletenessFactor > maxDataCompletenessFactor) { maxDataCompletenessFactor = dataCompletenessFactor; } @@ -114,25 +113,26 @@ private void resolveByDataCompleteness(CubeQueryContext cubeql) { } // We prune those candidate fact set, whose dataCompletenessFactor is less than maxDataCompletenessFactor - Iterator> iter = cubeql.getCandidateFactSets().iterator(); + Iterator iter = cubeql.getCandidates().iterator(); while (iter.hasNext()) { - Set facts = iter.next(); - float dataCompletenessFactor = computeDataCompletenessFactor(facts); + Candidate cand = iter.next(); + float dataCompletenessFactor = computeDataCompletenessFactor(cand); if (dataCompletenessFactor < maxDataCompletenessFactor) { - log.info("Not considering facts:{} from candidate fact tables as the dataCompletenessFactor for this:{} is " - + "less than the max:{}", facts, dataCompletenessFactor, maxDataCompletenessFactor); + log.info("Not considering Candidate :{} from the list as the dataCompletenessFactor for this:{} is " + + "less than the max:{}", cand, dataCompletenessFactor, maxDataCompletenessFactor); iter.remove(); + cubeql.addCandidatePruningMsg(cand, + new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.INCOMPLETE_PARTITION)); } } - cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCause.incompletePartitions(null)); } - private float computeDataCompletenessFactor(Set facts) { + private float computeDataCompletenessFactor(Candidate cand) { float completenessFactor = 0f; int numPartition = 0; - for (CandidateFact fact : facts) { - if (fact.getDataCompletenessMap() != null) { - Map> completenessMap = fact.getDataCompletenessMap(); + for (StorageCandidate sc : CandidateUtil.getStorageCandidates(cand)) { + if (sc.getDataCompletenessMap() != null) { + Map> completenessMap = sc.getDataCompletenessMap(); for (Map partitionCompleteness : completenessMap.values()) { for (Float value : partitionCompleteness.values()) { numPartition++; @@ -141,33 +141,30 @@ private float computeDataCompletenessFactor(Set facts) { } } } - return numPartition == 0 ? completenessFactor : completenessFactor/numPartition; + return numPartition == 0 ? completenessFactor : completenessFactor / numPartition; } /** * Returns time covered by fact set for each part column. - * @param facts + * + * @param cand * @return */ - private Map getTimeCoveredForEachPartCol(Set facts) { + private Map getTimeCoveredForEachPartCol(Candidate cand) { Map ret = Maps.newHashMap(); UpdatePeriod smallest = UpdatePeriod.values()[UpdatePeriod.values().length - 1]; - for (CandidateFact fact : facts) { - for (FactPartition part : fact.getPartsQueried()) { - if (part.getPeriod().compareTo(smallest) < 0) { - smallest = part.getPeriod(); - } + for (FactPartition part : cand.getParticipatingPartitions()) { + if (part.getPeriod().compareTo(smallest) < 0) { + smallest = part.getPeriod(); } } PartitionRangesForPartitionColumns partitionRangesForPartitionColumns = new PartitionRangesForPartitionColumns(); - for (CandidateFact fact : facts) { - for (FactPartition part : fact.getPartsQueried()) { - if (part.isFound()) { - try { - partitionRangesForPartitionColumns.add(part); - } catch (LensException e) { - log.error("invalid partition: ", e); - } + for (FactPartition part : cand.getParticipatingPartitions()) { + if (part.isFound()) { + try { + partitionRangesForPartitionColumns.add(part); + } catch (LensException e) { + log.error("invalid partition: ", e); } } } @@ -196,17 +193,8 @@ public static class TimeCovered { } public String toString() { - return new StringBuilder() - .append(days) - .append(" days, ") - .append(hours) - .append(" hours, ") - .append(minutes) - .append(" minutes, ") - .append(seconds) - .append(" seconds, ") - .append(milliseconds) - .append(" milliseconds.").toString(); + return String.valueOf(days) + " days, " + hours + " hours, " + minutes + + " minutes, " + seconds + " seconds, " + milliseconds + " milliseconds."; } } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java deleted file mode 100644 index 979c24bf5..000000000 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java +++ /dev/null @@ -1,238 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.cube.parse; - -import static org.apache.lens.cube.parse.HQLParser.*; - -import java.util.*; - -import org.apache.lens.cube.error.LensCubeErrorCode; -import org.apache.lens.cube.metadata.Dimension; -import org.apache.lens.server.api.error.LensException; - -import org.apache.hadoop.hive.ql.lib.Node; -import org.apache.hadoop.hive.ql.parse.ASTNode; -import org.apache.hadoop.hive.ql.parse.HiveParser; - -import org.antlr.runtime.CommonToken; - -import com.google.common.collect.Lists; -import lombok.extern.slf4j.Slf4j; - -/** - * Writes a join query with all the facts involved, with where, groupby and having expressions pushed down to the fact - * queries. - */ -@Slf4j -class MultiFactHQLContext extends SimpleHQLContext { - - private Set facts; - private CubeQueryContext query; - private Map factHQLContextMap = new HashMap<>(); - - MultiFactHQLContext(Set facts, Map dimsToQuery, - Map> factDimMap, CubeQueryContext query) throws LensException { - super(); - this.query = query; - this.facts = facts; - for (CandidateFact fact : facts) { - if (fact.getStorageTables().size() > 1) { - factHQLContextMap.put(fact, new SingleFactMultiStorageHQLContext(fact, dimsToQuery, query, fact)); - } else { - factHQLContextMap.put(fact, - new SingleFactSingleStorageHQLContext(fact, dimsToQuery, factDimMap.get(fact), query, - DefaultQueryAST.fromCandidateFact(fact, fact.getStorageTables().iterator().next(), fact))); - } - } - } - - protected void setMissingExpressions() throws LensException { - setSelect(getSelectString()); - setFrom(getFromString()); - setWhere(getWhereString()); - setGroupby(getGroupbyString()); - setHaving(getHavingString()); - setOrderby(getOrderbyString()); - } - - private String getOrderbyString() { - return query.getOrderByString(); - } - - private String getHavingString() { - return null; - } - - private String getGroupbyString() { - return null; - } - - private String getWhereString() { - return query.getWhereString(); - } - - public String toHQL() throws LensException { - return query.getInsertClause() + super.toHQL(); - } - - private String getSelectString() throws LensException { - Map> selectToFactIndex = new HashMap<>(query.getSelectAST().getChildCount()); - int fi = 1; - for (CandidateFact fact : facts) { - for (int ind : fact.getSelectIndices()) { - if (!selectToFactIndex.containsKey(ind)) { - selectToFactIndex.put(ind, Lists.newArrayList()); - } - selectToFactIndex.get(ind).add(fi); - } - fi++; - } - StringBuilder select = new StringBuilder(); - for (int i = 0; i < query.getSelectAST().getChildCount(); i++) { - if (selectToFactIndex.get(i) == null) { - throw new LensException(LensCubeErrorCode.EXPRESSION_NOT_IN_ANY_FACT.getLensErrorInfo(), - HQLParser.getString((ASTNode) query.getSelectAST().getChild(i))); - } - if (selectToFactIndex.get(i).size() == 1) { - select.append("mq").append(selectToFactIndex.get(i).get(0)).append(".") - .append(query.getSelectPhrases().get(i).getSelectAlias()).append(" "); - } else { - select.append("COALESCE("); - String sep = ""; - for (Integer factIndex : selectToFactIndex.get(i)) { - select.append(sep).append("mq").append(factIndex).append(".").append( - query.getSelectPhrases().get(i).getSelectAlias()); - sep = ", "; - } - select.append(") "); - } - select.append(query.getSelectPhrases().get(i).getFinalAlias()); - if (i != query.getSelectAST().getChildCount() - 1) { - select.append(", "); - } - } - return select.toString(); - } - - private String getMultiFactJoinCondition(int i, String dim) { - StringBuilder joinCondition = new StringBuilder(); - if (i <= 1) { - return "".toString(); - } else { - joinCondition.append("mq").append(i - 2).append(".").append(dim).append(" <=> "). - append("mq").append(i - 1).append(".").append(dim); - } - return joinCondition.toString(); - } - - private String getFromString() throws LensException { - StringBuilder fromBuilder = new StringBuilder(); - int aliasCount = 1; - String sep = ""; - for (CandidateFact fact : facts) { - SimpleHQLContext facthql = factHQLContextMap.get(fact); - fromBuilder.append(sep).append("(").append(facthql.toHQL()).append(")").append(" mq").append(aliasCount++); - sep = " full outer join "; - if (!fact.getDimFieldIndices().isEmpty() && aliasCount > 2) { - fromBuilder.append(" on "); - Iterator dimIter = fact.getDimFieldIndices().iterator(); - while (dimIter.hasNext()) { - String dim = query.getSelectPhrases().get(dimIter.next()).getSelectAlias(); - fromBuilder.append(getMultiFactJoinCondition(aliasCount, dim)); - if (dimIter.hasNext()) { - fromBuilder.append(" AND "); - } - } - } - } - return fromBuilder.toString(); - } - - - public static ASTNode convertHavingToWhere(ASTNode havingAST, CubeQueryContext context, Set cfacts, - AliasDecider aliasDecider) throws LensException { - if (havingAST == null) { - return null; - } - if (isAggregateAST(havingAST) || isTableColumnAST(havingAST) || isNonAggregateFunctionAST(havingAST)) { - // if already present in select, pick alias - String alias = null; - for (CandidateFact fact : cfacts) { - if (fact.isExpressionAnswerable(havingAST, context)) { - alias = fact.addAndGetAliasFromSelect(havingAST, aliasDecider); - return new ASTNode(new CommonToken(HiveParser.Identifier, alias)); - } - } - } - if (havingAST.getChildren() != null) { - for (int i = 0; i < havingAST.getChildCount(); i++) { - ASTNode replaced = convertHavingToWhere((ASTNode) havingAST.getChild(i), context, cfacts, aliasDecider); - havingAST.setChild(i, replaced); - } - } - return havingAST; - } - - public static ASTNode pushDownHaving(ASTNode ast, CubeQueryContext cubeQueryContext, Set cfacts) - throws LensException { - if (ast == null) { - return null; - } - if (ast.getType() == HiveParser.KW_AND || ast.getType() == HiveParser.TOK_HAVING) { - List children = Lists.newArrayList(); - for (Node child : ast.getChildren()) { - ASTNode newChild = pushDownHaving((ASTNode) child, cubeQueryContext, cfacts); - if (newChild != null) { - children.add(newChild); - } - } - if (children.size() == 0) { - return null; - } else if (children.size() == 1) { - return children.get(0); - } else { - ASTNode newASTNode = new ASTNode(ast.getToken()); - for (ASTNode child : children) { - newASTNode.addChild(child); - } - return newASTNode; - } - } - if (isPrimitiveBooleanExpression(ast)) { - CandidateFact fact = pickFactToPushDown(ast, cubeQueryContext, cfacts); - if (fact == null) { - return ast; - } - fact.addToHaving(ast); - return null; - } - return ast; - } - - private static CandidateFact pickFactToPushDown(ASTNode ast, CubeQueryContext cubeQueryContext, Set - cfacts) throws LensException { - for (CandidateFact fact : cfacts) { - if (fact.isExpressionAnswerable(ast, cubeQueryContext)) { - return fact; - } - } - return null; - } - -} diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java index 9b5a52f0b..0996db523 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java @@ -18,12 +18,17 @@ */ package org.apache.lens.cube.parse; +import static java.util.stream.Collectors.toMap; + +import static com.google.common.collect.Sets.newHashSet; + import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; -import org.apache.lens.cube.metadata.AbstractCubeTable; import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; import org.apache.commons.lang.StringUtils; @@ -36,7 +41,7 @@ import lombok.Getter; import lombok.NoArgsConstructor; -public class PruneCauses extends HashMap> { +public class PruneCauses extends HashMap> { @Getter(lazy = true) private final HashMap> reversed = reverse(); @Getter(lazy = true) @@ -48,10 +53,7 @@ private HashMap> computeCompact() { HashMap> detailedMessage = Maps.newHashMap(); for (Map.Entry> entry : getReversed().entrySet()) { String key = StringUtils.join(entry.getValue(), ","); - if (detailedMessage.get(key) == null) { - detailedMessage.put(key, new ArrayList()); - } - detailedMessage.get(key).add(entry.getKey()); + detailedMessage.computeIfAbsent(key, k -> new ArrayList<>()).add(entry.getKey()); } return detailedMessage; } @@ -66,14 +68,11 @@ public void addPruningMsg(T table, CandidateTablePruneCause msg) { get(table).add(msg); } - public HashMap> reverse() { + private HashMap> reverse() { HashMap> result = new HashMap>(); for (T key : keySet()) { for (CandidateTablePruneCause value : get(key)) { - if (result.get(value) == null) { - result.put(value, new ArrayList()); - } - result.get(value).add(key); + result.computeIfAbsent(value, k -> new ArrayList<>()).add(key); } } return result; @@ -94,14 +93,10 @@ private CandidateTablePruneCode computeMaxCause() { } public String getBriefCause() { - CandidateTablePruneCode maxCause = CandidateTablePruneCode.values()[0]; - for (CandidateTablePruneCause cause : getReversed().keySet()) { - if (cause.getCause().compareTo(maxCause) > 0) { - maxCause = cause.getCause(); - } - } + CandidateTablePruneCode maxCause = getReversed().keySet().stream() + .map(CandidateTablePruneCause::getCause).max(Comparator.naturalOrder()).get(); Map maxCauseMap = Maps.newHashMap(); - for (Map.Entry> entry: getReversed().entrySet()) { + for (Map.Entry> entry : getReversed().entrySet()) { if (entry.getKey().getCause().equals(maxCause)) { maxCauseMap.put(entry.getKey(), StringUtils.join(entry.getValue(), ",")); } @@ -120,5 +115,11 @@ public static void main(String[] args) { public static final class BriefAndDetailedError { private String brief; private HashMap> details; + + Map, List> enhanced() { + return getDetails().entrySet().stream().collect(toMap( + o -> newHashSet(o.getKey().split(",")), + Map.Entry::getValue)); + } } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java index 34a562d48..310a655cd 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java @@ -36,6 +36,8 @@ @EqualsAndHashCode(callSuper = true) @Slf4j class QueriedPhraseContext extends TracksQueriedColumns implements TrackQueriedCubeFields { + // position in org.apache.lens.cube.parse.CubeQueryContext.queriedPhrases + private int position; private final ASTNode exprAST; private Boolean aggregate; private String expr; @@ -98,89 +100,92 @@ public boolean hasMeasures(CubeQueryContext cubeQl) { return false; } - boolean isEvaluable(CubeQueryContext cubeQl, CandidateFact cfact) throws LensException { + /** + * @param cubeQl + * @param sc + * @return + * @throws LensException + */ + public boolean isEvaluable(CubeQueryContext cubeQl, StorageCandidate sc) throws LensException { // all measures of the queried phrase should be present for (String msr : queriedMsrs) { - if (!checkForColumnExistsAndValidForRange(cfact, msr, cubeQl)) { + if (!checkForColumnExistsAndValidForRange(sc, msr, cubeQl)) { return false; } } // all expression columns should be evaluable for (String exprCol : queriedExprColumns) { - if (!cubeQl.getExprCtx().isEvaluable(exprCol, cfact)) { - log.info("expression {} is not evaluable in fact table:{}", expr, cfact); + if (!cubeQl.getExprCtx().isEvaluable(exprCol, sc)) { + log.info("expression {} is not evaluable in fact table:{}", expr, sc); return false; } } // all dim-attributes should be present. for (String col : queriedDimAttrs) { - if (!cfact.getColumns().contains(col.toLowerCase())) { + if (!sc.getColumns().contains(col.toLowerCase())) { // check if it available as reference - if (!cubeQl.getDeNormCtx().addRefUsage(cubeQl, cfact, col, cubeQl.getCube().getName())) { - log.info("column {} is not available in fact table:{} ", col, cfact); + if (!cubeQl.getDeNormCtx().addRefUsage(cubeQl, sc, col, cubeQl.getCube().getName())) { + log.info("column {} is not available in fact table:{} ", col, sc); return false; } - } else if (!isFactColumnValidForRange(cubeQl, cfact, col)) { - log.info("column {} is not available in range queried in fact {}", col, cfact); + } else if (!isFactColumnValidForRange(cubeQl, sc, col)) { + log.info("column {} is not available in range queried in fact {}", col, sc); return false; } } return true; } - public static boolean isColumnAvailableInRange(final TimeRange range, Date startTime, Date endTime) { + private static boolean isColumnAvailableInRange(final TimeRange range, Date startTime, Date endTime) { return (isColumnAvailableFrom(range.getFromDate(), startTime) - && isColumnAvailableTill(range.getToDate(), endTime)); + && isColumnAvailableTill(range.getToDate(), endTime)); } - public static boolean isColumnAvailableFrom(@NonNull final Date date, Date startTime) { + private static boolean isColumnAvailableFrom(@NonNull final Date date, Date startTime) { return (startTime == null) || date.equals(startTime) || date.after(startTime); } - public static boolean isColumnAvailableTill(@NonNull final Date date, Date endTime) { + private static boolean isColumnAvailableTill(@NonNull final Date date, Date endTime) { return (endTime == null) || date.equals(endTime) || date.before(endTime); } - public static boolean isFactColumnValidForRange(CubeQueryContext cubeql, CandidateTable cfact, String col) { - for(TimeRange range : cubeql.getTimeRanges()) { - if (!isColumnAvailableInRange(range, getFactColumnStartTime(cfact, col), getFactColumnEndTime(cfact, col))) { + public static boolean isFactColumnValidForRange(CubeQueryContext cubeql, StorageCandidate sc, String col) { + for (TimeRange range : cubeql.getTimeRanges()) { + if (!isColumnAvailableInRange(range, getFactColumnStartTime(sc, col), getFactColumnEndTime(sc, col))) { return false; } } return true; } - public static Date getFactColumnStartTime(CandidateTable table, String factCol) { + public static Date getFactColumnStartTime(StorageCandidate sc, String factCol) { Date startTime = null; - if (table instanceof CandidateFact) { - for (String key : ((CandidateFact) table).fact.getProperties().keySet()) { - if (key.contains(MetastoreConstants.FACT_COL_START_TIME_PFX)) { - String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_START_TIME_PFX); - if (factCol.equals(propCol)) { - startTime = ((CandidateFact) table).fact.getDateFromProperty(key, false, true); - } + for (String key : sc.getTable().getProperties().keySet()) { + if (key.contains(MetastoreConstants.FACT_COL_START_TIME_PFX)) { + String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_START_TIME_PFX); + if (factCol.equals(propCol)) { + startTime = sc.getTable().getDateFromProperty(key, false, true); } } } return startTime; } - public static Date getFactColumnEndTime(CandidateTable table, String factCol) { + public static Date getFactColumnEndTime(StorageCandidate sc, String factCol) { Date endTime = null; - if (table instanceof CandidateFact) { - for (String key : ((CandidateFact) table).fact.getProperties().keySet()) { - if (key.contains(MetastoreConstants.FACT_COL_END_TIME_PFX)) { - String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_END_TIME_PFX); - if (factCol.equals(propCol)) { - endTime = ((CandidateFact) table).fact.getDateFromProperty(key, false, true); - } + for (String key : sc.getTable().getProperties().keySet()) { + if (key.contains(MetastoreConstants.FACT_COL_END_TIME_PFX)) { + String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_END_TIME_PFX); + if (factCol.equals(propCol)) { + endTime = sc.getTable().getDateFromProperty(key, false, true); } } } return endTime; } - static boolean checkForColumnExistsAndValidForRange(CandidateTable table, String column, CubeQueryContext cubeql) { - return (table.getColumns().contains(column) && isFactColumnValidForRange(cubeql, table, column)); + static boolean checkForColumnExistsAndValidForRange(StorageCandidate sc, String column, CubeQueryContext cubeql) { + return (sc.getColumns().contains(column) && isFactColumnValidForRange(cubeql, sc, column)); } + } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java index 7298604d1..b94f13165 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java @@ -83,4 +83,10 @@ public interface QueryAST { ASTNode getOrderByAST(); void setOrderByAST(ASTNode node); + + void setJoinAST(ASTNode node); + + void setFromString(String fromString); + void setWhereString(String whereString); + } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java index 62ceb12ff..77ebe82fa 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java @@ -18,14 +18,8 @@ */ package org.apache.lens.cube.parse; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - import org.apache.lens.server.api.error.LensException; -import org.apache.commons.lang.StringUtils; - import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -50,7 +44,7 @@ public abstract class SimpleHQLContext implements HQLContextInterface { } SimpleHQLContext(String select, String from, String where, String groupby, String orderby, String having, - Integer limit) { + Integer limit) { this.select = select; this.from = from; this.where = where; @@ -73,6 +67,7 @@ public abstract class SimpleHQLContext implements HQLContextInterface { *

* Leaving this empty implementation for the case of all expressions being passed in constructor. If other * constructors are used the missing expressions should be set here + * * @throws LensException */ protected void setMissingExpressions() throws LensException { @@ -80,57 +75,6 @@ protected void setMissingExpressions() throws LensException { public String toHQL() throws LensException { setMissingExpressions(); - String qfmt = getQueryFormat(); - Object[] queryTreeStrings = getQueryTreeStrings(); - if (log.isDebugEnabled()) { - log.debug("qfmt: {} Query strings: {}", qfmt, Arrays.toString(queryTreeStrings)); - } - String baseQuery = String.format(qfmt, queryTreeStrings); - return baseQuery; - } - - private String[] getQueryTreeStrings() throws LensException { - List qstrs = new ArrayList(); - qstrs.add(select); - qstrs.add(from); - if (!StringUtils.isBlank(where)) { - qstrs.add(where); - } - if (!StringUtils.isBlank(groupby)) { - qstrs.add(groupby); - } - if (!StringUtils.isBlank(having)) { - qstrs.add(having); - } - if (!StringUtils.isBlank(orderby)) { - qstrs.add(orderby); - } - if (limit != null) { - qstrs.add(String.valueOf(limit)); - } - return qstrs.toArray(new String[0]); - } - - private final String baseQueryFormat = "SELECT %s FROM %s"; - - private String getQueryFormat() { - StringBuilder queryFormat = new StringBuilder(); - queryFormat.append(baseQueryFormat); - if (!StringUtils.isBlank(where)) { - queryFormat.append(" WHERE %s"); - } - if (!StringUtils.isBlank(groupby)) { - queryFormat.append(" GROUP BY %s"); - } - if (!StringUtils.isBlank(having)) { - queryFormat.append(" HAVING %s"); - } - if (!StringUtils.isBlank(orderby)) { - queryFormat.append(" ORDER BY %s"); - } - if (limit != null) { - queryFormat.append(" LIMIT %s"); - } - return queryFormat.toString(); + return CandidateUtil.buildHQLString(select, from, where, groupby, orderby, having, limit); } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java deleted file mode 100644 index 9b4821301..000000000 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java +++ /dev/null @@ -1,259 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.lens.cube.parse; - -import static org.apache.lens.cube.parse.CubeQueryConfUtil.DEFAULT_ENABLE_STORAGES_UNION; -import static org.apache.lens.cube.parse.CubeQueryConfUtil.ENABLE_STORAGES_UNION; -import static org.apache.lens.cube.parse.HQLParser.*; - -import static org.apache.hadoop.hive.ql.parse.HiveParser.*; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; - -import org.apache.lens.cube.error.LensCubeErrorCode; -import org.apache.lens.cube.metadata.Dimension; -import org.apache.lens.cube.metadata.MetastoreUtil; -import org.apache.lens.server.api.error.LensException; - -import org.apache.hadoop.hive.ql.lib.Node; -import org.apache.hadoop.hive.ql.parse.ASTNode; -import org.apache.hadoop.hive.ql.parse.HiveParser; - -import org.antlr.runtime.CommonToken; - -public class SingleFactMultiStorageHQLContext extends UnionHQLContext { - - private final QueryAST ast; - - private Map innerToOuterASTs = new HashMap<>(); - private AliasDecider aliasDecider = new DefaultAliasDecider(); - - SingleFactMultiStorageHQLContext(CandidateFact fact, Map dimsToQuery, - CubeQueryContext query, QueryAST ast) - throws LensException { - super(query, fact); - if (!query.getConf().getBoolean(ENABLE_STORAGES_UNION, DEFAULT_ENABLE_STORAGES_UNION)) { - throw new LensException(LensCubeErrorCode.STORAGE_UNION_DISABLED.getLensErrorInfo()); - } - this.ast = ast; - processSelectAST(); - processGroupByAST(); - processHavingAST(); - processOrderByAST(); - processLimit(); - setHqlContexts(getUnionContexts(fact, dimsToQuery, query, ast)); - } - - private void processSelectAST() { - ASTNode originalSelectAST = MetastoreUtil.copyAST(ast.getSelectAST()); - ast.setSelectAST(new ASTNode(originalSelectAST.getToken())); - ASTNode outerSelectAST = processSelectExpression(originalSelectAST); - setSelect(getString(outerSelectAST)); - } - - private void processGroupByAST() { - if (ast.getGroupByAST() != null) { - setGroupby(getString(processExpression(ast.getGroupByAST()))); - } - } - - private void processHavingAST() throws LensException { - if (ast.getHavingAST() != null) { - setHaving(getString(processExpression(ast.getHavingAST()))); - ast.setHavingAST(null); - } - } - - - private void processOrderByAST() { - if (ast.getOrderByAST() != null) { - setOrderby(getString(processOrderbyExpression(ast.getOrderByAST()))); - ast.setOrderByAST(null); - } - } - - private void processLimit() { - setLimit(ast.getLimitValue()); - ast.setLimitValue(null); - } - - private ASTNode processExpression(ASTNode astNode) { - if (astNode == null) { - return null; - } - ASTNode outerExpression = new ASTNode(astNode); - // iterate over all children of the ast and get outer ast corresponding to it. - for (Node child : astNode.getChildren()) { - outerExpression.addChild(getOuterAST((ASTNode)child)); - } - return outerExpression; - } - - private ASTNode processSelectExpression(ASTNode astNode) { - if (astNode == null) { - return null; - } - ASTNode outerExpression = new ASTNode(astNode); - // iterate over all children of the ast and get outer ast corresponding to it. - for (Node node : astNode.getChildren()) { - ASTNode child = (ASTNode)node; - ASTNode outerSelect = new ASTNode(child); - ASTNode selectExprAST = (ASTNode)child.getChild(0); - ASTNode outerAST = getOuterAST(selectExprAST); - outerSelect.addChild(outerAST); - - // has an alias? add it - if (child.getChildCount() > 1) { - outerSelect.addChild(child.getChild(1)); - } - outerExpression.addChild(outerSelect); - } - return outerExpression; - } - - private ASTNode processOrderbyExpression(ASTNode astNode) { - if (astNode == null) { - return null; - } - ASTNode outerExpression = new ASTNode(astNode); - // sample orderby AST looks the following : - /* - TOK_ORDERBY - TOK_TABSORTCOLNAMEDESC - TOK_NULLS_LAST - . - TOK_TABLE_OR_COL - testcube - cityid - TOK_TABSORTCOLNAMEASC - TOK_NULLS_FIRST - . - TOK_TABLE_OR_COL - testcube - stateid - TOK_TABSORTCOLNAMEASC - TOK_NULLS_FIRST - . - TOK_TABLE_OR_COL - testcube - zipcode - */ - for (Node node : astNode.getChildren()) { - ASTNode child = (ASTNode)node; - ASTNode outerOrderby = new ASTNode(child); - ASTNode tokNullsChild = (ASTNode) child.getChild(0); - ASTNode outerTokNullsChild = new ASTNode(tokNullsChild); - outerTokNullsChild.addChild(getOuterAST((ASTNode)tokNullsChild.getChild(0))); - outerOrderby.addChild(outerTokNullsChild); - outerExpression.addChild(outerOrderby); - } - return outerExpression; - } - /* - - Perform a DFS on the provided AST, and Create an AST of similar structure with changes specific to the - inner query - outer query dynamics. The resultant AST is supposed to be used in outer query. - - Base cases: - 1. ast is null => null - 2. ast is aggregate_function(table.column) => add aggregate_function(table.column) to inner select expressions, - generate alias, return aggregate_function(cube.alias). Memoize the mapping - aggregate_function(table.column) => aggregate_function(cube.alias) - Assumption is aggregate_function is transitive i.e. f(a,b,c,d) = f(f(a,b), f(c,d)). SUM, MAX, MIN etc - are transitive, while AVG, COUNT etc are not. For non-transitive aggregate functions, the re-written - query will be incorrect. - 3. ast has aggregates - iterate over children and add the non aggregate nodes as is and recursively get outer ast - for aggregate. - 4. If no aggregates, simply select its alias in outer ast. - 5. If given ast is memorized as mentioned in the above cases, return the mapping. - */ - private ASTNode getOuterAST(ASTNode astNode) { - if (astNode == null) { - return null; - } - if (innerToOuterASTs.containsKey(new HashableASTNode(astNode))) { - return innerToOuterASTs.get(new HashableASTNode(astNode)); - } - if (isAggregateAST(astNode)) { - return processAggregate(astNode); - } else if (hasAggregate(astNode)) { - ASTNode outerAST = new ASTNode(astNode); - for (Node child : astNode.getChildren()) { - ASTNode childAST = (ASTNode) child; - if (hasAggregate(childAST)) { - outerAST.addChild(getOuterAST(childAST)); - } else { - outerAST.addChild(childAST); - } - } - return outerAST; - } else { - ASTNode innerSelectASTWithoutAlias = MetastoreUtil.copyAST(astNode); - ASTNode innerSelectExprAST = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR)); - innerSelectExprAST.addChild(innerSelectASTWithoutAlias); - String alias = aliasDecider.decideAlias(astNode); - ASTNode aliasNode = new ASTNode(new CommonToken(Identifier, alias)); - innerSelectExprAST.addChild(aliasNode); - addToInnerSelectAST(innerSelectExprAST); - ASTNode outerAST = getDotAST(query.getCube().getName(), alias); - innerToOuterASTs.put(new HashableASTNode(innerSelectASTWithoutAlias), outerAST); - return outerAST; - } - } - - private ASTNode processAggregate(ASTNode astNode) { - ASTNode innerSelectASTWithoutAlias = MetastoreUtil.copyAST(astNode); - ASTNode innerSelectExprAST = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR)); - innerSelectExprAST.addChild(innerSelectASTWithoutAlias); - String alias = aliasDecider.decideAlias(astNode); - ASTNode aliasNode = new ASTNode(new CommonToken(Identifier, alias)); - innerSelectExprAST.addChild(aliasNode); - addToInnerSelectAST(innerSelectExprAST); - ASTNode dotAST = getDotAST(query.getCube().getName(), alias); - ASTNode outerAST = new ASTNode(new CommonToken(TOK_FUNCTION)); - //TODO: take care or non-transitive aggregate functions - outerAST.addChild(new ASTNode(new CommonToken(Identifier, astNode.getChild(0).getText()))); - outerAST.addChild(dotAST); - innerToOuterASTs.put(new HashableASTNode(innerSelectASTWithoutAlias), outerAST); - return outerAST; - } - - private void addToInnerSelectAST(ASTNode selectExprAST) { - if (ast.getSelectAST() == null) { - ast.setSelectAST(new ASTNode(new CommonToken(TOK_SELECT))); - } - ast.getSelectAST().addChild(selectExprAST); - } - - private static ArrayList getUnionContexts(CandidateFact fact, Map - dimsToQuery, CubeQueryContext query, QueryAST ast) - throws LensException { - ArrayList contexts = new ArrayList<>(); - String alias = query.getAliasForTableName(query.getCube().getName()); - for (String storageTable : fact.getStorageTables()) { - SingleFactSingleStorageHQLContext ctx = new SingleFactSingleStorageHQLContext(fact, storageTable + " " + alias, - dimsToQuery, query, DefaultQueryAST.fromCandidateFact(fact, storageTable, ast)); - contexts.add(ctx); - } - return contexts; - } -} diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java deleted file mode 100644 index dbc84eda2..000000000 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.cube.parse; - -import java.util.Map; -import java.util.Set; - -import org.apache.lens.cube.metadata.Dimension; -import org.apache.lens.server.api.error.LensException; - -/** - * HQL context class which passes down all query strings to come from DimOnlyHQLContext and works with fact being - * queried. - *

- * Updates from string with join clause expanded - */ -class SingleFactSingleStorageHQLContext extends DimOnlyHQLContext { - - private final CandidateFact fact; - private String storageAlias; - - SingleFactSingleStorageHQLContext(CandidateFact fact, Map dimsToQuery, - CubeQueryContext query, QueryAST ast) - throws LensException { - this(fact, dimsToQuery, dimsToQuery.keySet(), query, ast); - } - - SingleFactSingleStorageHQLContext(CandidateFact fact, Map dimsToQuery, - Set dimsQueried, CubeQueryContext query, QueryAST ast) - throws LensException { - super(dimsToQuery, dimsQueried, query, ast); - this.fact = fact; - } - - SingleFactSingleStorageHQLContext(CandidateFact fact, String storageAlias, Map dimsToQuery, - CubeQueryContext query, QueryAST ast) throws LensException { - this(fact, dimsToQuery, query, ast); - this.storageAlias = storageAlias; - } - - @Override - protected String getFromTable() throws LensException { - if (getQuery().isAutoJoinResolved()) { - if (storageAlias != null) { - return storageAlias; - } else { - return fact.getStorageString(query.getAliasForTableName(query.getCube().getName())); - } - } else { - if (fact.getStorageTables().size() == 1) { - return getQuery().getQBFromString(fact, getDimsToQuery()); - } else { - return storageAlias; - } - } - } -} diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java new file mode 100644 index 000000000..d95cf273e --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -0,0 +1,1040 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + +import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipUpdatePeriodCode; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.timeDimNotSupported; +import static org.apache.lens.cube.parse.StorageUtil.getFallbackRange; +import static org.apache.lens.cube.parse.StorageUtil.joinWithAnd; +import static org.apache.lens.cube.parse.StorageUtil.processCubeColForDataCompleteness; +import static org.apache.lens.cube.parse.StorageUtil.processExpressionsForCompleteness; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.*; + +import org.apache.lens.cube.metadata.AbstractCubeTable; +import org.apache.lens.cube.metadata.CubeFactTable; +import org.apache.lens.cube.metadata.CubeInterface; +import org.apache.lens.cube.metadata.CubeMetastoreClient; +import org.apache.lens.cube.metadata.DateUtil; +import org.apache.lens.cube.metadata.Dimension; +import org.apache.lens.cube.metadata.FactPartition; +import org.apache.lens.cube.metadata.MetastoreUtil; +import org.apache.lens.cube.metadata.TimeRange; +import org.apache.lens.cube.metadata.UpdatePeriod; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.metastore.DataCompletenessChecker; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.HiveParser; +import org.apache.hadoop.hive.ql.session.SessionState; + +import org.antlr.runtime.CommonToken; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +/** + * Represents a fact on a storage table and the dimensions it needs to be joined with to answer the query + */ +@Slf4j +public class StorageCandidate implements Candidate, CandidateTable { + + // TODO union : Put comments on member variables. + @Getter + private final CubeQueryContext cubeql; + private final String processTimePartCol; + private final CubeMetastoreClient client; + private final String completenessPartCol; + private final float completenessThreshold; + + /** + * Name of this storage candidate = storageName_factName + */ + @Getter + @Setter + private String name; + + /** + * This is the storage table specific name. It is used while generating query from this candidate + */ + @Setter + private String resolvedName; + /** + * Valid update periods populated by Phase 1. + */ + @Getter + private TreeSet validUpdatePeriods = new TreeSet<>(); + + /** + * These are the update periods that finally participate in partitions. + * @see #getParticipatingPartitions() + */ + @Getter + private TreeSet participatingUpdatePeriods = new TreeSet<>(); + + @Getter + @Setter + Map updatePeriodRejectionCause; + private Configuration conf = null; + + /** + * This map holds Tags (A tag refers to one or more measures) that have incomplete (below configured threshold) data. + * Value is a map of date string and %completeness. + */ + @Getter + private Map> dataCompletenessMap = new HashMap<>(); + private SimpleDateFormat partWhereClauseFormat = null; + /** + * Participating fact, storage and dimensions for this StorageCandidate + */ + @Getter + private CubeFactTable fact; + @Getter + private String storageName; + @Getter + private String storageTable; + @Getter + @Setter + private QueryAST queryAst; + @Getter + private Map> rangeToPartitions = new LinkedHashMap<>(); + @Getter + private Map rangeToExtraWhereFallBack = new LinkedHashMap<>(); + @Getter + @Setter + private String whereString; + @Getter + private Set answerableMeasurePhraseIndices = Sets.newHashSet(); + @Getter + @Setter + private String fromString; + @Getter + private CubeInterface cube; + @Getter + private Map dimsToQuery; + @Getter + private Date startTime; + @Getter + private Date endTime; + /** + * Cached fact columns + */ + private Collection factColumns; + + /** + * Non existing partitions + */ + @Getter + private Set nonExistingPartitions = new HashSet<>(); + @Getter + private int numQueriedParts = 0; + + /** + * This will be true if this storage candidate has multiple storage tables (one per update period) + * https://issues.apache.org/jira/browse/LENS-1386 + */ + @Getter + private boolean isStorageTblsAtUpdatePeriodLevel; + + public StorageCandidate(StorageCandidate sc) throws LensException { + this(sc.getCube(), sc.getFact(), sc.getStorageName(), sc.getCubeql()); + this.validUpdatePeriods.addAll(sc.getValidUpdatePeriods()); + this.whereString = sc.whereString; + this.fromString = sc.fromString; + this.dimsToQuery = sc.dimsToQuery; + this.factColumns = sc.factColumns; + this.answerableMeasurePhraseIndices.addAll(sc.answerableMeasurePhraseIndices); + if (sc.getQueryAst() != null) { + this.queryAst = new DefaultQueryAST(); + CandidateUtil.copyASTs(sc.getQueryAst(), new DefaultQueryAST()); + } + for (Map.Entry> entry : sc.getRangeToPartitions().entrySet()) { + rangeToPartitions.put(entry.getKey(), new LinkedHashSet<>(entry.getValue())); + } + this.rangeToExtraWhereFallBack = sc.rangeToExtraWhereFallBack; + this.answerableMeasurePhraseIndices = sc.answerableMeasurePhraseIndices; + } + + public StorageCandidate(CubeInterface cube, CubeFactTable fact, String storageName, CubeQueryContext cubeql) + throws LensException { + if ((cube == null) || (fact == null) || (storageName == null)) { + throw new IllegalArgumentException("Cube,fact and storageName should be non null"); + } + this.cube = cube; + this.fact = fact; + this.cubeql = cubeql; + this.storageName = storageName; + this.storageTable = MetastoreUtil.getFactOrDimtableStorageTableName(fact.getName(), storageName); + this.conf = cubeql.getConf(); + this.name = fact.getName(); + this.processTimePartCol = conf.get(CubeQueryConfUtil.PROCESS_TIME_PART_COL); + String formatStr = conf.get(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT); + if (formatStr != null) { + this.partWhereClauseFormat = new SimpleDateFormat(formatStr); + } + completenessPartCol = conf.get(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL); + completenessThreshold = conf + .getFloat(CubeQueryConfUtil.COMPLETENESS_THRESHOLD, CubeQueryConfUtil.DEFAULT_COMPLETENESS_THRESHOLD); + client = cubeql.getMetastoreClient(); + Set storageTblNames = client.getStorageTables(fact.getName(), storageName); + if (storageTblNames.size() > 1) { + isStorageTblsAtUpdatePeriodLevel = true; + } else { + //if this.storageTable is equal to the storage table name it implies isStorageTblsAtUpdatePeriodLevel is false + isStorageTblsAtUpdatePeriodLevel = !storageTblNames.iterator().next().equalsIgnoreCase(storageTable); + } + setStorageStartAndEndDate(); + } + + /** + * Sets Storage candidates start and end time based on underlying storage-tables + * + * CASE 1 + * If has Storage has single storage table* + * Storage start time = max(storage start time , fact start time) + * Storage end time = min(storage end time , fact start time) + * + * CASE 2 + * If the Storage has multiple Storage Tables (one per update period)* + * update Period start Time = Max(update start time, fact start time) + * update Period end Time = Min(update end time, fact end time) + * Stoarge start and end time is derived form the underlying update period start and end times. + * Storage start time = min(update1 start time ,...., updateN start time) + * Storage end time = max(update1 end time ,...., updateN end time) + * + * Note in Case 2 its assumed that the time range supported by different update periods are either + * overlapping(Example 2) or form a non overlapping but continuous chain(Example 1) as illustrated + * in examples below + * + * Example 1 + * A Storage has 2 Non Oevralpping but continuous Update Periods. + * MONTHLY with start time as now.month -13 months and end time as now.month -2months and + * DAILY with start time as now.day and end time as now.month -2months + * Then this Sorage will have an implied start time as now.month -13 month and end time as now.day + * + * Example 2 + * A Storage has 2 Overlapping Update Periods. + * MONTHLY with start time as now.month -13 months and end time as now.month -1months and + * DAILY with start time as now.day and end time as now.month -2months + * Then this Sorage will have an implied start time as now.month -13 month and end time as now.day + * + * @throws LensException + */ + public void setStorageStartAndEndDate() throws LensException { + if (this.startTime != null && !this.isStorageTblsAtUpdatePeriodLevel) { + //If the times are already set and are not dependent of update period, no point setting times again. + return; + } + List startDates = new ArrayList<>(); + List endDates = new ArrayList<>(); + for (String storageTablePrefix : getValidStorageTableNames()) { + startDates.add(client.getStorageTableStartDate(storageTablePrefix, fact.getName())); + endDates.add(client.getStorageTableEndDate(storageTablePrefix, fact.getName())); + } + this.startTime = Collections.min(startDates); + this.endTime = Collections.max(endDates); + } + + private Set getValidStorageTableNames() throws LensException { + if (!validUpdatePeriods.isEmpty()) { + // In this case skip invalid update periods and get storage tables only for valid ones. + Set uniqueStorageTables = new HashSet<>(); + for (UpdatePeriod updatePeriod : validUpdatePeriods) { + uniqueStorageTables.add(client.getStorageTableName(fact.getName(), storageName, updatePeriod)); + } + return uniqueStorageTables; + } else { + //Get all storage tables. + return client.getStorageTables(fact.getName(), storageName); + } + } + + private void setMissingExpressions(Set queriedDims) throws LensException { + setFromString(String.format("%s", getFromTable())); + setWhereString(joinWithAnd( + genWhereClauseWithDimPartitions(whereString, queriedDims), cubeql.getConf().getBoolean( + CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL, CubeQueryConfUtil.DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL) + ? getPostSelectionWhereClause() : null)); + if (cubeql.getHavingAST() != null) { + queryAst.setHavingAST(MetastoreUtil.copyAST(cubeql.getHavingAST())); + } + } + + private String genWhereClauseWithDimPartitions(String originalWhere, Set queriedDims) { + StringBuilder whereBuf; + if (originalWhere != null) { + whereBuf = new StringBuilder(originalWhere); + } else { + whereBuf = new StringBuilder(); + } + + // add where clause for all dimensions + if (cubeql != null) { + boolean added = (originalWhere != null); + for (Dimension dim : queriedDims) { + CandidateDim cdim = dimsToQuery.get(dim); + String alias = cubeql.getAliasForTableName(dim.getName()); + if (!cdim.isWhereClauseAdded() && !StringUtils.isBlank(cdim.getWhereClause())) { + appendWhereClause(whereBuf, StorageUtil.getWhereClause(cdim, alias), added); + added = true; + } + } + } + if (whereBuf.length() == 0) { + return null; + } + return whereBuf.toString(); + } + + private static void appendWhereClause(StringBuilder filterCondition, String whereClause, boolean hasMore) { + // Make sure we add AND only when there are already some conditions in where + // clause + if (hasMore && !filterCondition.toString().isEmpty() && !StringUtils.isBlank(whereClause)) { + filterCondition.append(" AND "); + } + + if (!StringUtils.isBlank(whereClause)) { + filterCondition.append("("); + filterCondition.append(whereClause); + filterCondition.append(")"); + } + } + + private String getPostSelectionWhereClause() throws LensException { + return null; + } + + void setAnswerableMeasurePhraseIndices(int index) { + answerableMeasurePhraseIndices.add(index); + } + + public String toHQL(Set queriedDims) throws LensException { + setMissingExpressions(queriedDims); + // Check if the picked candidate is a StorageCandidate and in that case + // update the selectAST with final alias. + if (this == cubeql.getPickedCandidate()) { + CandidateUtil.updateFinalAlias(queryAst.getSelectAST(), cubeql); + updateOrderByWithFinalAlias(queryAst.getOrderByAST(), queryAst.getSelectAST()); + } else { + queryAst.setHavingAST(null); + } + return CandidateUtil + .buildHQLString(queryAst.getSelectString(), fromString, whereString, queryAst.getGroupByString(), + queryAst.getOrderByString(), queryAst.getHavingString(), queryAst.getLimitValue()); + } + + /** + * Update Orderby children with final alias used in select + * + * @param orderby Order by AST + * @param select Select AST + */ + private void updateOrderByWithFinalAlias(ASTNode orderby, ASTNode select) { + if (orderby == null) { + return; + } + for (Node orderbyNode : orderby.getChildren()) { + ASTNode orderBychild = (ASTNode) orderbyNode; + for (Node selectNode : select.getChildren()) { + ASTNode selectChild = (ASTNode) selectNode; + if (selectChild.getChildCount() == 2) { + if (HQLParser.getString((ASTNode) selectChild.getChild(0)) + .equals(HQLParser.getString((ASTNode) orderBychild.getChild(0)))) { + ASTNode alias = new ASTNode((ASTNode) selectChild.getChild(1)); + orderBychild.replaceChildren(0, 0, alias); + break; + } + } + } + } + } + + @Override + public String getStorageString(String alias) { + return storageName + " " + alias; + } + + @Override + public AbstractCubeTable getTable() { + return fact; + } + + @Override + public AbstractCubeTable getBaseTable() { + return (AbstractCubeTable) cube; + } + + @Override + public Collection getColumns() { + if (factColumns == null) { + factColumns = fact.getValidColumns(); + if (factColumns == null) { + factColumns = fact.getAllFieldNames(); + } + } + return factColumns; + } + + @Override + public double getCost() { + return fact.weight(); + } + + @Override + public boolean contains(Candidate candidate) { + return this.equals(candidate); + } + + @Override + public Collection getChildren() { + return null; + } + + private void updatePartitionStorage(FactPartition part) throws LensException { + try { + if (client.factPartitionExists(fact, part, storageTable)) { + part.getStorageTables().add(storageTable); + part.setFound(true); + } + } catch (HiveException e) { + log.warn("Hive exception while getting storage table partition", e); + } + } + + /** + * Gets FactPartitions for the given fact using the following logic + * + * 1. Find the max update interval that will be used for the query. Lets assume time + * range is 15 Sep to 15 Dec and the fact has two storage with update periods as MONTHLY,DAILY,HOURLY. + * In this case the data for [15 sep - 1 oct)U[1 Dec - 15 Dec) will be answered by DAILY partitions + * and [1 oct - 1Dec) will be answered by MONTHLY partitions. The max interavl for this query will be MONTHLY. + * + * 2.Prune Storgaes that do not fall in the queries time range. + * {@link CubeMetastoreClient#isStorageTableCandidateForRange(String, Date, Date)} + * + * 3. Iterate over max interavl . In out case it will give two months Oct and Nov. Find partitions for + * these two months.Check validity of FactPartitions for Oct and Nov + * via {@link #updatePartitionStorage(FactPartition)}. + * If the partition is missing, try getting partitions for the time range form other update periods (DAILY,HOURLY). + * This is achieved by calling getPartitions() recursively but passing only 2 update periods (DAILY,HOURLY) + * + * 4.If the monthly partitions are found, check for lookahead partitions and call getPartitions recursively for the + * remaining time intervals i.e, [15 sep - 1 oct) and [1 Dec - 15 Dec) + * + * TODO union : Move this into util. + */ + private boolean getPartitions(Date fromDate, Date toDate, String partCol, Set partitions, + TreeSet updatePeriods, boolean addNonExistingParts, boolean failOnPartialData, + PartitionRangesForPartitionColumns missingPartitions) throws LensException { + if (fromDate.equals(toDate) || fromDate.after(toDate)) { + return true; + } + if (updatePeriods == null | updatePeriods.isEmpty()) { + return false; + } + + UpdatePeriod maxInterval = CubeFactTable.maxIntervalInRange(fromDate, toDate, updatePeriods); + if (maxInterval == null) { + log.info("No max interval for range: {} to {}", fromDate, toDate); + return false; + } + + if (maxInterval == UpdatePeriod.CONTINUOUS + && cubeql.getRangeWriter().getClass().equals(BetweenTimeRangeWriter.class)) { + FactPartition part = new FactPartition(partCol, fromDate, maxInterval, null, partWhereClauseFormat); + partitions.add(part); + part.getStorageTables().add(storageName); + part = new FactPartition(partCol, toDate, maxInterval, null, partWhereClauseFormat); + partitions.add(part); + part.getStorageTables().add(storageName); + this.participatingUpdatePeriods.add(maxInterval); + log.info("Added continuous fact partition for storage table {}", storageName); + return true; + } + + if (!client.partColExists(this.getFact().getName(), storageName, partCol)) { + log.info("{} does not exist in {}", partCol, storageTable); + return false; + } + + Date maxIntervalStorageTblStartDate = getStorageTableStartDate(maxInterval); + Date maxIntervalStorageTblEndDate = getStorageTableEndDate(maxInterval); + + TreeSet remainingIntervals = new TreeSet<>(updatePeriods); + remainingIntervals.remove(maxInterval); + if (!CandidateUtil.isCandidatePartiallyValidForTimeRange( + maxIntervalStorageTblStartDate, maxIntervalStorageTblEndDate, fromDate, toDate)) { + //Check the time range in remainingIntervals as maxInterval is not useful + return getPartitions(fromDate, toDate, partCol, partitions, remainingIntervals, + addNonExistingParts, failOnPartialData, missingPartitions); + } + + Date ceilFromDate = DateUtil.getCeilDate(fromDate.after(maxIntervalStorageTblStartDate) + ? fromDate : maxIntervalStorageTblStartDate, maxInterval); + Date floorToDate = DateUtil.getFloorDate(toDate.before(maxIntervalStorageTblEndDate) + ? toDate : maxIntervalStorageTblEndDate, maxInterval); + if (ceilFromDate.equals(floorToDate) || floorToDate.before(ceilFromDate)) { + return getPartitions(fromDate, toDate, partCol, partitions, remainingIntervals, + addNonExistingParts, failOnPartialData, missingPartitions); + } + + int lookAheadNumParts = conf + .getInt(CubeQueryConfUtil.getLookAheadPTPartsKey(maxInterval), CubeQueryConfUtil.DEFAULT_LOOK_AHEAD_PT_PARTS); + TimeRange.Iterable.Iterator iter = TimeRange.iterable(ceilFromDate, floorToDate, maxInterval, 1).iterator(); + // add partitions from ceilFrom to floorTo + while (iter.hasNext()) { + Date dt = iter.next(); + Date nextDt = iter.peekNext(); + FactPartition part = new FactPartition(partCol, dt, maxInterval, null, partWhereClauseFormat); + updatePartitionStorage(part); + log.debug("Storage tables containing Partition {} are: {}", part, part.getStorageTables()); + if (part.isFound()) { + log.debug("Adding existing partition {}", part); + partitions.add(part); + this.participatingUpdatePeriods.add(maxInterval); + log.debug("Looking for look ahead process time partitions for {}", part); + if (processTimePartCol == null) { + log.debug("processTimePartCol is null"); + } else if (partCol.equals(processTimePartCol)) { + log.debug("part column is process time col"); + } else if (updatePeriods.first().equals(maxInterval)) { + log.debug("Update period is the least update period"); + } else if ((iter.getNumIters() - iter.getCounter()) > lookAheadNumParts) { + // see if this is the part of the last-n look ahead partitions + log.debug("Not a look ahead partition"); + } else { + log.debug("Looking for look ahead process time partitions for {}", part); + // check if finer partitions are required + // final partitions are required if no partitions from + // look-ahead + // process time are present + TimeRange.Iterable.Iterator processTimeIter = TimeRange.iterable(nextDt, lookAheadNumParts, maxInterval, 1) + .iterator(); + while (processTimeIter.hasNext()) { + Date pdt = processTimeIter.next(); + Date nextPdt = processTimeIter.peekNext(); + FactPartition processTimePartition = new FactPartition(processTimePartCol, pdt, maxInterval, null, + partWhereClauseFormat); + updatePartitionStorage(processTimePartition); + if (processTimePartition.isFound()) { + log.debug("Finer parts not required for look-ahead partition :{}", part); + } else { + log.debug("Looked ahead process time partition {} is not found", processTimePartition); + TreeSet newset = new TreeSet(); + newset.addAll(updatePeriods); + newset.remove(maxInterval); + log.debug("newset of update periods:{}", newset); + if (!newset.isEmpty()) { + // Get partitions for look ahead process time + log.debug("Looking for process time partitions between {} and {}", pdt, nextPdt); + Set processTimeParts = getPartitions( + TimeRange.getBuilder().fromDate(pdt).toDate(nextPdt).partitionColumn(processTimePartCol).build(), + newset, true, failOnPartialData, missingPartitions); + log.debug("Look ahead partitions: {}", processTimeParts); + TimeRange timeRange = TimeRange.getBuilder().fromDate(dt).toDate(nextDt).build(); + for (FactPartition pPart : processTimeParts) { + log.debug("Looking for finer partitions in pPart: {}", pPart); + for (Date date : timeRange.iterable(pPart.getPeriod(), 1)) { + FactPartition innerPart = new FactPartition(partCol, date, pPart.getPeriod(), pPart, + partWhereClauseFormat); + updatePartitionStorage(innerPart); + innerPart.setFound(pPart.isFound()); + if (innerPart.isFound()) { + partitions.add(innerPart); + } + } + log.debug("added all sub partitions blindly in pPart: {}", pPart); + } + } + } + } + } + } else { + log.info("Partition:{} does not exist in any storage table", part); + if (!getPartitions(dt, nextDt, partCol, partitions, remainingIntervals, false, failOnPartialData, + missingPartitions)) { + log.debug("Adding non existing partition {}", part); + if (addNonExistingParts) { + // Add non existing partitions for all cases of whether we populate all non existing or not. + this.participatingUpdatePeriods.add(maxInterval); + missingPartitions.add(part); + if (!failOnPartialData) { + partitions.add(part); + part.getStorageTables().add(storageName); + } + } else { + log.info("No finer granualar partitions exist for {}", part); + return false; + } + } else { + log.debug("Finer granualar partitions added for {}", part); + } + } + } + + return getPartitions(fromDate, ceilFromDate, partCol, partitions, remainingIntervals, + addNonExistingParts, failOnPartialData, missingPartitions) + && getPartitions(floorToDate, toDate, partCol, partitions, remainingIntervals, + addNonExistingParts, failOnPartialData, missingPartitions); + } + + @Override + public boolean evaluateCompleteness(TimeRange timeRange, TimeRange queriedTimeRange, boolean failOnPartialData) + throws LensException { + // Check the measure tags. + if (!evaluateMeasuresCompleteness(timeRange)) { + log.info("Storage candidate:{} has partitions with incomplete data: {} for given ranges: {}", this, + dataCompletenessMap, cubeql.getTimeRanges()); + if (failOnPartialData) { + return false; + } + } + PartitionRangesForPartitionColumns missingParts = new PartitionRangesForPartitionColumns(); + PruneCauses storagePruningMsgs = cubeql.getStoragePruningMsgs(); + Set unsupportedTimeDims = Sets.newHashSet(); + Set partColsQueried = Sets.newHashSet(); + partColsQueried.add(timeRange.getPartitionColumn()); + StringBuilder extraWhereClauseFallback = new StringBuilder(); + Set rangeParts = getPartitions(timeRange, validUpdatePeriods, true, failOnPartialData, missingParts); + String partCol = timeRange.getPartitionColumn(); + boolean partColNotSupported = rangeParts.isEmpty(); + String storageTableName = getStorageTable(); + + if (storagePruningMsgs.containsKey(this)) { + List causes = storagePruningMsgs.get(this); + // Find the PART_COL_DOES_NOT_EXISTS + for (CandidateTablePruneCause cause : causes) { + if (cause.getCause().equals(CandidateTablePruneCode.PART_COL_DOES_NOT_EXIST)) { + partColNotSupported &= cause.getNonExistantPartCols().contains(partCol); + } + } + } else { + partColNotSupported = false; + } + TimeRange prevRange = timeRange; + String sep = ""; + while (rangeParts.isEmpty()) { + String timeDim = cubeql.getBaseCube().getTimeDimOfPartitionColumn(partCol); + if (partColNotSupported && !CandidateUtil.factHasColumn(getFact(), timeDim)) { + unsupportedTimeDims.add(cubeql.getBaseCube().getTimeDimOfPartitionColumn(timeRange.getPartitionColumn())); + break; + } + TimeRange fallBackRange = getFallbackRange(prevRange, this.getFact().getName(), cubeql); + log.info("No partitions for range:{}. fallback range: {}", timeRange, fallBackRange); + if (fallBackRange == null) { + break; + } + partColsQueried.add(fallBackRange.getPartitionColumn()); + rangeParts = getPartitions(fallBackRange, validUpdatePeriods, true, failOnPartialData, missingParts); + extraWhereClauseFallback.append(sep) + .append(prevRange.toTimeDimWhereClause(cubeql.getAliasForTableName(cubeql.getCube()), timeDim)); + sep = " AND "; + prevRange = fallBackRange; + partCol = prevRange.getPartitionColumn(); + if (!rangeParts.isEmpty()) { + break; + } + } + // Add all the partitions. participatingPartitions contains all the partitions for previous time ranges also. + rangeToPartitions.put(queriedTimeRange, rangeParts); + numQueriedParts += rangeParts.size(); + if (!unsupportedTimeDims.isEmpty()) { + log.info("Not considering storage candidate:{} as it doesn't support time dimensions: {}", this, + unsupportedTimeDims); + cubeql.addStoragePruningMsg(this, timeDimNotSupported(unsupportedTimeDims)); + return false; + } + Set nonExistingParts = missingParts.toSet(partColsQueried); + // TODO union : Relook at this. + nonExistingPartitions.addAll(nonExistingParts); + if (rangeParts.size() == 0 || (failOnPartialData && !nonExistingParts.isEmpty())) { + log.info("Not considering storage candidate:{} as no partitions for fallback range:{}", this, timeRange); + return false; + } + String extraWhere = extraWhereClauseFallback.toString(); + if (!StringUtils.isEmpty(extraWhere)) { + rangeToExtraWhereFallBack.put(queriedTimeRange, extraWhere); + } + return true; + } + + @Override + public Set getParticipatingPartitions() { + Set allPartitions = new HashSet<>(numQueriedParts); + for (Set rangePartitions : rangeToPartitions.values()) { + allPartitions.addAll(rangePartitions); + } + return allPartitions; + } + + private boolean evaluateMeasuresCompleteness(TimeRange timeRange) throws LensException { + String factDataCompletenessTag = fact.getDataCompletenessTag(); + if (factDataCompletenessTag == null) { + log.info("Not checking completeness for the fact table:{} as the dataCompletenessTag is not set", fact); + return true; + } + Set measureTag = new HashSet<>(); + Map tagToMeasureOrExprMap = new HashMap<>(); + + processExpressionsForCompleteness(cubeql, measureTag, tagToMeasureOrExprMap); + + Set measures = cubeql.getQueriedMsrs(); + if (measures == null) { + measures = new HashSet<>(); + } + for (String measure : measures) { + processCubeColForDataCompleteness(cubeql, measure, measure, measureTag, tagToMeasureOrExprMap); + } + //Checking if dataCompletenessTag is set for the fact + if (measureTag.isEmpty()) { + log.info("No Queried measures with the dataCompletenessTag, hence skipping the availability check"); + return true; + } + boolean isDataComplete = false; + DataCompletenessChecker completenessChecker = client.getCompletenessChecker(); + DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + formatter.setTimeZone(TimeZone.getTimeZone("UTC")); + if (!timeRange.getPartitionColumn().equals(completenessPartCol)) { + log.info("Completeness check not available for partCol:{}", timeRange.getPartitionColumn()); + return true; + } + Date from = timeRange.getFromDate(); + Date to = timeRange.getToDate(); + Map> completenessMap = completenessChecker + .getCompleteness(factDataCompletenessTag, from, to, measureTag); + if (completenessMap != null && !completenessMap.isEmpty()) { + for (Map.Entry> measureCompleteness : completenessMap.entrySet()) { + String tag = measureCompleteness.getKey(); + for (Map.Entry completenessResult : measureCompleteness.getValue().entrySet()) { + if (completenessResult.getValue() < completenessThreshold) { + log.info("Completeness for the measure_tag {} is {}, threshold: {}, for the hour {}", tag, + completenessResult.getValue(), completenessThreshold, formatter.format(completenessResult.getKey())); + String measureorExprFromTag = tagToMeasureOrExprMap.get(tag); + dataCompletenessMap.computeIfAbsent(measureorExprFromTag, k -> new HashMap<>()) + .put(formatter.format(completenessResult.getKey()), completenessResult.getValue()); + isDataComplete = false; + } + } + } + } + return isDataComplete; + } + + private Set getPartitions(TimeRange timeRange, TreeSet updatePeriods, + boolean addNonExistingParts, boolean failOnPartialData, PartitionRangesForPartitionColumns missingParts) + throws LensException { + Set partitions = new TreeSet<>(); + if (timeRange != null && timeRange.isCoverableBy(updatePeriods)) { + getPartitions(timeRange.getFromDate(), timeRange.getToDate(), timeRange.getPartitionColumn(), + partitions, updatePeriods, addNonExistingParts, failOnPartialData, missingParts); + } + return partitions; + } + + @Override + public boolean isExpressionEvaluable(ExpressionResolver.ExpressionContext expr) { + return expr.isEvaluable(this); + } + + /** + * Update selectAST for StorageCandidate + * 1. Delete projected select expression if it's not answerable by StorageCandidate. + * 2. Replace the queried alias with select alias if both are different in a select expr. + * + * @param cubeql + * @throws LensException + */ + + public void updateAnswerableSelectColumns(CubeQueryContext cubeql) throws LensException { + // update select AST with selected fields + int currentChild = 0; + for (int i = 0; i < cubeql.getSelectAST().getChildCount(); i++) { + ASTNode selectExpr = (ASTNode) queryAst.getSelectAST().getChild(currentChild); + Set exprCols = HQLParser.getColsInExpr(cubeql.getAliasForTableName(cubeql.getCube()), selectExpr); + if (getColumns().containsAll(exprCols)) { + ASTNode aliasNode = HQLParser.findNodeByPath(selectExpr, HiveParser.Identifier); + String alias = cubeql.getSelectPhrases().get(i).getSelectAlias(); + if (aliasNode != null) { + String queryAlias = aliasNode.getText(); + if (!queryAlias.equals(alias)) { + // replace the alias node + ASTNode newAliasNode = new ASTNode(new CommonToken(HiveParser.Identifier, alias)); + queryAst.getSelectAST().getChild(currentChild) + .replaceChildren(selectExpr.getChildCount() - 1, selectExpr.getChildCount() - 1, newAliasNode); + } + } else { + // add column alias + ASTNode newAliasNode = new ASTNode(new CommonToken(HiveParser.Identifier, alias)); + queryAst.getSelectAST().getChild(currentChild).addChild(newAliasNode); + } + } else { + queryAst.getSelectAST().deleteChild(currentChild); + currentChild--; + } + currentChild++; + } + } + + @Override + public boolean equals(Object obj) { + if (super.equals(obj)) { + return true; + } + + if (obj == null || !(obj instanceof StorageCandidate)) { + return false; + } + + StorageCandidate storageCandidateObj = (StorageCandidate) obj; + //Assuming that same instance of cube and fact will be used across StorageCandidate s and hence relying directly + //on == check for these. + return (this.cube == storageCandidateObj.cube && this.fact == storageCandidateObj.fact && this.storageTable + .equals(storageCandidateObj.storageTable)); + } + + @Override + public int hashCode() { + return this.storageTable.hashCode(); + } + + @Override + public String toString() { + return getResolvedName(); + } + + void addValidUpdatePeriod(UpdatePeriod updatePeriod) { + this.validUpdatePeriods.add(updatePeriod); + } + + void updateFromString(CubeQueryContext query, Set queryDims, + Map dimsToQuery) throws LensException { + this.dimsToQuery = dimsToQuery; + String alias = cubeql.getAliasForTableName(cubeql.getCube().getName()); + fromString = getAliasForTable(alias); + if (query.isAutoJoinResolved()) { + fromString = query.getAutoJoinCtx().getFromString(fromString, this, queryDims, dimsToQuery, query, cubeql); + } + } + + private String getFromTable() throws LensException { + if (cubeql.isAutoJoinResolved()) { + return fromString; + } else { + return cubeql.getQBFromString(this, getDimsToQuery()); + } + } + + public String getAliasForTable(String alias) { + String database = SessionState.get().getCurrentDatabase(); + String ret; + if (alias == null || alias.isEmpty()) { + ret = getResolvedName(); + } else { + ret = getResolvedName() + " " + alias; + } + if (StringUtils.isNotBlank(database) && !"default".equalsIgnoreCase(database)) { + ret = database + "." + ret; + } + return ret; + } + + boolean isUpdatePeriodUseful(UpdatePeriod updatePeriod) { + return cubeql.getTimeRanges().stream().anyMatch(timeRange -> isUpdatePeriodUseful(timeRange, updatePeriod)); + } + + /** + * Is the update period useful for this time range. e.g. for a time range of hours and days, monthly + * and yearly update periods are useless. DAILY and HOURLY are useful. It further checks if the update + * period answers the range at least partially based on start and end times configured at update period + * level or at storage or fact level. + * @param timeRange The time range + * @param updatePeriod Update period + * @return Whether it's useless + */ + private boolean isUpdatePeriodUseful(TimeRange timeRange, UpdatePeriod updatePeriod) { + try { + if (!CandidateUtil.isCandidatePartiallyValidForTimeRange(getStorageTableStartDate(updatePeriod), + getStorageTableEndDate(updatePeriod), timeRange.getFromDate(), timeRange.getToDate())) { + return false; + } + Date storageTblStartDate = getStorageTableStartDate(updatePeriod); + Date storageTblEndDate = getStorageTableEndDate(updatePeriod); + TimeRange.getBuilder() //TODO date calculation to move to util method and resued + .fromDate(timeRange.getFromDate().after(storageTblStartDate) ? timeRange.getFromDate() : storageTblStartDate) + .toDate(timeRange.getToDate().before(storageTblEndDate) ? timeRange.getToDate() : storageTblEndDate) + .partitionColumn(timeRange.getPartitionColumn()) + .build() + .truncate(updatePeriod); + return true; + } catch (LensException e) { + return false; + } + } + + /** + * Is time range coverable based on valid update periods of this storage candidate + * + * @param timeRange + * @return + * @throws LensException + */ + public boolean isTimeRangeCoverable(TimeRange timeRange) throws LensException { + return isTimeRangeCoverable(timeRange.getFromDate(), timeRange.getToDate(), validUpdatePeriods); + } + + /** + * Is the time range coverable by given update periods. + * Extracts the max update period, then extracts maximum amount of range from the middle that this update + * period can cover. Then recurses on the remaining ranges on the left and right side of the extracted chunk + * using one less update period. + * + * @param timeRangeStart + * @param timeRangeEnd + * @param intervals Update periods to check + * @return Whether time range is coverable by provided update periods or not. + */ + private boolean isTimeRangeCoverable(Date timeRangeStart, Date timeRangeEnd, + Set intervals) throws LensException { + if (timeRangeStart.equals(timeRangeEnd) || timeRangeStart.after(timeRangeEnd)) { + return true; + } + if (intervals == null || intervals.isEmpty()) { + return false; + } + + UpdatePeriod maxInterval = CubeFactTable.maxIntervalInRange(timeRangeStart, timeRangeEnd, intervals); + if (maxInterval == null) { + return false; + } + + if (maxInterval == UpdatePeriod.CONTINUOUS + && cubeql.getRangeWriter().getClass().equals(BetweenTimeRangeWriter.class)) { + return true; + } + + Date maxIntervalStorageTableStartDate = getStorageTableStartDate(maxInterval); + Date maxIntervalStorageTableEndDate = getStorageTableEndDate(maxInterval); + Set remainingIntervals = Sets.difference(intervals, Sets.newHashSet(maxInterval)); + + if (!CandidateUtil.isCandidatePartiallyValidForTimeRange( + maxIntervalStorageTableStartDate, maxIntervalStorageTableEndDate, timeRangeStart, timeRangeEnd)) { + //Check the time range in remainingIntervals as maxInterval is not useful + return isTimeRangeCoverable(timeRangeStart, timeRangeEnd, remainingIntervals); + } + + Date ceilFromDate = DateUtil.getCeilDate(timeRangeStart.after(maxIntervalStorageTableStartDate) + ? timeRangeStart : maxIntervalStorageTableStartDate, maxInterval); + Date floorToDate = DateUtil.getFloorDate(timeRangeEnd.before(maxIntervalStorageTableEndDate) + ? timeRangeEnd : maxIntervalStorageTableEndDate, maxInterval); + if (ceilFromDate.equals(floorToDate) || floorToDate.before(ceilFromDate)) { + return isTimeRangeCoverable(timeRangeStart, timeRangeEnd, remainingIntervals); + } + + //ceilFromDate to floorToDate time range is covered by maxInterval (though there may be holes.. but that's ok) + //Check the remaining part of time range in remainingIntervals + return isTimeRangeCoverable(timeRangeStart, ceilFromDate, remainingIntervals) + && isTimeRangeCoverable(floorToDate, timeRangeEnd, remainingIntervals); + } + + private Date getStorageTableStartDate(UpdatePeriod interval) throws LensException { + if (!isStorageTblsAtUpdatePeriodLevel) { + //In this case the start time and end time is at Storage Level and will be same for all update periods. + return this.startTime; + } + return client.getStorageTableStartDate( + client.getStorageTableName(fact.getName(), storageName, interval), fact.getName()); + } + + private Date getStorageTableEndDate(UpdatePeriod interval) throws LensException { + if (!isStorageTblsAtUpdatePeriodLevel) { + //In this case the start time and end time is at Storage Level and will be same for all update periods. + return this.endTime; + } + return client.getStorageTableEndDate( + client.getStorageTableName(fact.getName(), storageName, interval), fact.getName()); + } + + + public String getResolvedName() { + if (resolvedName == null) { + return storageTable; + } + return resolvedName; + } + + /** + * Splits the Storage Candidates into multiple Storage Candidates if storage candidate has multiple + * storage tables (one per update period) + * + * @return + * @throws LensException + */ + public Collection splitAtUpdatePeriodLevelIfReq() throws LensException { + if (!isStorageTblsAtUpdatePeriodLevel) { + return Lists.newArrayList(this); // No need to explode in this case + } + return getPeriodSpecificStorageCandidates(); + } + + private Collection getPeriodSpecificStorageCandidates() throws LensException { + List periodSpecificScList = new ArrayList<>(participatingUpdatePeriods.size()); + StorageCandidate updatePeriodSpecificSc; + for (UpdatePeriod period : participatingUpdatePeriods) { + updatePeriodSpecificSc = new StorageCandidate(this); + updatePeriodSpecificSc.truncatePartitions(period); + updatePeriodSpecificSc.setResolvedName(client.getStorageTableName(fact.getName(), + storageName, period)); + periodSpecificScList.add(updatePeriodSpecificSc); + } + return periodSpecificScList; + } + + /** + * Truncates partitions in {@link #rangeToPartitions} such that only partitions belonging to + * the passed undatePeriod are retained. + * @param updatePeriod + */ + private void truncatePartitions(UpdatePeriod updatePeriod) { + Iterator>> rangeItr = rangeToPartitions.entrySet().iterator(); + while (rangeItr.hasNext()) { + Map.Entry> rangeEntry = rangeItr.next(); + Iterator partitionItr = rangeEntry.getValue().iterator(); + while (partitionItr.hasNext()) { + if (!partitionItr.next().getPeriod().equals(updatePeriod)) { + partitionItr.remove(); + } + } + if (rangeEntry.getValue().isEmpty()) { + rangeItr.remove(); + } + } + } + + +} diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java index 2b6319319..10c3bbe5a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -18,35 +18,21 @@ */ package org.apache.lens.cube.parse; -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.incompletePartitions; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.partitionColumnsMissing; import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static org.apache.lens.cube.metadata.DateUtil.WSPACE; -import static org.apache.lens.cube.metadata.MetastoreUtil.*; -import static org.apache.lens.cube.parse.CandidateTablePruneCause.*; -import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*; -import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode.*; import org.apache.lens.cube.metadata.*; -import org.apache.lens.cube.parse.CandidateTablePruneCause.*; +import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; +import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipUpdatePeriodCode; import org.apache.lens.server.api.error.LensException; -import org.apache.lens.server.api.metastore.*; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.util.ReflectionUtils; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; - import lombok.extern.slf4j.Slf4j; - /** * Resolve storages and partitions of all candidate tables and prunes candidate tables with missing storages or * partitions. @@ -57,35 +43,13 @@ class StorageTableResolver implements ContextRewriter { private final Configuration conf; private final List supportedStorages; private final boolean allStoragesSupported; - CubeMetastoreClient client; private final boolean failOnPartialData; private final List validDimTables; - private final Map>> validStorageMap = new HashMap<>(); - private String processTimePartCol = null; private final UpdatePeriod maxInterval; + // TODO union : Remove this. All partitions are stored in the StorageCandidate. private final Map> nonExistingPartitions = new HashMap<>(); - private TimeRangeWriter rangeWriter; - private DateFormat partWhereClauseFormat = null; + private CubeMetastoreClient client; private PHASE phase; - private HashMap> skipStorageCausesPerFact; - private float completenessThreshold; - private String completenessPartCol; - - enum PHASE { - FACT_TABLES, FACT_PARTITIONS, DIM_TABLE_AND_PARTITIONS; - - static PHASE first() { - return values()[0]; - } - - static PHASE last() { - return values()[values().length - 1]; - } - - PHASE next() { - return values()[(this.ordinal() + 1) % values().length]; - } - } StorageTableResolver(Configuration conf) { this.conf = conf; @@ -94,24 +58,13 @@ PHASE next() { this.failOnPartialData = conf.getBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false); String str = conf.get(CubeQueryConfUtil.VALID_STORAGE_DIM_TABLES); validDimTables = StringUtils.isBlank(str) ? null : Arrays.asList(StringUtils.split(str.toLowerCase(), ",")); - this.processTimePartCol = conf.get(CubeQueryConfUtil.PROCESS_TIME_PART_COL); String maxIntervalStr = conf.get(CubeQueryConfUtil.QUERY_MAX_INTERVAL); if (maxIntervalStr != null) { - this.maxInterval = UpdatePeriod.valueOf(maxIntervalStr); + this.maxInterval = UpdatePeriod.valueOf(maxIntervalStr.toUpperCase()); } else { this.maxInterval = null; } - rangeWriter = - ReflectionUtils.newInstance(conf.getClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, - CubeQueryConfUtil.DEFAULT_TIME_RANGE_WRITER, TimeRangeWriter.class), this.conf); - String formatStr = conf.get(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT); - if (formatStr != null) { - partWhereClauseFormat = new SimpleDateFormat(formatStr); - } this.phase = PHASE.first(); - completenessThreshold = conf.getFloat(CubeQueryConfUtil.COMPLETENESS_THRESHOLD, - CubeQueryConfUtil.DEFAULT_COMPLETENESS_THRESHOLD); - completenessPartCol = conf.get(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL); } private List getSupportedStorages(Configuration conf) { @@ -122,55 +75,88 @@ private List getSupportedStorages(Configuration conf) { return null; } - public boolean isStorageSupported(String storage) { + private boolean isStorageSupportedOnDriver(String storage) { return allStoragesSupported || supportedStorages.contains(storage); } - Map> storagePartMap = new HashMap>(); - @Override public void rewriteContext(CubeQueryContext cubeql) throws LensException { client = cubeql.getMetastoreClient(); switch (phase) { - case FACT_TABLES: - if (!cubeql.getCandidateFacts().isEmpty()) { - // resolve storage table names - resolveFactStorageTableNames(cubeql); + case STORAGE_TABLES: + if (!cubeql.getCandidates().isEmpty()) { + resolveStorageTable(cubeql); } - cubeql.pruneCandidateFactSet(CandidateTablePruneCode.NO_CANDIDATE_STORAGES); break; - case FACT_PARTITIONS: - if (!cubeql.getCandidateFacts().isEmpty()) { - // resolve storage partitions - resolveFactStoragePartitions(cubeql); - } - cubeql.pruneCandidateFactSet(CandidateTablePruneCode.NO_CANDIDATE_STORAGES); - if (client != null && client.isDataCompletenessCheckEnabled()) { - if (!cubeql.getCandidateFacts().isEmpty()) { - // resolve incomplete fact partition - resolveFactCompleteness(cubeql); - } - cubeql.pruneCandidateFactSet(CandidateTablePruneCode.INCOMPLETE_PARTITION); + case STORAGE_PARTITIONS: + if (!cubeql.getCandidates().isEmpty()) { + resolveStoragePartitions(cubeql); } break; case DIM_TABLE_AND_PARTITIONS: resolveDimStorageTablesAndPartitions(cubeql); if (cubeql.getAutoJoinCtx() != null) { // After all candidates are pruned after storage resolver, prune join paths. - cubeql.getAutoJoinCtx().pruneAllPaths(cubeql.getCube(), cubeql.getCandidateFacts(), null); + cubeql.getAutoJoinCtx() + .pruneAllPaths(cubeql.getCube(), CandidateUtil.getStorageCandidates(cubeql.getCandidates()), null); cubeql.getAutoJoinCtx().pruneAllPathsForCandidateDims(cubeql.getCandidateDimTables()); cubeql.getAutoJoinCtx().refreshJoinPathColumns(); } + // TODO union : What is this? We may not need this as it non existing partitions are stored in StorageCandidate + cubeql.setNonexistingParts(nonExistingPartitions); break; } - //Doing this on all three phases. Keep updating cubeql with the current identified missing partitions. - cubeql.setNonexistingParts(nonExistingPartitions); phase = phase.next(); } + /** + * Each candidate in the set is a complex candidate. We will evaluate each one to get + * all the partitions needed to answer the query. + * + * @param cubeql cube query context + */ + private void resolveStoragePartitions(CubeQueryContext cubeql) throws LensException { + Iterator candidateIterator = cubeql.getCandidates().iterator(); + while (candidateIterator.hasNext()) { + Candidate candidate = candidateIterator.next(); + boolean isComplete = true; + boolean isTimeRangeAnswerableByThisCandidate = true; + for (TimeRange range : cubeql.getTimeRanges()) { + if (!candidate.isTimeRangeCoverable(range)) { + isTimeRangeAnswerableByThisCandidate = false; + log.info("Not considering candidate:{} as it can not cover time range {}", candidate, range); + cubeql.addCandidatePruningMsg(candidate, + CandidateTablePruneCause.storageNotAvailableInRange(Lists.newArrayList(range))); + break; + } + isComplete &= candidate.evaluateCompleteness(range, range, failOnPartialData); + } + if (!isTimeRangeAnswerableByThisCandidate) { + candidateIterator.remove(); + } else if (failOnPartialData && !isComplete) { + candidateIterator.remove(); + log.info("Not considering candidate:{} as its data is not is not complete", candidate); + Set scSet = CandidateUtil.getStorageCandidates(candidate); + for (StorageCandidate sc : scSet) { + if (!sc.getNonExistingPartitions().isEmpty()) { + cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.missingPartitions(sc.getNonExistingPartitions())); + } else if (!sc.getDataCompletenessMap().isEmpty()) { + cubeql.addStoragePruningMsg(sc, incompletePartitions(sc.getDataCompletenessMap())); + } + } + } else if (candidate.getParticipatingPartitions().isEmpty() + && candidate instanceof StorageCandidate + && ((StorageCandidate) candidate).getNonExistingPartitions().isEmpty()) { + candidateIterator.remove(); + cubeql.addCandidatePruningMsg(candidate, + new CandidateTablePruneCause(CandidateTablePruneCode.NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE)); + } + } + } + private void resolveDimStorageTablesAndPartitions(CubeQueryContext cubeql) throws LensException { - Set allDims = new HashSet(cubeql.getDimensions()); + Set allDims = new HashSet<>(cubeql.getDimensions()); for (Aliased dim : cubeql.getOptionalDimensions()) { allDims.add(dim.getObject()); } @@ -184,21 +170,23 @@ private void resolveDimStorageTablesAndPartitions(CubeQueryContext cubeql) throw CandidateDim candidate = i.next(); CubeDimensionTable dimtable = candidate.dimtable; if (dimtable.getStorages().isEmpty()) { - cubeql.addDimPruningMsgs(dim, dimtable, new CandidateTablePruneCause( - CandidateTablePruneCode.MISSING_STORAGES)); + cubeql + .addDimPruningMsgs(dim, dimtable, new CandidateTablePruneCause(CandidateTablePruneCode.MISSING_STORAGES)); i.remove(); continue; } - Set storageTables = new HashSet(); + Set storageTables = new HashSet<>(); Map whereClauses = new HashMap(); boolean foundPart = false; - Map skipStorageCauses = new HashMap<>(); + // TODO union : We have to remove all usages of a deprecated class. + Map skipStorageCauses = new HashMap<>(); for (String storage : dimtable.getStorages()) { - if (isStorageSupported(storage)) { - String tableName = getFactOrDimtableStorageTableName(dimtable.getName(), storage).toLowerCase(); + if (isStorageSupportedOnDriver(storage)) { + String tableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimtable.getName(), + storage).toLowerCase(); if (validDimTables != null && !validDimTables.contains(tableName)) { log.info("Not considering dim storage table:{} as it is not a valid dim storage", tableName); - skipStorageCauses.put(tableName, new SkipStorageCause(SkipStorageCode.INVALID)); + skipStorageCauses.put(tableName, CandidateTablePruneCode.INVALID); continue; } @@ -212,13 +200,12 @@ private void resolveDimStorageTablesAndPartitions(CubeQueryContext cubeql) throw } if (!failOnPartialData || foundPart) { storageTables.add(tableName); - String whereClause = - StorageUtil.getWherePartClause(dim.getTimedDimension(), null, - StorageConstants.getPartitionsForLatest()); + String whereClause = StorageUtil + .getWherePartClause(dim.getTimedDimension(), null, StorageConstants.getPartitionsForLatest()); whereClauses.put(tableName, whereClause); } else { log.info("Not considering dim storage table:{} as no dim partitions exist", tableName); - skipStorageCauses.put(tableName, new SkipStorageCause(SkipStorageCode.NO_PARTITIONS)); + skipStorageCauses.put(tableName, CandidateTablePruneCode.NO_PARTITIONS); } } else { storageTables.add(tableName); @@ -226,7 +213,7 @@ private void resolveDimStorageTablesAndPartitions(CubeQueryContext cubeql) throw } } else { log.info("Storage:{} is not supported", storage); - skipStorageCauses.put(storage, new SkipStorageCause(SkipStorageCode.UNSUPPORTED)); + skipStorageCauses.put(storage, CandidateTablePruneCode.UNSUPPORTED_STORAGE); } } if (!foundPart) { @@ -234,7 +221,8 @@ private void resolveDimStorageTablesAndPartitions(CubeQueryContext cubeql) throw } if (storageTables.isEmpty()) { log.info("Not considering dim table:{} as no candidate storage tables eixst", dimtable); - cubeql.addDimPruningMsgs(dim, dimtable, noCandidateStorages(skipStorageCauses)); + cubeql.addDimPruningMsgs(dim, dimtable, + CandidateTablePruneCause.noCandidateStoragesForDimtable(skipStorageCauses)); i.remove(); continue; } @@ -245,619 +233,151 @@ private void resolveDimStorageTablesAndPartitions(CubeQueryContext cubeql) throw } } - // Resolves all the storage table names, which are valid for each updatePeriod - private void resolveFactStorageTableNames(CubeQueryContext cubeql) throws LensException { - Iterator i = cubeql.getCandidateFacts().iterator(); - skipStorageCausesPerFact = new HashMap<>(); - while (i.hasNext()) { - CubeFactTable fact = i.next().fact; - if (fact.getUpdatePeriods().isEmpty()) { - cubeql.addFactPruningMsgs(fact, new CandidateTablePruneCause(CandidateTablePruneCode.MISSING_STORAGES)); - i.remove(); + /** + * Following storages are removed: + * 1. The storage is not supported by driver. + * 2. The storage is not in the valid storage list. + * 3. The storage is not in any time range in the query. + * 4. The storage having no valid update period. + * + * This method also creates a list of valid update periods and stores them into {@link StorageCandidate}. + * + * TODO union : Do fourth point before 3. + */ + private void resolveStorageTable(CubeQueryContext cubeql) throws LensException { + Iterator it = cubeql.getCandidates().iterator(); + while (it.hasNext()) { + Candidate c = it.next(); + assert (c instanceof StorageCandidate); + StorageCandidate sc = (StorageCandidate) c; + String storageTable = sc.getStorageName(); + // first check: if the storage is supported on driver + if (!isStorageSupportedOnDriver(storageTable)) { + log.info("Skipping storage: {} as it is not supported", storageTable); + cubeql.addStoragePruningMsg(sc, new CandidateTablePruneCause(CandidateTablePruneCode.UNSUPPORTED_STORAGE)); + it.remove(); continue; } - Map> storageTableMap = new TreeMap>(); - validStorageMap.put(fact, storageTableMap); - String str = conf.get(CubeQueryConfUtil.getValidStorageTablesKey(fact.getName())); + String str = conf.get(CubeQueryConfUtil.getValidStorageTablesKey(sc.getFact().getName())); List validFactStorageTables = StringUtils.isBlank(str) ? null : Arrays.asList(StringUtils.split(str.toLowerCase(), ",")); - Map skipStorageCauses = new HashMap<>(); - - for (Map.Entry> entry : fact.getUpdatePeriods().entrySet()) { - String storage = entry.getKey(); - // skip storages that are not supported - if (!isStorageSupported(storage)) { - log.info("Skipping storage: {} as it is not supported", storage); - skipStorageCauses.put(storage, new SkipStorageCause(SkipStorageCode.UNSUPPORTED)); - continue; - } - String table = getStorageTableName(fact, storage, validFactStorageTables); - // skip the update period if the storage is not valid - if (table == null) { - skipStorageCauses.put(storage, new SkipStorageCause(SkipStorageCode.INVALID)); - continue; - } - List validUpdatePeriods = - CubeQueryConfUtil.getStringList(conf, CubeQueryConfUtil.getValidUpdatePeriodsKey(fact.getName(), storage)); - - boolean isStorageAdded = false; - Map skipUpdatePeriodCauses = new HashMap(); - for (UpdatePeriod updatePeriod : entry.getValue()) { - if (maxInterval != null && updatePeriod.compareTo(maxInterval) > 0) { - log.info("Skipping update period {} for fact {}", updatePeriod, fact); - skipUpdatePeriodCauses.put(updatePeriod.toString(), SkipUpdatePeriodCode.QUERY_INTERVAL_BIGGER); - continue; - } - if (validUpdatePeriods != null && !validUpdatePeriods.contains(updatePeriod.name().toLowerCase())) { - log.info("Skipping update period {} for fact {} for storage {}", updatePeriod, fact, storage); - skipUpdatePeriodCauses.put(updatePeriod.toString(), SkipUpdatePeriodCode.INVALID); - continue; - } - Set storageTables = storageTableMap.get(updatePeriod); - if (storageTables == null) { - storageTables = new LinkedHashSet<>(); - storageTableMap.put(updatePeriod, storageTables); - } - isStorageAdded = true; - log.debug("Adding storage table:{} for fact:{} for update period {}", table, fact, updatePeriod); - storageTables.add(table); - } - if (!isStorageAdded) { - skipStorageCauses.put(storage, SkipStorageCause.noCandidateUpdatePeriod(skipUpdatePeriodCauses)); - } - } - skipStorageCausesPerFact.put(fact, skipStorageCauses); - if (storageTableMap.isEmpty()) { - log.info("Not considering fact table:{} as it does not have any storage tables", fact); - cubeql.addFactPruningMsgs(fact, noCandidateStorages(skipStorageCauses)); - i.remove(); - } - } - } - - private TreeSet getValidUpdatePeriods(CubeFactTable fact) { - TreeSet set = new TreeSet(); - set.addAll(validStorageMap.get(fact).keySet()); - return set; - } - - String getStorageTableName(CubeFactTable fact, String storage, List validFactStorageTables) { - String tableName = getFactOrDimtableStorageTableName(fact.getName(), storage).toLowerCase(); - if (validFactStorageTables != null && !validFactStorageTables.contains(tableName)) { - log.info("Skipping storage table {} as it is not valid", tableName); - return null; - } - return tableName; - } - - private TimeRange getFallbackRange(TimeRange range, CandidateFact cfact, CubeQueryContext cubeql) - throws LensException { - Cube baseCube = cubeql.getBaseCube(); - ArrayList tableNames = Lists.newArrayList(cfact.fact.getName(), cubeql.getCube().getName()); - if (!cubeql.getCube().getName().equals(baseCube.getName())) { - tableNames.add(baseCube.getName()); - } - String fallBackString = null; - String timedim = baseCube.getTimeDimOfPartitionColumn(range.getPartitionColumn()); - for (String tableName : tableNames) { - fallBackString = cubeql.getMetastoreClient().getTable(tableName).getParameters() - .get(MetastoreConstants.TIMEDIM_RELATION + timedim); - if (StringUtils.isNotBlank(fallBackString)) { - break; - } - } - if (StringUtils.isBlank(fallBackString)) { - return null; - } - Matcher matcher = Pattern.compile("(.*?)\\+\\[(.*?),(.*?)\\]").matcher(fallBackString.replaceAll(WSPACE, "")); - if (!matcher.matches()) { - return null; - } - DateUtil.TimeDiff diff1 = DateUtil.TimeDiff.parseFrom(matcher.group(2).trim()); - DateUtil.TimeDiff diff2 = DateUtil.TimeDiff.parseFrom(matcher.group(3).trim()); - String relatedTimeDim = matcher.group(1).trim(); - String fallbackPartCol = baseCube.getPartitionColumnOfTimeDim(relatedTimeDim); - return TimeRange.getBuilder() - .fromDate(diff2.negativeOffsetFrom(range.getFromDate())) - .toDate(diff1.negativeOffsetFrom(range.getToDate())) - .partitionColumn(fallbackPartCol).build(); - } - - private void resolveFactStoragePartitions(CubeQueryContext cubeql) throws LensException { - // Find candidate tables wrt supported storages - Iterator i = cubeql.getCandidateFacts().iterator(); - while (i.hasNext()) { - CandidateFact cfact = i.next(); - Map whereClauseForFallback = new LinkedHashMap(); - List answeringParts = new ArrayList<>(); - Map skipStorageCauses = skipStorageCausesPerFact.get(cfact.fact); - if (skipStorageCauses == null) { - skipStorageCauses = new HashMap<>(); - } - PartitionRangesForPartitionColumns missingParts = new PartitionRangesForPartitionColumns(); - boolean noPartsForRange = false; - Set unsupportedTimeDims = Sets.newHashSet(); - Set partColsQueried = Sets.newHashSet(); - for (TimeRange range : cubeql.getTimeRanges()) { - partColsQueried.add(range.getPartitionColumn()); - StringBuilder extraWhereClause = new StringBuilder(); - Set rangeParts = getPartitions(cfact.fact, range, skipStorageCauses, missingParts); - // If no partitions were found, then we'll fallback. - String partCol = range.getPartitionColumn(); - boolean partColNotSupported = rangeParts.isEmpty(); - for (String storage : cfact.fact.getStorages()) { - String storageTableName = getFactOrDimtableStorageTableName(cfact.fact.getName(), storage).toLowerCase(); - partColNotSupported &= skipStorageCauses.containsKey(storageTableName) - && skipStorageCauses.get(storageTableName).getCause().equals(PART_COL_DOES_NOT_EXIST) - && skipStorageCauses.get(storageTableName).getNonExistantPartCols().contains(partCol); - } - TimeRange prevRange = range; - String sep = ""; - while (rangeParts.isEmpty()) { - // TODO: should we add a condition whether on range's partcol any missing partitions are not there - String timeDim = cubeql.getBaseCube().getTimeDimOfPartitionColumn(partCol); - if (partColNotSupported && !cfact.getColumns().contains(timeDim)) { - unsupportedTimeDims.add(cubeql.getBaseCube().getTimeDimOfPartitionColumn(range.getPartitionColumn())); - break; - } - TimeRange fallBackRange = getFallbackRange(prevRange, cfact, cubeql); - log.info("No partitions for range:{}. fallback range: {}", range, fallBackRange); - if (fallBackRange == null) { - break; - } - partColsQueried.add(fallBackRange.getPartitionColumn()); - rangeParts = getPartitions(cfact.fact, fallBackRange, skipStorageCauses, missingParts); - extraWhereClause.append(sep) - .append(prevRange.toTimeDimWhereClause(cubeql.getAliasForTableName(cubeql.getCube()), timeDim)); - sep = " AND "; - prevRange = fallBackRange; - partCol = prevRange.getPartitionColumn(); - if (!rangeParts.isEmpty()) { - break; - } - } - whereClauseForFallback.put(range, extraWhereClause.toString()); - if (rangeParts.isEmpty()) { - log.info("No partitions for fallback range:{}", range); - noPartsForRange = true; - continue; - } - // If multiple storage tables are part of the same fact, - // capture range->storage->partitions - Map> tablePartMap = new HashMap>(); - for (FactPartition factPart : rangeParts) { - for (String table : factPart.getStorageTables()) { - if (!tablePartMap.containsKey(table)) { - tablePartMap.put(table, new LinkedHashSet<>(Collections.singletonList(factPart))); - } else { - LinkedHashSet storagePart = tablePartMap.get(table); - storagePart.add(factPart); - } - } - } - cfact.getRangeToStoragePartMap().put(range, tablePartMap); - cfact.incrementPartsQueried(rangeParts.size()); - answeringParts.addAll(rangeParts); - cfact.getPartsQueried().addAll(rangeParts); - } - if (!unsupportedTimeDims.isEmpty()) { - log.info("Not considering fact table:{} as it doesn't support time dimensions: {}", cfact.fact, - unsupportedTimeDims); - cubeql.addFactPruningMsgs(cfact.fact, timeDimNotSupported(unsupportedTimeDims)); - i.remove(); - continue; - } - Set nonExistingParts = missingParts.toSet(partColsQueried); - if (!nonExistingParts.isEmpty()) { - addNonExistingParts(cfact.fact.getName(), nonExistingParts); - } - if (cfact.getNumQueriedParts() == 0 || (failOnPartialData && (noPartsForRange || !nonExistingParts.isEmpty()))) { - log.info("Not considering fact table:{} as it could not find partition for given ranges: {}", cfact.fact, - cubeql.getTimeRanges()); - /* - * This fact is getting discarded because of any of following reasons: - * 1. Has missing partitions - * 2. All Storage tables were skipped for some reasons. - * 3. Storage tables do not have the update period for the timerange queried. - */ - if (failOnPartialData && !nonExistingParts.isEmpty()) { - cubeql.addFactPruningMsgs(cfact.fact, missingPartitions(nonExistingParts)); - } else if (!skipStorageCauses.isEmpty()) { - CandidateTablePruneCause cause = noCandidateStorages(skipStorageCauses); - cubeql.addFactPruningMsgs(cfact.fact, cause); - } else { - CandidateTablePruneCause cause = - new CandidateTablePruneCause(NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE); - cubeql.addFactPruningMsgs(cfact.fact, cause); - } - i.remove(); + storageTable = sc.getStorageTable(); + // Check if storagetable is in the list of valid storages. + if (validFactStorageTables != null && !validFactStorageTables.contains(storageTable)) { + log.info("Skipping storage table {} as it is not valid", storageTable); + cubeql.addStoragePruningMsg(sc, new CandidateTablePruneCause(CandidateTablePruneCode.INVALID_STORAGE)); + it.remove(); continue; } - // Map from storage to covering parts - Map> minimalStorageTables = new LinkedHashMap>(); - StorageUtil.getMinimalAnsweringTables(answeringParts, minimalStorageTables); - if (minimalStorageTables.isEmpty()) { - log.info("Not considering fact table:{} as it does not have any storage tables", cfact); - cubeql.addFactPruningMsgs(cfact.fact, noCandidateStorages(skipStorageCauses)); - i.remove(); + List validUpdatePeriods = CubeQueryConfUtil + .getStringList(conf, CubeQueryConfUtil.getValidUpdatePeriodsKey(sc.getFact().getName(), sc.getStorageName())); + boolean isUpdatePeriodForStorageAdded = false; + Map skipUpdatePeriodCauses = new HashMap<>(); + + if (cubeql.getTimeRanges().stream().noneMatch(range -> CandidateUtil.isPartiallyValidForTimeRange(sc, range))) { + cubeql.addStoragePruningMsg(sc, + new CandidateTablePruneCause(CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE)); + it.remove(); continue; } - Set storageTables = new LinkedHashSet<>(); - storageTables.addAll(minimalStorageTables.keySet()); - cfact.setStorageTables(storageTables); - // Update range->storage->partitions with time range where clause - for (TimeRange trange : cfact.getRangeToStoragePartMap().keySet()) { - Map rangeToWhere = new HashMap<>(); - for (Map.Entry> entry : minimalStorageTables.entrySet()) { - String table = entry.getKey(); - Set minimalParts = entry.getValue(); - - LinkedHashSet rangeParts = cfact.getRangeToStoragePartMap().get(trange).get(table); - LinkedHashSet minimalPartsCopy = Sets.newLinkedHashSet(); - - if (rangeParts != null) { - minimalPartsCopy.addAll(minimalParts); - minimalPartsCopy.retainAll(rangeParts); - } - if (!StringUtils.isEmpty(whereClauseForFallback.get(trange))) { - rangeToWhere.put(table, "((" - + rangeWriter.getTimeRangeWhereClause(cubeql, cubeql.getAliasForTableName(cubeql.getCube().getName()), - minimalPartsCopy) + ") and (" + whereClauseForFallback.get(trange) + "))"); - } else { - rangeToWhere.put(table, rangeWriter.getTimeRangeWhereClause(cubeql, - cubeql.getAliasForTableName(cubeql.getCube().getName()), minimalPartsCopy)); - } - } - cfact.getRangeToStorageWhereMap().put(trange, rangeToWhere); - } - log.info("Resolved partitions for fact {}: {} storageTables:{}", cfact, answeringParts, storageTables); - } - } - private static boolean processCubeColForDataCompleteness(CubeQueryContext cubeql, String cubeCol, String alias, - Set measureTag, - Map tagToMeasureOrExprMap) { - CubeMeasure column = cubeql.getCube().getMeasureByName(cubeCol); - if (column != null && column.getTags() != null) { - String dataCompletenessTag = column.getTags().get(MetastoreConstants.MEASURE_DATACOMPLETENESS_TAG); - //Checking if dataCompletenessTag is set for queried measure - if (dataCompletenessTag != null) { - measureTag.add(dataCompletenessTag); - String value = tagToMeasureOrExprMap.get(dataCompletenessTag); - if (value == null) { - tagToMeasureOrExprMap.put(dataCompletenessTag, alias); + // Populate valid update periods abd check validity at update period level + for (UpdatePeriod updatePeriod : sc.getFact().getUpdatePeriods().get(sc.getStorageName())) { + if (maxInterval != null && updatePeriod.compareTo(maxInterval) > 0) { + // if user supplied max interval, all intervals larger than that are useless. + log.info("Skipping update period {} for candidate {} since it's more than max interval supplied({})", + updatePeriod, sc.getStorageTable(), maxInterval); + skipUpdatePeriodCauses.put(updatePeriod.toString(), SkipUpdatePeriodCode.UPDATE_PERIOD_BIGGER_THAN_MAX); + } else if (validUpdatePeriods != null && !validUpdatePeriods.contains(updatePeriod.name().toLowerCase())) { + // if user supplied valid update periods, other update periods are useless + log.info("Skipping update period {} for candidate {} for storage {} since it's invalid", + updatePeriod, sc.getStorageTable(), storageTable); + skipUpdatePeriodCauses.put(updatePeriod.toString(), SkipUpdatePeriodCode.INVALID); + } else if (!sc.isUpdatePeriodUseful(updatePeriod)) { + // if the storage candidate finds this update useful to keep looking at the time ranges queried + skipUpdatePeriodCauses.put(updatePeriod.toString(), + SkipUpdatePeriodCode.TIME_RANGE_NOT_ANSWERABLE_BY_UPDATE_PERIOD); } else { - value = value.concat(",").concat(alias); - tagToMeasureOrExprMap.put(dataCompletenessTag, value); - } - return true; - } - } - return false; - } - - private static void processMeasuresFromExprMeasures(CubeQueryContext cubeql, Set measureTag, - Map tagToMeasureOrExprMap) { - boolean isExprProcessed; - String cubeAlias = cubeql.getAliasForTableName(cubeql.getCube().getName()); - for (String expr : cubeql.getQueriedExprsWithMeasures()) { - isExprProcessed = false; - for (ExpressionResolver.ExprSpecContext esc : cubeql.getExprCtx().getExpressionContext(expr, cubeAlias) - .getAllExprs()) { - if (esc.getTblAliasToColumns().get(cubeAlias) != null) { - for (String cubeCol : esc.getTblAliasToColumns().get(cubeAlias)) { - if (processCubeColForDataCompleteness(cubeql, cubeCol, expr, measureTag, tagToMeasureOrExprMap)) { - /* This is done to associate the expression with one of the dataCompletenessTag for the measures. - So, even if the expression is composed of measures with different dataCompletenessTags, we will be - determining the dataCompleteness from one of the measure and this expression is grouped with the - other queried measures that have the same dataCompletenessTag. */ - isExprProcessed = true; - break; - } - } - } - if (isExprProcessed) { - break; + isUpdatePeriodForStorageAdded = true; + sc.addValidUpdatePeriod(updatePeriod); } } - } - } - - private void resolveFactCompleteness(CubeQueryContext cubeql) throws LensException { - if (client == null || client.getCompletenessChecker() == null || completenessPartCol == null) { - return; - } - DataCompletenessChecker completenessChecker = client.getCompletenessChecker(); - Set measureTag = new HashSet<>(); - Map tagToMeasureOrExprMap = new HashMap<>(); - - processMeasuresFromExprMeasures(cubeql, measureTag, tagToMeasureOrExprMap); - - Set measures = cubeql.getQueriedMsrs(); - if (measures == null) { - measures = new HashSet<>(); - } - for (String measure : measures) { - processCubeColForDataCompleteness(cubeql, measure, measure, measureTag, tagToMeasureOrExprMap); - } - //Checking if dataCompletenessTag is set for the fact - if (measureTag.isEmpty()) { - log.info("No Queried measures with the dataCompletenessTag, hence skipping the availability check"); - return; - } - Iterator i = cubeql.getCandidateFacts().iterator(); - DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - formatter.setTimeZone(TimeZone.getTimeZone("UTC")); - while (i.hasNext()) { - CandidateFact cFact = i.next(); - // Map from measure to the map from partition to %completeness - Map> incompleteMeasureData = new HashMap<>(); - - String factDataCompletenessTag = cFact.fact.getDataCompletenessTag(); - if (factDataCompletenessTag == null) { - log.info("Not checking completeness for the fact table:{} as the dataCompletenessTag is not set", cFact.fact); - continue; + // For DEBUG purpose only to see why some update periods are skipped. + if (!skipUpdatePeriodCauses.isEmpty()) { + sc.setUpdatePeriodRejectionCause(skipUpdatePeriodCauses); } - boolean isFactDataIncomplete = false; - for (TimeRange range : cubeql.getTimeRanges()) { - if (!range.getPartitionColumn().equals(completenessPartCol)) { - log.info("Completeness check not available for partCol:{}", range.getPartitionColumn()); - continue; + // if no update periods were added in previous section, we skip this storage candidate + if (!isUpdatePeriodForStorageAdded) { + if (skipUpdatePeriodCauses.values().stream().allMatch( + SkipUpdatePeriodCode.TIME_RANGE_NOT_ANSWERABLE_BY_UPDATE_PERIOD::equals)) { + // all update periods bigger than query range, it means time range not answerable. + cubeql.addStoragePruningMsg(sc, + new CandidateTablePruneCause(CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE)); + } else { // Update periods are rejected for multiple reasons. + cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.updatePeriodsRejected(skipUpdatePeriodCauses)); } - Date from = range.getFromDate(); - Date to = range.getToDate(); - Map> completenessMap = completenessChecker.getCompleteness(factDataCompletenessTag, - from, to, measureTag); - if (completenessMap != null && !completenessMap.isEmpty()) { - for (Map.Entry> measureCompleteness : completenessMap.entrySet()) { - String tag = measureCompleteness.getKey(); - for (Map.Entry completenessResult : measureCompleteness.getValue().entrySet()) { - if (completenessResult.getValue() < completenessThreshold) { - log.info("Completeness for the measure_tag {} is {}, threshold: {}, for the hour {}", tag, - completenessResult.getValue(), completenessThreshold, - formatter.format(completenessResult.getKey())); - String measureorExprFromTag = tagToMeasureOrExprMap.get(tag); - Map incompletePartition = incompleteMeasureData.get(measureorExprFromTag); - if (incompletePartition == null) { - incompletePartition = new HashMap<>(); - incompleteMeasureData.put(measureorExprFromTag, incompletePartition); + it.remove(); + } else { + //set the dates again as they can change based on ValidUpdatePeriod + sc.setStorageStartAndEndDate(); + Set allPruningCauses = new HashSet<>(cubeql.getTimeRanges().size()); + for (TimeRange range : cubeql.getTimeRanges()) { + CandidateTablePruneCause pruningCauseForThisTimeRange = null; + if (!CandidateUtil.isPartiallyValidForTimeRange(sc, range)) { + //This is the prune cause + pruningCauseForThisTimeRange = + new CandidateTablePruneCause(CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE); + } else if (cubeql.shouldReplaceTimeDimWithPart()) { + if (!client.partColExists(sc.getFact().getName(), sc.getStorageName(), range.getPartitionColumn())) { + pruningCauseForThisTimeRange = partitionColumnsMissing(range.getPartitionColumn()); + TimeRange fallBackRange = StorageUtil.getFallbackRange(range, sc.getFact().getName(), cubeql); + while (fallBackRange != null) { + pruningCauseForThisTimeRange = null; + if (!client.partColExists(sc.getFact().getName(), sc.getStorageName(), + fallBackRange.getPartitionColumn())) { + pruningCauseForThisTimeRange = partitionColumnsMissing(fallBackRange.getPartitionColumn()); + fallBackRange = StorageUtil.getFallbackRange(fallBackRange, sc.getFact().getName(), cubeql); + } else { + if (!CandidateUtil.isPartiallyValidForTimeRange(sc, fallBackRange)) { + pruningCauseForThisTimeRange = + new CandidateTablePruneCause(CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE); + } + break; } - incompletePartition.put(formatter.format(completenessResult.getKey()), completenessResult.getValue()); - isFactDataIncomplete = true; } } } + + if (pruningCauseForThisTimeRange != null) { + allPruningCauses.add(pruningCauseForThisTimeRange); + } } - } - if (isFactDataIncomplete) { - log.info("Fact table:{} has partitions with incomplete data: {} for given ranges: {}", cFact.fact, - incompleteMeasureData, cubeql.getTimeRanges()); - if (failOnPartialData) { - i.remove(); - cubeql.addFactPruningMsgs(cFact.fact, incompletePartitions(incompleteMeasureData)); - } else { - cFact.setDataCompletenessMap(incompleteMeasureData); + if (!allPruningCauses.isEmpty()) { + // TODO if this storage can answer atleast one time range , why prune it ? + it.remove(); + cubeql.addStoragePruningMsg(sc, allPruningCauses.toArray(new CandidateTablePruneCause[0])); } } } } - void addNonExistingParts(String name, Set nonExistingParts) { + private void addNonExistingParts(String name, Set nonExistingParts) { nonExistingPartitions.put(name, nonExistingParts); } - private Set getPartitions(CubeFactTable fact, TimeRange range, - Map skipStorageCauses, - PartitionRangesForPartitionColumns missingPartitions) throws LensException { - try { - return getPartitions(fact, range, getValidUpdatePeriods(fact), true, failOnPartialData, skipStorageCauses, - missingPartitions); - } catch (Exception e) { - throw new LensException(e); - } - } - - private Set getPartitions(CubeFactTable fact, TimeRange range, TreeSet updatePeriods, - boolean addNonExistingParts, boolean failOnPartialData, Map skipStorageCauses, - PartitionRangesForPartitionColumns missingPartitions) - throws Exception { - Set partitions = new TreeSet<>(); - if (range != null && range.isCoverableBy(updatePeriods) - && getPartitions(fact, range.getFromDate(), range.getToDate(), range.getPartitionColumn(), partitions, - updatePeriods, addNonExistingParts, failOnPartialData, skipStorageCauses, missingPartitions)) { - return partitions; - } else { - return new TreeSet<>(); - } - } - - private boolean getPartitions(CubeFactTable fact, Date fromDate, Date toDate, String partCol, - Set partitions, TreeSet updatePeriods, - boolean addNonExistingParts, boolean failOnPartialData, Map skipStorageCauses, - PartitionRangesForPartitionColumns missingPartitions) - throws Exception { - log.info("getPartitions for {} from fromDate:{} toDate:{}", fact, fromDate, toDate); - if (fromDate.equals(toDate) || fromDate.after(toDate)) { - return true; - } - UpdatePeriod interval = CubeFactTable.maxIntervalInRange(fromDate, toDate, updatePeriods); - if (interval == null) { - log.info("No max interval for range: {} to {}", fromDate, toDate); - return false; - } - log.debug("Max interval for {} is: {}", fact, interval); - Set storageTbls = new LinkedHashSet(); - storageTbls.addAll(validStorageMap.get(fact).get(interval)); - - if (interval == UpdatePeriod.CONTINUOUS && rangeWriter.getClass().equals(BetweenTimeRangeWriter.class)) { - for (String storageTbl : storageTbls) { - FactPartition part = new FactPartition(partCol, fromDate, interval, null, partWhereClauseFormat); - partitions.add(part); - part.getStorageTables().add(storageTbl); - part = new FactPartition(partCol, toDate, interval, null, partWhereClauseFormat); - partitions.add(part); - part.getStorageTables().add(storageTbl); - log.info("Added continuous fact partition for storage table {}", storageTbl); - } - return true; - } - - Iterator it = storageTbls.iterator(); - while (it.hasNext()) { - String storageTableName = it.next(); - if (!client.isStorageTableCandidateForRange(storageTableName, fromDate, toDate)) { - skipStorageCauses.put(storageTableName, new SkipStorageCause(RANGE_NOT_ANSWERABLE)); - it.remove(); - } else if (!client.partColExists(storageTableName, partCol)) { - log.info("{} does not exist in {}", partCol, storageTableName); - skipStorageCauses.put(storageTableName, SkipStorageCause.partColDoesNotExist(partCol)); - it.remove(); - } - } - - if (storageTbls.isEmpty()) { - return false; - } - Date ceilFromDate = DateUtil.getCeilDate(fromDate, interval); - Date floorToDate = DateUtil.getFloorDate(toDate, interval); - - int lookAheadNumParts = - conf.getInt(CubeQueryConfUtil.getLookAheadPTPartsKey(interval), CubeQueryConfUtil.DEFAULT_LOOK_AHEAD_PT_PARTS); - - TimeRange.Iterable.Iterator iter = TimeRange.iterable(ceilFromDate, floorToDate, interval, 1).iterator(); - // add partitions from ceilFrom to floorTo - while (iter.hasNext()) { - Date dt = iter.next(); - Date nextDt = iter.peekNext(); - FactPartition part = new FactPartition(partCol, dt, interval, null, partWhereClauseFormat); - log.debug("candidate storage tables for searching partitions: {}", storageTbls); - updateFactPartitionStorageTablesFrom(fact, part, storageTbls); - log.debug("Storage tables containing Partition {} are: {}", part, part.getStorageTables()); - if (part.isFound()) { - log.debug("Adding existing partition {}", part); - partitions.add(part); - log.debug("Looking for look ahead process time partitions for {}", part); - if (processTimePartCol == null) { - log.debug("processTimePartCol is null"); - } else if (partCol.equals(processTimePartCol)) { - log.debug("part column is process time col"); - } else if (updatePeriods.first().equals(interval)) { - log.debug("Update period is the least update period"); - } else if ((iter.getNumIters() - iter.getCounter()) > lookAheadNumParts) { - // see if this is the part of the last-n look ahead partitions - log.debug("Not a look ahead partition"); - } else { - log.debug("Looking for look ahead process time partitions for {}", part); - // check if finer partitions are required - // final partitions are required if no partitions from - // look-ahead - // process time are present - TimeRange.Iterable.Iterator processTimeIter = TimeRange.iterable(nextDt, lookAheadNumParts, - interval, 1).iterator(); - while (processTimeIter.hasNext()) { - Date pdt = processTimeIter.next(); - Date nextPdt = processTimeIter.peekNext(); - FactPartition processTimePartition = new FactPartition(processTimePartCol, pdt, interval, null, - partWhereClauseFormat); - updateFactPartitionStorageTablesFrom(fact, processTimePartition, - part.getStorageTables()); - if (processTimePartition.isFound()) { - log.debug("Finer parts not required for look-ahead partition :{}", part); - } else { - log.debug("Looked ahead process time partition {} is not found", processTimePartition); - TreeSet newset = new TreeSet(); - newset.addAll(updatePeriods); - newset.remove(interval); - log.debug("newset of update periods:{}", newset); - if (!newset.isEmpty()) { - // Get partitions for look ahead process time - log.debug("Looking for process time partitions between {} and {}", pdt, nextPdt); - Set processTimeParts = - getPartitions(fact, TimeRange.getBuilder().fromDate(pdt).toDate(nextPdt).partitionColumn( - processTimePartCol).build(), newset, true, false, skipStorageCauses, missingPartitions); - log.debug("Look ahead partitions: {}", processTimeParts); - TimeRange timeRange = TimeRange.getBuilder().fromDate(dt).toDate(nextDt).build(); - for (FactPartition pPart : processTimeParts) { - log.debug("Looking for finer partitions in pPart: {}", pPart); - for (Date date : timeRange.iterable(pPart.getPeriod(), 1)) { - FactPartition innerPart = new FactPartition(partCol, date, pPart.getPeriod(), pPart, - partWhereClauseFormat); - updateFactPartitionStorageTablesFrom(fact, innerPart, pPart); - if (innerPart.isFound()) { - partitions.add(innerPart); - } - } - log.debug("added all sub partitions blindly in pPart: {}", pPart); - } - } - } - } - } - } else { - log.info("Partition:{} does not exist in any storage table", part); - TreeSet newset = new TreeSet(); - newset.addAll(updatePeriods); - newset.remove(interval); - if (!getPartitions(fact, dt, nextDt, partCol, partitions, newset, false, failOnPartialData, skipStorageCauses, - missingPartitions)) { + enum PHASE { + STORAGE_TABLES, STORAGE_PARTITIONS, DIM_TABLE_AND_PARTITIONS; - log.debug("Adding non existing partition {}", part); - if (addNonExistingParts) { - // Add non existing partitions for all cases of whether we populate all non existing or not. - missingPartitions.add(part); - if (!failOnPartialData) { - Set st = getStorageTablesWithoutPartCheck(part, storageTbls); - if (st.isEmpty()) { - log.info("No eligible storage tables"); - return false; - } - partitions.add(part); - part.getStorageTables().addAll(st); - } - } else { - log.info("No finer granual partitions exist for {}", part); - return false; - } - } else { - log.debug("Finer granual partitions added for {}", part); - } - } + static PHASE first() { + return values()[0]; } - return getPartitions(fact, fromDate, ceilFromDate, partCol, partitions, - updatePeriods, addNonExistingParts, failOnPartialData, skipStorageCauses, missingPartitions) - && getPartitions(fact, floorToDate, toDate, partCol, partitions, - updatePeriods, addNonExistingParts, failOnPartialData, skipStorageCauses, missingPartitions); - } - private Set getStorageTablesWithoutPartCheck(FactPartition part, - Set storageTableNames) throws LensException, HiveException { - Set validStorageTbls = new HashSet<>(); - for (String storageTableName : storageTableNames) { - // skip all storage tables for which are not eligible for this partition - if (client.isStorageTablePartitionACandidate(storageTableName, part.getPartSpec())) { - validStorageTbls.add(storageTableName); - } else { - log.info("Skipping {} as it is not valid for part {}", storageTableName, part.getPartSpec()); - } + static PHASE last() { + return values()[values().length - 1]; } - return validStorageTbls; - } - private void updateFactPartitionStorageTablesFrom(CubeFactTable fact, - FactPartition part, Set storageTableNames) throws LensException, HiveException, ParseException { - for (String storageTableName : storageTableNames) { - // skip all storage tables for which are not eligible for this partition - if (client.isStorageTablePartitionACandidate(storageTableName, part.getPartSpec()) - && (client.factPartitionExists(fact, part, storageTableName))) { - part.getStorageTables().add(storageTableName); - part.setFound(true); - } + PHASE next() { + return values()[(this.ordinal() + 1) % values().length]; } } - - private void updateFactPartitionStorageTablesFrom(CubeFactTable fact, - FactPartition part, FactPartition pPart) throws LensException, HiveException, ParseException { - updateFactPartitionStorageTablesFrom(fact, part, pPart.getStorageTables()); - part.setFound(part.isFound() && pPart.isFound()); - } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java index f9636d130..f5cd540e4 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -18,13 +18,19 @@ */ package org.apache.lens.cube.parse; +import static org.apache.lens.cube.metadata.DateUtil.WSPACE; + import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; -import org.apache.lens.cube.metadata.FactPartition; -import org.apache.lens.cube.metadata.StorageConstants; +import org.apache.lens.cube.metadata.*; +import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.StringUtils; +import com.google.common.collect.Lists; + public final class StorageUtil { private StorageUtil() { @@ -69,8 +75,8 @@ public static String getNotLatestClauseForDimensions(String alias, Set t String sep = ""; for (String timePartCol : timedDimensions) { if (!timePartCol.equals(partCol)) { - sb.append(sep).append(alias).append(".").append(timePartCol) - .append(" != '").append(StorageConstants.LATEST_PARTITION_VALUE).append("'"); + sb.append(sep).append(alias).append(".").append(timePartCol).append(" != '") + .append(StorageConstants.LATEST_PARTITION_VALUE).append("'"); sep = " AND "; } } @@ -82,15 +88,11 @@ public static String joinWithAnd(String... clauses) { String sep = "(("; for (String clause : clauses) { if (clause != null && !clause.isEmpty()) { - sb - .append(sep) - .append(clause); + sb.append(sep).append(clause); sep = ") AND ("; } } - return sb - .append(sep.equals("((") ? "" : "))") - .toString(); + return sb.append(sep.equals("((") ? "" : "))").toString(); } /** @@ -161,4 +163,112 @@ public static String getWhereClause(CandidateDim dim, String alias) { return null; } } + + /** + * Get fallback range + * + * @param range + * @param factName + * @param cubeql + * @return + * @throws LensException + */ + public static TimeRange getFallbackRange(TimeRange range, String factName, CubeQueryContext cubeql) + throws LensException { + Cube baseCube = cubeql.getBaseCube(); + ArrayList tableNames = Lists.newArrayList(factName, cubeql.getCube().getName()); + if (!cubeql.getCube().getName().equals(baseCube.getName())) { + tableNames.add(baseCube.getName()); + } + String fallBackString = null; + String timedim = baseCube.getTimeDimOfPartitionColumn(range.getPartitionColumn()); + for (String tableName : tableNames) { + fallBackString = cubeql.getMetastoreClient().getTable(tableName).getParameters() + .get(MetastoreConstants.TIMEDIM_RELATION + timedim); + if (StringUtils.isNotBlank(fallBackString)) { + break; + } + } + if (StringUtils.isBlank(fallBackString)) { + return null; + } + Matcher matcher = Pattern.compile("(.*?)\\+\\[(.*?),(.*?)\\]").matcher(fallBackString.replaceAll(WSPACE, "")); + if (!matcher.matches()) { + return null; + } + DateUtil.TimeDiff diff1 = DateUtil.TimeDiff.parseFrom(matcher.group(2).trim()); + DateUtil.TimeDiff diff2 = DateUtil.TimeDiff.parseFrom(matcher.group(3).trim()); + String relatedTimeDim = matcher.group(1).trim(); + String fallbackPartCol = baseCube.getPartitionColumnOfTimeDim(relatedTimeDim); + return TimeRange.getBuilder().fromDate(diff2.negativeOffsetFrom(range.getFromDate())) + .toDate(diff1.negativeOffsetFrom(range.getToDate())).partitionColumn(fallbackPartCol).build(); + } + + /** + * Checks how much data is completed for a column. + * See this: {@link org.apache.lens.server.api.metastore.DataCompletenessChecker} + * + * @param cubeql + * @param cubeCol + * @param alias + * @param measureTag + * @param tagToMeasureOrExprMap + * @return + */ + public static boolean processCubeColForDataCompleteness(CubeQueryContext cubeql, String cubeCol, String alias, + Set measureTag, Map tagToMeasureOrExprMap) { + CubeMeasure column = cubeql.getCube().getMeasureByName(cubeCol); + if (column != null && column.getTags() != null) { + String dataCompletenessTag = column.getTags().get(MetastoreConstants.MEASURE_DATACOMPLETENESS_TAG); + //Checking if dataCompletenessTag is set for queried measure + if (dataCompletenessTag != null) { + measureTag.add(dataCompletenessTag); + String value = tagToMeasureOrExprMap.get(dataCompletenessTag); + if (value == null) { + tagToMeasureOrExprMap.put(dataCompletenessTag, alias); + } else { + value = value.concat(",").concat(alias); + tagToMeasureOrExprMap.put(dataCompletenessTag, value); + } + return true; + } + } + return false; + } + + /** + * This method extracts all the columns used in expressions (used in query) and evaluates each + * column separately for completeness + * + * @param cubeql + * @param measureTag + * @param tagToMeasureOrExprMap + */ + public static void processExpressionsForCompleteness(CubeQueryContext cubeql, Set measureTag, + Map tagToMeasureOrExprMap) { + boolean isExprProcessed; + String cubeAlias = cubeql.getAliasForTableName(cubeql.getCube().getName()); + for (String expr : cubeql.getQueriedExprsWithMeasures()) { + isExprProcessed = false; + for (ExpressionResolver.ExprSpecContext esc : cubeql.getExprCtx().getExpressionContext(expr, cubeAlias) + .getAllExprs()) { + if (esc.getTblAliasToColumns().get(cubeAlias) != null) { + for (String cubeCol : esc.getTblAliasToColumns().get(cubeAlias)) { + if (processCubeColForDataCompleteness(cubeql, cubeCol, expr, measureTag, tagToMeasureOrExprMap)) { + /* This is done to associate the expression with one of the dataCompletenessTag for the measures. + So, even if the expression is composed of measures with different dataCompletenessTags, we will be + determining the dataCompleteness from one of the measure and this expression is grouped with the + other queried measures that have the same dataCompletenessTag. */ + isExprProcessed = true; + break; + } + } + } + if (isExprProcessed) { + break; + } + } + } + } } + diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java new file mode 100644 index 000000000..7f07dbc84 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java @@ -0,0 +1,295 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + +import java.util.*; + +import org.apache.lens.cube.metadata.FactPartition; +import org.apache.lens.cube.metadata.TimeRange; +import org.apache.lens.server.api.error.LensException; + +import com.google.common.collect.Maps; + +/** + * Represents a union of two candidates + */ +public class UnionCandidate implements Candidate { + + /** + * Caching start and end time calculated for this candidate as it may have many child candidates. + */ + Date startTime = null; + Date endTime = null; + String toStr; + CubeQueryContext cubeql; + /** + * List of child candidates that will be union-ed + */ + private List childCandidates; + private QueryAST queryAst; + private Map> splitTimeRangeMap = Maps.newHashMap(); + public UnionCandidate(List childCandidates, CubeQueryContext cubeql) { + this.childCandidates = childCandidates; + //this.alias = alias; + this.cubeql = cubeql; + } + + @Override + public Set getAnswerableMeasurePhraseIndices() { + // All children in the UnionCandiate will be having common quriable measure + return getChildren().iterator().next().getAnswerableMeasurePhraseIndices(); + } + + @Override + public boolean isTimeRangeCoverable(TimeRange timeRange) throws LensException { + Map candidateRange = getTimeRangeSplit(timeRange); + for (Map.Entry entry : candidateRange.entrySet()) { + if (!entry.getKey().isTimeRangeCoverable(entry.getValue())) { + return false; + } + } + return true; + } + + @Override + public Collection getColumns() { + // In UnionCandidate all columns are same, return the columns + // of first child + return childCandidates.iterator().next().getColumns(); + } + + @Override + public Date getStartTime() { + //Note: concurrent calls not handled specifically (This should not be a problem even if we do + //get concurrent calls). + + if (startTime == null) { + Date minStartTime = childCandidates.get(0).getStartTime(); + for (Candidate child : childCandidates) { + if (child.getStartTime().before(minStartTime)) { + minStartTime = child.getStartTime(); + } + } + startTime = minStartTime; + } + return startTime; + } + + @Override + public Date getEndTime() { + if (endTime == null) { + Date maxEndTime = childCandidates.get(0).getEndTime(); + for (Candidate child : childCandidates) { + if (child.getEndTime().after(maxEndTime)) { + maxEndTime = child.getEndTime(); + } + } + endTime = maxEndTime; + } + return endTime; + } + + @Override + public double getCost() { + double cost = 0.0; + for (TimeRange timeRange : cubeql.getTimeRanges()) { + for (Map.Entry entry : getTimeRangeSplit(timeRange).entrySet()) { + cost += entry.getKey().getCost() * entry.getValue().milliseconds() / timeRange.milliseconds(); + } + } + return cost; + } + + @Override + public boolean contains(Candidate candidate) { + if (this.equals(candidate)) { + return true; + } + for (Candidate child : childCandidates) { + if (child.contains((candidate))) { + return true; + } + } + return false; + } + + @Override + public Collection getChildren() { + return childCandidates; + } + + /** + * @param timeRange + * @return + */ + @Override + public boolean evaluateCompleteness(TimeRange timeRange, TimeRange parentTimeRange, boolean failOnPartialData) + throws LensException { + Map candidateRange = getTimeRangeSplit(timeRange); + boolean ret = true; + for (Map.Entry entry : candidateRange.entrySet()) { + ret &= entry.getKey().evaluateCompleteness(entry.getValue(), parentTimeRange, failOnPartialData); + } + return ret; + } + + @Override + public Set getParticipatingPartitions() { + Set factPartitionSet = new HashSet<>(); + for (Candidate c : childCandidates) { + factPartitionSet.addAll(c.getParticipatingPartitions()); + } + return factPartitionSet; + } + + @Override + public boolean isExpressionEvaluable(ExpressionResolver.ExpressionContext expr) { + for (Candidate cand : childCandidates) { + if (!cand.isExpressionEvaluable(expr)) { + return false; + } + } + return true; + } + + @Override + public String toString() { + if (this.toStr == null) { + this.toStr = getToString(); + } + return this.toStr; + } + + private String getToString() { + StringBuilder builder = new StringBuilder(10 * childCandidates.size()); + builder.append("UNION["); + for (Candidate candidate : childCandidates) { + builder.append(candidate.toString()); + builder.append(", "); + } + builder.delete(builder.length() - 2, builder.length()); + builder.append("]"); + return builder.toString(); + } + + /** + * Splits the parent time range for each candidate. + * The candidates are sorted based on their costs. + * + * @param timeRange + * @return + */ + private Map splitTimeRangeForChildren(TimeRange timeRange) { + childCandidates.sort(Comparator.comparing(Candidate::getCost)); + Map childrenTimeRangeMap = new HashMap<>(); + // Sorted list based on the weights. + Set ranges = new HashSet<>(); + ranges.add(timeRange); + for (Candidate c : childCandidates) { + TimeRange.TimeRangeBuilder builder = getClonedBuiler(timeRange); + TimeRange tr = resolveTimeRangeForChildren(c, ranges, builder); + if (tr != null) { + // If the time range is not null it means this child candidate is valid for this union candidate. + childrenTimeRangeMap.put(c, tr); + } + } + return childrenTimeRangeMap; + } + private Map getTimeRangeSplit(TimeRange range) { + return splitTimeRangeMap.computeIfAbsent(range, this::splitTimeRangeForChildren); + } + + /** + * Resolves the time range for this candidate based on overlap. + * + * @param candidate : Candidate for which the time range is to be calculated + * @param ranges : Set of time ranges from which one has to be choosen. + * @param builder : TimeRange builder created by the common AST. + * @return Calculated timeRange for the candidate. If it returns null then there is no suitable time range split for + * this candidate. This is the correct behaviour because an union candidate can have non participating child + * candidates for the parent time range. + */ + private TimeRange resolveTimeRangeForChildren(Candidate candidate, Set ranges, + TimeRange.TimeRangeBuilder builder) { + Iterator it = ranges.iterator(); + Set newTimeRanges = new HashSet<>(); + TimeRange ret = null; + while (it.hasNext()) { + TimeRange range = it.next(); + // Check for out of range + if (candidate.getStartTime().getTime() >= range.getToDate().getTime() || candidate.getEndTime().getTime() <= range + .getFromDate().getTime()) { + continue; + } + // This means overlap. + if (candidate.getStartTime().getTime() <= range.getFromDate().getTime()) { + // Start time of the new time range will be range.getFromDate() + builder.fromDate(range.getFromDate()); + if (candidate.getEndTime().getTime() <= range.getToDate().getTime()) { + // End time is in the middle of the range is equal to c.getEndTime(). + builder.toDate(candidate.getEndTime()); + } else { + // End time will be range.getToDate() + builder.toDate(range.getToDate()); + } + } else { + builder.fromDate(candidate.getStartTime()); + if (candidate.getEndTime().getTime() <= range.getToDate().getTime()) { + builder.toDate(candidate.getEndTime()); + } else { + builder.toDate(range.getToDate()); + } + } + // Remove the time range and add more time ranges. + it.remove(); + ret = builder.build(); + if (ret.getFromDate().getTime() == range.getFromDate().getTime()) { + checkAndUpdateNewTimeRanges(ret, range, newTimeRanges); + } else { + TimeRange.TimeRangeBuilder b1 = getClonedBuiler(ret); + b1.fromDate(range.getFromDate()); + b1.toDate(ret.getFromDate()); + newTimeRanges.add(b1.build()); + checkAndUpdateNewTimeRanges(ret, range, newTimeRanges); + + } + break; + } + ranges.addAll(newTimeRanges); + return ret; + } + + private void checkAndUpdateNewTimeRanges(TimeRange ret, TimeRange range, Set newTimeRanges) { + if (ret.getToDate().getTime() < range.getToDate().getTime()) { + TimeRange.TimeRangeBuilder b2 = getClonedBuiler(ret); + b2.fromDate(ret.getToDate()); + b2.toDate(range.getToDate()); + newTimeRanges.add(b2.build()); + } + } + + private TimeRange.TimeRangeBuilder getClonedBuiler(TimeRange timeRange) { + TimeRange.TimeRangeBuilder builder = new TimeRange.TimeRangeBuilder(); + builder.astNode(timeRange.getAstNode()); + builder.childIndex(timeRange.getChildIndex()); + builder.parent(timeRange.getParent()); + builder.partitionColumn(timeRange.getPartitionColumn()); + return builder; + } +} diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionHQLContext.java deleted file mode 100644 index e6ee9895b..000000000 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionHQLContext.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.lens.cube.parse; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.lens.server.api.error.LensException; - -import org.apache.commons.lang.NotImplementedException; - -import lombok.AllArgsConstructor; -import lombok.RequiredArgsConstructor; - -@AllArgsConstructor -@RequiredArgsConstructor -public abstract class UnionHQLContext extends SimpleHQLContext { - protected final CubeQueryContext query; - protected final CandidateFact fact; - - List hqlContexts = new ArrayList<>(); - - public void setHqlContexts(List hqlContexts) throws LensException { - this.hqlContexts = hqlContexts; - StringBuilder queryParts = new StringBuilder("("); - String sep = ""; - for (HQLContextInterface ctx : hqlContexts) { - queryParts.append(sep).append(ctx.toHQL()); - sep = " UNION ALL "; - } - setFrom(queryParts.append(") ").append(query.getCube().getName()).toString()); - } - - @Override - public String getWhere() { - throw new NotImplementedException("Not Implemented"); - } -} diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java new file mode 100644 index 000000000..3ee817fc2 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java @@ -0,0 +1,730 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.cube.parse; + +import static org.apache.lens.cube.parse.HQLParser.*; + +import static org.apache.hadoop.hive.ql.parse.HiveParser.*; + +import java.util.*; + +import org.apache.lens.cube.metadata.Dimension; +import org.apache.lens.cube.metadata.MetastoreUtil; +import org.apache.lens.server.api.error.LensException; + +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.HiveParser; +import org.apache.hadoop.util.StringUtils; + +import org.antlr.runtime.CommonToken; + +import lombok.extern.slf4j.Slf4j; + +/** + * Utility class to write union query. Given any complex Join or Union Candidate, + * this class rewrites union query for all the participating StorageCandidates. + */ +@Slf4j +public class UnionQueryWriter { + + private QueryAST queryAst; + private Map innerToOuterSelectASTs = new HashMap<>(); + private Map innerToOuterHavingASTs = new HashMap<>(); + private Map storageCandidateToSelectAstMap = new HashMap<>(); + private AliasDecider aliasDecider = new DefaultAliasDecider(); + private CubeQueryContext cubeql; + Collection storageCandidates; + public static final String DEFAULT_MEASURE = "0.0"; + + public UnionQueryWriter(Collection storageCandidates, CubeQueryContext cubeql) { + if (storageCandidates == null || storageCandidates.size()<=1) { + throw new IllegalArgumentException("There should be atleast two storage candidates to write a union query"); + } + this.cubeql = cubeql; + this.storageCandidates = storageCandidates; + } + + public String toHQL(Map> factDimMap) throws LensException { + StorageCandidate firstCandidate = storageCandidates.iterator().next(); + // Set the default queryAST for the outer query + queryAst = DefaultQueryAST.fromStorageCandidate(firstCandidate, + firstCandidate.getQueryAst()); + updateAsts(); + updateInnterSelectASTWithDefault(); + processSelectAndHavingAST(); + processGroupByAST(); + processOrderByAST(); + CandidateUtil.updateFinalAlias(queryAst.getSelectAST(), cubeql); + return CandidateUtil.buildHQLString(queryAst.getSelectString(), getFromString(factDimMap), null, + queryAst.getGroupByString(), queryAst.getOrderByString(), + queryAst.getHavingString(), queryAst.getLimitValue()); + } + + /** + * Set having, order by and limit clauses to null for inner queries + * being constructed from StorageCandidate. + */ + private void updateAsts() { + for (StorageCandidate sc : storageCandidates) { + storageCandidateToSelectAstMap.put(sc.toString(), + new ASTNode(new CommonToken(TOK_SELECT, "TOK_SELECT"))); + if (sc.getQueryAst().getHavingAST() != null) { + sc.getQueryAst().setHavingAST(null); + } + if (sc.getQueryAst().getOrderByAST() != null) { + sc.getQueryAst().setOrderByAST(null); + } + if (sc.getQueryAst().getLimitValue() != null) { + sc.getQueryAst().setLimitValue(null); + } + } + } + + private void processGroupByAST() throws LensException { + if (queryAst.getGroupByAST() != null) { + queryAst.setGroupByAST(processGroupByExpression(queryAst.getGroupByAST())); + } + } + + /** + * Process havingAST for a StorageCandidate. Any column not projected and part of having clause + * project it in inner select + * + * @param innerAst + * @param aliasDecider + * @param sc + * @return ASTNode + * @throws LensException + */ + private ASTNode processHavingAST(ASTNode innerAst, AliasDecider aliasDecider, StorageCandidate sc) + throws LensException { + if (cubeql.getHavingAST() != null) { + ASTNode havingCopy = MetastoreUtil.copyAST(cubeql.getHavingAST()); + Set havingAggChildrenASTs = new LinkedHashSet<>(); + getAggregateChildrenInNode(havingCopy, havingAggChildrenASTs); + processHavingExpression(innerAst, havingAggChildrenASTs, aliasDecider, sc); + updateOuterHavingAST(havingCopy); + queryAst.setHavingAST(havingCopy); + HQLParser.getString(havingCopy); + } + return null; + } + + /** + * Update outer havingAST with proper alias name projected. + * + * @param node + * @return + */ + private ASTNode updateOuterHavingAST(ASTNode node) { + if (node.getToken().getType() == HiveParser.TOK_FUNCTION + && (HQLParser.isAggregateAST(node))) { + if (innerToOuterSelectASTs.containsKey(new HQLParser.HashableASTNode(node)) + || innerToOuterHavingASTs.containsKey(new HQLParser.HashableASTNode(node))) { + ASTNode expr = innerToOuterSelectASTs.containsKey(new HQLParser.HashableASTNode(node)) + ? innerToOuterSelectASTs.get(new HQLParser.HashableASTNode(node)) + : innerToOuterHavingASTs.get(new HQLParser.HashableASTNode(node)); + if (node.getChildCount() > 1) { + node.replaceChildren(1, 1, expr.getChild(1)); + } else { + node.replaceChildren(0, 0, expr); + } + } + } + for (int i = 0; i < node.getChildCount(); i++) { + ASTNode child = (ASTNode) node.getChild(i); + updateOuterHavingAST(child); + } + return node; + } + + + private void processOrderByAST() throws LensException { + if (queryAst.getOrderByAST() != null) { + queryAst.setOrderByAST(processOrderbyExpression(queryAst.getOrderByAST())); + } + } + + private ASTNode processOrderbyExpression(ASTNode astNode) throws LensException { + if (astNode == null) { + return null; + } + ASTNode outerExpression = new ASTNode(astNode); + // sample orderby AST looks the following : + /* + TOK_ORDERBY + TOK_TABSORTCOLNAMEDESC + TOK_NULLS_LAST + . + TOK_TABLE_OR_COL + testcube + cityid + TOK_TABSORTCOLNAMEASC + TOK_NULLS_FIRST + . + TOK_TABLE_OR_COL + testcube + stateid + TOK_TABSORTCOLNAMEASC + TOK_NULLS_FIRST + . + TOK_TABLE_OR_COL + testcube + zipcode + */ + for (Node node : astNode.getChildren()) { + ASTNode child = (ASTNode) node; + ASTNode outerOrderby = new ASTNode(child); + ASTNode tokNullsChild = (ASTNode) child.getChild(0); + ASTNode outerTokNullsChild = new ASTNode(tokNullsChild); + outerTokNullsChild.addChild(innerToOuterSelectASTs.get(new HQLParser.HashableASTNode((ASTNode) tokNullsChild))); + outerOrderby.addChild(outerTokNullsChild); + outerExpression.addChild(outerOrderby); + } + return outerExpression; + } + + /** + * Get the select expression. In case of node is default retunrs "0.0" with alias + * otherwise the select phrase with alias. + * + * @param nodeWithoutAlias + * @param aliasNode + * @param isDefault + * @return + * @throws LensException + */ + private ASTNode getSelectExpr(ASTNode nodeWithoutAlias, ASTNode aliasNode, boolean isDefault) + throws LensException { + ASTNode node = getSelectExprAST(); + if (nodeWithoutAlias == null && isDefault) { + node.addChild(HQLParser.parseExpr(DEFAULT_MEASURE)); + node.addChild(aliasNode); + } else { + node.addChild(nodeWithoutAlias); + node.addChild(aliasNode); + } + return node; + } + + + private ASTNode getSelectExprAST() { + return new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR, "TOK_SELEXPR")); + } + + + /** + * Get the aggregate node for the SelectPhrase index. A given measure might not be answerable + * for a StorageCanddate. In that case get the non default aggregate node wcich ideally not "0.0", + * from otherStorage candidate. + * + * @param position + * @return + */ + private ASTNode getAggregateNodesExpression(int position) { + ASTNode node = null; + for (StorageCandidate sc : storageCandidates) { + node = (ASTNode) sc.getQueryAst().getSelectAST().getChild(position).getChild(0); + if (HQLParser.isAggregateAST(node) || HQLParser.hasAggregate(node)) { + return MetastoreUtil.copyAST(node); + } + } + return MetastoreUtil.copyAST(node); + } + + /** + * Check if ASTNode is answerable by StorageCandidate + * @param sc + * @param node + * @return + */ + private boolean isNodeAnswerableForStorageCandidate(StorageCandidate sc, ASTNode node) { + Set cols = new LinkedHashSet<>(); + getAllColumnsOfNode(node, cols); + if (!sc.getColumns().containsAll(cols)) { + return true; + } + return false; + } + + /** + * Set the default value "0.0" in the non answerable aggreagte expressions. + * @param node + * @param sc + * @return + * @throws LensException + */ + private ASTNode setDefaultValueInExprForAggregateNodes(ASTNode node, StorageCandidate sc) throws LensException { + if (HQLParser.isAggregateAST(node) + && isNodeAnswerableForStorageCandidate(sc, node)) { + node.setChild(1, getSelectExpr(null, null, true)); + } + for (int i = 0; i < node.getChildCount(); i++) { + ASTNode child = (ASTNode) node.getChild(i); + setDefaultValueInExprForAggregateNodes(child, sc); + } + return node; + } + + private boolean isAggregateFunctionUsedInAST(ASTNode node) { + if (HQLParser.isAggregateAST(node) + || HQLParser.hasAggregate(node)) { + return true; + } + return false; + } + + private boolean isNodeDefault(ASTNode node) { + if (HQLParser.isAggregateAST((ASTNode) node.getChild(0))) { + if (HQLParser.getString((ASTNode) node.getChild(0).getChild(1)).equals(DEFAULT_MEASURE)) { + return true; + } + } + return false; + } + + private List getProjectedNonDefaultPhrases() { + List phrases = new ArrayList<>(); + for (int i = 0; i < storageCandidates.iterator().next().getQueryAst().getSelectAST().getChildCount(); i++) { + for (StorageCandidate sc : storageCandidates) { + ASTNode selectAST = sc.getQueryAst().getSelectAST(); + if (isNodeDefault((ASTNode) selectAST.getChild(i))) { + continue; + } else { + phrases.add((ASTNode) selectAST.getChild(i)); + break; + } + } + } + return phrases; + } + + private void removeRedundantProjectedPhrases() { + List phrases = getProjectedNonDefaultPhrases(); + List phrasesWithoutAlias = new ArrayList<>(); + // populate all phrases without alias + for (ASTNode node : phrases) { + phrasesWithoutAlias.add(HQLParser.getString((ASTNode) node.getChild(0))); + } + Map> phraseCountMap = new HashMap<>(); + Map> aliasMap = new HashMap<>(); + for (int i = 0; i < phrasesWithoutAlias.size(); i++) { + String phrase = phrasesWithoutAlias.get(i); + if (phraseCountMap.containsKey(phrase)) { + phraseCountMap.get(phrase).add(i); + } else { + List indices = new ArrayList<>(); + indices.add(i); + phraseCountMap.put(phrase, indices); + } + } + for (List values : phraseCountMap.values()) { + if (values.size() > 1) { + String aliasToKeep = HQLParser.findNodeByPath((ASTNode) + phrases.get(values.get(0)), Identifier).toString(); + ArrayList dupAliases = new ArrayList<>(); + for (int i : values.subList(1, values.size())) { + dupAliases.add(HQLParser.findNodeByPath((ASTNode) + phrases.get(i), Identifier).toString()); + } + aliasMap.put(aliasToKeep, dupAliases); + } + } + + for (String col : phraseCountMap.keySet()) { + if (phraseCountMap.get(col).size() > 1) { + List childenToDelete = phraseCountMap.get(col). + subList(1, phraseCountMap.get(col).size()); + int counter = 0; + for (int i : childenToDelete) { + for (StorageCandidate sc : storageCandidates) { + sc.getQueryAst().getSelectAST().deleteChild(i - counter); + } + counter++; + } + } + } + updateOuterASTDuplicateAliases(queryAst.getSelectAST(), aliasMap); + if (queryAst.getHavingAST() != null) { + updateOuterASTDuplicateAliases(queryAst.getHavingAST(), aliasMap); + } + } + + public void updateOuterASTDuplicateAliases(ASTNode node, + Map> aliasMap) { + if (node.getToken().getType() == HiveParser.DOT) { + String table = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier).toString(); + String col = node.getChild(1).toString(); + for (Map.Entry> entry : aliasMap.entrySet()) { + if (entry.getValue().contains(col)) { + try { + node.setChild(1, HQLParser.parseExpr(entry.getKey())); + } catch (LensException e) { + log.error("Unable to parse select expression: {}.", entry.getKey()); + } + } + + } + } + for (int i = 0; i < node.getChildCount(); i++) { + ASTNode child = (ASTNode) node.getChild(i); + updateOuterASTDuplicateAliases(child, aliasMap); + } + } + + + /** + * Set the default value for the non queriable measures. If a measure is not + * answerable from a StorageCandidate set it as 0.0 + * + * @throws LensException + */ + private void updateInnterSelectASTWithDefault() throws LensException { + for (int i = 0; i < cubeql.getSelectPhrases().size(); i++) { + SelectPhraseContext phrase = cubeql.getSelectPhrases().get(i); + ASTNode aliasNode = new ASTNode(new CommonToken(Identifier, phrase.getSelectAlias())); + + // Select phrase is dimension + if (!phrase.hasMeasures(cubeql)) { + for (StorageCandidate sc : storageCandidates) { + ASTNode exprWithOutAlias = (ASTNode) sc.getQueryAst().getSelectAST().getChild(i).getChild(0); + storageCandidateToSelectAstMap.get(sc.toString()). + addChild(getSelectExpr(exprWithOutAlias, aliasNode, false)); + } + + // Select phrase is measure + } else if (!phrase.getQueriedMsrs().isEmpty()) { + for (StorageCandidate sc : storageCandidates) { + if (sc.getAnswerableMeasurePhraseIndices().contains(phrase.getPosition())) { + ASTNode exprWithOutAlias = (ASTNode) sc.getQueryAst().getSelectAST().getChild(i).getChild(0); + storageCandidateToSelectAstMap.get(sc.toString()). + addChild(getSelectExpr(exprWithOutAlias, aliasNode, false)); + } else { + ASTNode resolvedExprNode = getAggregateNodesExpression(i); + if (isAggregateFunctionUsedInAST(resolvedExprNode)) { + setDefaultValueInExprForAggregateNodes(resolvedExprNode, sc); + } else { + resolvedExprNode = getSelectExpr(null, null, true); + } + storageCandidateToSelectAstMap.get(sc.toString()). + addChild(getSelectExpr(resolvedExprNode, aliasNode, false)); + } + } + + // Select phrase is expression + } else { + for (StorageCandidate sc : storageCandidates) { + if (phrase.isEvaluable(cubeql, sc) + || sc.getAnswerableMeasurePhraseIndices().contains(phrase.getPosition())) { + ASTNode exprWithOutAlias = (ASTNode) sc.getQueryAst().getSelectAST().getChild(i).getChild(0); + storageCandidateToSelectAstMap.get(sc.toString()). + addChild(getSelectExpr(exprWithOutAlias, aliasNode, false)); + } else { + ASTNode resolvedExprNode = getAggregateNodesExpression(i); + if (isAggregateFunctionUsedInAST(resolvedExprNode)) { + setDefaultValueInExprForAggregateNodes(resolvedExprNode, sc); + } else { + resolvedExprNode = getSelectExpr(null, null, true); + } + storageCandidateToSelectAstMap.get(sc.toString()). + addChild(getSelectExpr(resolvedExprNode, aliasNode, false)); + } + } + } + } + } + + /** + * Update Select and Having clause of outer query. + * + * @throws LensException + */ + private void processSelectAndHavingAST() throws LensException { + ASTNode outerSelectAst = new ASTNode(queryAst.getSelectAST()); + DefaultAliasDecider aliasDecider = new DefaultAliasDecider(); + int selectAliasCounter = 0; + for (StorageCandidate sc : storageCandidates) { + aliasDecider.setCounter(0); + ASTNode innerSelectAST = new ASTNode(new CommonToken(TOK_SELECT, "TOK_SELECT")); + processSelectExpression(sc, outerSelectAst, innerSelectAST, aliasDecider); + selectAliasCounter = aliasDecider.getCounter(); + } + queryAst.setSelectAST(outerSelectAst); + + // Iterate over the StorageCandidates and add non projected having columns in inner select ASTs + for (StorageCandidate sc : storageCandidates) { + aliasDecider.setCounter(selectAliasCounter); + processHavingAST(sc.getQueryAst().getSelectAST(), aliasDecider, sc); + } + removeRedundantProjectedPhrases(); + } + + /** + * Get the inner and outer AST with alias for each child of StorageCandidate + * + * @param sc + * @param outerSelectAst + * @param innerSelectAST + * @param aliasDecider + * @throws LensException + */ + private void processSelectExpression(StorageCandidate sc, ASTNode outerSelectAst, ASTNode innerSelectAST, + AliasDecider aliasDecider) throws LensException { + //ASTNode selectAST = sc.getQueryAst().getSelectAST(); + ASTNode selectAST = storageCandidateToSelectAstMap.get(sc.toString()); + if (selectAST == null) { + return; + } + // iterate over all children of the ast and get outer ast corresponding to it. + for (int i = 0; i < selectAST.getChildCount(); i++) { + ASTNode child = (ASTNode) selectAST.getChild(i); + ASTNode outerSelect = new ASTNode(child); + ASTNode selectExprAST = (ASTNode) child.getChild(0); + ASTNode outerAST = getOuterAST(selectExprAST, innerSelectAST, aliasDecider, sc, true, + cubeql.getBaseCube().getDimAttributeNames()); + outerSelect.addChild(outerAST); + // has an alias? add it + if (child.getChildCount() > 1) { + outerSelect.addChild(child.getChild(1)); + } + if (outerSelectAst.getChildCount() <= selectAST.getChildCount()) { + if (outerSelectAst.getChild(i) == null) { + outerSelectAst.addChild(outerSelect); + } else if (HQLParser.getString((ASTNode) outerSelectAst.getChild(i).getChild(0)).equals(DEFAULT_MEASURE)) { + outerSelectAst.replaceChildren(i, i, outerSelect); + } + } + } + sc.getQueryAst().setSelectAST(innerSelectAST); + } + + /* + Perform a DFS on the provided AST, and Create an AST of similar structure with changes specific to the + inner query - outer query dynamics. The resultant AST is supposed to be used in outer query. + + Base cases: + 1. ast is null => null + 2. ast is aggregate_function(table.column) => add aggregate_function(table.column) to inner select expressions, + generate alias, return aggregate_function(cube.alias). Memoize the mapping + aggregate_function(table.column) => aggregate_function(cube.alias) + Assumption is aggregate_function is transitive i.e. f(a,b,c,d) = f(f(a,b), f(c,d)). SUM, MAX, MIN etc + are transitive, while AVG, COUNT etc are not. For non-transitive aggregate functions, the re-written + query will be incorrect. + 3. ast has aggregates - iterate over children and add the non aggregate nodes as is and recursively get outer ast + for aggregate. + 4. If no aggregates, simply select its alias in outer ast. + 5. If given ast is memorized as mentioned in the above cases, return the mapping. + */ + private ASTNode getOuterAST(ASTNode astNode, ASTNode innerSelectAST, + AliasDecider aliasDecider, StorageCandidate sc, boolean isSelectAst, Set dimensionSet) + throws LensException { + if (astNode == null) { + return null; + } + Set msrCols = new HashSet<>(); + getAllColumnsOfNode(astNode, msrCols); + msrCols.removeAll(dimensionSet); + if (isAggregateAST(astNode) && sc.getColumns().containsAll(msrCols)) { + return processAggregate(astNode, innerSelectAST, aliasDecider, isSelectAst); + } else if (isAggregateAST(astNode) && !sc.getColumns().containsAll(msrCols)) { + ASTNode outerAST = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR, "TOK_SELEXPR")); + ASTNode exprCopy = MetastoreUtil.copyAST(astNode); + setDefaultValueInExprForAggregateNodes(exprCopy, sc); + outerAST.addChild(getOuterAST(getSelectExpr(exprCopy, null, true), + innerSelectAST, aliasDecider, sc, isSelectAst, dimensionSet)); + return outerAST; + } else { + if (hasAggregate(astNode)) { + ASTNode outerAST = new ASTNode(astNode); + for (Node child : astNode.getChildren()) { + ASTNode childAST = (ASTNode) child; + if (hasAggregate(childAST) && sc.getColumns().containsAll(msrCols)) { + outerAST.addChild(getOuterAST(childAST, innerSelectAST, aliasDecider, sc, isSelectAst, dimensionSet)); + } else if (hasAggregate(childAST) && !sc.getColumns().containsAll(msrCols)) { + childAST.replaceChildren(1, 1, getSelectExpr(null, null, true)); + outerAST.addChild(getOuterAST(childAST, innerSelectAST, aliasDecider, sc, isSelectAst, dimensionSet)); + } else { + outerAST.addChild(childAST); + } + } + return outerAST; + } else { + ASTNode innerSelectASTWithoutAlias = MetastoreUtil.copyAST(astNode); + ASTNode innerSelectExprAST = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR, "TOK_SELEXPR")); + innerSelectExprAST.addChild(innerSelectASTWithoutAlias); + String alias = aliasDecider.decideAlias(astNode); + ASTNode aliasNode = new ASTNode(new CommonToken(Identifier, alias)); + innerSelectExprAST.addChild(aliasNode); + innerSelectAST.addChild(innerSelectExprAST); + if (astNode.getText().equals(DEFAULT_MEASURE)) { + ASTNode outerAST = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR, "TOK_SELEXPR")); + outerAST.addChild(astNode); + return outerAST; + } else { + ASTNode outerAST = getDotAST(cubeql.getCube().getName(), alias); + if (isSelectAst) { + innerToOuterSelectASTs.put(new HashableASTNode(innerSelectASTWithoutAlias), outerAST); + } else { + innerToOuterHavingASTs.put(new HashableASTNode(innerSelectASTWithoutAlias), outerAST); + } + return outerAST; + } + } + } + } + + private ASTNode processAggregate(ASTNode astNode, ASTNode innerSelectAST, + AliasDecider aliasDecider, boolean isSelectAst) { + ASTNode innerSelectASTWithoutAlias = MetastoreUtil.copyAST(astNode); + ASTNode innerSelectExprAST = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR, "TOK_SELEXPR")); + innerSelectExprAST.addChild(innerSelectASTWithoutAlias); + String alias = aliasDecider.decideAlias(astNode); + ASTNode aliasNode = new ASTNode(new CommonToken(Identifier, alias)); + innerSelectExprAST.addChild(aliasNode); + innerSelectAST.addChild(innerSelectExprAST); + ASTNode dotAST = getDotAST(cubeql.getCube().getName(), alias); + ASTNode outerAST = new ASTNode(new CommonToken(TOK_FUNCTION, "TOK_FUNCTION")); + //TODO: take care or non-transitive aggregate functions + outerAST.addChild(new ASTNode(new CommonToken(Identifier, astNode.getChild(0).getText()))); + outerAST.addChild(dotAST); + if (isSelectAst) { + innerToOuterSelectASTs.put(new HashableASTNode(innerSelectASTWithoutAlias), outerAST); + } else { + innerToOuterHavingASTs.put(new HashableASTNode(innerSelectASTWithoutAlias), outerAST); + } + return outerAST; + } + + /** + * GroupbyAST is having dim only columns all the columns should have been + * projected. Get the alias for the projected columns and add to group by clause. + * + * @param astNode + * @return + * @throws LensException + */ + + private ASTNode processGroupByExpression(ASTNode astNode) throws LensException { + ASTNode outerExpression = new ASTNode(astNode); + // iterate over all children of the ast and get outer ast corresponding to it. + for (Node child : astNode.getChildren()) { + // Columns in group by should have been projected as they are dimension columns + if (innerToOuterSelectASTs.containsKey(new HQLParser.HashableASTNode((ASTNode) child))) { + outerExpression.addChild(innerToOuterSelectASTs.get(new HQLParser.HashableASTNode((ASTNode) child))); + } + } + return outerExpression; + } + + /** + * Process having clause, if a columns is not projected add it + * to the projected columns of inner selectAST. + * + * @param innerSelectAst + * @param havingAggASTs + * @param aliasDecider + * @param sc + * @throws LensException + */ + + private void processHavingExpression(ASTNode innerSelectAst, Set havingAggASTs, + AliasDecider aliasDecider, StorageCandidate sc) throws LensException { + // iterate over all children of the ast and get outer ast corresponding to it. + for (ASTNode child : havingAggASTs) { + if (!innerToOuterSelectASTs.containsKey(new HQLParser.HashableASTNode(child))) { + getOuterAST(child, innerSelectAst, aliasDecider, sc, false, cubeql.getBaseCube().getDimAttributeNames()); + } + } + } + + /** + * Gets all aggreage nodes used in having + * + * @param node + * @param havingClauses + * @return + */ + private Set getAggregateChildrenInNode(ASTNode node, Set havingClauses) { + if (node.getToken().getType() == HiveParser.TOK_FUNCTION && (HQLParser.isAggregateAST(node))) { + havingClauses.add(node); + } + for (int i = 0; i < node.getChildCount(); i++) { + ASTNode child = (ASTNode) node.getChild(i); + getAggregateChildrenInNode(child, havingClauses); + } + return havingClauses; + } + + /** + * Get columns used in ASTNode + * + * @param node + * @param msrs + * @return + */ + private Set getAllColumnsOfNode(ASTNode node, Set msrs) { + if (node.getToken().getType() == HiveParser.DOT) { + String table = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier).toString(); + msrs.add(node.getChild(1).toString()); + } + for (int i = 0; i < node.getChildCount(); i++) { + ASTNode child = (ASTNode) node.getChild(i); + getAllColumnsOfNode(child, msrs); + } + return msrs; + } + + /** + * Gets from string of the ouer query, this is a union query of all + * StorageCandidates participated. + * + * @return + * @throws LensException + */ + private String getFromString(Map> factDimMap) throws LensException { + StringBuilder from = new StringBuilder(); + List hqlQueries = new ArrayList<>(); + for (StorageCandidate sc : storageCandidates) { + removeAggreagateFromDefaultColumns(sc.getQueryAst().getSelectAST()); + Set queriedDims = factDimMap.get(sc); + hqlQueries.add(sc.toHQL(queriedDims)); + } + return from.append(" ( ") + .append(StringUtils.join(" UNION ALL ", hqlQueries)) + .append(" ) as " + cubeql.getBaseCube()).toString(); + } + + private void removeAggreagateFromDefaultColumns(ASTNode node) throws LensException { + for (int i = 0; i < node.getChildCount(); i++) { + ASTNode selectExpr = (ASTNode) node.getChild(i); + if (selectExpr.getChildCount() == 2) { + ASTNode column = (ASTNode) selectExpr.getChild(0); + if (HQLParser.isAggregateAST(column) + && column.getChildCount() == 2) { + if (HQLParser.getString((ASTNode) column.getChild(1)).equals("0.0")) { + selectExpr.getParent().setChild(i, getSelectExpr(null, (ASTNode) selectExpr.getChild(1), true)); + } + } + } + } + + } +} diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java index 3d5c5acc8..aab671e23 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java @@ -64,7 +64,7 @@ public class AutoJoinContext { // there can be separate join clause for each fact in-case of multi fact queries @Getter - Map factClauses = new HashMap<>(); + Map factClauses = new HashMap<>(); @Getter @Setter JoinClause minCostClause; @@ -99,11 +99,11 @@ public AbstractCubeTable getAutoJoinTarget() { return autoJoinTarget; } - public JoinClause getJoinClause(CandidateFact fact) { - if (fact == null || !factClauses.containsKey(fact)) { + public JoinClause getJoinClause(StorageCandidate sc) { + if (sc == null || !factClauses.containsKey(sc)) { return minCostClause; } - return factClauses.get(fact); + return factClauses.get(sc); } // Populate map of tables to their columns which are present in any of the @@ -169,7 +169,7 @@ public void removeJoinedTable(Aliased dim) { joinPathFromColumns.remove(dim); } - public String getFromString(String fromTable, CandidateFact fact, Set qdims, + public String getFromString(String fromTable, StorageCandidate sc, Set qdims, Map dimsToQuery, CubeQueryContext cubeql, QueryAST ast) throws LensException { String fromString = fromTable; log.info("All paths dump:{} Queried dims:{}", cubeql.getAutoJoinCtx().getAllPaths(), qdims); @@ -177,15 +177,15 @@ public String getFromString(String fromTable, CandidateFact fact, Set return fromString; } // Compute the merged join clause string for the min cost joinClause - String clause = getMergedJoinClause(cubeql, fact, ast, - cubeql.getAutoJoinCtx().getJoinClause(fact), dimsToQuery); + String clause = getMergedJoinClause(cubeql, sc, ast, + cubeql.getAutoJoinCtx().getJoinClause(sc), dimsToQuery); fromString += clause; return fromString; } // Some refactoring needed to account for multiple join paths - public String getMergedJoinClause(CubeQueryContext cubeql, CandidateFact fact, QueryAST ast, JoinClause joinClause, + public String getMergedJoinClause(CubeQueryContext cubeql, StorageCandidate sc, QueryAST ast, JoinClause joinClause, Map dimsToQuery) throws LensException { Set clauses = new LinkedHashSet<>(); String joinTypeStr = ""; @@ -198,7 +198,7 @@ public String getMergedJoinClause(CubeQueryContext cubeql, CandidateFact fact, Q Iterator iter = joinClause.getJoinTree().dft(); boolean hasBridgeTable = false; - BridgeTableJoinContext bridgeTableJoinContext = new BridgeTableJoinContext(cubeql, fact, ast, bridgeTableFieldAggr, + BridgeTableJoinContext bridgeTableJoinContext = new BridgeTableJoinContext(cubeql, sc, ast, bridgeTableFieldAggr, bridgeTableFieldArrayFilter, doFlatteningEarly); while (iter.hasNext()) { @@ -347,22 +347,30 @@ public Set getAllJoinPathColumnsOfTable(AbstractCubeTable table) { return allPaths; } - public void pruneAllPaths(CubeInterface cube, final Set cfacts, + /** + * Prunes the join chains defined in Cube whose starting column is not there in any of the candidate facts. + * Same is done in case of join paths defined in Dimensions. + * + * @param cube + * @param scSet picked StorageCandidates + * @param dimsToQuery + * @throws LensException + */ + public void pruneAllPaths(CubeInterface cube, Collection scSet, final Map dimsToQuery) throws LensException { // Remove join paths which cannot be satisfied by the resolved candidate // fact and dimension tables - if (cfacts != null) { - // include columns from all picked facts - Set factColumns = new HashSet<>(); - for (CandidateFact cFact : cfacts) { - factColumns.addAll(cFact.getColumns()); + if (scSet != null) { + // include columns from picked candidate + Set candColumns = new HashSet<>(); + for (StorageCandidate sc : scSet) { + candColumns.addAll(sc.getColumns()); } - for (List paths : allPaths.values()) { for (int i = 0; i < paths.size(); i++) { JoinPath jp = paths.get(i); List cubeCols = jp.getColumnsForTable((AbstractCubeTable) cube); - if (cubeCols != null && !factColumns.containsAll(cubeCols)) { + if (cubeCols != null && !candColumns.containsAll(cubeCols)) { // This path requires some columns from the cube which are not // present in the candidate fact // Remove this path @@ -433,7 +441,7 @@ private void pruneEmptyPaths(Map, List> allPaths) t } private Map, List> pruneFactPaths(CubeInterface cube, - final CandidateFact cFact) throws LensException { + final StorageCandidate sc) throws LensException { Map, List> prunedPaths = new HashMap<>(); // Remove join paths which cannot be satisfied by the candidate fact for (Map.Entry, List> ppaths : allPaths.entrySet()) { @@ -442,7 +450,7 @@ private Map, List> pruneFactPaths(CubeInterface cub for (int i = 0; i < paths.size(); i++) { JoinPath jp = paths.get(i); List cubeCols = jp.getColumnsForTable((AbstractCubeTable) cube); - if (cubeCols != null && !cFact.getColumns().containsAll(cubeCols)) { + if (cubeCols != null && !sc.getColumns().containsAll(cubeCols)) { // This path requires some columns from the cube which are not // present in the candidate fact // Remove this path @@ -485,12 +493,12 @@ private void pruneAllPaths(final Map dimsToQuery) throw * There can be multiple join paths between a dimension and the target. Set of all possible join clauses is the * cartesian product of join paths of all dimensions */ - private Iterator getJoinClausesForAllPaths(final CandidateFact fact, + private Iterator getJoinClausesForAllPaths(final StorageCandidate sc, final Set qDims, final CubeQueryContext cubeql) throws LensException { Map, List> allPaths; // if fact is passed only look at paths possible from fact to dims - if (fact != null) { - allPaths = pruneFactPaths(cubeql.getCube(), fact); + if (sc != null) { + allPaths = pruneFactPaths(cubeql.getCube(), sc); } else { allPaths = new LinkedHashMap<>(this.allPaths); } @@ -573,7 +581,7 @@ private void pruneAllPathsWithQueriedDims(Map, List } } - public Set pickOptionalTables(final CandidateFact fact, + public Set pickOptionalTables(final StorageCandidate sc, Set qdims, CubeQueryContext cubeql) throws LensException { // Find the min cost join clause and add dimensions in the clause as optional dimensions Set joiningOptionalTables = new HashSet<>(); @@ -581,7 +589,7 @@ public Set pickOptionalTables(final CandidateFact fact, return joiningOptionalTables; } // find least cost path - Iterator itr = getJoinClausesForAllPaths(fact, qdims, cubeql); + Iterator itr = getJoinClausesForAllPaths(sc, qdims, cubeql); JoinClause minCostClause = null; while (itr.hasNext()) { JoinClause clause = itr.next(); @@ -595,9 +603,9 @@ public Set pickOptionalTables(final CandidateFact fact, qdims.toString(), autoJoinTarget.getName()); } - log.info("Fact: {} minCostClause:{}", fact, minCostClause); - if (fact != null) { - cubeql.getAutoJoinCtx().getFactClauses().put(fact, minCostClause); + log.info("Fact: {} minCostClause:{}", sc, minCostClause); + if (sc != null) { + cubeql.getAutoJoinCtx().getFactClauses().put(sc, minCostClause); } else { cubeql.getAutoJoinCtx().setMinCostClause(minCostClause); } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/BridgeTableJoinContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/BridgeTableJoinContext.java index cf746344d..ab5c4f9bd 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/BridgeTableJoinContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/BridgeTableJoinContext.java @@ -41,7 +41,7 @@ public class BridgeTableJoinContext { private final String bridgeTableFieldAggr; private final String arrayFilter; private final CubeQueryContext cubeql; - private final CandidateFact fact; + private final StorageCandidate sc; private final QueryAST queryAST; private final boolean doFlatteningEarly; private boolean initedBridgeClauses = false; @@ -51,11 +51,11 @@ public class BridgeTableJoinContext { private final StringBuilder bridgeJoinClause = new StringBuilder(); private final StringBuilder bridgeGroupbyClause = new StringBuilder(); - public BridgeTableJoinContext(CubeQueryContext cubeql, CandidateFact fact, QueryAST queryAST, + public BridgeTableJoinContext(CubeQueryContext cubeql, StorageCandidate sc, QueryAST queryAST, String bridgeTableFieldAggr, String arrayFilter, boolean doFlatteningEarly) { this.cubeql = cubeql; this.queryAST = queryAST; - this.fact = fact; + this.sc = sc; this.bridgeTableFieldAggr = bridgeTableFieldAggr; this.arrayFilter = arrayFilter; this.doFlatteningEarly = doFlatteningEarly; @@ -139,10 +139,10 @@ public String generateJoinClause(String joinTypeStr, String toAlias) throws Lens // iterate over all select expressions and add them for select clause if do_flattening_early is disabled if (!doFlatteningEarly) { BridgeTableSelectCtx selectCtx = new BridgeTableSelectCtx(bridgeTableFieldAggr, arrayFilter, toAlias); - selectCtx.processSelectAST(queryAST.getSelectAST()); - selectCtx.processWhereClauses(fact); - selectCtx.processGroupbyAST(queryAST.getGroupByAST()); - selectCtx.processOrderbyAST(queryAST.getOrderByAST()); + selectCtx.processSelectAST(sc.getQueryAst().getSelectAST()); + selectCtx.processWhereClauses(sc); + selectCtx.processGroupbyAST(sc.getQueryAst().getGroupByAST()); + selectCtx.processOrderbyAST(sc.getQueryAst().getOrderByAST()); clause.append(",").append(StringUtils.join(selectCtx.getSelectedBridgeExprs(), ",")); } else { for (String col : cubeql.getTblAliasToColumns().get(toAlias)) { @@ -236,12 +236,8 @@ void processOrderbyAST(ASTNode ast) } } - void processWhereClauses(CandidateFact fact) throws LensException { - - for (Map.Entry whereEntry : fact.getStorgeWhereClauseMap().entrySet()) { - ASTNode whereAST = whereEntry.getValue(); - processWhereAST(whereAST, null, 0); - } + void processWhereClauses(StorageCandidate sc) throws LensException { + processWhereAST(sc.getQueryAst().getWhereAST(), null, 0); } void processWhereAST(ASTNode ast, ASTNode parent, int childPos) diff --git a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java index fd6c30dcd..caf8770ab 100644 --- a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java +++ b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java @@ -23,8 +23,7 @@ import java.util.Set; import org.apache.lens.cube.metadata.FactPartition; -import org.apache.lens.cube.parse.CandidateTable; -import org.apache.lens.cube.parse.CubeQueryContext; +import org.apache.lens.cube.parse.*; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.driver.DriverQueryPlan; import org.apache.lens.server.api.error.LensException; @@ -49,23 +48,23 @@ void extractPlan(Collection cubeQueries) { for (CubeQueryContext ctx : cubeQueries) { if (ctx.getPickedDimTables() != null && !ctx.getPickedDimTables().isEmpty()) { - for (CandidateTable dim : ctx.getPickedDimTables()) { - addTablesQueried(dim.getStorageTables()); + for (CandidateDim dim : ctx.getPickedDimTables()) { + addTablesQueried(dim.getStorageTable()); if (partitions.get(dim.getName()) == null || partitions.get(dim.getName()).isEmpty()) { // puts storage table to latest part - partitions.put(dim.getName(), dim.getPartsQueried()); + partitions.put(dim.getName(), dim.getParticipatingPartitions()); } } } - if (ctx.getPickedFacts() != null && !ctx.getPickedFacts().isEmpty()) { - for (CandidateTable fact : ctx.getPickedFacts()) { - addTablesQueried(fact.getStorageTables()); - Set factParts = (Set) partitions.get(fact.getName()); + if (ctx.getPickedCandidate() != null) { + for (StorageCandidate sc : CandidateUtil.getStorageCandidates(ctx.getPickedCandidate())) { + addTablesQueried(sc.getAliasForTable("")); + Set factParts = (Set) partitions.get(sc.getStorageTable()); if (factParts == null) { factParts = new HashSet(); - partitions.put(fact.getName(), factParts); + partitions.put(sc.getStorageTable(), factParts); } - factParts.addAll((Set) fact.getPartsQueried()); + factParts.addAll((Set) sc.getParticipatingPartitions()); } } for (String table : getTablesQueried()) { diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java index 3bdc047c3..855f54a3a 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java @@ -65,11 +65,16 @@ public Date get(Object key) { } public static class GeneralDateOffsetProvider extends HashMap { + boolean truncate; + public GeneralDateOffsetProvider(boolean truncate) { + this.truncate = truncate; + } + @Override public DateOffsetProvider get(Object key) { if (!containsKey(key) && key instanceof UpdatePeriod) { UpdatePeriod up = (UpdatePeriod) key; - put(up, new DateOffsetProvider(up)); + put(up, new DateOffsetProvider(up, truncate)); } return super.get(key); } @@ -79,13 +84,19 @@ public Date get(UpdatePeriod updatePeriod, int offset) { } } - public static final GeneralDateOffsetProvider GENERAL_DATE_OFFSET_PROVIDER = new GeneralDateOffsetProvider(); + public static final GeneralDateOffsetProvider GENERAL_DATE_OFFSET_PROVIDER = new GeneralDateOffsetProvider(false); + public static final GeneralDateOffsetProvider GENERAL_TRUNCATED_DATE_OFFSET_PROVIDER + = new GeneralDateOffsetProvider(true); public static Date getDateWithOffset(UpdatePeriod up, int offset) { return GENERAL_DATE_OFFSET_PROVIDER.get(up, offset); } + public static Date getTruncatedDateWithOffset(UpdatePeriod up, int offset) { + return GENERAL_TRUNCATED_DATE_OFFSET_PROVIDER.get(up, offset); + } + public static String getDateStringWithOffset(UpdatePeriod up, int offset) { return getDateStringWithOffset(up, offset, up); } @@ -141,6 +152,10 @@ public static String getTimeRangeString(UpdatePeriod updatePeriod, int startOffs public static final Date NOW; public static final Date TWODAYS_BACK; public static final Date TWO_MONTHS_BACK; + public static final Date THIS_MONTH_TRUNCATED; + public static final Date ONE_MONTH_BACK_TRUNCATED; + public static final Date TWO_MONTHS_BACK_TRUNCATED; + public static final Date THREE_MONTHS_BACK_TRUNCATED; public static final Date BEFORE_6_DAYS; public static final Date BEFORE_4_DAYS; @@ -159,6 +174,8 @@ public static String getTimeRangeString(UpdatePeriod updatePeriod, int startOffs public static final String TWO_MONTHS_RANGE_UPTO_DAYS; public static final String TWO_MONTHS_RANGE_UPTO_HOURS; public static final String TWO_DAYS_RANGE_BEFORE_4_DAYS; + public static final String THREE_MONTHS_RANGE_UPTO_DAYS; + public static final String THREE_MONTHS_RANGE_UPTO_MONTH; private static boolean zerothHour; @@ -179,6 +196,12 @@ public static boolean isZerothHour() { TWO_MONTHS_BACK = getDateWithOffset(MONTHLY, -2); System.out.println("Test TWO_MONTHS_BACK:" + TWO_MONTHS_BACK); + THIS_MONTH_TRUNCATED = getTruncatedDateWithOffset(MONTHLY, 0); + ONE_MONTH_BACK_TRUNCATED = getTruncatedDateWithOffset(MONTHLY, -1); + TWO_MONTHS_BACK_TRUNCATED = getTruncatedDateWithOffset(MONTHLY, -2); + THREE_MONTHS_BACK_TRUNCATED = getTruncatedDateWithOffset(MONTHLY, -3); + + // Before 4days BEFORE_4_DAYS = getDateWithOffset(DAILY, -4); BEFORE_6_DAYS = getDateWithOffset(DAILY, -6); @@ -196,6 +219,8 @@ public static boolean isZerothHour() { TWO_MONTHS_RANGE_UPTO_MONTH = getTimeRangeString(MONTHLY, -2, 0); TWO_MONTHS_RANGE_UPTO_DAYS = getTimeRangeString(MONTHLY, -2, 0, DAILY); TWO_MONTHS_RANGE_UPTO_HOURS = getTimeRangeString(MONTHLY, -2, 0, HOURLY); + THREE_MONTHS_RANGE_UPTO_DAYS = getTimeRangeString(MONTHLY, -3, 0, DAILY); + THREE_MONTHS_RANGE_UPTO_MONTH = getTimeRangeString(MONTHLY, -3, 0, MONTHLY); // calculate LAST_HOUR_TIME_RANGE LAST_HOUR_TIME_RANGE = getTimeRangeString(HOURLY, -1, 0); diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 9b290832f..033264c81 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -27,29 +27,36 @@ import static org.testng.Assert.*; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.StringReader; import java.util.*; +import java.util.stream.Collectors; +import javax.xml.bind.JAXBException; + +import org.apache.lens.api.ToXMLString; +import org.apache.lens.api.jaxb.LensJAXBContext; +import org.apache.lens.api.metastore.SchemaTraverser; import org.apache.lens.cube.metadata.*; -import org.apache.lens.cube.metadata.ExprColumn.ExprSpec; -import org.apache.lens.cube.metadata.ReferencedDimAttribute.ChainRefCol; import org.apache.lens.cube.metadata.timeline.EndsAndHolesPartitionTimeline; import org.apache.lens.cube.metadata.timeline.PartitionTimeline; import org.apache.lens.cube.metadata.timeline.StoreAllPartitionTimeline; -import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.text.StrLookup; +import org.apache.commons.lang3.text.StrSubstitutor; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.parse.ParseException; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.mapred.TextInputFormat; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -92,9 +99,6 @@ public class CubeTestSetup { public static final String TEST_CUBE_NAME = "testCube"; public static final String DERIVED_CUBE_NAME = "derivedCube"; public static final String BASE_CUBE_NAME = "baseCube"; - public static final String DERIVED_CUBE_NAME1 = "der1"; - public static final String DERIVED_CUBE_NAME2 = "der2"; - public static final String DERIVED_CUBE_NAME3 = "der3"; private static String c0 = "C0"; private static String c1 = "C1"; @@ -137,7 +141,7 @@ public static String getExpectedUnionQuery(String cubeName, List storage innerWhere, innerPostWhere, null, provider.providePartitionsForStorage(storage))); sep = " UNION ALL "; } - return sb.append(") ").append(cubeName).append(" ").append(outerWhere == null ? "" : outerWhere) + return sb.append(") ").append(" as ").append(cubeName).append(" ").append(outerWhere == null ? "" : outerWhere) .append(" ").append(outerPostWhere == null ? "" : outerPostWhere).toString(); } public static String getExpectedUnionQuery(String cubeName, List storages, StoragePartitionProvider provider, @@ -329,9 +333,9 @@ public static Map getWhereForDailyAndHourly2daysWithTimeDimUnion return updatePeriodToWhereMap; } - // storageTables[0] is hourly - // storageTables[1] is daily - // storageTables[2] is monthly + // storageName[0] is hourly + // storageName[1] is daily + // storageName[2] is monthly public static Map getWhereForMonthlyDailyAndHourly2months(String... storageTables) { Map storageTableToWhereClause = new LinkedHashMap(); List hourlyparts = new ArrayList(); @@ -448,6 +452,16 @@ public static Map getWhereForDays(String dailyTable, Date startD StorageUtil.getWherePartClause("dt", TEST_CUBE_NAME, parts)); return storageTableToWhereClause; } + + public static Map getWhereForMonthly(String monthlyTable, Date startMonth, Date endMonth) { + Map storageTableToWhereClause = new LinkedHashMap(); + List parts = new ArrayList(); + addParts(parts, MONTHLY, startMonth, endMonth); + storageTableToWhereClause.put(getDbName() + monthlyTable, + StorageUtil.getWherePartClause("dt", TEST_CUBE_NAME, parts)); + return storageTableToWhereClause; + } + public static Map getWhereForHourly2days(String hourlyTable) { return getWhereForHourly2days(TEST_CUBE_NAME, hourlyTable); } @@ -511,1143 +525,14 @@ public static String getExpectedQuery(String dimName, String selExpr, String joi return expected.toString(); } - Set exprs; - - private void createCube(CubeMetastoreClient client) throws HiveException, ParseException, LensException { - cubeMeasures = new HashSet(); - Map tags = new HashMap<>(); - tags.put(MetastoreConstants.MEASURE_DATACOMPLETENESS_TAG, "tag1"); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr1", "int", "first measure"), null, null, null, null, null, - null, null, null, null, tags)); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr2", "float", "second measure"), "Measure2", null, "SUM", - "RS")); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr21", "float", "second measure"), "Measure22", null, "SUM", - "RS")); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr22", "float", "second measure"), "Measure22", null, "SUM", - "RS")); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr3", "double", "third measure"), "Measure3", null, "MAX", - null)); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr4", "bigint", "fourth measure"), "Measure4", null, "COUNT", - null)); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr9", "bigint", "ninth measure"), null, null, null, null, - null, null, null, null, null, tags)); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("noAggrMsr", "bigint", "measure without a default aggregate"), - "No aggregateMsr", null, null, null)); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("newmeasure", "bigint", "measure available from now"), - "New measure", null, null, null, NOW, null, 100.0)); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr15", "int", "fifteenth measure"), "Measure15", null, "SUM", - "RS")); - - cubeDimensions = new HashSet(); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("d_time", "timestamp", "d time"))); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("processing_time", "timestamp", "processing time"))); - List locationHierarchy = new ArrayList(); - locationHierarchy.add(new BaseDimAttribute(new FieldSchema("zipcode", "int", "zip"))); - locationHierarchy.add(new BaseDimAttribute(new FieldSchema("cityid", "int", "city"))); - locationHierarchy.add(new BaseDimAttribute(new FieldSchema("stateid", "int", "state"))); - locationHierarchy.add(new BaseDimAttribute(new FieldSchema("countryid", "int", "country"))); - List regions = Arrays.asList("APAC", "EMEA", "USA"); - locationHierarchy.add(new BaseDimAttribute(new FieldSchema("regionname", "string", "region"), "regionname", null, - null, null, null, regions)); - - cubeDimensions.add(new HierarchicalDimAttribute("location", "Location hierarchy", locationHierarchy)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("dim1", "string", "basedim"))); - // Added for ambiguity test - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("ambigdim1", "string", "used in testColumnAmbiguity"))); - cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("dim2", "int", "ref dim"), "dim2 refer", - "dim2chain", "id", null, null, 0.0)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("cdim2", "int", "ref dim"), "Dim2 refer", NOW, null, null)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("urdimid", "int", "ref dim"), "urdim refer", - null, null, 10.0)); - cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("unreachableName", "string", ""), "urdim name", - "unreachableDim_chain", "name", null, null, 10.0)); - // denormalized reference - cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("dim2big1", "bigint", "ref dim"), "dim2 refer", - "dim2chain", "bigid1", null, null, 0.0)); - cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("dim2big2", "bigint", "ref dim"), "dim2 refer", - "dim2chain", "bigid2", null, null, 0.0)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("dim2bignew", "bigint", "ref dim"), "Dim2 refer", - NOW, null, null)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("test_time_dim_hour_id", "int", "ref dim"), - "Timedim reference", null, null, null)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("test_time_dim_day_id", "int", "ref dim"), - "Timedim reference", null, null, null)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("test_time_dim_hour_id2", "int", "ref dim"))); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("test_time_dim_day_id2", "int", "ref dim"))); - cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("testDim3id", "string", "direct id to testdim3"), - "dim3 refer", "dim3chain", "id", null, null, 0.0)); - cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("cityname", "string", "city name"), - "city name", "cubecity", "name", null, null, 0.0)); - cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("statename_cube", "string", "state name"), - "state name", "cubestate", "name", null, null, 0.0)); - List references = new ArrayList<>(); - references.add(new ChainRefCol("timedatechain1", "full_date")); - references.add(new ChainRefCol("timehourchain1", "full_hour")); - cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("test_time_dim", "date", "ref dim"), - "Timedim full date", references, null, null, null, null)); - List chainRefs = new ArrayList<>(); - chainRefs.add(new ChainRefCol("timehourchain2", "full_hour")); - chainRefs.add(new ChainRefCol("timedatechain2", "full_date")); - cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("test_time_dim2", "date", "chained dim"), - "Timedim full date", chainRefs, null, null, null, null)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("cityid1", "int", "id to city"), - "City1", null, null, null)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("cityid2", "int", "id to city"), - "City2", null, null, null)); - cubeDimensions.add(new BaseDimAttribute(new FieldSchema("concatedcitystate", "string", "citystate"), - "CityState", null, null, null)); - - Map joinChains = new HashMap<>(); - addCubeChains(joinChains, TEST_CUBE_NAME); - - exprs = new HashSet(); - exprs.add(new ExprColumn(new FieldSchema("avgmsr", "double", "avg measure"), "Avg Msr", "avg(msr1 + msr2)")); - exprs.add(new ExprColumn(new FieldSchema("singlecolmsr2expr", "double", "measure2"), "Msr2", "msr2)")); - exprs.add(new ExprColumn(new FieldSchema("singlecolmsr2qualifiedexpr", "double", "testcube.measure2"), - "Msr2", "testcube.msr2")); - exprs.add(new ExprColumn(new FieldSchema("singlecoldim1expr", "string", "dim1"), "dim1", "dim1)")); - exprs.add(new ExprColumn(new FieldSchema("singlecoldim1qualifiedexpr", "string", "testcube.dim1"), - "dim1", "testcube.dim1")); - exprs.add(new ExprColumn(new FieldSchema("singlecolchainid", "string", "dim3chain.id"), - "dim3chainid", "dim3chain.id)")); - exprs.add(new ExprColumn(new FieldSchema("singlecolchainrefexpr", "string", "testcube.testDim3id"), - "dim3chainid", "testcube.testDim3id")); - exprs.add(new ExprColumn(new FieldSchema("singlecolchainfield", "string", "cubecity.name"), - "cubecityname", "cubecity.name")); - exprs.add(new ExprColumn(new FieldSchema("summsrs", "double", "sum measures"), "Sum Msrs", - "(1000 + sum(msr1) + sum(msr2))/100")); - exprs.add(new ExprColumn(new FieldSchema("msr5", "double", "materialized in some facts"), "Fifth Msr", - "msr2 + msr3")); - exprs.add(new ExprColumn(new FieldSchema("msr8", "double", "measure expression"), "Sixth Msr", - "msr2 + msr3")); - exprs.add(new ExprColumn(new FieldSchema("msr7", "double", "measure expression"), "Seventh Msr", - "case when sum(msr2) = 0 then 0 else sum(case when cityid='x' then msr21 else msr22 end)/sum(msr2) end")); - exprs.add(new ExprColumn(new FieldSchema("equalsums", "double", "sums are equals"), "equalsums", - new ExprSpec("msr3 + msr4", null, null), new ExprSpec("(msr3 + msr2)/100", null, null))); - exprs.add(new ExprColumn(new FieldSchema("roundedmsr1", "double", "rounded measure1"), "Rounded msr1", - "round(msr1/1000)")); - exprs.add(new ExprColumn(new FieldSchema("roundedmsr2", "double", "rounded measure2"), "Rounded msr2", - "round(msr2/1000)")); - exprs.add(new ExprColumn(new FieldSchema("flooredmsr12", "double", "floored measure12"), "Floored msr12", - "floor(msr12)")); - exprs.add(new ExprColumn(new FieldSchema("nestedexpr", "double", "nested expr"), "Nested expr", - new ExprSpec("avg(roundedmsr2)", null, null), new ExprSpec("avg(equalsums)", null, null), - new ExprSpec("case when substrexpr = 'xyz' then avg(msr5) when substrexpr = 'abc' then avg(msr4)/100 end", - null, null))); - exprs.add(new ExprColumn(new FieldSchema("msr2expr", "double", "nested expr"), "Nested expr", - new ExprSpec("case when cityStateName = 'xyz' then msr2 else 0 end", null, null))); - exprs.add(new ExprColumn(new FieldSchema("nestedExprWithTimes", "double", "nested expr"), "Nested expr", - new ExprSpec("avg(roundedmsr2)", null, null), new ExprSpec("avg(equalsums)", null, null), - new ExprSpec("case when substrexpr = 'xyz' then avg(msr5) when substrexpr = 'abc' then avg(msr4)/100 end", - NOW, null), new ExprSpec("avg(newmeasure)", null, null))); - exprs.add(new ExprColumn(new FieldSchema("msr6", "bigint", "sixth measure"), "Measure6", - "sum(msr2) + max(msr3)/ count(msr4)")); - exprs.add(new ExprColumn(new FieldSchema("booleancut", "boolean", "a boolean expression"), "Boolean cut", - "(dim1 != 'x' AND dim2 != 10)")); - exprs.add(new ExprColumn(new FieldSchema("substrexpr", "string", "a sub-string expression"), "Substr expr", - new ExprSpec("substr(dim1, 3))", null, null), new ExprSpec("substr(ascii(dim2chain.name), 3)", null, null))); - exprs.add(new ExprColumn(new FieldSchema("substrexprdim2", "string", "a sub-string expression"), "Substr expr", - new ExprSpec("substr(dim2, 3))", null, null), new ExprSpec("substr(ascii(dim2chain.name), 3)", null, null))); - exprs.add(new ExprColumn(new FieldSchema("indiasubstr", "boolean", "nested sub string expression"), "Nested expr", - "substrexpr = 'INDIA'")); - exprs.add(new ExprColumn(new FieldSchema("refexpr", "string", "expression which facts and dimensions"), - "Expr with cube and dim fields", "concat(dim1, \":\", citydim.name)")); - exprs.add(new ExprColumn(new FieldSchema("nocolexpr", "string", "expression which non existing colun"), - "No col expr", "myfun(nonexist)")); - exprs.add(new ExprColumn(new FieldSchema("newexpr", "string", "expression which non existing colun"), - "new measure expr", "myfun(newmeasure)")); - exprs.add(new ExprColumn(new FieldSchema("cityAndState", "String", "city and state together"), "City and State", - new ExprSpec("concat(cityname, \":\", statename_cube)", null, null), - new ExprSpec("substr(concatedcitystate, 10)", null, null))); - exprs.add(new ExprColumn(new FieldSchema("cityAndStateNew", "String", "city and state together"), "City and State", - new ExprSpec("concat(cityname, \":\", statename_cube)", null, TWO_MONTHS_BACK), - new ExprSpec("substr(concatedcitystate, 10)", null, null))); - exprs.add(new ExprColumn(new FieldSchema("cityStateName", "String", "city state"), "City State", - "concat('CityState:', cubecity.statename)")); - exprs.add(new ExprColumn(new FieldSchema("isIndia", "String", "is indian city/state"), "Is Indian City/state", - "cubecity.name == 'DELHI' OR cubestate.name == 'KARNATAKA' OR cubestate.name == 'MAHARASHTRA'")); - exprs.add(new ExprColumn(new FieldSchema("cubeStateName", "String", "statename from cubestate"), "CubeState Name", - "substr(cubestate.name, 5)")); - exprs.add(new ExprColumn(new FieldSchema("substrdim2big1", "String", "substr of dim2big1"), "dim2big1 substr", - "substr(dim2big1, 5)")); - exprs.add(new ExprColumn(new FieldSchema("asciicity", "String", "ascii cityname"), "ascii cityname substr", - "ascii(cityname)")); - exprs.add(new ExprColumn(new FieldSchema("countofdistinctcityid", "int", "Count of Distinct CityId"), - "Count of Distinct CityId Expr", "count(distinct(cityid))")); - exprs.add(new ExprColumn(new FieldSchema("notnullcityid", "int", "Not null cityid"), - "Not null cityid Expr", "case when cityid is null then 0 else cityid end")); - - Map cubeProperties = new HashMap(); - cubeProperties.put(MetastoreUtil.getCubeTimedDimensionListKey(TEST_CUBE_NAME), - "d_time,pt,it,et,test_time_dim,test_time_dim2"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "test_time_dim", "ttd"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "test_time_dim2", "ttd2"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "d_time", "dt"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "it", "it"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "et", "et"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "pt", "pt"); - cubeProperties.put(MetastoreConstants.TIMEDIM_RELATION + "d_time", "test_time_dim+[-10 days,10 days]"); - - client.createCube(TEST_CUBE_NAME, cubeMeasures, cubeDimensions, exprs, Sets.newHashSet(joinChains.values()), - cubeProperties); - - Set measures = new HashSet(); - measures.add("msr1"); - measures.add("msr2"); - measures.add("msr3"); - measures.add("msr9"); - Set dimensions = new HashSet(); - dimensions.add("dim1"); - dimensions.add("dim2"); - dimensions.add("dim2big1"); - dimensions.add("dim2big2"); - dimensions.add("dim2bignew"); - // Try creating derived cube with non existant dim/measures - try{ - client.createDerivedCube(TEST_CUBE_NAME, DERIVED_CUBE_NAME, - Sets.newHashSet("random_measure"), Sets.newHashSet("random_dim_attribute"), - new HashMap(), 5L); - } catch(LensException e) { - assertTrue(e.getMessage().contains("random_measure")); - assertTrue(e.getMessage().contains("random_dim_attribute")); - assertTrue(e.getMessage().contains("not present")); - } - client.createDerivedCube(TEST_CUBE_NAME, DERIVED_CUBE_NAME, - measures, dimensions, new HashMap(), 5L); - } - - private void addCubeChains(Map joinChains, final String cubeName) { - joinChains.put("timehourchain1", new JoinChain("timehourchain1", "time chain", "time dim thru hour dim") { - { - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "test_time_dim_hour_id")); - add(new TableReference("hourdim", "id")); - } - }); - } - }); - joinChains.put("timedatechain1", new JoinChain("timedatechain1", "time chain", "time dim thru date dim") { - { - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "test_time_dim_day_id")); - add(new TableReference("daydim", "id")); - } - }); - } - }); - joinChains.put("timehourchain2", new JoinChain("timehourchain2", "time chain", "time dim thru hour dim") { - { - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "test_time_dim_hour_id2")); - add(new TableReference("hourdim", "id")); - } - }); - } - }); - joinChains.put("timedatechain2", new JoinChain("timedatechain2", "time chain", "time dim thru date dim") { - { - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "test_time_dim_day_id2")); - add(new TableReference("daydim", "id")); - } - }); - } - }); - joinChains.put("cubeCity", new JoinChain("cubeCity", "cube-city", "city thru cube") { - { - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "cityid")); - add(new TableReference("citydim", "id")); - } - }); - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "dim2")); - add(new TableReference("testdim2", "id")); - add(new TableReference("testdim2", "cityid")); - add(new TableReference("citydim", "id")); - } - }); - } - }); - joinChains.put("cubeCity1", new JoinChain("cubeCity1", "cube-city", "city thru cube") { - { - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "cityid1")); - add(new TableReference("citydim", "id")); - } - }); - } - }); - joinChains.put("cubeCity2", new JoinChain("cubeCity2", "cube-city", "city thru cube") { - { - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "cityid2")); - add(new TableReference("citydim", "id")); - } - }); - } - }); - joinChains.put("cubeState", new JoinChain("cubeState", "cube-state", "state thru cube") { - { - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "stateid")); - add(new TableReference("statedim", "id")); - } - }); - } - }); - joinChains.put("cubeZip", new JoinChain("cubeZip", "cube-zip", "Zipcode thru cube") { - { - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "zipcode")); - add(new TableReference("zipdim", "code")); - } - }); - } - }); - joinChains.put("cubeCountry", new JoinChain("cubeCountry", "cube-country", "country thru cube") { - { - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "countryid")); - add(new TableReference("countrydim", "id")); - } - }); - } - }); - joinChains.put("dim2chain", new JoinChain("dim2chain", "cube-testdim2", "testdim2 thru cube") { - { - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "dim2")); - add(new TableReference("testdim2", "id")); - } - }); - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "dim2big1")); - add(new TableReference("testdim2", "bigid1")); - } - }); - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "dim2big2")); - add(new TableReference("testdim2", "bigid2")); - } - }); - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "dim2bignew")); - add(new TableReference("testdim2", "bigidnew")); - } - }); - } - }); - joinChains.put("dim3chain", new JoinChain("dim3chain", "cube-testdim3", "cyclicdim thru cube") { - { - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "dim2")); - add(new TableReference("testdim2", "id")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - } - }); - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "dim2big1")); - add(new TableReference("testdim2", "bigid1")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - } - }); - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "dim2big2")); - add(new TableReference("testdim2", "bigid2")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - } - }); - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "dim2bignew")); - add(new TableReference("testdim2", "bigidnew")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - } - }); - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "testdim3id")); - add(new TableReference("testdim3", "id")); - } - }); - } - }); - joinChains.put("dim4chain", new JoinChain("dim4chain", "cube-testdim3", "cyclicdim thru cube") { - { - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "dim2")); - add(new TableReference("testdim2", "id")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - add(new TableReference("testdim3", "testdim4id")); - add(new TableReference("testdim4", "id")); - } - }); - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "dim2big1")); - add(new TableReference("testdim2", "bigid1")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - add(new TableReference("testdim3", "testdim4id")); - add(new TableReference("testdim4", "id")); - } - }); - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "dim2big2")); - add(new TableReference("testdim2", "bigid2")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - add(new TableReference("testdim3", "testdim4id")); - add(new TableReference("testdim4", "id")); - } - }); - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "dim2bignew")); - add(new TableReference("testdim2", "bigidnew")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - add(new TableReference("testdim3", "testdim4id")); - add(new TableReference("testdim4", "id")); - } - }); - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "testdim3id")); - add(new TableReference("testdim3", "id")); - add(new TableReference("testdim3", "testdim4id")); - add(new TableReference("testdim4", "id")); - } - }); - } - }); - joinChains.put("cdimChain", new JoinChain("cdimChain", "cube-cyclicdim", "cyclicdim thru cube") { - { - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "cdim2")); - add(new TableReference("cycledim1", "id")); - } - }); - } - }); - joinChains.put("unreachableDim_chain", new JoinChain("unreachableDim_chain", "cube-unreachableDim", - "unreachableDim thru cube") { - { - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "urdimid")); - add(new TableReference("unreachableDim", "id")); - } - }); - } - }); - joinChains.put("cubeCountry", new JoinChain("cubeCountry", "cube-country", "country thru cube") { - { - addPath(new ArrayList() { - { - add(new TableReference(cubeName, "countryid")); - add(new TableReference("countrydim", "id")); - } - }); - } - }); - } - private void createBaseAndDerivedCubes(CubeMetastoreClient client) - throws HiveException, ParseException, LensException { - Set cubeMeasures2 = new HashSet<>(cubeMeasures); - Set cubeDimensions2 = new HashSet<>(cubeDimensions); - cubeMeasures2.add(new ColumnMeasure(new FieldSchema("msr11", "int", "first measure"))); - cubeMeasures2.add(new ColumnMeasure(new FieldSchema("msr12", "float", "second measure"), "Measure2", null, "SUM", - "RS")); - cubeMeasures2.add(new ColumnMeasure(new FieldSchema("msr13", "double", "third measure"), "Measure3", null, "MAX", - null)); - cubeMeasures2.add(new ColumnMeasure(new FieldSchema("msr14", "bigint", "fourth measure"), "Measure4", null, - "COUNT", null)); - cubeMeasures2.add(new ColumnMeasure(new FieldSchema("directMsr", "bigint", "fifth measure"), "Direct Measure", - null, "SUM", null)); - - cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("dim11", "string", "basedim"))); - cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("dim12", "int", "ref dim"), "Dim2 refer", - "dim2chain", "id", null, null, null)); // used as key in the chains - cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("dim22", "int", "ref dim"), "Dim2 refer", - "dim2chain", "id", null, null, null)); // not used as key in the chains - cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("dim13", "string", "basedim"))); - cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("userid", "int", "userid"))); - cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("xuserid", "int", "userid"))); - cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("yuserid", "int", "userid"))); - cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("user_id_added_in_past", "int", "user_id_added_in_past"))); - cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("user_id_added_far_future", "int", - "user_id_added_far_future"))); - cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("user_id_deprecated", "int", "user_id_deprecated"))); - - cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("xsports", "array", ""), - "xuser sports", "xusersports", "name", null, null, null)); - cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("ysports", "array", ""), - "yuser sports", "yusersports", "name", null, null, null)); - cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("sports", "array", ""), - "user sports", "usersports", "name", null, null, null)); - cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("sportids", "array", ""), - "user sports", "userInterestIds", "sport_id", null, null, null)); - cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("statecountry", "string", ""), - "state country", "cubestatecountry", "name", null, null, null)); - cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("citycountry", "string", ""), - "city country", "cubecitystatecountry", "name", null, null, null)); - List refCols = new ArrayList<>(); - refCols.add(new ChainRefCol("cubeState", "countrycapital")); - refCols.add(new ChainRefCol("cubeCityStateCountry", "capital")); - cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("cubeCountryCapital", "String", "ref dim"), - "Country capital", refCols, null, null, null, null)); - Map cubeProperties = new HashMap<>(); - cubeProperties.put(MetastoreUtil.getCubeTimedDimensionListKey(BASE_CUBE_NAME), - "d_time,pt,it,et,test_time_dim,test_time_dim2"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "test_time_dim", "ttd"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "test_time_dim2", "ttd2"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "d_time", "dt"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "it", "it"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "et", "et"); - cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "processing_time", "pt"); - cubeProperties.put(MetastoreConstants.TIMEDIM_RELATION + "d_time", "processing_time+[-5 days,5 days]"); - cubeProperties.put(MetastoreConstants.TIMEDIM_RELATION + "processing_time", "test_time_dim+[-5 days,5 days]"); - cubeProperties.put(MetastoreConstants.CUBE_ALL_FIELDS_QUERIABLE, "false"); - - Map joinChainMap = new HashMap<>(); - addCubeChains(joinChainMap, "basecube"); - // update new paths - joinChainMap.get("dim2chain").addPath(new ArrayList() { - { - add(new TableReference("basecube", "dim12")); - add(new TableReference("testdim2", "id")); - } - }); - joinChainMap.get("dim3chain").addPath(new ArrayList() { - { - add(new TableReference("basecube", "dim12")); - add(new TableReference("testdim2", "id")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - } - }); - joinChainMap.get("dim4chain").addPath(new ArrayList() { - { - add(new TableReference("basecube", "dim12")); - add(new TableReference("testdim2", "id")); - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - add(new TableReference("testdim3", "testdim4id")); - add(new TableReference("testdim4", "id")); - } - }); - Set joinChains = Sets.newHashSet(joinChainMap.values()); - joinChains.add(new JoinChain("cityState", "city-state", "state thru city") { - { - addPath(new ArrayList() { - { - add(new TableReference("basecube", "cityid")); - add(new TableReference("citydim", "id")); - add(new TableReference("citydim", "stateid")); - add(new TableReference("statedim", "id")); - } - }); - addPath(new ArrayList() { - { - add(new TableReference("basecube", "cityid")); - add(new TableReference("citydim", "id")); - add(new TableReference("citydim", "statename")); - add(new TableReference("statedim", "name")); - } - }); - } - }); - joinChains.add(new JoinChain("cityZip", "city-zip", "zip thru city") { - { - addPath(new ArrayList() { - { - add(new TableReference("basecube", "cityid")); - add(new TableReference("citydim", "id")); - add(new TableReference("citydim", "zipcode")); - add(new TableReference("zipdim", "code")); - } - }); - } - }); - joinChains.add(new JoinChain("cubeStateCountry", "cube-state-country", "country through state") { - { - addPath(new ArrayList() { - { - add(new TableReference("basecube", "stateid")); - add(new TableReference("statedim", "id")); - add(new TableReference("statedim", "countryid")); - add(new TableReference("countrydim", "id")); - } - }); - } - }); - joinChains.add(new JoinChain("cubeCityStateCountry", "cube-city-state-country", "country through state thru city") { - { - addPath(new ArrayList() { - { - add(new TableReference("basecube", "cityid")); - add(new TableReference("citydim", "id")); - add(new TableReference("citydim", "stateid")); - add(new TableReference("statedim", "id")); - add(new TableReference("statedim", "countryid")); - add(new TableReference("countrydim", "id")); - } - }); - } - }); - joinChains.add(new JoinChain("userchain", "user-chain", "user chain") { - { - addPath(new ArrayList() { - { - add(new TableReference("basecube", "userid")); - add(new TableReference("userdim", "id")); - } - }); - } - }); - joinChains.add(new JoinChain("user_id_added_far_future_chain", "user_id_added_far_future_chain", - "user_id_added_far_future_chain") { - { - addPath(new ArrayList() { - { - add(new TableReference("basecube", "user_id_added_far_future")); - add(new TableReference("userdim", "user_id_added_far_future")); - } - }); - } - }); - joinChains.add(new JoinChain("userSports", "user-sports", "user sports") { - { - addPath(new ArrayList() { - { - add(new TableReference("basecube", "userid")); - add(new TableReference("userdim", "id")); - add(new TableReference("userdim", "id")); - add(new TableReference("user_interests", "user_id", true)); - add(new TableReference("user_interests", "sport_id")); - add(new TableReference("sports", "id")); - } - }); - } - }); - joinChains.add(new JoinChain("userInterestIds", "user-interestsIds", "user interest ids") { - { - addPath(new ArrayList() { - { - add(new TableReference("basecube", "userid")); - add(new TableReference("userdim", "id")); - add(new TableReference("userdim", "id")); - add(new TableReference("user_interests", "user_id", true)); - } - }); - } - }); - joinChains.add(new JoinChain("xuserSports", "xuser-sports", "xuser sports") { - { - addPath(new ArrayList() { - { - add(new TableReference("basecube", "xuserid")); - add(new TableReference("userdim", "id")); - add(new TableReference("userdim", "id")); - add(new TableReference("user_interests", "user_id", true)); - add(new TableReference("user_interests", "sport_id")); - add(new TableReference("sports", "id")); - } - }); - } - }); - joinChains.add(new JoinChain("yuserSports", "user-sports", "user sports") { - { - addPath(new ArrayList() { - { - add(new TableReference("basecube", "yuserid")); - add(new TableReference("userdim", "id")); - add(new TableReference("userdim", "id")); - add(new TableReference("user_interests", "user_id", true)); - add(new TableReference("user_interests", "sport_id")); - add(new TableReference("sports", "id")); - } - }); - } - }); - - // add ref dim through chain - cubeDimensions2.add( - new ReferencedDimAttribute(new FieldSchema("cityStateCapital", "string", "State's capital thru city"), - "State's capital thru city", "cityState", "capital", null, null, null)); - Set baseExprs = new HashSet<>(exprs); - baseExprs.add(new ExprColumn(new FieldSchema("substrsprorts", "String", "substr of sports"), "substr sports", - "substr(sports, 10)")); - baseExprs.add(new ExprColumn(new FieldSchema("xsports_abbr", "array", ""), - "xuser sports", "substr(xsports, 3)")); - baseExprs.add(new ExprColumn(new FieldSchema("ysports_abbr", "array", ""), - "yuser sports", "substr(ysports, 3)")); - baseExprs.add(new ExprColumn(new FieldSchema("sports_abbr", "array", ""), - "user sports", "substr(sports, 3)")); - baseExprs.add(new ExprColumn(new FieldSchema("sportids_abbr", "array", ""), - "user sports", "case when sportids == 1 then 'CKT' when sportids == 2 then 'FTB' else 'NON' end")); - baseExprs.add(new ExprColumn(new FieldSchema("directMsrExpr", "bigint", ""), - "Direct Measure", new ExprSpec("directMsr + 0", null, null), new ExprSpec("msr13 + msr14", null, null))); - client.createCube(BASE_CUBE_NAME, cubeMeasures2, cubeDimensions2, baseExprs, joinChains, cubeProperties); - - Map derivedProperties = new HashMap<>(); - derivedProperties.put(MetastoreConstants.CUBE_ALL_FIELDS_QUERIABLE, "true"); - Set measures = new HashSet<>(); - measures.add("msr1"); - measures.add("msr9"); - measures.add("msr11"); - Set dimensions = new HashSet<>(); - dimensions.add("dim1"); - dimensions.add("dim11"); - dimensions.add("d_time"); - client.createDerivedCube(BASE_CUBE_NAME, DERIVED_CUBE_NAME1, measures, dimensions, derivedProperties, 5L); - - measures = new HashSet<>(); - measures.add("msr2"); - measures.add("msr12"); - measures.add("msr13"); - measures.add("msr14"); - measures.add("directmsr"); - dimensions = new HashSet<>(); - dimensions.add("cityid"); - dimensions.add("stateid"); - dimensions.add("userid"); - dimensions.add("xuserid"); - dimensions.add("yuserid"); - dimensions.add("dim1"); - dimensions.add("dim2"); - dimensions.add("dim2big1"); - dimensions.add("dim2big2"); - dimensions.add("dim2bignew"); - dimensions.add("dim11"); - dimensions.add("dim13"); - dimensions.add("dim12"); - dimensions.add("dim22"); - dimensions.add("d_time"); - dimensions.add("test_time_dim"); - dimensions.add("test_time_dim2"); - dimensions.add("test_time_dim_hour_id"); - dimensions.add("test_time_dim_day_id"); - dimensions.add("test_time_dim_hour_id2"); - dimensions.add("test_time_dim_day_id2"); - client.createDerivedCube(BASE_CUBE_NAME, DERIVED_CUBE_NAME2, measures, dimensions, derivedProperties, 10L); - measures = new HashSet<>(); - measures.add("msr3"); - measures.add("msr13"); - dimensions = new HashSet<>(); - dimensions.add("dim1"); - dimensions.add("location"); - dimensions.add("d_time"); - dimensions.add("test_time_dim"); - dimensions.add("test_time_dim2"); - dimensions.add("test_time_dim_hour_id"); - dimensions.add("test_time_dim_day_id"); - dimensions.add("test_time_dim_hour_id2"); - dimensions.add("test_time_dim_day_id2"); - client.createDerivedCube(BASE_CUBE_NAME, DERIVED_CUBE_NAME3, measures, dimensions, derivedProperties, 20L); - - // create base cube facts - createBaseCubeFacts(client); - } - - private void createBaseCubeFacts(CubeMetastoreClient client) throws HiveException, LensException { - - Map> storageAggregatePeriods = new HashMap>(); - Set updates = new HashSet(); - updates.add(MINUTELY); - updates.add(HOURLY); - updates.add(DAILY); - updates.add(MONTHLY); - updates.add(QUARTERLY); - updates.add(YEARLY); - - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - ArrayList s2PartCols = new ArrayList(); - s2PartCols.add(new FieldSchema("ttd", serdeConstants.STRING_TYPE_NAME, "test date partition")); - s2PartCols.add(new FieldSchema("ttd2", serdeConstants.STRING_TYPE_NAME, "test date partition")); - s2.setPartCols(s2PartCols); - s2.setTimePartCols(Arrays.asList("ttd", "ttd2")); - - storageAggregatePeriods.put(c1, updates); - storageAggregatePeriods.put(c2, updates); - storageAggregatePeriods.put(c3, updates); - storageAggregatePeriods.put(c4, updates); - - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - storageTables.put(c4, s2); - storageTables.put(c2, s1); - storageTables.put(c3, s1); - - String factName = "testFact1_BASE"; - List factColumns = new ArrayList(cubeMeasures.size()); - for (CubeMeasure measure : cubeMeasures) { - factColumns.add(measure.getColumn()); - } - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("zipcode", "int", "zip")); - factColumns.add(new FieldSchema("cityid", "int", "city id")); - factColumns.add(new FieldSchema("stateid", "int", "state id")); - factColumns.add(new FieldSchema("userid", "int", "user id")); - factColumns.add(new FieldSchema("xuserid", "int", "user id")); - factColumns.add(new FieldSchema("yuserid", "int", "user id")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - factColumns.add(new FieldSchema("dim11", "string", "base dim")); - factColumns.add(new FieldSchema("test_time_dim_hour_id", "int", "time id")); - - // create cube fact with materialized expressions - client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L, - factValidityProperties, storageTables); - - factName = "testFact5_BASE"; - factColumns = new ArrayList<>(cubeMeasures.size()); - for (CubeMeasure measure : cubeMeasures) { - factColumns.add(measure.getColumn()); - } - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - factColumns.add(new FieldSchema("booleancut", "boolean", "expr dim")); - - // create cube fact - client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 150L, - factValidityProperties, storageTables); - - // create fact only with extra measures - factName = "testFact2_BASE"; - factColumns = new ArrayList(); - factColumns.add(new FieldSchema("msr12", "float", "second measure")); - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - factColumns.add(new FieldSchema("dim11", "string", "base dim")); - factColumns.add(new FieldSchema("dim2", "int", "dim2 id")); - factColumns.add(new FieldSchema("userid", "int", "user id")); - factColumns.add(new FieldSchema("xuserid", "int", "user id")); - factColumns.add(new FieldSchema("yuserid", "int", "user id")); - // create cube fact - client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L, - factValidityProperties, storageTables); - Map properties = Maps.newHashMap(factValidityProperties); - properties.put(MetastoreConstants.FACT_ABSOLUTE_END_TIME, DateUtil.relativeToAbsolute("now.day - 2 days")); - properties.put(MetastoreConstants.FACT_ABSOLUTE_START_TIME, DateUtil.relativeToAbsolute("now.day - 3 days")); - client.createCubeFactTable(BASE_CUBE_NAME, "testfact_deprecated", factColumns, storageAggregatePeriods, 5L, - properties, storageTables); - - // create fact only with extra measures - factName = "testFact3_BASE"; - factColumns = new ArrayList(); - factColumns.add(new FieldSchema("msr13", "double", "third measure")); - factColumns.add(new FieldSchema("msr14", "bigint", "fourth measure")); - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - factColumns.add(new FieldSchema("dim11", "string", "base dim")); - - // create cube fact - client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L, - factValidityProperties, storageTables); - - // create fact with materialized expression - factName = "testFact6_BASE"; - factColumns = new ArrayList<>(); - factColumns.add(new FieldSchema("msr13", "double", "third measure")); - factColumns.add(new FieldSchema("msr14", "bigint", "fourth measure")); - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - factColumns.add(new FieldSchema("booleancut", "boolean", "expr dim")); - - // create cube fact - client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 150L, - factValidityProperties, storageTables); - - // create raw fact only with extra measures - factName = "testFact2_RAW_BASE"; - factColumns = new ArrayList(); - factColumns.add(new FieldSchema("msr11", "int", "first measure")); - factColumns.add(new FieldSchema("msr12", "float", "second measure")); - factColumns.add(new FieldSchema("msr9", "bigint", "ninth measure")); - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - factColumns.add(new FieldSchema("dim11", "string", "base dim")); - factColumns.add(new FieldSchema("dim13", "string", "base dim")); - factColumns.add(new FieldSchema("dim12", "string", "base dim")); - factColumns.add(new FieldSchema("dim22", "string", "base dim")); - factColumns.add(new FieldSchema("cityid", "int", "city id")); - - storageAggregatePeriods = new HashMap>(); - updates = new HashSet(); - updates.add(HOURLY); - storageAggregatePeriods.put(c1, updates); - - storageTables = new HashMap(); - storageTables.put(c1, s1); - - // create cube fact - properties.clear(); - properties.putAll(factValidityProperties); - properties.put(MetastoreConstants.FACT_AGGREGATED_PROPERTY, "false"); - properties.put(MetastoreConstants.FACT_DATA_COMPLETENESS_TAG, "f2"); - - client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 100L, properties, - storageTables); - - // create raw fact only with extra measures - factName = "testFact3_RAW_BASE"; - factColumns = new ArrayList(); - factColumns.add(new FieldSchema("msr13", "double", "third measure")); - factColumns.add(new FieldSchema("msr14", "bigint", "fourth measure")); - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - factColumns.add(new FieldSchema("dim11", "string", "base dim")); - factColumns.add(new FieldSchema("dim12", "string", "base dim")); - - storageAggregatePeriods = new HashMap>(); - updates = new HashSet(); - updates.add(HOURLY); - storageAggregatePeriods.put(c1, updates); - - storageTables = new HashMap(); - storageTables.put(c1, s1); - properties.put(MetastoreConstants.FACT_COL_START_TIME_PFX.concat("user_id_added_in_past"), "2016-01-01"); - properties.put(MetastoreConstants.FACT_COL_END_TIME_PFX.concat("user_id_deprecated"), "2016-01-01"); - properties.put(MetastoreConstants.FACT_COL_START_TIME_PFX.concat("user_id_added_far_future"), "2099-01-01"); - client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 100L, properties, - storageTables); - - factName = "testFact4_RAW_BASE"; - factColumns = new ArrayList(); - factColumns.add(new FieldSchema("msr13", "double", "third measure")); - factColumns.add(new FieldSchema("msr14", "bigint", "fourth measure")); - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - factColumns.add(new FieldSchema("user_id_added_in_past", "int", "user id")); - factColumns.add(new FieldSchema("user_id_added_far_future", "int", "user id")); - factColumns.add(new FieldSchema("user_id_deprecated", "int", "user id")); - - storageTables = new HashMap(); - storageTables.put(c1, s1); - properties.put(MetastoreConstants.FACT_COL_START_TIME_PFX.concat("user_id_added_in_past"), "2016-01-01"); - properties.put(MetastoreConstants.FACT_COL_END_TIME_PFX.concat("user_id_deprecated"), "2016-01-01"); - properties.put(MetastoreConstants.FACT_COL_START_TIME_PFX.concat("user_id_added_far_future"), "2099-01-01"); - client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 100L, properties, - storageTables); - - factName = "testFact5_RAW_BASE"; - factColumns = new ArrayList(); - factColumns.add(new FieldSchema("msr9", "bigint", "ninth measure")); - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - - properties.clear(); - properties.putAll(factValidityProperties); - properties.put(MetastoreConstants.FACT_AGGREGATED_PROPERTY, "false"); - properties.put(MetastoreConstants.FACT_DATA_COMPLETENESS_TAG, "f2"); - client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 100L, properties, - storageTables); - - CubeFactTable fact = client.getFactTable(factName); - // Add all hourly partitions for two days - Calendar cal = Calendar.getInstance(); - cal.setTime(TWODAYS_BACK); - Date temp = cal.getTime(); - while (!(temp.after(NOW))) { - Map timeParts = new HashMap(); - timeParts.put("dt", temp); - StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); - client.addPartition(sPartSpec, c1, CubeTableType.FACT); - cal.add(HOUR_OF_DAY, 1); - temp = cal.getTime(); - } - } - - private void createCubeContinuousFact(CubeMetastoreClient client) throws Exception { - // create continuous raw fact only with extra measures - String factName = "testFact_CONTINUOUS"; - List factColumns = new ArrayList(); - factColumns.add(new FieldSchema("msr11", "double", "third measure")); - factColumns.add(new FieldSchema("msr15", "int", "fifteenth measure")); - - // add dimensions of the cube - factColumns.add(new FieldSchema("d_time", "timestamp", "event time")); - factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); - factColumns.add(new FieldSchema("dim1", "string", "base dim")); - factColumns.add(new FieldSchema("dim11", "string", "base dim")); - factColumns.add(new FieldSchema("dim12", "string", "base dim")); - - Map> storageAggregatePeriods = new HashMap>(); - Set updates = new HashSet(); - updates.add(CONTINUOUS); - storageAggregatePeriods.put(c0, updates); - - StorageTableDesc s0 = new StorageTableDesc(); - s0.setInputFormat(TextInputFormat.class.getCanonicalName()); - s0.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - - Map storageTables = new HashMap(); - storageTables.put(c0, s0); - Map properties = Maps.newHashMap(factValidityProperties); - properties.put(MetastoreConstants.FACT_ABSOLUTE_START_TIME, DateUtil.relativeToAbsolute("now.day - 3 days")); - - client.createCubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 100L, properties, - storageTables); - } - - private void createCubeFact(CubeMetastoreClient client) throws Exception { + private void assertTestFactTimelineClass(CubeMetastoreClient client) throws Exception { String factName = "testFact"; - List factColumns = new ArrayList(cubeMeasures.size()); - for (CubeMeasure measure : cubeMeasures) { - if (!measure.getColumn().getName().equals("msr15")) { //do not add msr15 - factColumns.add(measure.getColumn()); - } - } - factColumns.add(new FieldSchema("msr5", "double", "msr5")); - - // add dimensions of the cube - factColumns.add(new FieldSchema("zipcode", "int", "zip")); - factColumns.add(new FieldSchema("cityid", "int", "city id")); - factColumns.add(new FieldSchema("cityid1", "int", "city id")); - factColumns.add(new FieldSchema("stateid", "int", "city id")); - factColumns.add(new FieldSchema("test_time_dim_day_id", "int", "time id")); - factColumns.add(new FieldSchema("test_time_dim_day_id2", "int", "time id")); - factColumns.add(new FieldSchema("ambigdim1", "string", "used in" + " testColumnAmbiguity")); - - Map> storageAggregatePeriods = new HashMap>(); - Set updates = new HashSet(); - updates.add(MINUTELY); - updates.add(HOURLY); - updates.add(DAILY); - updates.add(MONTHLY); - updates.add(QUARTERLY); - updates.add(YEARLY); - - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - ArrayList s2PartCols = new ArrayList(); - s2PartCols.add(new FieldSchema("ttd", serdeConstants.STRING_TYPE_NAME, "test date partition")); - s2PartCols.add(new FieldSchema("ttd2", serdeConstants.STRING_TYPE_NAME, "test date partition")); - s2.setPartCols(s2PartCols); - s2.setTimePartCols(Arrays.asList("ttd", "ttd2")); - - StorageTableDesc s3 = new StorageTableDesc(); - s3.setInputFormat(TextInputFormat.class.getCanonicalName()); - s3.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s3.setPartCols(partCols); - s3.setTimePartCols(timePartCols); - s3.getTblProps().put(MetastoreUtil.getStoragetableStartTimesKey(), "now.day - 90 days"); - s3.getTblProps().put(MetastoreUtil.getStoragetableEndTimesKey(), "now.day - 10 days"); - - StorageTableDesc s5 = new StorageTableDesc(); - s5.setInputFormat(TextInputFormat.class.getCanonicalName()); - s5.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s5.setPartCols(partCols); - s5.setTimePartCols(timePartCols); - s5.getTblProps().put(MetastoreUtil.getStoragetableStartTimesKey(), "now.day - 10 days"); - - storageAggregatePeriods.put(c1, updates); - storageAggregatePeriods.put(c2, updates); - storageAggregatePeriods.put(c3, updates); - storageAggregatePeriods.put(c4, updates); - storageAggregatePeriods.put(c5, updates); - - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - storageTables.put(c4, s2); - storageTables.put(c2, s1); - storageTables.put(c3, s3); - storageTables.put(c5, s5); - - //add storage with continuous update period - updates.add(CONTINUOUS); - storageAggregatePeriods.put(c0, updates); - StorageTableDesc s0 = new StorageTableDesc(); - s0.setInputFormat(TextInputFormat.class.getCanonicalName()); - s0.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - storageTables.put(c0, s0); - - // create cube fact - client.createCubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L, - factValidityProperties, storageTables); + client.getTimelines(factName, c1, null, null); client.getTimelines(factName, c4, null, null); + client.clearHiveTableCache(); + CubeFactTable fact = client.getFactTable(factName); Table table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(), Storage.getPrefix(c1))); assertEquals(table.getParameters().get(MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true"); @@ -1695,62 +580,8 @@ private void assertTimeline(CubeMetastoreClient client, String factName, String assertTimeline(client, factName, storageName, updatePeriod, timeDim, expectedTimeline); } - private void createCubeCheapFact(CubeMetastoreClient client) throws HiveException, LensException { + private void createCubeCheapFactPartitions(CubeMetastoreClient client) throws HiveException, LensException { String factName = "cheapFact"; - List factColumns = new ArrayList(cubeMeasures.size()); - for (CubeMeasure measure : cubeMeasures) { - factColumns.add(measure.getColumn()); - } - - // add dimensions of the cube - factColumns.add(new FieldSchema("zipcode", "int", "zip")); - factColumns.add(new FieldSchema("cityid", "int", "city id")); - factColumns.add(new FieldSchema("stateid", "int", "city id")); - factColumns.add(new FieldSchema("test_time_dim_hour_id", "int", "time id")); - factColumns.add(new FieldSchema("ambigdim1", "string", "used in" + " testColumnAmbiguity")); - - Map> storageAggregatePeriods = new HashMap>(); - Set updates = new HashSet(); - updates.add(MINUTELY); - updates.add(HOURLY); - updates.add(DAILY); - updates.add(MONTHLY); - updates.add(QUARTERLY); - updates.add(YEARLY); - - ArrayList partCols = new ArrayList<>(); - List timePartCols = new ArrayList<>(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - s1.setTblProps(new HashMap()); - s1.getTblProps().put(MetastoreUtil.getStoragetableStartTimesKey(), "2000, now - 10 years"); - s1.getTblProps().put(MetastoreUtil.getStoragetableEndTimesKey(), "now - 5 years, 2010"); - - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - ArrayList s2PartCols = new ArrayList<>(); - s2PartCols.add(new FieldSchema("ttd", serdeConstants.STRING_TYPE_NAME, "test date partition")); - s2PartCols.add(new FieldSchema("ttd2", serdeConstants.STRING_TYPE_NAME, "test date partition")); - s2.setPartCols(s2PartCols); - s2.setTimePartCols(Arrays.asList("ttd", "ttd2")); - - storageAggregatePeriods.put(c99, updates); - storageAggregatePeriods.put(c0, updates); - - Map storageTables = new HashMap<>(); - storageTables.put(c99, s2); - storageTables.put(c0, s1); - // create cube fact - client.createCubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 0L, - factValidityProperties, storageTables); - CubeFactTable fact = client.getFactTable(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); @@ -1780,88 +611,9 @@ private void createCubeCheapFact(CubeMetastoreClient client) throws HiveExceptio } } - private void createCubeFactWeekly(CubeMetastoreClient client) throws Exception { - String factName = "testFactWeekly"; - List factColumns = new ArrayList(cubeMeasures.size()); - for (CubeMeasure measure : cubeMeasures) { - factColumns.add(measure.getColumn()); - } - // add dimensions of the cube - factColumns.add(new FieldSchema("zipcode", "int", "zip")); - - Map> storageAggregatePeriods = new HashMap>(); - Set updates = new HashSet(); - updates.add(WEEKLY); - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - - storageAggregatePeriods.put(c1, updates); - - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - // create cube fact - client.createCubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L, - factValidityProperties, storageTables); - } - - private void createCubeFactOnlyHourly(CubeMetastoreClient client) throws Exception { + private void createTestFact2Partitions(CubeMetastoreClient client) throws Exception { String factName = "testFact2"; - List factColumns = new ArrayList(cubeMeasures.size()); - for (CubeMeasure measure : cubeMeasures) { - if (!measure.getName().equals("msr4")) { - factColumns.add(measure.getColumn()); - } - } - - // add dimensions of the cube - factColumns.add(new FieldSchema("zipcode", "int", "zip")); - factColumns.add(new FieldSchema("cityid", "int", "city id")); - factColumns.add(new FieldSchema("cityid2", "int", "city id")); - factColumns.add(new FieldSchema("test_time_dim_hour_id", "int", "time id")); - factColumns.add(new FieldSchema("test_time_dim_hour_id2", "int", "time id")); - factColumns.add(new FieldSchema("cdim2", "int", "cycledim id")); - - Map> storageAggregatePeriods = new HashMap>(); - Set updates = new HashSet(); - updates.add(HOURLY); - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - ArrayList s2PartCols = new ArrayList(); - s2PartCols.add(new FieldSchema("ttd", serdeConstants.STRING_TYPE_NAME, "test date partition")); - s2PartCols.add(new FieldSchema("ttd2", serdeConstants.STRING_TYPE_NAME, "test date partition")); - s2.setPartCols(s2PartCols); - s2.setTimePartCols(Arrays.asList("ttd", "ttd2")); - - storageAggregatePeriods.put(c1, updates); - storageAggregatePeriods.put(c4, updates); - - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - storageTables.put(c4, s2); - - // create cube fact - client - .createCubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 10L, - factValidityProperties, storageTables); CubeFactTable fact = client.getFactTable(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); @@ -1948,54 +700,8 @@ private void createCubeFactOnlyHourly(CubeMetastoreClient client) throws Excepti } } - private void createCubeFactOnlyHourlyRaw(CubeMetastoreClient client) throws HiveException, LensException { + private void createTestFact2RawPartitions(CubeMetastoreClient client) throws HiveException, LensException { String factName = "testFact2_raw"; - String factName2 = "testFact1_raw_BASE"; - List factColumns = new ArrayList(cubeMeasures.size()); - for (CubeMeasure measure : cubeMeasures) { - factColumns.add(measure.getColumn()); - } - - // add dimensions of the cube - factColumns.add(new FieldSchema("zipcode", "int", "zip")); - factColumns.add(new FieldSchema("cityid", "int", "city id")); - factColumns.add(new FieldSchema("cityid1", "int", "city id")); - factColumns.add(new FieldSchema("cityid2", "int", "city id")); - factColumns.add(new FieldSchema("stateid", "int", "state id")); - factColumns.add(new FieldSchema("countryid", "int", "country id")); - factColumns.add(new FieldSchema("dim1", "string", "dim1")); - factColumns.add(new FieldSchema("dim2", "int", "dim2")); - factColumns.add(new FieldSchema("concatedCityState", "string", "citystate")); - - Map> storageAggregatePeriods = new HashMap>(); - Set updates = new HashSet(); - updates.add(HOURLY); - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - storageAggregatePeriods.put(c1, updates); - storageAggregatePeriods.put(c3, updates); - - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - storageTables.put(c3, s1); - - // create cube fact - Map properties = new HashMap(); - properties.putAll(factValidityProperties); - properties.put(MetastoreConstants.FACT_AGGREGATED_PROPERTY, "false"); - properties.put(MetastoreConstants.FACT_DATA_COMPLETENESS_TAG, "f1"); - - client.createCubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 100L, properties, - storageTables); - client.createCubeFactTable(BASE_CUBE_NAME, factName2, factColumns, storageAggregatePeriods, 100L, properties, - storageTables); CubeFactTable fact2 = client.getFactTable(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); @@ -2011,917 +717,6 @@ private void createCubeFactOnlyHourlyRaw(CubeMetastoreClient client) throws Hive } } - private void createCubeFactMonthly(CubeMetastoreClient client) throws Exception { - String factName = "testFactMonthly"; - List factColumns = new ArrayList(cubeMeasures.size()); - for (CubeMeasure measure : cubeMeasures) { - factColumns.add(measure.getColumn()); - } - - // add one dimension of the cube - factColumns.add(new FieldSchema("countryid", "int", "country id")); - - Map> storageAggregatePeriods = new HashMap>(); - Set updates = new HashSet(); - updates.add(MONTHLY); - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - - storageAggregatePeriods.put(c2, updates); - - Map storageTables = new HashMap(); - storageTables.put(c2, s1); - - // create cube fact - client.createCubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L, - factValidityProperties, storageTables); - } - - // DimWithTwoStorages - private void createCityTable(CubeMetastoreClient client) throws Exception { - Set cityAttrs = new HashSet(); - cityAttrs.add(new BaseDimAttribute(new FieldSchema("id", "int", "code"))); - cityAttrs.add(new BaseDimAttribute(new FieldSchema("name", "string", "city name"))); - cityAttrs.add(new BaseDimAttribute(new FieldSchema("ambigdim1", "string", "used in testColumnAmbiguity"))); - cityAttrs.add(new BaseDimAttribute(new FieldSchema("ambigdim2", "string", "used in testColumnAmbiguity"))); - cityAttrs.add(new BaseDimAttribute(new FieldSchema("nocandidatecol", "string", "used in testing no" - + " candidate available"))); - cityAttrs.add(new BaseDimAttribute(new FieldSchema("stateid", "int", "state id"))); - cityAttrs.add(new ReferencedDimAttribute(new FieldSchema("statename", "string", "state name"), "State name", - "citystate", "name", null, null, null, null)); - cityAttrs.add(new BaseDimAttribute(new FieldSchema("zipcode", "int", "zip code"))); - Map dimProps = new HashMap(); - dimProps.put(MetastoreUtil.getDimTimedDimensionKey("citydim"), TestCubeMetastoreClient.getDatePartitionKey()); - Set exprs = new HashSet(); - exprs.add(new ExprColumn(new FieldSchema("CityAddress", "string", "city with state and city and zip"), - "City Address", - new ExprSpec("concat(citydim.name, \":\", citystate.name, \":\", citycountry.name, \":\", cityzip.code)", null, - null), new ExprSpec("concat(citydim.name, \":\", citystate.name)", null, null))); - exprs.add(new ExprColumn(new FieldSchema("CityState", "string", "city's state"), - "City State", new ExprSpec("concat(citydim.name, \":\", citydim.statename)", null, null))); - exprs.add(new ExprColumn(new FieldSchema("AggrExpr", "int", "count(name)"), "city count", - new ExprSpec("count(name)", null, null))); - Set joinchains = new HashSet() { - { - add(new JoinChain("cityState", "city-state", "state thru city") { - { - addPath(new ArrayList() { - { - add(new TableReference("citydim", "stateid")); - add(new TableReference("statedim", "id")); - } - }); - addPath(new ArrayList() { - { - add(new TableReference("citydim", "statename")); - add(new TableReference("statedim", "name")); - } - }); - } - }); - } - }; - joinchains.add(new JoinChain("cityCountry", "cube-zip", "country thru city") { - { - addPath(new ArrayList() { - { - add(new TableReference("citydim", "stateid")); - add(new TableReference("statedim", "id")); - add(new TableReference("statedim", "countryid")); - add(new TableReference("countrydim", "id")); - } - }); - addPath(new ArrayList() { - { - add(new TableReference("citydim", "statename")); - add(new TableReference("statedim", "name")); - add(new TableReference("statedim", "countryid")); - add(new TableReference("countrydim", "id")); - } - }); - } - }); - joinchains.add(new JoinChain("cityZip", "city-zip", "Zipcode thru city") { - { - addPath(new ArrayList() { - { - add(new TableReference("citydim", "zipcode")); - add(new TableReference("zipdim", "code")); - } - }); - } - }); - Dimension cityDim = new Dimension("citydim", cityAttrs, exprs, joinchains, dimProps, 0L); - client.createDimension(cityDim); - - String dimName = "citytable"; - - List dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "code")); - dimColumns.add(new FieldSchema("name", "string", "field1")); - dimColumns.add(new FieldSchema("stateid", "int", "state id")); - dimColumns.add(new FieldSchema("zipcode", "int", "zip code")); - dimColumns.add(new FieldSchema("ambigdim1", "string", "used in" + " testColumnAmbiguity")); - dimColumns.add(new FieldSchema("ambigdim2", "string", "used in " + "testColumnAmbiguity")); - - Map dumpPeriods = new HashMap(); - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - Map tblPros = Maps.newHashMap(); - tblPros.put(LensConfConstants.STORAGE_COST, "100"); - s1.setTblProps(tblPros); - dumpPeriods.put(c1, HOURLY); - - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - dumpPeriods.put(c2, null); - - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - storageTables.put(c2, s2); - - client.createCubeDimensionTable(cityDim.getName(), dimName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - - dimName = "citytable2"; - - dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "code")); - dimColumns.add(new FieldSchema("stateid", "int", "state id")); - - dumpPeriods = new HashMap(); - storageTables = new HashMap(); - storageTables.put(c4, s2); - dumpPeriods.put(c4, null); - - client.createCubeDimensionTable(cityDim.getName(), dimName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - - dimName = "citytable3"; - - dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "code")); - dimColumns.add(new FieldSchema("name", "string", "name")); - - client.createCubeDimensionTable(cityDim.getName(), dimName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - - dimName = "citytable4"; - - dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "code")); - - client.createCubeDimensionTable(cityDim.getName(), dimName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - - } - - private void createTestDim2(CubeMetastoreClient client) throws Exception { - String dimName = "testDim2"; - Set dimAttrs = new HashSet(); - - Set joinchains = new HashSet<>(); - JoinChain cityState = new JoinChain("cityState", "city-state", "state thru city"); - List statePaths1 = new ArrayList<>(); - statePaths1.add(new TableReference("testDim2", "cityid")); - statePaths1.add(new TableReference("citydim", "id")); - statePaths1.add(new TableReference("citydim", "stateid")); - statePaths1.add(new TableReference("statedim", "id")); - cityState.addPath(statePaths1); - List statePaths2 = new ArrayList(); - statePaths2.add(new TableReference("testDim2", "cityid")); - statePaths2.add(new TableReference("citydim", "id")); - statePaths2.add(new TableReference("citydim", "statename")); - statePaths2.add(new TableReference("statedim", "name")); - cityState.addPath(statePaths2); - joinchains.add(cityState); - joinchains.add(new JoinChain("dim2city", "dim2-city", "city thru dim2") { - { - addPath(new ArrayList() { - { - add(new TableReference("testdim2", "cityid")); - add(new TableReference("citydim", "id")); - } - }); - } - }); - joinchains.add(new JoinChain("dim3chain", "dim3-chain", "dim3 thru dim2") { - { - addPath(new ArrayList() { - { - add(new TableReference("testdim2", "testDim3id")); - add(new TableReference("testdim3", "id")); - } - }); - } - }); - joinchains.add(new JoinChain("unreachableDim_chain", "dim2-unreachableDim", "unreachableDim thru dim2") { - { - addPath(new ArrayList() { - { - add(new TableReference("testdim2", "urdimid")); - add(new TableReference("unreachableDim", "id")); - } - }); - } - }); - joinchains.add(new JoinChain("dim4chain", "cube-testdim3", "cyclicdim thru cube") { - { - addPath(new ArrayList() { - { - add(new TableReference("testdim2", "testdim3id")); - add(new TableReference("testdim3", "id")); - add(new TableReference("testdim3", "testdim4id")); - add(new TableReference("testdim4", "id")); - } - }); - } - }); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("id", "int", "code"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("bigid1", "bigint", "big id"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("bigid2", "bigint", "big id"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("bigidnew", "bigint", "big id"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("name", "string", "name"))); - dimAttrs.add(new ReferencedDimAttribute(new FieldSchema("testDim3id", "string", "f-key to testdim3"), "dim3 refer", - "dim3chain", "id", null, null, 0.0)); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("cityId", "string", "f-key to citydim"))); - dimAttrs.add(new ReferencedDimAttribute(new FieldSchema("cityname", "string", "name"), "cityname", - "dim2city", "name", null, null, 0.0)); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("urdimid", "int", "ref dim"), "urdim refer", - null, null, 10.0)); - dimAttrs.add(new ReferencedDimAttribute(new FieldSchema("unreachableName", "string", ""), "urdim name", - "unreachableDim_chain", "name", null, null, 10.0)); - // add ref dim through chain - dimAttrs.add(new ReferencedDimAttribute( - new FieldSchema("cityStateCapital", "string", "State's capital thru city"), "State's capital thru city", - "cityState", "capital", null, null, null)); - - Map dimProps = new HashMap(); - dimProps.put(MetastoreUtil.getDimTimedDimensionKey(dimName), TestCubeMetastoreClient.getDatePartitionKey()); - Dimension testDim2 = new Dimension(dimName, dimAttrs, null, joinchains, dimProps, 0L); - client.createDimension(testDim2); - - String dimTblName = "testDim2Tbl"; - List dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "code")); - dimColumns.add(new FieldSchema("name", "string", "field1")); - dimColumns.add(new FieldSchema("cityId", "string", "f-key to cityDim")); - dimColumns.add(new FieldSchema("testDim3id", "string", "f-key to testdim3")); - - Map dumpPeriods = new HashMap(); - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - dumpPeriods.put(c1, HOURLY); - - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - dumpPeriods.put(c2, null); - - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - storageTables.put(c2, s2); - - client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - - // create table2 - dimTblName = "testDim2Tbl2"; - dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "code")); - dimColumns.add(new FieldSchema("bigid1", "int", "code")); - dimColumns.add(new FieldSchema("name", "string", "field1")); - dimColumns.add(new FieldSchema("cityId", "string", "f-key to cityDim")); - storageTables.put(c3, s1); - dumpPeriods.put(c3, HOURLY); - - client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 10L, dumpPeriods, dimProps, storageTables); - - // create table2 - dimTblName = "testDim2Tbl3"; - dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "code")); - dimColumns.add(new FieldSchema("bigid1", "int", "code")); - dimColumns.add(new FieldSchema("name", "string", "field1")); - dimColumns.add(new FieldSchema("testDim3id", "string", "f-key to testdim3")); - - client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 20L, dumpPeriods, dimProps, storageTables); - } - - private void createTimeDims(CubeMetastoreClient client) throws Exception { - String dimName = "dayDim"; - Set dimAttrs = new HashSet(); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("id", "int", "code"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("full_date", "string", "full date"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("calendar_quarter", "int", "quarter id"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("day_number_of_year", "int", "day number in year"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("is_weekend", "boolean", "is weekend?"))); - Map dimProps = new HashMap(); - dimProps.put(MetastoreUtil.getDimTimedDimensionKey(dimName), TestCubeMetastoreClient.getDatePartitionKey()); - Dimension testDim = new Dimension(dimName, dimAttrs, dimProps, 0L); - client.createDimension(testDim); - - String dimTblName = "dayDimTbl"; - List dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "code")); - dimColumns.add(new FieldSchema("full_date", "string", "field1")); - - Map dumpPeriods = new HashMap(); - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - dumpPeriods.put(c3, HOURLY); - - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - dumpPeriods.put(c4, null); - - Map storageTables = new HashMap(); - storageTables.put(c3, s1); - storageTables.put(c4, s2); - - client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - - String dimName2 = "hourDim"; - dimAttrs = new HashSet(); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("id", "int", "code"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("full_hour", "string", "full date"))); - dimProps = new HashMap(); - dimProps.put(MetastoreUtil.getDimTimedDimensionKey(dimName2), TestCubeMetastoreClient.getDatePartitionKey()); - testDim = new Dimension(dimName2, dimAttrs, dimProps, 0L); - client.createDimension(testDim); - - String dimTblName2 = "hourDimTbl"; - dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "code")); - dimColumns.add(new FieldSchema("full_hour", "string", "field1")); - - client.createCubeDimensionTable(dimName2, dimTblName2, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - - } - - private void createTestDim3(CubeMetastoreClient client) throws Exception { - String dimName = "testDim3"; - - Set dimAttrs = new HashSet(); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("id", "int", "code"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("name", "string", "name"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("testDim4id", "string", "f-key to testdim4"))); - Map dimProps = new HashMap(); - dimProps.put(MetastoreUtil.getDimTimedDimensionKey(dimName), TestCubeMetastoreClient.getDatePartitionKey()); - Set joinchains = new HashSet() { - { - add(new JoinChain("dim4chain", "dim4-chain", "dim4 thru dim3") { - { - addPath(new ArrayList() { - { - add(new TableReference("testdim3", "testDim4id")); - add(new TableReference("testdim4", "id")); - } - }); - } - }); - } - }; - Dimension testDim3 = new Dimension(dimName, dimAttrs, null, joinchains, dimProps, 0L); - client.createDimension(testDim3); - - String dimTblName = "testDim3Tbl"; - List dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "code")); - dimColumns.add(new FieldSchema("name", "string", "field1")); - dimColumns.add(new FieldSchema("testDim4id", "string", "f-key to testDim4")); - - Map dumpPeriods = new HashMap(); - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - dumpPeriods.put(c1, HOURLY); - - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - dumpPeriods.put(c2, null); - - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - storageTables.put(c2, s2); - - client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - } - - private void createTestDim4(CubeMetastoreClient client) throws Exception { - String dimName = "testDim4"; - - Set dimAttrs = new HashSet(); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("id", "int", "code"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("name", "string", "name"))); - Map dimProps = new HashMap(); - dimProps.put(MetastoreUtil.getDimTimedDimensionKey(dimName), TestCubeMetastoreClient.getDatePartitionKey()); - Dimension testDim4 = new Dimension(dimName, dimAttrs, dimProps, 0L); - client.createDimension(testDim4); - - String dimTblName = "testDim4Tbl"; - - List dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "code")); - dimColumns.add(new FieldSchema("name", "string", "field1")); - - Map dumpPeriods = new HashMap(); - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - dumpPeriods.put(c1, HOURLY); - - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - dumpPeriods.put(c2, null); - - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - storageTables.put(c2, s2); - - client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - } - - private void createCyclicDim1(CubeMetastoreClient client) throws Exception { - String dimName = "cycleDim1"; - - Set dimAttrs = new HashSet(); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("id", "int", "code"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("name", "string", "name"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("cyleDim2Id", "string", "link to cyclic dim 2"))); - Map dimProps = new HashMap(); - dimProps.put(MetastoreUtil.getDimTimedDimensionKey(dimName), TestCubeMetastoreClient.getDatePartitionKey()); - Set joinchains = new HashSet() { - { - add(new JoinChain("cycledim2chain", "cycledim2chain", "cycledim2chain") { - { - addPath(new ArrayList() { - { - add(new TableReference("cycledim1", "cyleDim2Id")); - add(new TableReference("cycleDim2", "id")); - } - }); - } - }); - } - }; - Dimension cycleDim1 = new Dimension(dimName, dimAttrs, null, joinchains, dimProps, 0L); - client.createDimension(cycleDim1); - - String dimTblName = "cycleDim1Tbl"; - - List dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "code")); - dimColumns.add(new FieldSchema("name", "string", "field1")); - dimColumns.add(new FieldSchema("cyleDim2Id", "string", "link to cyclic dim 2")); - - - Map dumpPeriods = new HashMap(); - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - dumpPeriods.put(c1, HOURLY); - - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - dumpPeriods.put(c2, null); - - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - storageTables.put(c2, s2); - - client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - } - - private void createCyclicDim2(CubeMetastoreClient client) throws Exception { - String dimName = "cycleDim2"; - - Set dimAttrs = new HashSet(); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("id", "int", "code"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("name", "string", "name"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("cyleDim1Id", "string", "link to cyclic dim 1"))); - Map dimProps = new HashMap(); - dimProps.put(MetastoreUtil.getDimTimedDimensionKey(dimName), TestCubeMetastoreClient.getDatePartitionKey()); - Set joinchains = new HashSet() { - { - add(new JoinChain("cycledim1chain", "cycledim1chain", "cycledim1chain") { - { - addPath(new ArrayList() { - { - add(new TableReference("cycledim2", "cyleDim1Id")); - add(new TableReference("cycleDim1", "id")); - } - }); - } - }); - } - }; - Dimension cycleDim2 = new Dimension(dimName, dimAttrs, null, joinchains, dimProps, 0L); - client.createDimension(cycleDim2); - - String dimTblName = "cycleDim2Tbl"; - List dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "code")); - dimColumns.add(new FieldSchema("name", "string", "field1")); - dimColumns.add(new FieldSchema("cyleDim1Id", "string", "link to cyclic dim 1")); - - Map> dimensionReferences = new HashMap>(); - dimensionReferences.put("cyleDim1Id", Arrays.asList(new TableReference("cycleDim1", "id"))); - - Map dumpPeriods = new HashMap(); - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - dumpPeriods.put(c1, HOURLY); - - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - dumpPeriods.put(c2, null); - - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - storageTables.put(c2, s2); - - client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - } - - private void createZiptable(CubeMetastoreClient client) throws Exception { - String dimName = "zipdim"; - - Set dimAttrs = new HashSet(); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("code", "int", "code"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("f1", "string", "name"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("f2", "string", "name"))); - Map dimProps = new HashMap(); - dimProps.put(MetastoreUtil.getDimTimedDimensionKey(dimName), TestCubeMetastoreClient.getDatePartitionKey()); - Dimension zipDim = new Dimension(dimName, dimAttrs, dimProps, 0L); - client.createDimension(zipDim); - - String dimTblName = "ziptable"; - List dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("code", "int", "code")); - dimColumns.add(new FieldSchema("f1", "string", "field1")); - dimColumns.add(new FieldSchema("f2", "string", "field2")); - - Map dumpPeriods = new HashMap(); - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - dumpPeriods.put(c1, HOURLY); - - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - - client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - } - - private void createUnReachabletable(CubeMetastoreClient client) throws Exception { - String dimName = "unreachableDim"; - - Set dimAttrs = new HashSet(); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("id", "int", "code"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("name", "int", "code"))); - Map dimProps = new HashMap(); - dimProps.put(MetastoreUtil.getDimTimedDimensionKey(dimName), TestCubeMetastoreClient.getDatePartitionKey()); - Dimension urDim = new Dimension(dimName, dimAttrs, dimProps, 0L); - client.createDimension(urDim); - - String dimTblName = "unreachableDimTable"; - List dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "code")); - dimColumns.add(new FieldSchema("name", "string", "field1")); - - Map dumpPeriods = new HashMap(); - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - dumpPeriods.put(c1, HOURLY); - - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - - client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - } - - private void createCountryTable(CubeMetastoreClient client) throws Exception { - String dimName = "countrydim"; - - Set dimAttrs = new HashSet(); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("id", "int", "code"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("name", "string", "name"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("captial", "string", "field2"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("region", "string", "region name"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("ambigdim2", "string", "used in testColumnAmbiguity"))); - Map dimProps = new HashMap(); - dimProps.put(MetastoreUtil.getDimTimedDimensionKey(dimName), TestCubeMetastoreClient.getDatePartitionKey()); - Dimension countryDim = new Dimension(dimName, dimAttrs, dimProps, 0L); - client.createDimension(countryDim); - - String dimTblName = "countrytable"; - List dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "code")); - dimColumns.add(new FieldSchema("name", "string", "field1")); - dimColumns.add(new FieldSchema("capital", "string", "field2")); - dimColumns.add(new FieldSchema("region", "string", "region name")); - dimColumns.add(new FieldSchema("ambigdim2", "string", "used in" + " testColumnAmbiguity")); - - - Map dumpPeriods = new HashMap(); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - dumpPeriods.put(c1, null); - - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - - client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - dimTblName = "countrytable_partitioned"; - - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - ArrayList partCols = Lists.newArrayList(); - partCols.add(dimColumns.remove(dimColumns.size() - 2)); - s2.setPartCols(partCols); - dumpPeriods.clear(); - dumpPeriods.put(c3, HOURLY); - storageTables.clear(); - storageTables.put(c3, s2); - dimProps.put(MetastoreUtil.getDimTablePartsKey(dimTblName), partCols.get(0).getName()); - client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - } - - private void createStateTable(CubeMetastoreClient client) throws Exception { - String dimName = "statedim"; - - Set dimAttrs = new HashSet(); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("id", "int", "code"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("name", "string", "name"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("capital", "string", "field2"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("countryid", "string", "link to country table"))); - Map dimProps = new HashMap(); - dimProps.put(MetastoreUtil.getDimTimedDimensionKey(dimName), TestCubeMetastoreClient.getDatePartitionKey()); - Set joinchains = new HashSet() { - { - add(new JoinChain("countrychain", "countrychain", "countrychain") { - { - addPath(new ArrayList() { - { - add(new TableReference("statedim", "countryid")); - add(new TableReference("country", "id")); - } - }); - } - }); - } - }; - Dimension stateDim = new Dimension(dimName, dimAttrs, null, joinchains, dimProps, 0L); - client.createDimension(stateDim); - - String dimTblName = "statetable"; - List dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "code")); - dimColumns.add(new FieldSchema("name", "string", "field1")); - dimColumns.add(new FieldSchema("capital", "string", "field2")); - dimColumns.add(new FieldSchema("countryid", "string", "region name")); - - Map dumpPeriods = new HashMap(); - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - dumpPeriods.put(c1, HOURLY); - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - - client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - - // In this, country id will be a partition - dimTblName = "statetable_partitioned"; - - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - partCols.add(dimColumns.remove(dimColumns.size() - 1)); - s2.setPartCols(partCols); - s2.setTimePartCols(timePartCols); - dumpPeriods.clear(); - dumpPeriods.put(c3, HOURLY); - storageTables.clear(); - storageTables.put(c3, s2); - dimProps.put(MetastoreUtil.getDimTablePartsKey(dimTblName), partCols.get(1).getName()); - client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - } - - private void createUserTable(CubeMetastoreClient client) throws Exception { - String dimName = "userdim"; - - Set dimAttrs = new HashSet(); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("id", "int", "id"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("name", "string", "name"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("age", "string", "age"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("gender", "string", "gender"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("gender", "string", "gender"))); - - Map dimProps = new HashMap(); - dimProps.put(MetastoreUtil.getDimTimedDimensionKey(dimName), TestCubeMetastoreClient.getDatePartitionKey()); - Set joinChains = new HashSet(); - joinChains.add(new JoinChain("userSports", "user-sports", "user sports") { - { - addPath(new ArrayList() { - { - add(new TableReference("userdim", "id")); - add(new TableReference("user_interests", "user_id", true)); - add(new TableReference("user_interests", "sport_id")); - add(new TableReference("sports", "id")); - } - }); - } - }); - Dimension userDim = new Dimension(dimName, dimAttrs, null, joinChains, dimProps, 0L); - client.createDimension(userDim); - - String dimTblName = "usertable"; - List dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "id")); - dimColumns.add(new FieldSchema("name", "string", "name")); - dimColumns.add(new FieldSchema("age", "string", "age")); - dimColumns.add(new FieldSchema("gender", "string", "gender")); - dimColumns.add(new FieldSchema("user_id_added_in_past", "int", "user_id_added_in_past")); - dimColumns.add(new FieldSchema("user_id_added_far_future", "int", "user_id_added_far_future")); - - Map dumpPeriods = new HashMap(); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - dumpPeriods.put(c1, null); - - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s2.setPartCols(partCols); - s2.setTimePartCols(timePartCols); - dumpPeriods.put(c2, HOURLY); - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - storageTables.put(c2, s2); - - client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - } - - private void createUserInterests(CubeMetastoreClient client) throws Exception { - String dimName = "user_interests"; - - Set dimAttrs = new HashSet(); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("id", "int", "id"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("user_id", "int", "user id"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("sport_id", "int", "sport id"))); - Map dimProps = new HashMap(); - dimProps.put(MetastoreUtil.getDimTimedDimensionKey(dimName), TestCubeMetastoreClient.getDatePartitionKey()); - Dimension interestDim = new Dimension(dimName, dimAttrs, dimProps, 0L); - client.createDimension(interestDim); - - String dimTblName = "user_interests_tbl"; - List dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "id")); - dimColumns.add(new FieldSchema("user_id", "int", "user id")); - dimColumns.add(new FieldSchema("sport_id", "int", "sport id")); - - Map dumpPeriods = new HashMap(); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - dumpPeriods.put(c1, null); - - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s2.setPartCols(partCols); - s2.setTimePartCols(timePartCols); - dumpPeriods.put(c2, HOURLY); - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - storageTables.put(c2, s2); - client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - } - - private void createSports(CubeMetastoreClient client) throws Exception { - String dimName = "sports"; - - Set dimAttrs = new HashSet(); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("id", "int", "id"))); - dimAttrs.add(new BaseDimAttribute(new FieldSchema("name", "string", "name"))); - Map dimProps = new HashMap(); - dimProps.put(MetastoreUtil.getDimTimedDimensionKey(dimName), TestCubeMetastoreClient.getDatePartitionKey()); - Dimension interestDim = new Dimension(dimName, dimAttrs, dimProps, 0L); - client.createDimension(interestDim); - - String dimTblName = "sports_tbl"; - List dimColumns = new ArrayList(); - dimColumns.add(new FieldSchema("id", "int", "id")); - dimColumns.add(new FieldSchema("name", "string", "name")); - - Map dumpPeriods = new HashMap(); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - dumpPeriods.put(c1, null); - - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s2.setPartCols(partCols); - s2.setTimePartCols(timePartCols); - dumpPeriods.put(c2, HOURLY); - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - storageTables.put(c2, s2); - - client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables); - } public void createSources(HiveConf conf, String dbName) throws Exception { try { @@ -2931,149 +726,129 @@ public void createSources(HiveConf conf, String dbName) throws Exception { Hive.get(conf).createDatabase(database); SessionState.get().setCurrentDatabase(dbName); CubeMetastoreClient client = CubeMetastoreClient.getInstance(conf); - client.createStorage(new HDFSStorage(c0)); - client.createStorage(new HDFSStorage(c1)); - client.createStorage(new HDFSStorage(c2)); - client.createStorage(new HDFSStorage(c3)); - client.createStorage(new HDFSStorage(c4)); - client.createStorage(new HDFSStorage(c5)); - client.createStorage(new HDFSStorage(c99)); - createCube(client); - createBaseAndDerivedCubes(client); - createCubeFact(client); - createCubeContinuousFact(client); - createCubeCheapFact(client); + createFromXML(client); + assertTestFactTimelineClass(client); + createCubeCheapFactPartitions(client); // commenting this as the week date format throws IllegalPatternException // createCubeFactWeekly(client); - createCubeFactOnlyHourly(client); - createCubeFactOnlyHourlyRaw(client); - - createCityTable(client); - // For join resolver test - createTestDim2(client); - createTestDim3(client); - createTestDim4(client); - createTimeDims(client); - - // For join resolver cyclic links in dimension tables - createCyclicDim1(client); - createCyclicDim2(client); - - createCubeFactMonthly(client); - createZiptable(client); - createCountryTable(client); - createStateTable(client); - createCubeFactsWithValidColumns(client); - createUnReachabletable(client); - createUserTable(client); - createSports(client); - createUserInterests(client); + createTestFact2Partitions(client); + createTestFact2RawPartitions(client); + createBaseCubeFactPartitions(client); + createSummaryPartitions(client); +// dump(client); } catch (Exception exc) { log.error("Exception while creating sources.", exc); throw exc; } } + StrSubstitutor substitutor = new StrSubstitutor(new StrLookup() { + @Override + public String lookup(String s) { + try { + return JAXBUtils.getXMLGregorianCalendar(DateUtil.resolveDate(s, NOW)).toString(); + } catch (LensException e) { + throw new RuntimeException(e); + } + } + }, "$gregorian{", "}", '$'); + StrSubstitutor substitutor2 = new StrSubstitutor(new StrLookup() { + @Override + public String lookup(String s) { + try { + return DateUtil.relativeToAbsolute(s, NOW); + } catch (LensException e) { + throw new RuntimeException(e); + } + } + }, "$absolute{", "}", '$'); + private void createFromXML(CubeMetastoreClient client) { + SchemaTraverser.SchemaEntityProcessor processor = (file, aClass) -> { + try { + BufferedReader br = new BufferedReader(new FileReader(file)); + String replaced = br.lines().map(s->substitutor2.replace(substitutor.replace(s))) + .collect(Collectors.joining("\n")); + StringReader sr = new StringReader(replaced); + client.createEntity(LensJAXBContext.unmarshall(sr)); + } catch (LensException | JAXBException | IOException e) { + throw new RuntimeException(e); + } + }; + new SchemaTraverser(new File(getClass().getResource("/schema").getFile()), processor).run(); + } + + private void dump(CubeMetastoreClient client) throws LensException, IOException { +// for (CubeInterface cubeInterface : client.getAllCubes()) { +// String path = getClass().getResource("/schema/cubes/" + ((cubeInterface instanceof Cube) ? "base" +// : "derived")).getPath() + "/" + cubeInterface.getName() + ".xml"; +// try(BufferedWriter bw = new BufferedWriter(new FileWriter(path))) { +// bw.write(ToXMLString.toString(JAXBUtils.xCubeFromHiveCube(cubeInterface))); +// } +// } + for (CubeFactTable cubeFactTable : client.getAllFacts()) { + try(BufferedWriter bw = new BufferedWriter(new FileWriter(getClass() + .getResource("/schema/facts").getPath()+"/"+cubeFactTable.getName()+".xml"))) { + bw.write(ToXMLString.toString(client.getXFactTable(cubeFactTable))); + } + } +// for (Dimension dim : client.getAllDimensions()) { +// try(BufferedWriter bw = new BufferedWriter(new FileWriter(getClass() +// .getResource("/schema/dimensions").getPath()+"/"+dim.getName()+".xml"))) { +// bw.write(ToXMLString.toString(JAXBUtils.xdimensionFromDimension(dim))); +// } +// } + for (CubeDimensionTable dim : client.getAllDimensionTables()) { + try(BufferedWriter bw = new BufferedWriter(new FileWriter(getClass() + .getResource("/schema/dimtables").getPath()+"/"+dim.getName()+".xml"))) { + bw.write(ToXMLString.toString(client.getXDimensionTable(dim))); + } + } +// for (Storage storage : client.getAllStorages()) { +// try(BufferedWriter bw = new BufferedWriter(new FileWriter(getClass() +// .getResource("/schema/storages").getPath()+"/"+storage.getName()+".xml"))) { +// bw.write(ToXMLString.toString(JAXBUtils.xstorageFromStorage(storage))); +// } +// } + } public void dropSources(HiveConf conf, String dbName) throws Exception { Hive metastore = Hive.get(conf); metastore.dropDatabase(dbName, true, true, true); } - private void createCubeFactsWithValidColumns(CubeMetastoreClient client) throws Exception { + private void createSummaryPartitions(CubeMetastoreClient client) throws Exception { String factName = "summary1"; - StringBuilder commonCols = new StringBuilder(); - List factColumns = new ArrayList(cubeMeasures.size()); - for (CubeMeasure measure : cubeMeasures) { - factColumns.add(measure.getColumn()); - commonCols.append(measure.getName()); - commonCols.append(","); - } - - // add dimensions of the cube - factColumns.add(new FieldSchema("dim1", "string", "dim1")); - factColumns.add(new FieldSchema("dim2", "string", "dim2")); - factColumns.add(new FieldSchema("testdim3id", "string", "dim2")); - factColumns.add(new FieldSchema("dim2big", "string", "dim2")); - factColumns.add(new FieldSchema("zipcode", "int", "zip")); - factColumns.add(new FieldSchema("cityid", "int", "city id")); - Set updates = new HashSet(); - updates.add(MINUTELY); - updates.add(HOURLY); - updates.add(DAILY); - - ArrayList partCols = new ArrayList(); - List timePartCols = new ArrayList(); - partCols.add(TestCubeMetastoreClient.getDatePartition()); - timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey()); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - - ArrayList partCols2 = new ArrayList(); - List timePartCols2 = new ArrayList(); - partCols2.add(new FieldSchema("pt", "string", "p time")); - partCols2.add(new FieldSchema("it", "string", "i time")); - partCols2.add(new FieldSchema("et", "string", "e time")); - timePartCols2.add("pt"); - timePartCols2.add("it"); - timePartCols2.add("et"); - StorageTableDesc s2 = new StorageTableDesc(); - s2.setInputFormat(TextInputFormat.class.getCanonicalName()); - s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s2.setPartCols(partCols2); - s2.setTimePartCols(timePartCols2); - - Map> storageUpdatePeriods = new HashMap>(); - storageUpdatePeriods.put(c1, updates); - storageUpdatePeriods.put(c2, updates); - - Map storageTables = new HashMap(); - storageTables.put(c1, s1); - storageTables.put(c2, s2); - - // create cube fact summary1 - Map properties = new HashMap(); - properties.putAll(factValidityProperties); - String validColumns = commonCols.toString() + ",dim1,testdim3id"; - properties.put(MetastoreUtil.getValidColumnsKey(factName), validColumns); - CubeFactTable fact1 = - new CubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageUpdatePeriods, 10L, properties); - client.createCubeTable(fact1, storageTables); + CubeFactTable fact1 = client.getFactTable(factName); createPIEParts(client, fact1, c2); - // create summary2 - same schema, different valid columns factName = "summary2"; - validColumns = commonCols.toString() + ",dim1,dim2"; - properties.put(MetastoreUtil.getValidColumnsKey(factName), validColumns); - CubeFactTable fact2 = - new CubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageUpdatePeriods, 20L, properties); - client.createCubeTable(fact2, storageTables); + CubeFactTable fact2 = client.getFactTable(factName); createPIEParts(client, fact2, c2); factName = "summary3"; - validColumns = commonCols.toString() + ",dim1,dim2,cityid,stateid"; - properties.put(MetastoreUtil.getValidColumnsKey(factName), validColumns); - CubeFactTable fact3 = - new CubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageUpdatePeriods, 30L, properties); - client.createCubeTable(fact3, storageTables); + CubeFactTable fact3 = client.getFactTable(factName); createPIEParts(client, fact3, c2); - // create summary4 only on c2 - storageUpdatePeriods = new HashMap>(); - storageUpdatePeriods.put(c2, updates); - storageTables = new HashMap(); - storageTables.put(c2, s2); factName = "summary4"; - validColumns = commonCols.toString() + ",dim1,dim2big1,dim2big2,cityid"; - properties.put(MetastoreUtil.getValidColumnsKey(factName), validColumns); - CubeFactTable fact4 = - new CubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageUpdatePeriods, 15L, properties); - client.createCubeTable(fact4, storageTables); + CubeFactTable fact4 = client.getFactTable(factName); createPIEParts(client, fact4, c2); } + private void createBaseCubeFactPartitions(CubeMetastoreClient client) throws HiveException, LensException { + String factName = "testFact5_RAW_BASE"; + CubeFactTable fact = client.getFactTable(factName); + // Add all hourly partitions for two days + Calendar cal = Calendar.getInstance(); + cal.setTime(TWODAYS_BACK); + Date temp = cal.getTime(); + while (!(temp.after(NOW))) { + Map timeParts = new HashMap(); + timeParts.put("dt", temp); + StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); + client.addPartition(sPartSpec, c1, CubeTableType.FACT); + cal.add(HOUR_OF_DAY, 1); + temp = cal.getTime(); + } + } private void createPIEParts(CubeMetastoreClient client, CubeFactTable fact, String storageName) throws Exception { diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java index 1a5bd0d1f..44bf51237 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestAggregateResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestAggregateResolver.java index dd0b6dc89..3e08740bf 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestAggregateResolver.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestAggregateResolver.java @@ -97,42 +97,47 @@ public void testAggregateResolver() throws Exception { String q11 = "SELECT cityid from testCube where " + TWO_DAYS_RANGE + " having (testCube.msr2 > 100)"; String expectedq1 = - getExpectedQuery(cubeName, "SELECT testcube.cityid, sum(testCube.msr2) from ", null, + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, sum(testCube.msr2) as `msr2` from ", null, "group by testcube.cityid", getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); String expectedq2 = - getExpectedQuery(cubeName, "SELECT testcube.cityid, sum(testCube.msr2) * max(testCube.msr3) from ", null, + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, sum(testCube.msr2) * max(testCube.msr3) " + + "as `testCube.msr2 * testCube.msr3` from ", null, "group by testcube.cityid", getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); String expectedq3 = - getExpectedQuery(cubeName, "SELECT testcube.cityid, sum(testCube.msr2) from ", null, - "group by testcube.cityid", getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, sum(testCube.msr2) as `sum(testCube.msr2)` " + + "from ", null, "group by testcube.cityid", + getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); String expectedq4 = - getExpectedQuery(cubeName, "SELECT testcube.cityid, sum(testCube.msr2) from ", null, - "group by testcube.cityid having" + " sum(testCube.msr2) > 100", + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, sum(testCube.msr2) as `sum(testCube.msr2)` " + + "from ", null, "group by testcube.cityid having" + " sum(testCube.msr2) > 100", getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); String expectedq5 = - getExpectedQuery(cubeName, "SELECT testcube.cityid, sum(testCube.msr2) from ", null, + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, sum(testCube.msr2) as `msr2` from ", null, "group by testcube.cityid having" + " sum(testCube.msr2) + max(testCube.msr3) > 100", getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); String expectedq6 = - getExpectedQuery(cubeName, "SELECT testcube.cityid, sum(testCube.msr2) from ", null, + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, sum(testCube.msr2) as `msr2`from ", null, "group by testcube.cityid having" + " sum(testCube.msr2) > 100 and sum(testCube.msr2) < 1000", getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); String expectedq7 = - getExpectedQuery(cubeName, "SELECT testcube.cityid, sum(testCube.msr2) from ", null, - "group by testcube.cityid having" + " sum(testCube.msr2) > 100 OR (sum(testCube.msr2) < 100 AND" + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, sum(testCube.msr2) as `sum(testCube.msr2)` " + + "from ", null, "group by testcube.cityid having" + + " sum(testCube.msr2) > 100 OR (sum(testCube.msr2) < 100 AND" + " max(testcube.msr3) > 1000)", getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); String expectedq8 = - getExpectedQuery(cubeName, "SELECT testcube.cityid, sum(testCube.msr2) * max(testCube.msr3) from ", null, + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, sum(testCube.msr2) * max(testCube.msr3) " + + "as `sum(testCube.msr2) * max(testCube.msr3)` from ", null, "group by testcube.cityid", getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); String expectedq9 = getExpectedQuery(cubeName, "SELECT testcube.cityid as `c1`, max(testCube.msr3) as `m3` from ", "c1 > 100", "group by testcube.cityid" + " having sum(testCube.msr2) < 100 AND (m3 > 1000)", getWhereForDailyAndHourly2days(cubeName, "c2_testfact")); String expectedq10 = - getExpectedQuery(cubeName, "SELECT testcube.cityid, round(sum(testCube.msr2)) from ", null, + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, round(sum(testCube.msr2)) " + + "as `round(testCube.msr2)` from ", null, "group by testcube.cityid", getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); String expectedq11 = - getExpectedQuery(cubeName, "SELECT testcube.cityid from ", null, + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`from ", null, "group by testcube.cityid" + "having sum(testCube.msr2) > 100", getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); String[] tests = { @@ -162,7 +167,8 @@ public void testDimOnlyDistinctQuery() throws ParseException, LensException { String query1 = "SELECT testcube.cityid,testcube.zipcode,testcube.stateid from testCube where " + TWO_DAYS_RANGE; String hQL1 = rewrite(query1, conf); String expectedQL1 = - getExpectedQuery(cubeName, "SELECT distinct testcube.cityid, testcube.zipcode, testcube.stateid" + " from ", null, + getExpectedQuery(cubeName, "SELECT distinct testcube.cityid as `cityid`, testcube.zipcode as `zipcode`, " + + "testcube.stateid as `stateid`" + " from ", null, null, getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); compareQueries(hQL1, expectedQL1); @@ -170,7 +176,8 @@ public void testDimOnlyDistinctQuery() throws ParseException, LensException { String query2 = "SELECT count (distinct testcube.cityid) from testcube where " + TWO_DAYS_RANGE; String hQL2 = rewrite(query2, conf); String expectedQL2 = - getExpectedQuery(cubeName, "SELECT count (distinct testcube.cityid)" + " from ", null, null, + getExpectedQuery(cubeName, "SELECT count (distinct testcube.cityid) as `count(distinct testcube.cityid)`" + + " from ", null, null, getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); compareQueries(hQL2, expectedQL2); @@ -178,7 +185,8 @@ public void testDimOnlyDistinctQuery() throws ParseException, LensException { String query3 = "SELECT testcube.cityid, count(distinct testcube.stateid) from testcube where " + TWO_DAYS_RANGE; String hQL3 = rewrite(query3, conf); String expectedQL3 = - getExpectedQuery(cubeName, "SELECT testcube.cityid, count(distinct testcube.stateid)" + " from ", null, + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, count(distinct testcube.stateid) " + + "as `count(distinct testcube.stateid)` " + " from ", null, "group by testcube.cityid", getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); compareQueries(hQL3, expectedQL3); @@ -186,7 +194,7 @@ public void testDimOnlyDistinctQuery() throws ParseException, LensException { String query4 = "SELECT count(testcube.stateid) from testcube where " + TWO_DAYS_RANGE; String hQL4 = rewrite(query4, conf); String expectedQL4 = - getExpectedQuery(cubeName, "SELECT count(testcube.stateid)" + " from ", null, + getExpectedQuery(cubeName, "SELECT count(testcube.stateid) as `count(testcube.stateid)`" + " from ", null, null, getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); compareQueries(hQL4, expectedQL4); @@ -195,7 +203,7 @@ public void testDimOnlyDistinctQuery() throws ParseException, LensException { String query5 = "SELECT testcube.stateid from testcube where " + TWO_DAYS_RANGE; String hQL5 = rewrite(query5, conf); String expectedQL5 = - getExpectedQuery(cubeName, "SELECT testcube.stateid" + " from ", null, + getExpectedQuery(cubeName, "SELECT testcube.stateid as `stateid`" + " from ", null, null, getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); compareQueries(hQL5, expectedQL5); @@ -212,11 +220,13 @@ public void testAggregateResolverOff() throws ParseException, LensException { String query = "SELECT cityid, testCube.msr2 FROM testCube WHERE " + TWO_DAYS_RANGE; CubeQueryContext cubeql = rewriteCtx(query, conf2); String hQL = cubeql.toHQL(); - Assert.assertEquals(1, cubeql.getCandidateFacts().size()); - CandidateFact candidateFact = cubeql.getCandidateFacts().iterator().next(); - Assert.assertEquals("testFact2_raw".toLowerCase(), candidateFact.fact.getName().toLowerCase()); + Assert.assertEquals(1, cubeql.getCandidates().size()); + Candidate candidate = cubeql.getCandidates().iterator().next(); + Assert.assertTrue(candidate instanceof StorageCandidate); + Assert.assertEquals("c1_testFact2_raw".toLowerCase(), + ((StorageCandidate) candidate).getStorageTable().toLowerCase()); String expectedQL = - getExpectedQuery(cubeName, "SELECT testcube.cityid, testCube.msr2 from ", null, null, + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, testCube.msr2 as `msr2` from ", null, null, getWhereForHourly2days("c1_testfact2_raw")); compareQueries(hQL, expectedQL); conf2.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); @@ -230,14 +240,14 @@ private void aggregateFactSelectionTests(Configuration conf) throws ParseExcepti CubeQueryContext cubeql = rewriteCtx(query, conf); String hQL = cubeql.toHQL(); String expectedQL = - getExpectedQuery(cubeName, "SELECT count(distinct testcube.cityid) from ", null, null, - getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); + getExpectedQuery(cubeName, "SELECT count(distinct testcube.cityid) as `count( distinct cityid)` from ", + null, null, getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); compareQueries(hQL, expectedQL); query = "SELECT distinct cityid from testcube where " + TWO_DAYS_RANGE; hQL = rewrite(query, conf); expectedQL = - getExpectedQuery(cubeName, "SELECT distinct testcube.cityid from ", null, null, + getExpectedQuery(cubeName, "SELECT distinct testcube.cityid as `cityid` from ", null, null, getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); compareQueries(hQL, expectedQL); @@ -247,15 +257,15 @@ private void aggregateFactSelectionTests(Configuration conf) throws ParseExcepti cubeql = rewriteCtx(query, conf); hQL = cubeql.toHQL(); expectedQL = - getExpectedQuery(cubeName, "SELECT testcube.cityid, sum(testCube.msr2) from ", null, - "group by testcube.cityid", getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, sum(testCube.msr2) as `sum(testCube.msr2)` " + + "from ", null, "group by testcube.cityid", getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); compareQueries(hQL, expectedQL); query = "SELECT cityid, sum(testCube.msr2) m2 FROM testCube WHERE " + TWO_DAYS_RANGE + " order by m2"; cubeql = rewriteCtx(query, conf); hQL = cubeql.toHQL(); expectedQL = - getExpectedQuery(cubeName, "SELECT testcube.cityid, sum(testCube.msr2) as `m2` from ", null, + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, sum(testCube.msr2) as `m2` from ", null, "group by testcube.cityid order by m2 asc", getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); compareQueries(hQL, expectedQL); @@ -263,8 +273,8 @@ private void aggregateFactSelectionTests(Configuration conf) throws ParseExcepti cubeql = rewriteCtx(query, conf); hQL = cubeql.toHQL(); expectedQL = - getExpectedQuery(cubeName, "SELECT testcube.cityid, sum(testCube.msr2) from ", null, - "group by testcube.cityid having max(testcube.msr3) > 100", + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, sum(testCube.msr2) as `sum(testCube.msr2)` " + + "from ", null, "group by testcube.cityid having max(testcube.msr3) > 100", getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); compareQueries(hQL, expectedQL); } @@ -274,153 +284,185 @@ private void rawFactSelectionTests(Configuration conf) throws ParseException, Le String query = "SELECT cityid, avg(testCube.msr2) FROM testCube WHERE " + TWO_DAYS_RANGE; CubeQueryContext cubeql = rewriteCtx(query, conf); String hQL = cubeql.toHQL(); - Assert.assertEquals(1, cubeql.getCandidateFacts().size()); - CandidateFact candidateFact = cubeql.getCandidateFacts().iterator().next(); - Assert.assertEquals("testFact2_raw".toLowerCase(), candidateFact.fact.getName().toLowerCase()); + Assert.assertEquals(1, cubeql.getCandidates().size()); + Candidate candidate = cubeql.getCandidates().iterator().next(); + Assert.assertTrue(candidate instanceof StorageCandidate); + Assert.assertEquals("c1_testFact2_raw".toLowerCase(), ((StorageCandidate) candidate) + .getStorageTable().toLowerCase()); String expectedQL = - getExpectedQuery(cubeName, "SELECT testcube.cityid, avg(testCube.msr2) from ", null, - "group by testcube.cityid", getWhereForHourly2days("c1_testfact2_raw")); + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, avg(testCube.msr2) as `avg(testCube.msr2)` " + + "from ", null, "group by testcube.cityid", getWhereForHourly2days("c1_testfact2_raw")); compareQueries(hQL, expectedQL); // query with measure in a where clause query = "SELECT cityid, sum(testCube.msr2) FROM testCube WHERE testCube.msr1 < 100 and " + TWO_DAYS_RANGE; cubeql = rewriteCtx(query, conf); - Assert.assertEquals(1, cubeql.getCandidateFacts().size()); - candidateFact = cubeql.getCandidateFacts().iterator().next(); - Assert.assertEquals("testFact2_raw".toLowerCase(), candidateFact.fact.getName().toLowerCase()); + Assert.assertEquals(1, cubeql.getCandidates().size()); + candidate = cubeql.getCandidates().iterator().next(); + Assert.assertTrue(candidate instanceof StorageCandidate); + Assert.assertEquals("c1_testFact2_raw".toLowerCase(), ((StorageCandidate) candidate) + .getStorageTable().toLowerCase()); hQL = cubeql.toHQL(); expectedQL = - getExpectedQuery(cubeName, "SELECT testcube.cityid, sum(testCube.msr2) from ", "testcube.msr1 < 100", - "group by testcube.cityid", getWhereForHourly2days("c1_testfact2_raw")); + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, sum(testCube.msr2) as `sum(testCube.msr2)` " + + "from ", "testcube.msr1 < 100", "group by testcube.cityid", getWhereForHourly2days("c1_testfact2_raw")); compareQueries(hQL, expectedQL); query = "SELECT cityid, testCube.msr2 FROM testCube WHERE testCube.msr2 < 100 and " + TWO_DAYS_RANGE; cubeql = rewriteCtx(query, conf); - Assert.assertEquals(1, cubeql.getCandidateFacts().size()); - candidateFact = cubeql.getCandidateFacts().iterator().next(); - Assert.assertEquals("testFact2_raw".toLowerCase(), candidateFact.fact.getName().toLowerCase()); + Assert.assertEquals(1, cubeql.getCandidates().size()); + candidate = cubeql.getCandidates().iterator().next(); + Assert.assertTrue(candidate instanceof StorageCandidate); + Assert.assertEquals("c1_testFact2_raw".toLowerCase(), ((StorageCandidate) candidate) + .getStorageTable().toLowerCase()); hQL = cubeql.toHQL(); expectedQL = - getExpectedQuery(cubeName, "SELECT testcube.cityid, testCube.msr2 from ", "testcube.msr2 < 100", null, - getWhereForHourly2days("c1_testfact2_raw")); + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, testCube.msr2 as `msr2` from ", + "testcube.msr2 < 100", null, getWhereForHourly2days("c1_testfact2_raw")); compareQueries(hQL, expectedQL); query = "SELECT cityid, sum(testCube.msr2) FROM testCube WHERE " + TWO_DAYS_RANGE + " group by testCube.msr1"; cubeql = rewriteCtx(query, conf); - Assert.assertEquals(1, cubeql.getCandidateFacts().size()); - candidateFact = cubeql.getCandidateFacts().iterator().next(); - Assert.assertEquals("testFact2_raw".toLowerCase(), candidateFact.fact.getName().toLowerCase()); + Assert.assertEquals(1, cubeql.getCandidates().size()); + candidate = cubeql.getCandidates().iterator().next(); + Assert.assertTrue(candidate instanceof StorageCandidate); + Assert.assertEquals("c1_testFact2_raw".toLowerCase(), ((StorageCandidate) candidate) + .getStorageTable().toLowerCase()); hQL = cubeql.toHQL(); expectedQL = - getExpectedQuery(cubeName, "SELECT testcube.cityid, sum(testCube.msr2) from ", null, - " group by testCube.msr1", getWhereForHourly2days("c1_testfact2_raw")); + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, sum(testCube.msr2) as `sum(testCube.msr2)` " + + "from ", null, " group by testCube.msr1", getWhereForHourly2days("c1_testfact2_raw")); compareQueries(hQL, expectedQL); query = "SELECT cityid, sum(testCube.msr2) FROM testCube WHERE " + TWO_DAYS_RANGE + " group by testCube.msr3"; cubeql = rewriteCtx(query, conf); - Assert.assertEquals(1, cubeql.getCandidateFacts().size()); - candidateFact = cubeql.getCandidateFacts().iterator().next(); - Assert.assertEquals("testFact2_raw".toLowerCase(), candidateFact.fact.getName().toLowerCase()); + Assert.assertEquals(1, cubeql.getCandidates().size()); + candidate = cubeql.getCandidates().iterator().next(); + Assert.assertTrue(candidate instanceof StorageCandidate); + Assert.assertEquals("c1_testFact2_raw".toLowerCase(), ((StorageCandidate) candidate) + .getStorageTable().toLowerCase()); hQL = cubeql.toHQL(); expectedQL = - getExpectedQuery(cubeName, "SELECT testcube.cityid, sum(testCube.msr2) from ", null, - " group by testCube.msr3", getWhereForHourly2days("c1_testfact2_raw")); + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, sum(testCube.msr2) as `sum(testCube.msr2)` " + + "from ", null, " group by testCube.msr3", getWhereForHourly2days("c1_testfact2_raw")); compareQueries(hQL, expectedQL); query = "SELECT cityid, sum(testCube.msr2) FROM testCube WHERE " + TWO_DAYS_RANGE + " order by testCube.msr1"; cubeql = rewriteCtx(query, conf); - Assert.assertEquals(1, cubeql.getCandidateFacts().size()); - candidateFact = cubeql.getCandidateFacts().iterator().next(); - Assert.assertEquals("testFact2_raw".toLowerCase(), candidateFact.fact.getName().toLowerCase()); + Assert.assertEquals(1, cubeql.getCandidates().size()); + candidate = cubeql.getCandidates().iterator().next(); + Assert.assertTrue(candidate instanceof StorageCandidate); + Assert.assertEquals("c1_testFact2_raw".toLowerCase(), ((StorageCandidate) candidate) + .getStorageTable().toLowerCase()); hQL = cubeql.toHQL(); expectedQL = - getExpectedQuery(cubeName, "SELECT testcube.cityid, sum(testCube.msr2) from ", null, - " group by testcube.cityid order by testcube.msr1 asc", getWhereForHourly2days("c1_testfact2_raw")); + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, sum(testCube.msr2) as `sum(testCube.msr2)` " + + "from ", null, " group by testcube.cityid order by testcube.msr1 asc", + getWhereForHourly2days("c1_testfact2_raw")); compareQueries(hQL, expectedQL); query = "SELECT cityid, sum(testCube.msr2) FROM testCube WHERE " + TWO_DAYS_RANGE + " order by testCube.msr3"; cubeql = rewriteCtx(query, conf); - Assert.assertEquals(1, cubeql.getCandidateFacts().size()); - candidateFact = cubeql.getCandidateFacts().iterator().next(); - Assert.assertEquals("testFact2_raw".toLowerCase(), candidateFact.fact.getName().toLowerCase()); + Assert.assertEquals(1, cubeql.getCandidates().size()); + candidate = cubeql.getCandidates().iterator().next(); + Assert.assertTrue(candidate instanceof StorageCandidate); + Assert.assertEquals("c1_testFact2_raw".toLowerCase(), ((StorageCandidate) candidate) + .getStorageTable().toLowerCase()); hQL = cubeql.toHQL(); expectedQL = - getExpectedQuery(cubeName, "SELECT testcube.cityid, sum(testCube.msr2) from ", null, - " group by testcube.cityid order by testcube.msr3 asc", getWhereForHourly2days("c1_testfact2_raw")); + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, sum(testCube.msr2) as `sum(testCube.msr2)` " + + "from ", null, " group by testcube.cityid order by testcube.msr3 asc", + getWhereForHourly2days("c1_testfact2_raw")); compareQueries(hQL, expectedQL); query = "SELECT distinct cityid, round(testCube.msr2) from testCube where " + TWO_DAYS_RANGE; cubeql = rewriteCtx(query, conf); - Assert.assertEquals(1, cubeql.getCandidateFacts().size()); - candidateFact = cubeql.getCandidateFacts().iterator().next(); - Assert.assertEquals("testFact2_raw".toLowerCase(), candidateFact.fact.getName().toLowerCase()); + Assert.assertEquals(1, cubeql.getCandidates().size()); + candidate = cubeql.getCandidates().iterator().next(); + Assert.assertTrue(candidate instanceof StorageCandidate); + Assert.assertEquals("c1_testFact2_raw".toLowerCase(), ((StorageCandidate) candidate) + .getStorageTable().toLowerCase()); hQL = cubeql.toHQL(); expectedQL = - getExpectedQuery(cubeName, "SELECT distinct testcube.cityid, round(testCube.msr2) from ", null, null, - getWhereForHourly2days("c1_testfact2_raw")); + getExpectedQuery(cubeName, "SELECT distinct testcube.cityid as `cityid`, round(testCube.msr2) " + + "as `round(testCube.msr2)` from ", null, null, getWhereForHourly2days("c1_testfact2_raw")); compareQueries(hQL, expectedQL); query = "SELECT cityid, count(distinct(testCube.msr2)) from testCube where " + TWO_DAYS_RANGE; cubeql = rewriteCtx(query, conf); - Assert.assertEquals(1, cubeql.getCandidateFacts().size()); - candidateFact = cubeql.getCandidateFacts().iterator().next(); - Assert.assertEquals("testFact2_raw".toLowerCase(), candidateFact.fact.getName().toLowerCase()); + Assert.assertEquals(1, cubeql.getCandidates().size()); + candidate = cubeql.getCandidates().iterator().next(); + Assert.assertTrue(candidate instanceof StorageCandidate); + Assert.assertEquals("c1_testFact2_raw".toLowerCase(), ((StorageCandidate) candidate) + .getStorageTable().toLowerCase()); hQL = cubeql.toHQL(); expectedQL = - getExpectedQuery(cubeName, "SELECT testcube.cityid, count(distinct testCube.msr2) from ", null, - "group by testcube.cityid", getWhereForHourly2days("c1_testfact2_raw")); + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, count(distinct testCube.msr2) " + + "as `count(distinct(testCube.msr2))` from ", null, "group by testcube.cityid", + getWhereForHourly2days("c1_testfact2_raw")); compareQueries(hQL, expectedQL); // query with no default aggregate measure query = "SELECT cityid, round(testCube.msr1) from testCube where " + TWO_DAYS_RANGE; cubeql = rewriteCtx(query, conf); - Assert.assertEquals(1, cubeql.getCandidateFacts().size()); - candidateFact = cubeql.getCandidateFacts().iterator().next(); - Assert.assertEquals("testFact2_raw".toLowerCase(), candidateFact.fact.getName().toLowerCase()); + Assert.assertEquals(1, cubeql.getCandidates().size()); + candidate = cubeql.getCandidates().iterator().next(); + Assert.assertTrue(candidate instanceof StorageCandidate); + Assert.assertEquals("c1_testFact2_raw".toLowerCase(), ((StorageCandidate) candidate) + .getStorageTable().toLowerCase()); hQL = cubeql.toHQL(); expectedQL = - getExpectedQuery(cubeName, "SELECT testcube.cityid, round(testCube.msr1) from ", null, null, - getWhereForHourly2days("c1_testfact2_raw")); + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, round(testCube.msr1) as `round(testCube.msr1)` " + + "from ", null, null, getWhereForHourly2days("c1_testfact2_raw")); compareQueries(hQL, expectedQL); query = "SELECT distinct cityid, round(testCube.msr1) from testCube where " + TWO_DAYS_RANGE; cubeql = rewriteCtx(query, conf); - Assert.assertEquals(1, cubeql.getCandidateFacts().size()); - candidateFact = cubeql.getCandidateFacts().iterator().next(); - Assert.assertEquals("testFact2_raw".toLowerCase(), candidateFact.fact.getName().toLowerCase()); + Assert.assertEquals(1, cubeql.getCandidates().size()); + candidate = cubeql.getCandidates().iterator().next(); + Assert.assertTrue(candidate instanceof StorageCandidate); + Assert.assertEquals("c1_testFact2_raw".toLowerCase(), ((StorageCandidate) candidate) + .getStorageTable().toLowerCase()); hQL = cubeql.toHQL(); expectedQL = - getExpectedQuery(cubeName, "SELECT distinct testcube.cityid, round(testCube.msr1) from ", null, null, - getWhereForHourly2days("c1_testfact2_raw")); + getExpectedQuery(cubeName, "SELECT distinct testcube.cityid as `cityid`, round(testCube.msr1) " + + "as `round(testCube.msr1)` from ", null, null, getWhereForHourly2days("c1_testfact2_raw")); compareQueries(hQL, expectedQL); query = "SELECT cityid, count(distinct(testCube.msr1)) from testCube where " + TWO_DAYS_RANGE; cubeql = rewriteCtx(query, conf); - Assert.assertEquals(1, cubeql.getCandidateFacts().size()); - candidateFact = cubeql.getCandidateFacts().iterator().next(); - Assert.assertEquals("testFact2_raw".toLowerCase(), candidateFact.fact.getName().toLowerCase()); + Assert.assertEquals(1, cubeql.getCandidates().size()); + candidate = cubeql.getCandidates().iterator().next(); + Assert.assertTrue(candidate instanceof StorageCandidate); + Assert.assertEquals("c1_testFact2_raw".toLowerCase(), ((StorageCandidate) candidate) + .getStorageTable().toLowerCase()); hQL = cubeql.toHQL(); expectedQL = - getExpectedQuery(cubeName, "SELECT testcube.cityid, count(distinct testCube.msr1) from ", null, - "group by testcube.cityid", getWhereForHourly2days("c1_testfact2_raw")); + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, count(distinct testCube.msr1) " + + "as ` count(distinct testCube.msr1)` from ", null, "group by testcube.cityid", + getWhereForHourly2days("c1_testfact2_raw")); compareQueries(hQL, expectedQL); query = "SELECT cityid, sum(testCube.msr1) from testCube where " + TWO_DAYS_RANGE; cubeql = rewriteCtx(query, conf); - Assert.assertEquals(1, cubeql.getCandidateFacts().size()); - candidateFact = cubeql.getCandidateFacts().iterator().next(); - Assert.assertEquals("testFact2_raw".toLowerCase(), candidateFact.fact.getName().toLowerCase()); + Assert.assertEquals(1, cubeql.getCandidates().size()); + candidate = cubeql.getCandidates().iterator().next(); + Assert.assertTrue(candidate instanceof StorageCandidate); + Assert.assertEquals("c1_testFact2_raw".toLowerCase(), ((StorageCandidate) candidate) + .getStorageTable().toLowerCase()); hQL = cubeql.toHQL(); expectedQL = - getExpectedQuery(cubeName, "SELECT testcube.cityid, sum(testCube.msr1) from ", null, - "group by testcube.cityid", getWhereForHourly2days("c1_testfact2_raw")); + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, sum(testCube.msr1) as `sum(testCube.msr1)` " + + "from ", null, "group by testcube.cityid", + getWhereForHourly2days("c1_testfact2_raw")); compareQueries(hQL, expectedQL); query = "SELECT cityid, sum(testCube.msr2) FROM testCube WHERE " + TWO_DAYS_RANGE + " having max(msr1) > 100"; cubeql = rewriteCtx(query, conf); hQL = cubeql.toHQL(); expectedQL = - getExpectedQuery(cubeName, "SELECT testcube.cityid, sum(testCube.msr2) from ", null, - "group by testcube.cityid having max(testcube.msr1) > 100", getWhereForHourly2days("c1_testfact2_raw")); + getExpectedQuery(cubeName, "SELECT testcube.cityid as `cityid`, sum(testCube.msr2) as `sum(testCube.msr2)` " + + "from ", null, "group by testcube.cityid having max(testcube.msr1) > 100", + getWhereForHourly2days("c1_testfact2_raw")); compareQueries(hQL, expectedQL); } } diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java index dbb8fa302..93dbfc3f3 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java @@ -31,15 +31,12 @@ import static org.testng.Assert.*; import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.lens.api.error.ErrorCollectionFactory; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.error.NoCandidateFactAvailableException; import org.apache.lens.cube.metadata.TimeRange; import org.apache.lens.cube.metadata.UpdatePeriod; -import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.time.DateUtils; @@ -52,7 +49,6 @@ import org.testng.annotations.Test; import com.google.common.base.Splitter; -import com.google.common.collect.Sets; import lombok.Getter; public class TestBaseCubeQueries extends TestQueryRewrite { @@ -71,21 +67,17 @@ public void setupDriver() throws Exception { } @Test - public void testNoCandidateFactAvailableExceptionCompareTo() throws Exception { - //maxCause : COLUMN_NOT_FOUND, Ordinal : 9 - NoCandidateFactAvailableException ne1 =(NoCandidateFactAvailableException) - getLensExceptionInRewrite("select dim1, test_time_dim, msr3, msr13 from basecube where " - + TWO_DAYS_RANGE, conf); - //maxCause : FACT_NOT_AVAILABLE_IN_RANGE, Ordinal : 1 - NoCandidateFactAvailableException ne2 = (NoCandidateFactAvailableException) - getLensExceptionInRewrite("select dim1 from " + cubeName + " where " + LAST_YEAR_RANGE, getConf()); - assertEquals(ne1.compareTo(ne2), 8); + public void testNoUnionCandidateAndNoJoinCandidateErrorWeight() throws Exception { + LensException e1 = getLensExceptionInRewrite( + "select dim1, test_time_dim, msr3, msr13 from basecube where " + TWO_DAYS_RANGE, conf); + LensException e2 = getLensExceptionInRewrite("select dim1 from " + cubeName + + " where " + LAST_YEAR_RANGE, getConf()); + assertEquals(e1.getErrorWeight() - e2.getErrorWeight(), 1); } @Test public void testColumnErrors() throws Exception { LensException e; - e = getLensExceptionInRewrite("select msr11 + msr2 from basecube" + " where " + TWO_DAYS_RANGE, conf); e.buildLensErrorResponse(new ErrorCollectionFactory().createErrorCollection(), null, "testid"); assertEquals(e.getErrorCode(), @@ -96,43 +88,9 @@ public void testColumnErrors() throws Exception { e = getLensExceptionInRewrite("select dim1, test_time_dim, msr3, msr13 from basecube where " + TWO_DAYS_RANGE, conf); assertEquals(e.getErrorCode(), - LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo().getErrorCode()); - NoCandidateFactAvailableException ne = (NoCandidateFactAvailableException) e; - PruneCauses.BriefAndDetailedError pruneCauses = ne.getJsonMessage(); - String regexp = String.format(CandidateTablePruneCause.CandidateTablePruneCode.COLUMN_NOT_FOUND.errorFormat, - "Column Sets: (.*?)", "queriable together"); - Matcher matcher = Pattern.compile(regexp).matcher(pruneCauses.getBrief()); - assertTrue(matcher.matches(), pruneCauses.getBrief()); - assertEquals(matcher.groupCount(), 1); - String columnSetsStr = matcher.group(1); - assertNotEquals(columnSetsStr.indexOf("test_time_dim"), -1, columnSetsStr); - assertNotEquals(columnSetsStr.indexOf("msr3, msr13"), -1); - - /** - * Verifying the BriefAndDetailedError: - * 1. Check for missing columns(COLUMN_NOT_FOUND) - * and check the respective tables for each COLUMN_NOT_FOUND - * 2. check for ELEMENT_IN_SET_PRUNED - * - */ - boolean columnNotFound = false; - List testTimeDimFactTables = Arrays.asList("testfact3_base", "testfact1_raw_base", "testfact3_raw_base", - "testfact5_base", "testfact6_base", "testfact4_raw_base"); - List factTablesForMeasures = Arrays.asList("testfact_deprecated", "testfact2_raw_base", "testfact2_base", - "testfact5_raw_base"); - for (Map.Entry> entry : pruneCauses.getDetails().entrySet()) { - if (entry.getValue().contains(CandidateTablePruneCause.columnNotFound("test_time_dim"))) { - columnNotFound = true; - compareStrings(testTimeDimFactTables, entry); - } - if (entry.getValue().contains(CandidateTablePruneCause.columnNotFound("msr3", "msr13"))) { - columnNotFound = true; - compareStrings(factTablesForMeasures, entry); - } - } - Assert.assertTrue(columnNotFound); - assertEquals(pruneCauses.getDetails().get("testfact1_base"), - Arrays.asList(new CandidateTablePruneCause(CandidateTablePruneCode.ELEMENT_IN_SET_PRUNED))); + LensCubeErrorCode.NO_JOIN_CANDIDATE_AVAILABLE.getLensErrorInfo().getErrorCode()); + assertTrue(e.getMessage().contains("[msr3, msr13]")); + } private void compareStrings(List factTablesList, Map.Entry> entry) { @@ -147,117 +105,116 @@ private void compareStrings(List factTablesList, Map.Entry candidateFacts = new HashSet(); - for (CandidateFact cfact : ctx.getCandidateFacts()) { - candidateFacts.add(cfact.getName().toLowerCase()); + Set storageCandidates = new HashSet(); + Set scSet = CandidateUtil.getStorageCandidates(ctx.getCandidates()); + for (StorageCandidate sc : scSet) { + storageCandidates.add(sc.getStorageTable()); } - Assert.assertTrue(candidateFacts.contains("testfact1_base")); - Assert.assertTrue(candidateFacts.contains("testfact2_base")); + Assert.assertTrue(storageCandidates.contains("c1_testfact1_base")); + Assert.assertTrue(storageCandidates.contains("c1_testfact2_base")); String hqlQuery = ctx.toHQL(); String expected1 = - getExpectedQuery(cubeName, "select sum(basecube.msr12) as `msr12` FROM ", null, + getExpectedQuery(cubeName, "SELECT 0.0 as `alias0`, sum((basecube.msr12)) as `alias1` FROM ", null, null, getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); String expected2 = - getExpectedQuery(cubeName, "select round(sum(basecube.msr2)/1000) as `roundedmsr2` FROM ", null, + getExpectedQuery(cubeName, "SELECT sum((basecube.msr2)) as `alias0`, 0.0 as `alias1` FROM ", null, null, getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); String lower = hqlQuery.toLowerCase(); - assertTrue(lower.startsWith("select mq2.roundedmsr2 roundedmsr2, mq1.msr12 msr12 from ") - || lower.startsWith("select mq1.roundedmsr2 roundedmsr2, mq2.msr12 msr12 from "), hqlQuery); - assertTrue(lower.contains("mq1 full outer join") && lower.endsWith("mq2"), hqlQuery); - assertFalse(lower.contains("mq2 on"), hqlQuery); - assertFalse(lower.contains("<=>"), hqlQuery); + assertTrue(lower.startsWith("select round((sum((basecube.alias0)) / 1000)) as `roundedmsr2`, " + + "sum((basecube.alias1)) as `msr12` from "), hqlQuery); + assertFalse(lower.contains("UNION ALL"), hqlQuery); } @Test public void testMoreThanTwoFactQueryWithNoDimensionsSelected() throws Exception { CubeQueryContext ctx = rewriteCtx("select roundedmsr2, msr14, msr12 from basecube" + " where " + TWO_DAYS_RANGE, conf); - Set candidateFacts = new HashSet(); - for (CandidateFact cfact : ctx.getCandidateFacts()) { - candidateFacts.add(cfact.getName().toLowerCase()); + Set storageCandidates = new HashSet(); + Set scSet = CandidateUtil.getStorageCandidates(ctx.getCandidates()); + for (StorageCandidate sc : scSet) { + storageCandidates.add(sc.getStorageTable()); } - Assert.assertEquals(candidateFacts.size(), 3); - Assert.assertTrue(candidateFacts.contains("testfact1_base")); - Assert.assertTrue(candidateFacts.contains("testfact2_base")); - Assert.assertTrue(candidateFacts.contains("testfact3_base")); + Assert.assertEquals(storageCandidates.size(), 3); + Assert.assertTrue(storageCandidates.contains("c1_testfact1_base")); + Assert.assertTrue(storageCandidates.contains("c1_testfact2_base")); + Assert.assertTrue(storageCandidates.contains("c1_testfact3_base")); String hqlQuery = ctx.toHQL(); - String expected1 = getExpectedQuery(cubeName, "select sum(basecube.msr12) as `msr12` FROM ", null, null, + String expected1 = getExpectedQuery(cubeName, "SELECT 0.0 as `alias0`, 0.0 as `alias1`, " + + "sum((basecube.msr12)) as `alias2` FROM ", null, null, getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); - String expected2 = getExpectedQuery(cubeName, "select round(sum(basecube.msr2)/1000) as `roundedmsr2` FROM ", null, + String expected2 = getExpectedQuery(cubeName, "SELECT sum((basecube.msr2)) as `alias0`, 0.0 as `alias1`, " + + "0.0 as `alias2` FROM ", null, null, getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); - String expected3 = getExpectedQuery(cubeName, "select count((basecube.msr14)) as `msr14` FROM ", null, null, + String expected3 = getExpectedQuery(cubeName, "SELECT 0.0 as `alias0`, count((basecube.msr14)) as `alias1`, " + + "0.0 as `alias2` FROM ", null, null, getWhereForDailyAndHourly2days(cubeName, "C1_testFact3_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); compareContains(expected3, hqlQuery); String lower = hqlQuery.toLowerCase(); - assertTrue(lower.startsWith("select mq1.roundedmsr2 roundedmsr2, mq3.msr14 msr14, mq2.msr12 msr12 from ") || lower - .startsWith("select mq3.roundedmsr2 roundedmsr2, mq1.msr14 msr14, mq2.msr12 msr12 from ") || lower - .startsWith("select mq2.roundedmsr2 roundedmsr2, mq3.msr14 msr14, mq1.msr12 msr12 from ") || lower - .startsWith("select mq3.roundedmsr2 roundedmsr2, mq2.msr14 msr14, mq1.msr12 msr12 from ") || lower - .startsWith("select mq1.roundedmsr2 roundedmsr2, mq2.msr14 msr14, mq3.msr12 msr12 from ") || lower - .startsWith("select mq2.roundedmsr2 roundedmsr2, mq1.msr14 msr14, mq3.msr12 msr12 from "), hqlQuery); - assertTrue(lower.contains("mq1 full outer join") && lower.endsWith("mq3")); - assertFalse(lower.contains("mq3 on"), hqlQuery); - assertFalse(lower.contains("mq2 on"), hqlQuery); - assertFalse(lower.contains("<=>"), hqlQuery); + assertTrue(lower.startsWith("select round((sum((basecube.alias0)) / 1000)) as `roundedmsr2`, " + + "count((basecube.alias1)) as `msr14`, sum((basecube.alias2)) as `msr12` from"), hqlQuery); + assertTrue(lower.contains("union all")); } @Test public void testMultiFactQueryWithSingleCommonDimension() throws Exception { String hqlQuery = rewrite("select dim1, roundedmsr2, msr12 from basecube" + " where " + TWO_DAYS_RANGE, conf); String expected1 = - getExpectedQuery(cubeName, "select basecube.dim1 as `dim1`, sum(basecube.msr12) as `msr12` FROM ", null, - " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); - String expected2 = getExpectedQuery(cubeName, - "select basecube.dim1 as `dim1`, round(sum(basecube.msr2)/1000) as `roundedmsr2` FROM ", null, - " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); + getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum((basecube.msr2)) as `alias1`, " + + "0.0 as `alias2` FROM ", null, " group by basecube.dim1", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); + String expected2 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, " + + "sum((basecube.msr12)) as `alias2` FROM ", null, " group by basecube.dim1", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); String lower = hqlQuery.toLowerCase(); assertTrue( - lower.startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, mq2.roundedmsr2 roundedmsr2, mq1.msr12 msr12 from ") - || lower.startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, mq1.roundedmsr2 roundedmsr2, mq2.msr12 msr12" - + " from "), hqlQuery); - - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.dim1 <=> mq2.dim1"), + lower.startsWith("select (basecube.alias0) as `dim1`, round((sum((basecube.alias1)) / 1000)) as `roundedmsr2`, " + + "sum((basecube.alias2)) as `msr12` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @@ -266,21 +223,18 @@ public void testMultiFactQueryWithSingleCommonDimensionWithLightestFactFirst() t Configuration tConf = new Configuration(conf); tConf.setBoolean(CubeQueryConfUtil.LIGHTEST_FACT_FIRST, true); String hqlQuery = rewrite("select dim1, roundedmsr2, msr12 from basecube" + " where " + TWO_DAYS_RANGE, tConf); - String expected1 = - getExpectedQuery(cubeName, "select basecube.dim1 as `dim1`, sum(basecube.msr12) as `msr12` FROM ", null, - " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); - String expected2 = getExpectedQuery(cubeName, - "select basecube.dim1 as `dim1`, round(sum(basecube.msr2)/1000) as `roundedmsr2` FROM ", null, - " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); + String expected1 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, " + + "sum((basecube.msr12)) as `alias2` FROM ", null, " group by basecube.dim1", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); + String expected2 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum((basecube.msr2)) " + + "as `alias1`, 0.0 as `alias2` FROM ", null, " group by basecube.dim1", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); String lower = hqlQuery.toLowerCase(); - assertTrue( - lower.startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, mq2.roundedmsr2 roundedmsr2, mq1.msr12 msr12 from ") - || lower.startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, mq1.roundedmsr2 roundedmsr2, mq2.msr12 msr12" - + " from "), hqlQuery); - - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.dim1 <=> mq2.dim1"), + assertTrue(lower.startsWith("select (basecube.alias0) as `dim1`, round((sum((basecube.alias1)) / 1000)) " + + "as `roundedmsr2`, sum((basecube.alias2)) as `msr12` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @@ -290,25 +244,19 @@ public void testMultiFactQueryWithExpressionsFromMultipleFacts() throws Exceptio tConf.setBoolean(CubeQueryConfUtil.LIGHTEST_FACT_FIRST, true); String hqlQuery = rewrite("select dim1, roundedmsr2, flooredmsr12 from basecube" + " where " + TWO_DAYS_RANGE, tConf); - String expected1 = - getExpectedQuery(cubeName, "select basecube.dim1 as `dim1`, " - + "floor(sum(( basecube . msr12 ))) as `flooredmsr12` FROM ", null, - " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); - String expected2 = getExpectedQuery(cubeName, - "select basecube.dim1 as `dim1`, round(sum(basecube.msr2)/1000) as `roundedmsr2` FROM ", null, - " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); + String expected1 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, " + + "sum((basecube.msr12)) as `alias2` FROM ", null, " group by basecube.dim1", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); + String expected2 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum((basecube.msr2)) " + + "as `alias1`, 0.0 as `alias2` FROM ", null, " group by basecube.dim1", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); String lower = hqlQuery.toLowerCase(); - assertTrue( - lower.startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, mq2.roundedmsr2 roundedmsr2, " - + "mq1.flooredmsr12 flooredmsr12 from ") - || lower.startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, mq1.roundedmsr2 roundedmsr2, " - + "mq2.flooredmsr12 flooredmsr12" - + " from "), hqlQuery); - - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.dim1 <=> mq2.dim1"), - hqlQuery); + assertTrue(lower.startsWith("select (basecube.alias0) as `dim1`, round((sum((basecube.alias1)) / 1000)) " + + "as `roundedmsr2`, floor(sum((basecube.alias2))) as `flooredmsr12` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), + hqlQuery); } @Test @@ -316,21 +264,20 @@ public void testMultiFactQueryWithSingleCommonDimensionWithColumnsSwapped() thro // columns in select interchanged String hqlQuery = rewrite("select dim1, msr12, roundedmsr2 from basecube" + " where " + TWO_DAYS_RANGE, conf); String expected1 = - getExpectedQuery(cubeName, "select basecube.dim1 as `dim1`, sum(basecube.msr12) as `msr12` FROM ", null, - " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); + getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum((basecube.msr12)) as `alias1`, " + + "0.0 as `alias2` FROM", null, " group by basecube.dim1", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); String expected2 = getExpectedQuery(cubeName, - "select basecube.dim1 as `dim1`, round(sum(basecube.msr2)/1000) as `roundedmsr2` FROM ", null, + "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, sum((basecube.msr2)) as `alias2` FROM ", null, " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); String lower = hqlQuery.toLowerCase(); assertTrue( - lower.startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, mq2.msr12 msr12, mq1.roundedmsr2 roundedmsr2 from ") - || lower.startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, mq1.msr12 msr12, mq2.roundedmsr2 roundedmsr2" - + " from "), hqlQuery); + lower.startsWith("select (basecube.alias0) as `dim1`, sum((basecube.alias1)) as `msr12`, " + + "round((sum((basecube.alias2)) / 1000)) as `roundedmsr2` from"), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.dim1 <=> mq2.dim1"), - hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @Test @@ -339,67 +286,49 @@ public void testMultiFactQueryInvolvingThreeFactTables() throws Exception { String hqlQuery = rewrite("select dim1, d_time, msr12, roundedmsr2, msr13, msr3 from basecube where " + TWO_DAYS_RANGE, conf); String expected1 = - getExpectedQuery(cubeName, "select basecube.dim1 as `dim1`, basecube.d_time as `d_time`, " - + "sum(basecube.msr12) as `msr12` FROM ", null, " group by basecube.dim1", + getExpectedQuery(cubeName, " SELECT (basecube.dim1) as `alias0`, (basecube.d_time) as `alias1`, " + + "sum((basecube.msr12)) as `alias2`, 0.0 as `alias3`, 0.0 as `alias4`, 0.0 as `alias5` FROM ", + null, " group by basecube.dim1, (basecube.d_time)", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); String expected2 = getExpectedQuery( cubeName, - "select basecube.dim1 as `dim1`, basecube.d_time as `d_time`, round(sum(basecube.msr2)/1000) " - + "as `roundedmsr2`, max(basecube.msr3) as `msr3` FROM ", null, - " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); + "SELECT (basecube.dim1) as `alias0`, (basecube.d_time) as `alias1`, 0.0 as `alias2`, " + + "sum((basecube.msr2)) as `alias3`, 0.0 as `alias4`, max((basecube.msr3)) as `alias5` FROM ", null, + " group by basecube.dim1, (basecube.d_time)", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); String expected3 = - getExpectedQuery(cubeName, "select basecube.dim1 as `dim1`, basecube.d_time as `d_time`, " - + "max(basecube.msr13) as `msr13` FROM ", null, - " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "c1_testfact3_base")); + getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, (basecube.d_time) as `alias1`, 0.0 " + + "as `alias2`, 0.0 as `alias3`, max((basecube.msr13)) as `alias4`, 0.0 as `alias5` FROM ", null, + " group by basecube.dim1, (basecube.d_time)", getWhereForDailyAndHourly2days(cubeName, "c1_testfact3_base")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); compareContains(expected3, hqlQuery); assertTrue( hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim1, mq2.dim1, mq3.dim1) dim1, coalesce(mq1.d_time, mq2.d_time, mq3.d_time) d_time, " - + "mq1.msr12 msr12, mq2.roundedmsr2 roundedmsr2, mq3.msr13 msr13, mq2.msr3 msr3 from ") - || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim1, mq2.dim1, mq3.dim1) dim1, coalesce(mq1.d_time, mq2.d_time, mq3.d_time) d_time," - + " mq1.msr12 msr12, mq3.roundedmsr2 roundedmsr2, mq2.msr13 msr13, mq3.msr3 msr3 from ") - || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim1, mq2.dim1, mq3.dim1) dim1, coalesce(mq1.d_time, mq2.d_time, mq3.d_time) d_time," - + " mq2.msr12 msr12, mq1.roundedmsr2 roundedmsr2, mq3.msr13 msr13, mq1.msr3 msr3 from ") - || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim1, mq2.dim1, mq3.dim1) dim1, coalesce(mq1.d_time, mq2.d_time, mq3.d_time) d_time, " - + "mq2.msr12 msr12, mq3.roundedmsr2 roundedmsr2, mq1.msr13 msr13, mq3.msr3 msr3 from ") - || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim1, mq2.dim1, mq3.dim1) dim1, coalesce(mq1.d_time, mq2.d_time, mq3.d_time) d_time," - + " mq3.msr12 msr12, mq1.roundedmsr2 roundedmsr2, mq2.msr13 msr13, mq1.msr3 msr3 from ") - || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim1, mq2.dim1, mq3.dim1) dim1, coalesce(mq1.d_time, mq2.d_time, mq3.d_time) d_time, " - + "mq3.msr12 msr12, mq2.roundedmsr2 roundedmsr2, mq1.msr13 msr13, mq2.msr3 msr3 from "), hqlQuery); - assertTrue(hqlQuery.toLowerCase().contains("mq1 full outer join ") - && hqlQuery.toLowerCase().contains("mq2 on mq1.dim1 <=> mq2.dim1 and mq1.d_time <=> mq2.d_time") - && hqlQuery.toLowerCase().endsWith("mq3 on mq2.dim1 <=> mq3.dim1 and mq2.d_time <=> mq3.d_time"), hqlQuery); + "select (basecube.alias0) as `dim1`, (basecube.alias1) as `d_time`, sum((basecube.alias2)) as `msr12`, " + + "round((sum((basecube.alias3)) / 1000)) as `roundedmsr2`, max((basecube.alias4)) as `msr13`, " + + "max((basecube.alias5)) as `msr3` from "), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0), (basecube.alias1)"), + hqlQuery); } @Test public void testMultiFactQueryWithTwoCommonDimensions() throws Exception { // query two dim attributes String hqlQuery = rewrite("select dim1, dim11, msr12, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, conf); - String expected1 = getExpectedQuery(cubeName, - "select basecube.dim1 as `dim1`, basecube.dim11 as `dim11`, sum(basecube.msr12) as `msr12` FROM ", null, - " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); - String expected2 = getExpectedQuery( - cubeName, - "select basecube.dim1 as `dim1`, basecube.dim11 as `dim11`, round(sum(basecube.msr2)/1000) as `roundedmsr2` " - + "FROM ", null, " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); + String expected1 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, " + + "sum((basecube.msr12)) as `alias2`, 0.0 as `alias3` FROM ", null, " group by basecube.dim1", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); + String expected2 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, " + + "0.0 as `alias2`, sum((basecube.msr2)) as `alias3` FROM ", null, " group by basecube.dim1", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); assertTrue(hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim1, mq2.dim1) dim1, coalesce(mq1.dim11, mq2.dim11) dim11," - + " mq1.msr12 msr12, mq2.roundedmsr2 roundedmsr2 from ") - || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim1, mq2.dim1) dim1, coalesce(mq1.dim11, mq2.dim11) dim11," - + " mq2.msr12 msr12, mq1.roundedmsr2 roundedmsr2 from "), hqlQuery); - - assertTrue(hqlQuery.contains("mq1 full outer join ") - && hqlQuery.endsWith("mq2 on mq1.dim1 <=> mq2.dim1 AND mq1.dim11 <=> mq2.dim11"), hqlQuery); + "select (basecube.alias0) as `dim1`, (basecube.alias1) as `dim11`, sum((basecube.alias2)) as `msr12`, " + + "round((sum((basecube.alias3)) / 1000)) as `roundedmsr2` from"), hqlQuery); + + assertTrue(hqlQuery.contains("UNION ALL") + && hqlQuery.endsWith("GROUP BY (basecube.alias0), (basecube.alias1)"), hqlQuery); } @Test @@ -407,19 +336,18 @@ public void testMultiFactQueryWithNoAggregates() throws Exception { // no aggregates in the query String hqlQuery = rewrite("select dim1, msr11, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, conf); String expected1 = - getExpectedQuery(cubeName, "select basecube.dim1 as `dim1`, basecube.msr11 as `msr11` FROM ", null, null, - getWhereForHourly2days(cubeName, "C1_testfact2_raw_base")); + getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, (basecube.msr11) as `alias1`, " + + "0.0 as `alias2` FROM ", null, null, getWhereForHourly2days(cubeName, "C1_testfact2_raw_base")); String expected2 = getExpectedQuery(cubeName, - "select basecube.dim1 as `dim1`, round(basecube.msr2/1000) as `roundedmsr2` FROM ", null, null, + "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, round(((basecube.msr2) / 1000)) " + + "as `alias2` FROM ", null, null, getWhereForHourly2days(cubeName, "C1_testfact1_raw_base")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); assertTrue(hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim1, mq2.dim1) dim1, mq1.msr11 msr11, mq2.roundedmsr2 roundedmsr2 from ") - || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim1, mq2.dim1) dim1, mq2.msr11 msr11, mq1.roundedmsr2 roundedmsr2 from "), hqlQuery); - - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.dim1 <=> mq2.dim1"), + "select (basecube.alias0) as `dim1`, (basecube.alias1) as `msr11`, " + + "(basecube.alias2) as `roundedmsr2` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("as basecube"), hqlQuery); } @@ -429,18 +357,19 @@ public void testMultiFactQueryWithColumnAliases() throws Exception { String hqlQuery = rewrite("select dim1 d1, msr12 `my msr12`, roundedmsr2 m2 from basecube where " + TWO_DAYS_RANGE, conf); String expected1 = - getExpectedQuery(cubeName, "select basecube.dim1 as `expr1`, sum(basecube.msr12) as `expr2` FROM ", null, - " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); + getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum((basecube.msr12)) as `alias1`, " + + "0.0 as `alias2` FROM ", null, " group by basecube.dim1", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); String expected2 = - getExpectedQuery(cubeName, "select basecube.dim1 as `expr1`, round(sum(basecube.msr2)/1000) as `expr3` FROM ", - null, " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); + getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, sum((basecube.msr2)) " + + "as `alias2` FROM ", null, " group by basecube.dim1", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); assertTrue(hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.expr1, mq2.expr1) `d1`, mq2.expr2 `my msr12`, mq1.expr3 `m2` from ") - || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.expr1, mq2.expr1) `d1`, mq1.expr2 `my msr12`, mq2.expr3 `m2` from "), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.expr1 <=> mq2.expr1"), + "select (basecube.alias0) as `d1`, sum((basecube.alias1)) as `my msr12`, " + + "round((sum((basecube.alias2)) / 1000)) as `m2` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @@ -450,19 +379,19 @@ public void testMultiFactQueryWithColumnAliasesAsFunctions() throws Exception { rewrite("select dim1 d1, msr12 `sum(msr12)`, roundedmsr2 as `round(sum(msr2)/1000)` from basecube where " + TWO_DAYS_RANGE, conf); String expected1 = - getExpectedQuery(cubeName, "select basecube.dim1 as `expr1`, sum(basecube.msr12) as `expr2` FROM ", null, - " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); + getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum((basecube.msr12)) as `alias1`, " + + "0.0 as `alias2` FROM ", null, " group by basecube.dim1", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); String expected2 = - getExpectedQuery(cubeName, "select basecube.dim1 as `expr1`, round(sum(basecube.msr2)/1000) as `expr3` FROM ", - null, " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); + getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, sum((basecube.msr2)) " + + "as `alias2` FROM ", null, " group by basecube.dim1", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); assertTrue(hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.expr1, mq2.expr1) `d1`, mq2.expr2 `sum(msr12)`, mq1.expr3 `round(sum(msr2)/1000)` from ") - || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.expr1, mq2.expr1) `d1`, mq1.expr2 `sum(msr12)`, mq2.expr3 `round(sum(msr2)/1000)` from "), - hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.expr1 <=> mq2.expr1"), + "select (basecube.alias0) as `d1`, sum((basecube.alias1)) as `sum(msr12)`, " + + "round((sum((basecube.alias2)) / 1000)) as `round(sum(msr2)/1000)` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @@ -472,19 +401,19 @@ public void testMultiFactQueryWithAliasAsColumnName() throws Exception { rewrite("select dim1 d1, msr12 `my msr12`, roundedmsr2 as `msr2` from basecube where " + TWO_DAYS_RANGE, conf); String expected1 = - getExpectedQuery(cubeName, "select basecube.dim1 as `expr1`, sum(basecube.msr12) as `expr2` FROM ", null, - " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); + getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum((basecube.msr12)) as `alias1`, " + + "0.0 as `alias2` FROM ", null, " group by basecube.dim1", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); String expected2 = - getExpectedQuery(cubeName, "select basecube.dim1 as `expr1`, round(sum(basecube.msr2)/1000) as `expr3` FROM ", - null, " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); + getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, sum((basecube.msr2)) " + + "as `alias2` FROM ", null, " group by basecube.dim1", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); assertTrue(hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.expr1, mq2.expr1) `d1`, mq2.expr2 `my msr12`, mq1.expr3 `msr2` from ") - || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.expr1, mq2.expr1) `d1`, mq1.expr2 `my msr12`, mq2.expr3 `msr2` from "), - hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.expr1 <=> mq2.expr1"), + "select (basecube.alias0) as `d1`, sum((basecube.alias1)) as `my msr12`," + + " round((sum((basecube.alias2)) / 1000)) as `msr2` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("(basecube.alias0)"), hqlQuery); } @@ -495,19 +424,19 @@ public void testMultiFactQueryWithAliasAsExpressionName() throws Exception { conf); String expected1 = - getExpectedQuery(cubeName, "select basecube.dim1 as `expr1`, sum(basecube.msr12) as `expr2` FROM ", null, - " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); + getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum((basecube.msr12)) as `alias1`, " + + "0.0 as `alias2` FROM", null, " group by basecube.dim1", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); String expected2 = - getExpectedQuery(cubeName, "select basecube.dim1 as `expr1`, round(sum(basecube.msr2)/1000) as `expr3` FROM ", - null, " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); + getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, sum((basecube.msr2)) " + + "as `alias2` FROM", null, + " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); assertTrue(hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.expr1, mq2.expr1) `d1`, mq2.expr2 `my msr12`, mq1.expr3 `roundedmsr2` from ") - || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.expr1, mq2.expr1) `d1`, mq1.expr2 `my msr12`, mq2.expr3 `roundedmsr2` from "), - hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.expr1 <=> mq2.expr1"), + "select (basecube.alias0) as `d1`, sum((basecube.alias1)) as `my msr12`, " + + "round((sum((basecube.alias2)) / 1000)) as `roundedmsr2` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @@ -516,24 +445,23 @@ public void testMultiFactQueryWithExprOnDimsWithoutAliases() throws Exception { String hqlQuery = rewrite("select reverse(dim1), ltrim(dim1), msr12, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, conf); String expected1 = - getExpectedQuery(cubeName, "select reverse(basecube.dim1) as `expr1`, ltrim(basecube.dim1) as `expr2`," - + " sum(basecube.msr12) as `msr12` FROM ", null, - " group by reverse(basecube.dim1), ltrim(basecube.dim1)", + getExpectedQuery(cubeName, "SELECT reverse((basecube.dim1)) as `alias0`, ltrim((basecube.dim1)) as `alias1`, " + + "sum((basecube.msr12)) as `alias2`, 0.0 as `alias3` FROM ", null, + " group by reverse(basecube.dim1), ltrim(basecube.dim1)", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); String expected2 = - getExpectedQuery(cubeName, "select reverse(basecube.dim1) as `expr1`, ltrim(basecube.dim1) as `expr2`," - + " round(sum(basecube.msr2)/1000) as `roundedmsr2` FROM ", null, + getExpectedQuery(cubeName, "SELECT reverse((basecube.dim1)) as `alias0`, ltrim((basecube.dim1)) as `alias1`, " + + "0.0 as `alias2`, sum((basecube.msr2)) as `alias3` FROM ", null, " group by reverse(basecube.dim1), ltrim(basecube.dim1)", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); - assertTrue(hqlQuery.toLowerCase().startsWith("select coalesce(mq1.expr1, mq2.expr1) `reverse(dim1)`," - + " coalesce(mq1.expr2, mq2.expr2) `ltrim(dim1)`, mq2.msr12 msr12, mq1.roundedmsr2 roundedmsr2 from ") - || hqlQuery.toLowerCase().startsWith("select coalesce(mq1.expr1, mq2.expr1) `reverse(dim1)`," - + " coalesce(mq1.expr2, mq2.expr2) `ltrim(dim1)`, mq1.msr12 msr12, mq2.roundedmsr2 roundedmsr2 from "), + assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `reverse(dim1)`, (basecube.alias1) " + + "as `ltrim(dim1)`, sum((basecube.alias2)) as `msr12`, round((sum((basecube.alias3)) / 1000)) " + + "as `roundedmsr2` from"), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") - && hqlQuery.endsWith("mq2 on mq1.expr1 <=> mq2.expr1 AND mq1.expr2 <=> mq2.expr2"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") + && hqlQuery.endsWith("GROUP BY (basecube.alias0), (basecube.alias1)"), hqlQuery); } @Test @@ -542,23 +470,20 @@ public void testMultiFactQueryWithDirectMsr() throws Exception { rewrite("select reverse(dim1), directMsrExpr as directMsr, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, conf); String expected1 = - getExpectedQuery(cubeName, "select reverse(basecube.dim1) as `expr1`, " - + "max(basecube.msr13) + count(basecube . msr14) as `expr2` FROM ", null, + getExpectedQuery(cubeName, "SELECT reverse((basecube.dim1)) as `alias0`, max((basecube.msr13)) as `alias1`, " + + "count((basecube.msr14)) as `alias2`, 0.0 as `alias3` FROM", null, " group by reverse(basecube.dim1)", getWhereForDailyAndHourly2days(cubeName, "C1_testFact3_BASE")); String expected2 = - getExpectedQuery(cubeName, "select reverse(basecube.dim1) as expr1, " - + "round(sum(basecube.msr2)/1000) as `roundedmsr2` FROM ", null, " group by reverse(basecube.dim1)", - getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); + getExpectedQuery(cubeName, "SELECT reverse((basecube.dim1)) as `alias0`, 0.0 as `alias1`, " + + "0.0 as `alias2`, sum((basecube.msr2)) as `alias3` FROM", null, + " group by reverse(basecube.dim1)", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); assertTrue(hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.expr1, mq2.expr1) `reverse(dim1)`, mq2.expr2 `directmsr`, mq1.roundedmsr2 roundedmsr2 " - + "from ") - || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.expr1, mq2.expr1) `reverse(dim1)`, mq1.expr2 `directmsr`, mq2.roundedmsr2 roundedmsr2 " - + "from "), + "select (basecube.alias0) as `reverse(dim1)`, (max((basecube.alias1)) + count((basecube.alias2))) " + + "as `directmsr`, round((sum((basecube.alias3)) / 1000)) as `roundedmsr2` from"), hqlQuery.toLowerCase()); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.expr1 <=> mq2.expr1"), + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @@ -567,20 +492,19 @@ public void testMultiFactQueryWithNoDefaultAggregates() throws Exception { // query with non default aggregate String hqlQuery = rewrite("select dim1, avg(msr12), avg(msr2) from basecube where " + TWO_DAYS_RANGE, conf); String expected1 = - getExpectedQuery(cubeName, "select basecube.dim1 as `dim1`, avg(basecube.msr12) as `expr2` FROM ", null, - " group by basecube.dim1", getWhereForHourly2days(cubeName, "C1_testfact2_raw_base")); + getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, avg((basecube.msr12)) as `alias1`," + + " 0.0 as `alias2` FROM ", null, " group by basecube.dim1", + getWhereForHourly2days(cubeName, "C1_testfact2_raw_base")); String expected2 = - getExpectedQuery(cubeName, "select basecube.dim1 as `dim1`, avg(basecube.msr2)) as `expr3` FROM ", null, - " group by basecube.dim1", getWhereForHourly2days(cubeName, "C1_testfact1_raw_base")); + getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, avg((basecube.msr2)) " + + "as `alias2` FROM ", null, " group by basecube.dim1", + getWhereForHourly2days(cubeName, "C1_testfact1_raw_base")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); assertTrue(hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim1, mq2.dim1) dim1, mq2.expr2 `avg(msr12)`, mq1.expr3 `avg(msr2)` from ") - || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim1, mq2.dim1) dim1, mq1.expr2 `avg(msr12)`, mq2.expr3 `avg(msr2)` from "), hqlQuery); - - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.dim1 <=> mq2.dim1"), - hqlQuery); + "select (basecube.alias0) as `dim1`, avg((basecube.alias1)) as `avg(msr12)`, avg((basecube.alias2)) " + + "as `avg(msr2)` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @Test @@ -588,20 +512,23 @@ public void testMultiFactQueryWithJoins() throws Exception { // query with join String hqlQuery = rewrite("select dim2chain.name, msr12, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, conf); String expected1 = getExpectedQuery(cubeName, - "select dim2chain.name as `name`, sum(basecube.msr12) as `msr12` FROM ", " JOIN " + getDbName() - + "c1_testdim2tbl dim2chain ON basecube.dim2 = " + " dim2chain.id and (dim2chain.dt = 'latest') ", null, + "SELECT (dim2chain.name) as `alias0`, sum((basecube.msr12)) as `alias1`, 0.0 as `alias2` FROM ", + " JOIN " + getDbName() + + "c1_testdim2tbl dim2chain ON basecube.dim2 = " + + " dim2chain.id and (dim2chain.dt = 'latest') ", null, " group by dim2chain.name", null, getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); String expected2 = getExpectedQuery(cubeName, - "select dim2chain.name as `name`, round(sum(basecube.msr2)/1000) as `roundedmsr2` FROM ", " JOIN " + getDbName() - + "c1_testdim2tbl dim2chain ON basecube.dim2 = " + " dim2chain.id and (dim2chain.dt = 'latest') ", null, + "SELECT (dim2chain.name) as `alias0`, 0.0 as `alias1`, sum((basecube.msr2)) as `alias2` FROM ", " JOIN " + + getDbName() + + "c1_testdim2tbl dim2chain ON basecube.dim2 = " + + " dim2chain.id and (dim2chain.dt = 'latest') ", null, " group by dim2chain.name", null, getWhereForHourly2days(cubeName, "C1_testfact1_raw_base")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); assertTrue(hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.name, mq2.name) name, mq2.msr12 msr12, mq1.roundedmsr2 roundedmsr2 from ") - || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.name, mq2.name) name, mq1.msr12 msr12, mq2.roundedmsr2 roundedmsr2 from "), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.name <=> mq2.name"), + "select (basecube.alias0) as `name`, sum((basecube.alias1)) as `msr12`, " + + "round((sum((basecube.alias2)) / 1000)) as `roundedmsr2` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @@ -609,20 +536,20 @@ public void testMultiFactQueryWithJoins() throws Exception { public void testMultiFactQueryWithDenormColumn() throws Exception { // query with denorm variable String hqlQuery = rewrite("select dim2, msr13, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, conf); - String expected1 = getExpectedQuery(cubeName, "select dim2chain.id as `dim2`, max(basecube.msr13) as `msr13` FROM ", - " JOIN " + getDbName() + "c1_testdim2tbl dim2chain ON basecube.dim12 = " - + " dim2chain.id and (dim2chain.dt = 'latest') ", null, " group by dim2chain.id", null, + String expected1 = getExpectedQuery(cubeName, "SELECT (dim2chain.id) as `alias0`, max((basecube.msr13)) " + + "as `alias1`, 0.0 as `alias2` FROM ", " JOIN " + getDbName() + + "c1_testdim2tbl dim2chain ON basecube.dim12 = " + + " dim2chain.id and (dim2chain.dt = 'latest') ", null, " group by dim2chain.id", null, getWhereForHourly2days(cubeName, "C1_testFact3_RAW_BASE")); String expected2 = getExpectedQuery(cubeName, - "select basecube.dim2 as `dim2`, round(sum(basecube.msr2)/1000) as `roundedmsr2` FROM ", null, + "SELECT (basecube.dim2) as `alias0`, 0.0 as `alias1`, sum((basecube.msr2)) as `alias2` FROM ", null, " group by basecube.dim2", getWhereForHourly2days(cubeName, "C1_testfact1_raw_base")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); assertTrue(hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim2, mq2.dim2) dim2, mq2.msr13 msr13, mq1.roundedmsr2 roundedmsr2 from ") - || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim2, mq2.dim2) dim2, mq1.msr13 msr13, mq2.roundedmsr2 roundedmsr2 from "), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.dim2 <=> mq2.dim2"), + "select (basecube.alias0) as `dim2`, max((basecube.alias1)) as `msr13`, " + + "round((sum((basecube.alias2)) / 1000)) as `roundedmsr2` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @@ -631,22 +558,24 @@ public void testMultiFactQueryWithDenormColumnInWhere() throws Exception { // query with denorm variable String hqlQuery = rewrite("select dim2, msr13, roundedmsr2 from basecube where dim2 == 10 and " + TWO_DAYS_RANGE, conf); - String expected1 = getExpectedQuery(cubeName, "select dim2chain.id as `dim2`, max(basecube.msr13) as `msr13` FROM ", - " JOIN " + getDbName() + "c1_testdim2tbl dim2chain ON basecube.dim12 = " + String expected1 = getExpectedQuery(cubeName, "SELECT (dim2chain.id) as `alias0`, max((basecube.msr13)) " + + "as `alias1`, 0.0 as `alias2` FROM ", " JOIN " + getDbName() + + "c1_testdim2tbl dim2chain ON basecube.dim12 = " + " dim2chain.id and (dim2chain.dt = 'latest') ", "dim2chain.id == 10", " group by dim2chain.id", null, getWhereForHourly2days(cubeName, "C1_testFact3_RAW_BASE")); String expected2 = getExpectedQuery(cubeName, - "select basecube.dim2 as `dim2`, round(sum(basecube.msr2)/1000) as `roundedmsr2` FROM ", "basecube.dim2 == 10", - " group by basecube.dim2", getWhereForHourly2days(cubeName, "C1_testfact1_raw_base")); + "SELECT (basecube.dim2) as `alias0`, 0.0 as `alias1`, sum((basecube.msr2)) as `alias2` FROM ", + "basecube.dim2 == 10", " group by basecube.dim2", + getWhereForHourly2days(cubeName, "C1_testfact1_raw_base")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); assertTrue(hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim2, mq2.dim2) dim2, mq2.msr13 msr13, mq1.roundedmsr2 roundedmsr2 from ") - || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim2, mq2.dim2) dim2, mq1.msr13 msr13, mq2.roundedmsr2 roundedmsr2 from "), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.dim2 <=> mq2.dim2"), + "select (basecube.alias0) as `dim2`, max((basecube.alias1)) as `msr13`, " + + "round((sum((basecube.alias2)) / 1000)) as `roundedmsr2` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } + @Test public void testMultiFactQueryWithExpressionInvolvingDenormVariable() throws Exception { // query with expression @@ -656,24 +585,24 @@ public void testMultiFactQueryWithExpressionInvolvingDenormVariable() throws Exc "select booleancut, round(sum(msr2)/1000), avg(msr13 + msr14) from basecube where " + TWO_DAYS_RANGE, conf); String expected1 = - getExpectedQuery(cubeName, "select basecube.dim1 != 'x' AND dim2chain.id != 10 as `booleancut`," - + " avg(basecube.msr13 + basecube.msr14) as `expr3` FROM ", " JOIN " + getDbName() - + "c1_testdim2tbl dim2chain ON basecube.dim12 = " + " dim2chain.id and (dim2chain.dt = 'latest') ", null, + getExpectedQuery(cubeName, "SELECT (((basecube.dim1) != 'x') and ((dim2chain.id) != 10)) as `alias0`, " + + "0.0 as `alias1`, avg(((basecube.msr13) + (basecube.msr14))) as `alias2` FROM ", " JOIN " + + getDbName() + "c1_testdim2tbl dim2chain ON basecube.dim12 = " + + " dim2chain.id and (dim2chain.dt = 'latest') ", null, " group by basecube.dim1 != 'x' AND dim2chain.id != 10", null, getWhereForHourly2days(cubeName, "C1_testfact3_raw_base")); String expected2 = - getExpectedQuery(cubeName, "select basecube.dim1 != 'x' AND basecube.dim2 != 10 as `booleancut`," - + " round(sum(basecube.msr2)/1000) as `expr2` FROM ", null, - " group by basecube.dim1 != 'x' AND basecube.dim2 != 10", + getExpectedQuery(cubeName, "SELECT (((basecube.dim1) != 'x') and ((basecube.dim2) != 10)) as `alias0`, " + + "sum((basecube.msr2)) as `alias1`, 0.0 as `alias2` FROM", null, + " group by basecube.dim1 != 'x' AND basecube.dim2 != 10", getWhereForHourly2days(cubeName, "C1_testfact1_raw_base")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); - assertTrue(hqlQuery.toLowerCase().startsWith("select coalesce(mq1.booleancut, mq2.booleancut) booleancut, " - + "mq2.expr2 `round((sum(msr2) / 1000))`, mq1.expr3 `avg((msr13 + msr14))` from ") - || hqlQuery.toLowerCase().startsWith("select coalesce(mq1.booleancut, mq2.booleancut) booleancut, " - + "mq1.expr2 `round((sum(msr2) / 1000))`, mq2.expr3 `avg((msr13 + msr14))` from "), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") - && hqlQuery.endsWith("mq2 on mq1.booleancut <=> mq2.booleancut"), + assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `booleancut`, " + + "round((sum((basecube.alias1)) / 1000)) as `round((sum(msr2) / 1000))`, " + + "avg((basecube.alias2)) as `avg((msr13 + msr14))` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") + && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @@ -686,26 +615,25 @@ public void testMultiFactQueryWithExpressionInvolvingDenormVariableInWhereClause "select booleancut, round(sum(msr2)/1000), avg(msr13 + msr14) from basecube where booleancut == 'true' and " + TWO_DAYS_RANGE, conf); String expected1 = - getExpectedQuery(cubeName, "select basecube.dim1 != 'x' AND dim2chain.id != 10 as `booleancut`," - + " avg(basecube.msr13 + basecube.msr14) as `expr3` FROM ", " JOIN " + getDbName() + getExpectedQuery(cubeName, "SELECT (((basecube.dim1) != 'x') and ((dim2chain.id) != 10)) as `alias0`, " + + "0.0 as `alias1`, avg(((basecube.msr13) + (basecube.msr14))) as `alias2` FROM ", " JOIN " + getDbName() + "c1_testdim2tbl dim2chain ON basecube.dim12 = " + " dim2chain.id and (dim2chain.dt = 'latest') ", "(basecube.dim1 != 'x' AND dim2chain.id != 10) == true", " group by basecube.dim1 != 'x' AND dim2chain.id != 10", null, getWhereForHourly2days(cubeName, "C1_testfact3_raw_base")); String expected2 = - getExpectedQuery(cubeName, "select basecube.dim1 != 'x' AND basecube.dim2 != 10 as `booleancut`," - + " round(sum(basecube.msr2)/1000) as `expr2` FROM ", - "(basecube.dim1 != 'x' AND basecube.dim2 != 10) == true", - " group by basecube.dim1 != 'x' AND basecube.dim2 != 10", + getExpectedQuery(cubeName, "SELECT (((basecube.dim1) != 'x') and ((basecube.dim2) != 10)) as `alias0`, " + + "sum((basecube.msr2)) as `alias1`, 0.0 as `alias2` FROM ", + "(basecube.dim1 != 'x' AND basecube.dim2 != 10) == true", + " group by basecube.dim1 != 'x' AND basecube.dim2 != 10", getWhereForHourly2days(cubeName, "C1_testfact1_raw_base")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); - assertTrue(hqlQuery.toLowerCase().startsWith("select coalesce(mq1.booleancut, mq2.booleancut) booleancut, " - + "mq2.expr2 `round((sum(msr2) / 1000))`, mq1.expr3 `avg((msr13 + msr14))` from ") - || hqlQuery.toLowerCase().startsWith("select coalesce(mq1.booleancut, mq2.booleancut) booleancut, " - + "mq1.expr2 `round((sum(msr2) / 1000))`, mq2.expr3 `avg((msr13 + msr14))` from "), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") - && hqlQuery.endsWith("mq2 on mq1.booleancut <=> mq2.booleancut"), + assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `booleancut`, " + + "round((sum((basecube.alias1)) / 1000)) as `round((sum(msr2) / 1000))`, " + + "avg((basecube.alias2)) as `avg((msr13 + msr14))` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") + && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @@ -714,23 +642,22 @@ public void testMultiFactQueryWithMaterializedExpressions() throws Exception { Configuration tconf = new Configuration(conf); tconf.set(CubeQueryConfUtil.getValidFactTablesKey("basecube"), "testfact5_base,testfact6_base"); String hqlQuery = - rewrite( - "select booleancut, round(sum(msr2)/1000), msr13 from basecube where " + TWO_DAYS_RANGE, tconf); + rewrite("select booleancut, round(sum(msr2)/1000), msr13 from basecube where " + TWO_DAYS_RANGE, tconf); String expected1 = - getExpectedQuery(cubeName, "select basecube.booleancut as `booleancut`,max(basecube.msr13) as `msr13` FROM ", - null, " group by basecube.booleancut", getWhereForDailyAndHourly2days(cubeName, "C1_testfact6_base")); + getExpectedQuery(cubeName, "SELECT (basecube.booleancut) as `alias0`, 0.0 as `alias1`, " + + "max((basecube.msr13)) as `alias2` FROM", null, " " + + "group by basecube.booleancut", getWhereForDailyAndHourly2days(cubeName, "C1_testfact6_base")); String expected2 = - getExpectedQuery(cubeName, "select basecube.booleancut as `booleancut`," - + " round(sum(basecube.msr2)/1000) as `expr2` FROM ", null, " group by basecube.booleancut", + getExpectedQuery(cubeName, "SELECT (basecube.booleancut) as `alias0`, sum((basecube.msr2)) as `alias1`, " + + "0.0 as `alias2` FROM ", null, " group by basecube.booleancut", getWhereForDailyAndHourly2days(cubeName, "C1_testfact5_base")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); - assertTrue(hqlQuery.toLowerCase().startsWith("select coalesce(mq1.booleancut, mq2.booleancut) booleancut, " - + "mq2.expr2 `round((sum(msr2) / 1000))`, mq1.msr13 msr13 from ") - || hqlQuery.toLowerCase().startsWith("select coalesce(mq1.booleancut, mq2.booleancut) booleancut, " - + "mq1.expr2 `round((sum(msr2) / 1000))`, mq2.msr13 msr13 from "), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") - && hqlQuery.endsWith("mq2 on mq1.booleancut <=> mq2.booleancut"), + assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `booleancut`, " + + "round((sum((basecube.alias1)) / 1000)) as `round((sum(msr2) / 1000))`, " + + "max((basecube.alias2)) as `msr13` from "), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") + && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @@ -741,16 +668,17 @@ public void testMultiFactQueryCaseWhenExpressionWithChainField() throws Exceptio rewrite("select sum(case when dim22 = 'x' then msr12 else 0 end) as case_expr, sum(msr1) from basecube where " + TWO_DAYS_RANGE, tconf); String expected1 = - getExpectedQuery(cubeName, "select sum(case when basecube.dim22 = 'x' then basecube.msr12 else 0 end) as " - + "`expr1` FROM ", null, null, getWhereForHourly2days(cubeName, "C1_testfact2_raw_base")); + getExpectedQuery(cubeName, "SELECT sum(case when ((basecube.dim22) = 'x') then (basecube.msr12) else 0 end) " + + "as `alias0`, 0.0 as `alias1` FROM ", null, null, + getWhereForHourly2days(cubeName, "C1_testfact2_raw_base")); String expected2 = - getExpectedQuery(cubeName, "select sum(basecube.msr1) as `expr2` FROM ", null, null, + getExpectedQuery(cubeName, "SELECT 0.0 as `alias0`, sum((basecube.msr1)) as `alias1` FROM ", null, null, getWhereForHourly2days(cubeName, "c1_testfact1_raw_base")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); - assertTrue(hqlQuery.toLowerCase().startsWith("select mq2.expr1 `case_expr`, mq1.expr2 `sum(msr1)` from ") - || hqlQuery.toLowerCase().startsWith("select mq1.expr1 `case_expr`, mq2.expr2 `sum(msr1)` from "), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2"), hqlQuery); + assertTrue(hqlQuery.toLowerCase().startsWith("select sum((basecube.alias0)) as `case_expr`, " + + "sum((basecube.alias1)) as `sum(msr1)` from "), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("basecube"), hqlQuery); } @Test @@ -760,16 +688,17 @@ public void testMultiFactQueryCaseWhenExpression() throws Exception { rewrite("select sum(case when dim13 = 'x' then msr12 else 0 end) as case_expr, sum(msr1) from basecube where " + TWO_DAYS_RANGE, tconf); String expected1 = - getExpectedQuery(cubeName, "select sum(case when basecube.dim13 = 'x' then basecube.msr12 else 0 end) as " - + "`expr1` FROM ", null, null, getWhereForHourly2days(cubeName, "C1_testfact2_raw_base")); + getExpectedQuery(cubeName, "SELECT sum(case when ((basecube.dim13) = 'x') then (basecube.msr12) else 0 end) " + + "as `alias0`, 0.0 as `alias1` FROM ", null, null, + getWhereForHourly2days(cubeName, "C1_testfact2_raw_base")); String expected2 = - getExpectedQuery(cubeName, "select sum(basecube.msr1) as `expr2` FROM ", null, null, + getExpectedQuery(cubeName, "SELECT 0.0 as `alias0`, sum((basecube.msr1)) as `alias1` FROM ", null, null, getWhereForHourly2days(cubeName, "c1_testfact1_raw_base")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); - assertTrue(hqlQuery.toLowerCase().startsWith("select mq2.expr1 `case_expr`, mq1.expr2 `sum(msr1)` from ") - || hqlQuery.toLowerCase().startsWith("select mq1.expr1 `case_expr`, mq2.expr2 `sum(msr1)` from "), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2"), hqlQuery); + assertTrue(hqlQuery.toLowerCase().startsWith("select sum((basecube.alias0)) as `case_expr`, " + + "sum((basecube.alias1)) as `sum(msr1)` from "), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("basecube"), hqlQuery); } @Test @@ -779,19 +708,20 @@ public void testMultiFactQueryCaseWhenExpressionWithGroupby() throws Exception { rewrite("select dim1, sum(case when dim13 = 'x' then msr12 else 0 end) as case_expr, sum(msr1) from basecube " + "where " + TWO_DAYS_RANGE, tconf); String expected1 = - getExpectedQuery(cubeName, "select basecube.dim1 as `dim1`, sum(case when basecube.dim13 = 'x' then basecube" - + ".msr12 else 0 end) as `expr2` FROM ", null, " group by basecube.dim1 ", + getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum(case when ((basecube.dim13) = 'x') " + + "then (basecube.msr12) else 0 end) as `alias1`, 0.0 as `alias2` FROM ", null, + " group by basecube.dim1 ", getWhereForHourly2days(cubeName, "C1_testfact2_raw_base")); String expected2 = - getExpectedQuery(cubeName, "select basecube.dim1 as `dim1`, sum(basecube.msr1) as `expr3` FROM ", null, + getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, sum((basecube.msr1)) " + + "as `alias2` FROM", null, " group by basecube.dim1 ", getWhereForHourly2days(cubeName, "c1_testfact1_raw_base")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); assertTrue(hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim1, mq2.dim1) dim1, mq2.expr2 `case_expr`, mq1.expr3 `sum(msr1)` from ") - || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim1, mq2.dim1) dim1, mq1.expr2 `case_expr`, mq2.expr3 `sum(msr1)` from "), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.dim1 <=> mq2.dim1"), + "select (basecube.alias0) as `dim1`, sum((basecube.alias1)) as `case_expr`, " + + "sum((basecube.alias2)) as `sum(msr1)` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @@ -801,19 +731,21 @@ public void testMultiFactQueryCaseWhenExpressionWithHavingClause() throws Except String hqlQuery = rewrite("select sum(case when dim13 = 'x' then msr12 else 0 end) as case_expr, sum(msr1) from basecube " + "where " - + TWO_DAYS_RANGE + " having sum(case when dim13 = 'x' then msr12 else 0 end) > 100 and sum(msr1) > 500", tconf); + + TWO_DAYS_RANGE + " having sum(case when dim13 = 'x' then msr12 else 0 end) > 100 " + + "and sum(msr1) > 500", tconf); String expected1 = - getExpectedQuery(cubeName, "select sum(case when basecube.dim13 = 'x' then basecube.msr12 else 0 end) as " - + "`expr1` FROM ", null, " having sum(case when basecube.dim13 = 'x' then basecube.msr12 else 0 end) > 100", - getWhereForHourly2days(cubeName, "C1_testfact2_raw_base")); + getExpectedQuery(cubeName, "SELECT sum(case when ((basecube.dim13) = 'x') then (basecube.msr12) else 0 end) " + + "as `alias0`, 0.0 as `alias1` FROM ", null, "", + getWhereForHourly2days(cubeName, "C1_testfact2_raw_base")); String expected2 = - getExpectedQuery(cubeName, "select sum(basecube.msr1) as `expr2` FROM ", null, " having sum(basecube.msr1) > 500", + getExpectedQuery(cubeName, "SELECT 0.0 as `alias0`, sum((basecube.msr1)) as `alias1` FROM ", null, "", getWhereForHourly2days(cubeName, "c1_testfact1_raw_base")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); - assertTrue(hqlQuery.toLowerCase().startsWith("select mq2.expr1 `case_expr`, mq1.expr2 `sum(msr1)` from ") - || hqlQuery.toLowerCase().startsWith("select mq1.expr1 `case_expr`, mq2.expr2 `sum(msr1)` from "), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2"), hqlQuery); + assertTrue(hqlQuery.toLowerCase().startsWith("select sum((basecube.alias0)) as `case_expr`, sum((basecube.alias1)) " + + "as `sum(msr1)` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("HAVING ((sum((basecube.alias0)) > 100) " + + "and (sum((basecube.alias1)) > 500))"), hqlQuery); } @Test @@ -821,25 +753,23 @@ public void testMultiFactQueryCaseWhenExpressionWithGroubyAndHavingClause() thro Configuration tconf = new Configuration(conf); String hqlQuery = rewrite("select dim1, sum(case when dim13 = 'x' then msr12 else 0 end) as case_expr, sum(msr1) from basecube " - + "where " - + TWO_DAYS_RANGE + " having sum(case when dim13 = 'x' then msr12 else 0 end) > 100 and sum(msr1) > 500", tconf); + + "where " + TWO_DAYS_RANGE + " having sum(case when dim13 = 'x' then msr12 else 0 end) > 100 " + + "and sum(msr1) > 500", tconf); String expected1 = - getExpectedQuery(cubeName, "select basecube.dim1 as `dim1`, sum(case when basecube.dim13 = 'x' then basecube" - + ".msr12 else 0 end) as `expr2` FROM ", null, - " group by basecube.dim1 having sum(case when basecube.dim13 = 'x' then basecube.msr12 else 0 end) > 100", - getWhereForHourly2days(cubeName, "C1_testfact2_raw_base")); + getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, sum(case when ((basecube.dim13) = 'x') then " + + "(basecube.msr12) else 0 end) as `alias1`, 0.0 as `alias2` FROM", null, " group by basecube.dim1", + getWhereForHourly2days(cubeName, "C1_testfact2_raw_base")); String expected2 = - getExpectedQuery(cubeName, "select basecube.dim1 as `dim1`, sum(basecube.msr1) as `expr3` FROM ", null, - " group by basecube.dim1 having sum(basecube.msr1) > 500", - getWhereForHourly2days(cubeName, "c1_testfact1_raw_base")); + getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, 0.0 as `alias1`, sum((basecube.msr1)) " + + "as `alias2` FROM", null, " group by basecube.dim1", + getWhereForHourly2days(cubeName, "c1_testfact1_raw_base")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); assertTrue(hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim1, mq2.dim1) dim1, mq2.expr2 `case_expr`, mq1.expr3 `sum(msr1)` from ") - || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.dim1, mq2.dim1) dim1, mq1.expr2 `case_expr`, mq2.expr3 `sum(msr1)` from "), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.dim1 <=> mq2.dim1"), - hqlQuery); + "select (basecube.alias0) as `dim1`, sum((basecube.alias1)) as `case_expr`, " + + "sum((basecube.alias2)) as `sum(msr1)` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") + && hqlQuery.endsWith("HAVING ((sum((basecube.alias1)) > 100) and (sum((basecube.alias2)) > 500))"), hqlQuery); } @Test @@ -849,10 +779,9 @@ public void testFallbackPartCol() throws Exception { String hql, expected; // Prefer fact that has a storage with part col on queried time dim hql = rewrite("select msr12 from basecube where " + TWO_DAYS_RANGE, conf); - expected = getExpectedQuery(BASE_CUBE_NAME, "select sum(basecube.msr12) FROM ", null, null, + expected = getExpectedQuery(BASE_CUBE_NAME, "select sum(basecube.msr12) as `msr12` FROM ", null, null, getWhereForDailyAndHourly2days(BASE_CUBE_NAME, "c1_testfact2_base")); compareQueries(hql, expected); - // If going to fallback timedim, and partitions are missing, then error should be missing partition on that conf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C4"); conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, true); @@ -860,11 +789,11 @@ public void testFallbackPartCol() throws Exception { getLensExceptionInRewrite("select msr12 from basecube where " + TWO_DAYS_RANGE, conf); NoCandidateFactAvailableException ne = (NoCandidateFactAvailableException) exc; PruneCauses.BriefAndDetailedError pruneCause = ne.getJsonMessage(); - assertTrue(pruneCause.getBrief().contains("Missing partitions")); - assertEquals(pruneCause.getDetails().get("testfact2_base").iterator().next().getCause(), MISSING_PARTITIONS); - assertEquals(pruneCause.getDetails().get("testfact2_base").iterator().next().getMissingPartitions().size(), 1); + assertTrue(pruneCause.getBrief().contains("Missing partitions"), pruneCause.getBrief()); + assertEquals(pruneCause.getDetails().get("c4_testfact2_base").iterator().next().getCause(), MISSING_PARTITIONS); + assertEquals(pruneCause.getDetails().get("c4_testfact2_base").iterator().next().getMissingPartitions().size(), 1); assertEquals( - pruneCause.getDetails().get("testfact2_base").iterator().next().getMissingPartitions().iterator().next(), + pruneCause.getDetails().get("c4_testfact2_base").iterator().next().getMissingPartitions().iterator().next(), "ttd:[" + UpdatePeriod.SECONDLY.format(DateUtils.addDays(DateUtils.truncate(TWODAYS_BACK, Calendar.HOUR), -10)) + ", " + UpdatePeriod.SECONDLY.format(DateUtils.addDays(DateUtils.truncate(NOW, Calendar.HOUR), 10)) @@ -884,7 +813,7 @@ public void testFallbackPartCol() throws Exception { DateUtils.addDays(TWODAYS_BACK, -5))))) + "' and " + "basecube.processing_time < '" + HIVE_QUERY_DATE_PARSER.get().format(ABSDATE_PARSER.get().parse( getAbsDateFormatString(getDateUptoHours(DateUtils.addDays(NOW, 5))))); - expected = getExpectedQuery(BASE_CUBE_NAME, "select sum(basecube.msr12) FROM ", null, + expected = getExpectedQuery(BASE_CUBE_NAME, "select sum(basecube.msr12) as `msr12` FROM ", null, " and " + dTimeWhereClause + " and " + pTimeWhereClause, getWhereForDailyAndHourly2daysWithTimeDim(BASE_CUBE_NAME, "ttd", DateUtils.addDays(TWODAYS_BACK, -10), DateUtils.addDays(NOW, 10), "c4_testfact2_base")); @@ -893,32 +822,27 @@ public void testFallbackPartCol() throws Exception { // Multiple timedims in single query. test that CubeQueryContext ctx = rewriteCtx("select msr12 from basecube where " + TWO_DAYS_RANGE + " and " + TWO_DAYS_RANGE_TTD, conf); - assertEquals(ctx.getCandidateFactSets().size(), 1); - assertEquals(ctx.getCandidateFactSets().iterator().next().size(), 1); - CandidateFact cfact = ctx.getCandidateFactSets().iterator().next().iterator().next(); - - assertEquals(cfact.getRangeToStoragePartMap().size(), 2); - Set storages = Sets.newHashSet(); - for(Map entry: cfact.getRangeToStorageWhereMap().values()) { - storages.addAll(entry.keySet()); - } - assertEquals(storages.size(), 1); - String storage = storages.iterator().next(); - for(Map.Entry> entry: cfact.getRangeToStorageWhereMap().entrySet()) { - if (entry.getKey().getPartitionColumn().equals("dt")) { - ASTNode parsed = HQLParser.parseExpr(entry.getValue().get(storage)); + assertEquals(ctx.getCandidates().size(), 1); + assertEquals(CandidateUtil.getStorageCandidates(ctx.getCandidates().iterator().next()).size(), 1); + StorageCandidate sc = CandidateUtil.getStorageCandidates(ctx.getCandidates().iterator().next()).iterator().next(); + assertEquals(sc.getRangeToPartitions().size(), 2); + for(TimeRange range: sc.getRangeToPartitions().keySet()) { + String rangeWhere = CandidateUtil.getTimeRangeWhereClasue(ctx.getRangeWriter(), sc, range); + if (range.getPartitionColumn().equals("dt")) { + ASTNode parsed = HQLParser.parseExpr(rangeWhere); assertEquals(parsed.getToken().getType(), KW_AND); - assertTrue(entry.getValue().get(storage).substring(((CommonToken) parsed.getToken()).getStopIndex() + 1) + assertTrue(rangeWhere.substring(((CommonToken) parsed.getToken()).getStopIndex() + 1) .toLowerCase().contains(dTimeWhereClause)); - assertFalse(entry.getValue().get(storage).substring(0, ((CommonToken) parsed.getToken()).getStartIndex()) + assertFalse(rangeWhere.substring(0, ((CommonToken) parsed.getToken()).getStartIndex()) .toLowerCase().contains("and")); - } else if (entry.getKey().getPartitionColumn().equals("ttd")) { - assertFalse(entry.getValue().get(storage).toLowerCase().contains("and")); + } else if (range.getPartitionColumn().equals("ttd")) { + assertFalse(rangeWhere.toLowerCase().contains("and")); } else { throw new LensException("Unexpected"); } } } + @Test public void testMultiFactQueryWithHaving() throws Exception { @@ -930,198 +854,205 @@ public void testMultiFactQueryWithHaving() throws Exception { hqlQuery = rewrite("select dim1, dim11, msr12 from basecube where " + TWO_DAYS_RANGE + "having roundedmsr2 > 0", conf); expected1 = getExpectedQuery(cubeName, - "select basecube.dim1 as dim1, basecube.dim11 as dim11, sum(basecube.msr12) as msr12 FROM ", - null, " group by basecube.dim1, basecube.dim11", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) " + + "as `alias2`, 0.0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); expected2 = getExpectedQuery(cubeName, - "select basecube.dim1 as dim1, basecube.dim11 as dim11 FROM ", - null, " group by basecube.dim1, basecube.dim11 having round(sum(basecube.msr2)/1000) > 0", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, " + + "sum((basecube.msr2)) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); - assertTrue(hqlQuery.toLowerCase().contains("having")); + assertTrue(hqlQuery.toLowerCase().contains("group by (basecube.alias0), (basecube.alias1)")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); - assertTrue(hqlQuery.toLowerCase().startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, " - + "coalesce(mq1.dim11, mq2.dim11) dim11, mq2.msr12 msr12 from ") - || hqlQuery.toLowerCase().startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, coalesce(mq1.dim11, mq2.dim11) " - + "dim11, mq1.msr12 msr12 from "), hqlQuery); - assertTrue(hqlQuery.contains(joinSubString) - && hqlQuery.endsWith(endSubString), hqlQuery); + assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `dim1`, (basecube.alias1) as `dim11`, " + + "sum((basecube.alias2)) as `msr12` from"), hqlQuery); + assertTrue(hqlQuery.endsWith("HAVING (round((sum((basecube.alias3)) / 1000)) > 0)")); // Two having clause, one from each fact. hqlQuery = rewrite("select dim1, dim11, msr12, roundedmsr2 from basecube where " + TWO_DAYS_RANGE + "having msr12 > 2 and roundedmsr2 > 0", conf); expected1 = getExpectedQuery(cubeName, - "select basecube.dim1 as dim1, basecube.dim11 as dim11, sum(basecube.msr12) as msr12 FROM ", - null, " group by basecube.dim1, basecube.dim11 HAVING sum(basecube.msr12) > 2", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, " + + "0.0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); expected2 = getExpectedQuery(cubeName, - "select basecube.dim1 as dim1, basecube.dim11 as dim11, round(sum(basecube.msr2)/1000) as roundedmsr2 FROM ", - null, " group by basecube.dim1, basecube.dim11 HAVING round(sum(basecube.msr2)/1000) > 0", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, " + + "sum((basecube.msr2)) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); - assertTrue(hqlQuery.toLowerCase().startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, " - + "coalesce(mq1.dim11, mq2.dim11) dim11, mq1.msr12 msr12, mq2.roundedmsr2 roundedmsr2 from ") - || hqlQuery.toLowerCase().startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, " - + "coalesce(mq1.dim11, mq2.dim11) dim11, mq2.msr12 msr12, mq1.roundedmsr2 roundedmsr2 from "), hqlQuery); - assertTrue(hqlQuery.contains(joinSubString) - && hqlQuery.endsWith(endSubString), hqlQuery); - - // Two having clauses and one complex expression in having which needs to be split over the two facts - // And added as where clause outside + assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `dim1`, (basecube.alias1) as `dim11`, " + + "sum((basecube.alias2)) as `msr12`, round((sum((basecube.alias3)) / 1000)) as `roundedmsr2` from"), + hqlQuery); + assertTrue(hqlQuery.endsWith("HAVING ((sum((basecube.alias2)) > 2) " + + "and (round((sum((basecube.alias3)) / 1000)) > 0))")); + hqlQuery = rewrite("select dim1, dim11, msr12, roundedmsr2 from basecube where " + TWO_DAYS_RANGE - + "having flooredmsr12+roundedmsr2 <= 1000 and msr12 > 2 and roundedmsr2 > 0", conf); + + "having msr12+roundedmsr2 <= 1000 and msr12 > 2 and roundedmsr2 > 0", conf); expected1 = getExpectedQuery(cubeName, - "select basecube.dim1 as dim1, basecube.dim11 as dim11, sum(basecube.msr12) as msr12 , " - + "floor(sum(basecube.msr12)) as alias0 FROM ", - null, " group by basecube.dim1, basecube.dim11 HAVING sum(basecube.msr12) > 2", - getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, " + + "0.0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); + + expected2 = getExpectedQuery(cubeName, + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, " + + "sum((basecube.msr2)) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); - assertTrue(hqlQuery.toLowerCase().startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, " - + "coalesce(mq1.dim11, mq2.dim11) dim11, mq2.msr12 msr12, mq1.roundedmsr2 roundedmsr2 from ") - || hqlQuery.toLowerCase().startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, " - + "coalesce(mq1.dim11, mq2.dim11) dim11, mq1.msr12 msr12, mq2.roundedmsr2 roundedmsr2 from "), hqlQuery); - assertTrue(hqlQuery.contains(joinSubString) - && hqlQuery.endsWith(endSubString + " WHERE ((alias0 + roundedmsr2) <= 1000)"), hqlQuery); + assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `dim1`, (basecube.alias1) as `dim11`, " + + "sum((basecube.alias2)) as `msr12`, round((sum((basecube.alias3)) / 1000)) as `roundedmsr2` from"), hqlQuery); + assertTrue(hqlQuery.endsWith("(((sum((basecube.alias2)) + round((sum((basecube.alias3)) / 1000))) <= 1000) " + + "and (sum((basecube.alias2)) > 2) and (round((sum((basecube.alias3)) / 1000)) > 0))"), hqlQuery); // No push-down-able having clauses. hqlQuery = rewrite("select dim1, dim11, msr12, roundedmsr2 from basecube where " + TWO_DAYS_RANGE - + "having flooredmsr12+roundedmsr2 <= 1000", conf); + + "having msr12+roundedmsr2 <= 1000", conf); expected1 = getExpectedQuery(cubeName, - "select basecube.dim1 as dim1, basecube.dim11 as dim11, sum(basecube.msr12) as msr12, " - + "floor(sum(( basecube . msr12 ))) as `alias0` FROM ", - null, " group by basecube.dim1, basecube.dim11", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, " + + "0.0 as `alias3` FROM", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); expected2 = getExpectedQuery(cubeName, - "select basecube.dim1 as dim1, basecube.dim11 as dim11, round(sum(basecube.msr2)/1000) as roundedmsr2 FROM ", - null, " group by basecube.dim1, basecube.dim11", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, sum((basecube.msr2)) " + + "as `alias3` FROM", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); - assertFalse(hqlQuery.toLowerCase().contains("having")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); - assertTrue(hqlQuery.toLowerCase().startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, " - + "coalesce(mq1.dim11, mq2.dim11) dim11, mq2.msr12 msr12, mq1.roundedmsr2 roundedmsr2 from ") - || hqlQuery.toLowerCase().startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, coalesce(mq1.dim11, mq2.dim11) " - + "dim11, mq1.msr12 msr12, mq2.roundedmsr2 roundedmsr2 from "), hqlQuery); - assertTrue(hqlQuery.contains(joinSubString) - && hqlQuery.endsWith(endSubString + " WHERE ((alias0 + roundedmsr2) <= 1000)"), hqlQuery); + assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `dim1`, (basecube.alias1) as `dim11`, " + + "sum((basecube.alias2)) as `msr12`, round((sum((basecube.alias3)) / 1000)) as `roundedmsr2` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") + && hqlQuery.endsWith("HAVING ((sum((basecube.alias2)) + " + + "round((sum((basecube.alias3)) / 1000))) <= 1000)"), hqlQuery); // function over expression of two functions over measures hqlQuery = rewrite("select dim1, dim11, msr12, roundedmsr2 from basecube where " + TWO_DAYS_RANGE - + "having round(flooredmsr12+roundedmsr2) <= 1000", conf); + + "having round(msr12+roundedmsr2) <= 1000", conf); expected1 = getExpectedQuery(cubeName, - "select basecube.dim1 as dim1, basecube.dim11 as dim11, sum(basecube.msr12) as msr12, " - + "floor(sum(( basecube . msr12 ))) as `alias0` FROM ", - null, " group by basecube.dim1, basecube.dim11", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, " + + "0.0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); expected2 = getExpectedQuery(cubeName, - "select basecube.dim1 as dim1, basecube.dim11 as dim11, round(sum(basecube.msr2)/1000) as roundedmsr2 FROM ", - null, " group by basecube.dim1, basecube.dim11", + " SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, " + + "sum((basecube.msr2)) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); - assertFalse(hqlQuery.toLowerCase().contains("having")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); - assertTrue(hqlQuery.toLowerCase().startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, " - + "coalesce(mq1.dim11, mq2.dim11) dim11, mq2.msr12 msr12, mq1.roundedmsr2 roundedmsr2 from ") - || hqlQuery.toLowerCase().startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, coalesce(mq1.dim11, mq2.dim11) " - + "dim11, mq1.msr12 msr12, mq2.roundedmsr2 roundedmsr2 from "), hqlQuery); - assertTrue(hqlQuery.contains(joinSubString) - && hqlQuery.endsWith(endSubString + " WHERE (round((alias0 + roundedmsr2)) <= 1000)"), hqlQuery); + assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `dim1`, (basecube.alias1) as `dim11`, " + + "sum((basecube.alias2)) as `msr12`, round((sum((basecube.alias3)) / 1000)) " + + "as `roundedmsr2` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") + && hqlQuery.endsWith(" HAVING (round((sum((basecube.alias2)) + " + + "round((sum((basecube.alias3)) / 1000)))) <= 1000)"), hqlQuery); // Following test cases only select dimensions, and all the measures are in having. // Mostly tests follow the same pattern as the above tests, // The extra thing to test is the inclusion of sub-expressions in select clauses. - hqlQuery = rewrite("select dim1, dim11 from basecube where " + TWO_DAYS_RANGE + "having msr12 > 2 and roundedmsr2 > 0", conf); expected1 = getExpectedQuery(cubeName, - "select basecube.dim1 as dim1, basecube.dim11 as dim11 FROM ", - null, " group by basecube.dim1, basecube.dim11 HAVING sum(basecube.msr12) > 2", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, " + + "0.0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); expected2 = getExpectedQuery(cubeName, - "select basecube.dim1 as dim1, basecube.dim11 as dim11 FROM ", - null, " group by basecube.dim1, basecube.dim11 HAVING round(sum(basecube.msr2)/1000) > 0", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, " + + "sum((basecube.msr2)) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); - String begin = "select coalesce(mq1.dim1, mq2.dim1) dim1, coalesce(mq1.dim11, mq2.dim11) dim11 from "; + String begin = "select (basecube.alias0) as `dim1`, (basecube.alias1) as `dim11` from"; compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); assertTrue(hqlQuery.toLowerCase().startsWith(begin), hqlQuery); - assertTrue(hqlQuery.contains(joinSubString) && hqlQuery.endsWith(endSubString), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") + && hqlQuery.endsWith("HAVING ((sum((basecube.alias2)) > 2) " + + "and (round((sum((basecube.alias3)) / 1000)) > 0))"), hqlQuery); hqlQuery = rewrite("select dim1, dim11 from basecube where " + TWO_DAYS_RANGE + "having msr12 > 2 and roundedmsr2 > 0 and msr2 > 100", conf); + expected1 = getExpectedQuery(cubeName, + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, " + + "0.0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", + getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); expected2 = getExpectedQuery(cubeName, - "select basecube.dim1 as dim1, basecube.dim11 as dim11 FROM ", null, - " group by basecube.dim1, basecube.dim11 HAVING round(sum(basecube.msr2)/1000) > 0 and sum(basecube.msr2) > 100", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, " + + "sum((basecube.msr2)) as `alias3` FROM ", null, + " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); + compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); - assertTrue(hqlQuery.toLowerCase().startsWith(begin), hqlQuery); - assertTrue(hqlQuery.contains(joinSubString) && hqlQuery.endsWith(endSubString), hqlQuery); + assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `dim1`, " + + "(basecube.alias1) as `dim11` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("HAVING ((sum((basecube.alias2)) > 2) " + + "and (round((sum((basecube.alias3)) / 1000)) > 0) and (sum((basecube.alias3)) > 100))"), hqlQuery); hqlQuery = rewrite("select dim1, dim11 from basecube where " + TWO_DAYS_RANGE - + "having flooredmsr12+roundedmsr2 <= 1000", conf); + + "having msr12+roundedmsr2 <= 1000", conf); expected1 = getExpectedQuery(cubeName, - "select basecube.dim1 as dim1, basecube.dim11 as dim11, " - + "floor(sum(basecube.msr12)) as alias0 FROM ", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) " + + "as `alias2`, 0.0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); expected2 = getExpectedQuery(cubeName, - "select basecube.dim1 as dim1, basecube.dim11 as dim11, round(sum(basecube.msr2/1000)) as alias1 FROM ", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, " + + "sum((basecube.msr2)) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); - assertFalse(hqlQuery.toLowerCase().contains("having")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); - assertTrue(hqlQuery.toLowerCase().startsWith(begin), hqlQuery); - assertTrue(hqlQuery.contains(joinSubString) - && hqlQuery.endsWith(endSubString + " WHERE ((alias0 + alias1) <= 1000)"), hqlQuery); + assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `dim1`, (basecube.alias1) " + + "as `dim11` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") + && hqlQuery.endsWith("HAVING ((sum((basecube.alias2)) + round((sum((basecube.alias3)) / 1000))) <= 1000)"), + hqlQuery); hqlQuery = rewrite("select dim1, dim11 from basecube where " + TWO_DAYS_RANGE - + "having msr12 > 2 and roundedmsr2 > 0 and flooredmsr12+roundedmsr2 <= 1000", conf); + + "having msr12 > 2 and roundedmsr2 > 0 and msr12+roundedmsr2 <= 1000", conf); expected1 = getExpectedQuery(cubeName, - "select basecube.dim1 as dim1, basecube.dim11 as dim11, " - + "floor(sum(( basecube . msr12 ))) as `alias0` FROM ", - null, " group by basecube.dim1, basecube.dim11 having sum(basecube.msr12) > 2", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, " + + "0.0 as `alias3` FROM ", + null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); expected2 = getExpectedQuery(cubeName, - "select basecube.dim1 as dim1, basecube.dim11 as dim11, round(sum(basecube.msr2)/1000) as alias1 FROM ", - null, " group by basecube.dim1, basecube.dim11 having round(sum(basecube.msr2)/1000) > 0", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, " + + "sum((basecube.msr2)) as `alias3` FROM ", + null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); - assertTrue(hqlQuery.toLowerCase().startsWith(begin), hqlQuery); - assertTrue(hqlQuery.contains(joinSubString) - && hqlQuery.endsWith(endSubString + " WHERE ((alias0 + alias1) <= 1000)"), hqlQuery); + assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `dim1`, (basecube.alias1) " + + "as `dim11` from "), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") + && hqlQuery.endsWith("HAVING ((sum((basecube.alias2)) > 2) and (round((sum((basecube.alias3)) / 1000)) > 0) " + + "and ((sum((basecube.alias2)) + round((sum((basecube.alias3)) / 1000))) <= 1000))"), hqlQuery); + hqlQuery = rewrite("select dim1, dim11 from basecube where " + TWO_DAYS_RANGE - + "having msr12 > 2 or roundedmsr2 > 0 or flooredmsr12+roundedmsr2 <= 1000", conf); + + "having msr12 > 2 or roundedmsr2 > 0 or msr12+roundedmsr2 <= 1000", conf); expected1 = getExpectedQuery(cubeName, - "select basecube.dim1 as dim1, basecube.dim11 as dim11, " - + "sum(basecube.msr12) as alias0, floor(sum(basecube.msr12)) as alias2 FROM ", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, " + + "0.0 as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); expected2 = getExpectedQuery(cubeName, - "select basecube.dim1 as dim1, basecube.dim11 as dim11, round(sum(basecube.msr2)/1000) as alias1 FROM ", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, " + + "sum((basecube.msr2)) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); - String havingToWhere = " WHERE ((alias0 > 2) or (alias1 > 0) or ((alias2 + alias1) <= 1000))"; - assertFalse(hqlQuery.toLowerCase().contains("having")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); - assertTrue(hqlQuery.toLowerCase().startsWith(begin), hqlQuery); - assertTrue(hqlQuery.contains(joinSubString) - && hqlQuery.endsWith(endSubString + havingToWhere), hqlQuery); + assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `dim1`, (basecube.alias1) " + + "as `dim11` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") + && hqlQuery.endsWith("HAVING ((sum((basecube.alias2)) > 2) or (round((sum((basecube.alias3)) / 1000)) > 0) " + + "or ((sum((basecube.alias2)) + round((sum((basecube.alias3)) / 1000))) <= 1000))"), hqlQuery); } } diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBetweenTimeRangeWriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBetweenTimeRangeWriter.java index cabb95ee8..5c577819c 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBetweenTimeRangeWriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBetweenTimeRangeWriter.java @@ -19,6 +19,8 @@ package org.apache.lens.cube.parse; +import static java.util.Optional.*; + import static org.apache.lens.cube.metadata.DateFactory.*; import static org.apache.lens.cube.metadata.UpdatePeriod.DAILY; @@ -37,19 +39,19 @@ public class TestBetweenTimeRangeWriter extends TestTimeRangeWriter { - public static final String CLOSED = "CLOSED"; - public static final String OPEN = "OPEN"; + private static final String CLOSED = "CLOSED"; + private static final String OPEN = "OPEN"; - public static final int START_DATE_OFFSET = 1; - public static final int END_DATE_OFFSET = 2; + private static final int START_DATE_OFFSET = 1; + private static final int END_DATE_OFFSET = 2; - public static final DateFormat DAY_DB_FORMAT = new SimpleDateFormat("yyyy-MM-dd"); + private static final DateFormat DAY_DB_FORMAT = new SimpleDateFormat("yyyy-MM-dd"); private static Set answeringParts; private static Set answeringPartsWithFormat; static { - answeringParts = new LinkedHashSet(); + answeringParts = new LinkedHashSet<>(); answeringParts.add(new FactPartition("dt", getDateWithOffset(DAILY, START_DATE_OFFSET), DAILY, null, null)); answeringParts.add(new FactPartition("dt", getDateWithOffset(DAILY, END_DATE_OFFSET), DAILY, null, null)); @@ -108,7 +110,7 @@ public void testSinglePartBetweenOnly() throws LensException { } - public void validateBetweenOnlySingle(String whereClause, DateFormat format) { + private void validateBetweenOnlySingle(String whereClause, DateFormat format) { String expected = null; if (format == null) { expected = @@ -121,22 +123,7 @@ public void validateBetweenOnlySingle(String whereClause, DateFormat format) { @DataProvider public Object[][] getBoundTypes() { - - Object[][] data = new Object[4][2]; - - data[0][0] = OPEN; - data[0][1] = OPEN; - - data[1][0] = OPEN; - data[1][1] = CLOSED; - - data[2][0] = CLOSED; - data[2][1] = OPEN; - - data[3][0] = CLOSED; - data[3][1] = CLOSED; - - return data; + return new Object[][]{{OPEN, OPEN}, {OPEN, CLOSED}, {CLOSED, OPEN}, {CLOSED, CLOSED}}; } @Test(dataProvider = "getBoundTypes") @@ -148,10 +135,10 @@ public void testBetweenBoundTypes(String startBoundType, String endBoundType) int testStartOffset = START_DATE_OFFSET; int testEndOffset = END_DATE_OFFSET; - if (startBoundType.equals(OPEN)) { + if (startBoundType.equalsIgnoreCase(OPEN)) { testStartOffset = START_DATE_OFFSET - 1; } - if (endBoundType.equals(OPEN)) { + if (endBoundType.equalsIgnoreCase(OPEN)) { testEndOffset = END_DATE_OFFSET + 1; } validateBetweenBoundTypes(whereClause, null, testStartOffset, testEndOffset); @@ -163,18 +150,11 @@ public void testBetweenBoundTypes(String startBoundType, String endBoundType) validateBetweenBoundTypes(whereClause, DAY_DB_FORMAT, testStartOffset, testEndOffset); } + private void validateBetweenBoundTypes(String whereClause, DateFormat format, - int testStartOffset, int testEndOffset) { - String expected = null; - if (format == null) { - expected = - getBetweenClause("test", "dt", getDateWithOffset(DAILY, testStartOffset), - getDateWithOffset(DAILY, testEndOffset), DAILY.format()); - } else { - expected = - getBetweenClause("test", "dt", getDateWithOffset(DAILY, testStartOffset), - getDateWithOffset(DAILY, testEndOffset), format); - } + int testStartOffset, int testEndOffset) { + String expected = getBetweenClause("test", "dt", getDateWithOffset(DAILY, testStartOffset), + getDateWithOffset(DAILY, testEndOffset), ofNullable(format).orElseGet(DAILY::format)); Assert.assertEquals(expected, whereClause); } } diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java index 2f002444f..9fbeb4163 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java @@ -48,7 +48,8 @@ public void setupInstance() throws Exception { public void testBridgeTablesWithoutDimtablePartitioning() throws Exception { String query = "select usersports.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE; String hqlQuery = rewrite(query, hConf); - String expected = getExpectedQuery("basecube", "select usersports.balias0, sum(basecube.msr2) FROM ", + String expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `name`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from " + getDbName() + "c1_user_interests_tbl user_interests" @@ -60,6 +61,16 @@ public void testBridgeTablesWithoutDimtablePartitioning() throws Exception { TestCubeRewriter.compareQueries(hqlQuery, expected); // run with chain ref column query = "select sports, sum(msr2) from basecube where " + TWO_DAYS_RANGE; + expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `sports`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", + " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + + " from " + getDbName() + "c1_user_interests_tbl user_interests" + + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id" + + " group by user_interests.user_id) usersports" + + " on userdim.id = usersports.user_id ", + null, "group by usersports.balias0", null, + getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); hqlQuery = rewrite(query, hConf); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -68,8 +79,9 @@ public void testBridgeTablesWithoutDimtablePartitioning() throws Exception { public void testBridgeTablesForExprFieldWithoutDimtablePartitioning() throws Exception { String query = "select substr(usersports.name, 10), sum(msr2) from basecube where " + TWO_DAYS_RANGE; String hqlQuery = rewrite(query, hConf); - String expected = getExpectedQuery("basecube", "select usersports.balias0, sum(basecube.msr2) FROM ", - " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + String expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `substr((usersports.name), 10)`, " + + "sum((basecube.msr2)) as `sum(msr2)` FROM", " join " + + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(substr(usersports.name, 10)) as balias0" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id" @@ -81,6 +93,16 @@ public void testBridgeTablesForExprFieldWithoutDimtablePartitioning() throws Exc // run with chain ref column query = "select substrsprorts, sum(msr2) from basecube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hConf); + expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `substrsprorts`, " + + "sum((basecube.msr2)) as `sum(msr2)` FROM", " join " + + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + + " join (select user_interests.user_id as user_id,collect_set(substr(usersports.name, 10)) as balias0" + + " from " + getDbName() + "c1_user_interests_tbl user_interests" + + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id" + + " group by user_interests.user_id) usersports" + + " on userdim.id = usersports.user_id ", + null, "group by usersports.balias0", null, + getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -90,8 +112,9 @@ public void testFlattenBridgeTablesOFF() throws Exception { conf.setBoolean(CubeQueryConfUtil.ENABLE_FLATTENING_FOR_BRIDGETABLES, false); String query = "select usersports.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE; String hqlQuery = rewrite(query, conf); - String expected = getExpectedQuery("basecube", "select usersports.name, sum(basecube.msr2) FROM ", - " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + String expected = getExpectedQuery("basecube", "SELECT (usersports.name) as `name`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", " join " + getDbName() + + "c1_usertable userdim ON basecube.userid = userdim.id " + " join " + getDbName() + "c1_user_interests_tbl user_interests on userdim.id = user_interests.user_id" + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id", null, "group by usersports.name", null, @@ -99,6 +122,13 @@ public void testFlattenBridgeTablesOFF() throws Exception { TestCubeRewriter.compareQueries(hqlQuery, expected); // run with chain ref column query = "select sports, sum(msr2) from basecube where " + TWO_DAYS_RANGE; + expected = getExpectedQuery("basecube", "SELECT (usersports.name) as `sports`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", " join " + getDbName() + + "c1_usertable userdim ON basecube.userid = userdim.id " + + " join " + getDbName() + "c1_user_interests_tbl user_interests on userdim.id = user_interests.user_id" + + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id", + null, "group by usersports.name", null, + getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); hqlQuery = rewrite(query, conf); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -109,8 +139,9 @@ public void testFlattenBridgeTablesWithCustomAggregate() throws Exception { conf.set(CubeQueryConfUtil.BRIDGE_TABLE_FIELD_AGGREGATOR, "custom_aggr"); String query = "select usersports.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE; String hqlQuery = rewrite(query, conf); - String expected = getExpectedQuery("basecube", "select usersports.balias0, sum(basecube.msr2) FROM ", - " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + String expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `name`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", " join " + + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,custom_aggr(usersports.name) as balias0" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id" @@ -121,6 +152,16 @@ public void testFlattenBridgeTablesWithCustomAggregate() throws Exception { TestCubeRewriter.compareQueries(hqlQuery, expected); // run with chain ref column query = "select sports, sum(msr2) from basecube where " + TWO_DAYS_RANGE; + expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `sports`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", " join " + + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + + " join (select user_interests.user_id as user_id,custom_aggr(usersports.name) as balias0" + + " from " + getDbName() + "c1_user_interests_tbl user_interests" + + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id" + + " group by user_interests.user_id) usersports" + + " on userdim.id = usersports.user_id ", + null, "group by usersports.balias0", null, + getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); hqlQuery = rewrite(query, conf); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -129,8 +170,8 @@ public void testFlattenBridgeTablesWithCustomAggregate() throws Exception { public void testBridgeTablesWithMegringChains() throws Exception { String query = "select userInterestIds.sport_id, usersports.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE; String hqlQuery = rewrite(query, hConf); - String expected = getExpectedQuery("basecube", "select userInterestIds.balias0, usersports.balias0," - + " sum(basecube.msr2) FROM ", + String expected = getExpectedQuery("basecube", "SELECT (userinterestids.balias0) as `sport_id`, " + + "(usersports.balias0) as `name`, sum((basecube.msr2)) as `sum(msr2)` FROM ", " join " + getDbName() + "c1_usertable userdim on basecube.userid = userdim.id join (select userinterestids" + ".user_id as user_id,collect_set(userinterestids.sport_id) as balias0 from " + getDbName() + "c1_user_interests_tbl userinterestids group by userinterestids.user_id) userinterestids on userdim.id = " @@ -144,6 +185,18 @@ public void testBridgeTablesWithMegringChains() throws Exception { TestCubeRewriter.compareQueries(hqlQuery, expected); // run with chain ref column query = "select sportids, sports, sum(msr2) from basecube where " + TWO_DAYS_RANGE; + expected = getExpectedQuery("basecube", "SELECT (userinterestids.balias0) as `sportids`, " + + "(usersports.balias0) as `sports`, sum((basecube.msr2)) as `sum(msr2)` FROM ", + " join " + getDbName() + "c1_usertable userdim on basecube.userid = userdim.id join (select userinterestids" + + ".user_id as user_id,collect_set(userinterestids.sport_id) as balias0 from " + getDbName() + + "c1_user_interests_tbl userinterestids group by userinterestids.user_id) userinterestids on userdim.id = " + + "userinterestids.user_id " + + "join (select userinterestids.user_id as user_id,collect_set(usersports.name) as balias0 from " + + getDbName() + "c1_user_interests_tbl userinterestids join " + + getDbName() + "c1_sports_tbl usersports on userinterestids.sport_id = usersports.id" + + " group by userinterestids.user_id) usersports on userdim.id = usersports.user_id", + null, "group by userinterestids.balias0, usersports.balias0", null, + getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); hqlQuery = rewrite(query, hConf); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -153,7 +206,8 @@ public void testBridgeTablesWithMultipleFacts() throws Exception { String query = "select usersports.name, msr2, msr12 from basecube where " + TWO_DAYS_RANGE; String hqlQuery = rewrite(query, hConf); String expected1 = getExpectedQuery("basecube", - "select usersports.balias0 as `name`, sum(basecube.msr2) as `msr2` FROM ", " join " + getDbName() + "SELECT (usersports.balias0) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 as `alias2` FROM ", + " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() @@ -161,7 +215,8 @@ public void testBridgeTablesWithMultipleFacts() throws Exception { + " group by user_interests.user_id) usersports" + " on userdim.id = usersports.user_id ", null, "group by usersports.balias0", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); String expected2 = getExpectedQuery("basecube", - "select usersports.balias0 as `name`, sum(basecube.msr12) as `msr12` FROM ", " join " + getDbName() + "SELECT (usersports.balias0) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) as `alias2` FROM ", + " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() @@ -173,25 +228,26 @@ public void testBridgeTablesWithMultipleFacts() throws Exception { TestCubeRewriter.compareContains(expected2, hqlQuery); String lower = hqlQuery.toLowerCase(); assertTrue( - lower.startsWith("select coalesce(mq1.name, mq2.name) name, mq2.msr2 msr2, mq1.msr12 msr12 from ") - || lower.startsWith("select coalesce(mq1.name, mq2.name) name, mq1.msr2 msr2, mq2.msr12 msr12 from "), hqlQuery); + lower.startsWith("select (basecube.alias0) as `name`, sum((basecube.alias1)) as `msr2`, " + + "sum((basecube.alias2)) as `msr12` from"), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.name <=> mq2.name"), + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); // run with chain ref column query = "select sports, msr2, msr12 from basecube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hConf); expected1 = getExpectedQuery("basecube", - "select usersports.balias0 as `sports`, sum(basecube.msr2) as `msr2` FROM ", " join " + getDbName() - + "c1_usertable userdim ON basecube.userid = userdim.id " + "SELECT (usersports.balias0) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 as `alias2` FROM ", + " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id" + " group by user_interests.user_id) usersports" + " on userdim.id = usersports.user_id ", null, "group by usersports.balias0", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); expected2 = getExpectedQuery("basecube", - "select usersports.balias0 as `sports`, sum(basecube.msr12) as `msr12` FROM ", " join " + getDbName() + "SELECT (usersports.balias0) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) " + + "as `alias2` FROM ", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() @@ -203,11 +259,10 @@ public void testBridgeTablesWithMultipleFacts() throws Exception { TestCubeRewriter.compareContains(expected2, hqlQuery); lower = hqlQuery.toLowerCase(); assertTrue( - lower.startsWith("select coalesce(mq1.sports, mq2.sports) sports, mq2.msr2 msr2, mq1.msr12 msr12 from ") - || lower.startsWith("select coalesce(mq1.sports, mq2.sports) sports, mq1.msr2 msr2, mq2.msr12 msr12 from "), - hqlQuery); + lower.startsWith("select (basecube.alias0) as `sports`, sum((basecube.alias1)) as `msr2`, " + + "sum((basecube.alias2)) as `msr12` from"), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.sports <=> mq2.sports"), + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @@ -216,8 +271,8 @@ public void testBridgeTablesWithMultipleChains() throws Exception { String query = "select usersports.name, xusersports.name, yusersports.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE; String hqlQuery = rewrite(query, hConf); - String expected = getExpectedQuery("basecube", "select usersports.balias0, xusersports.balias0, " - + "yusersports.balias0, sum(basecube.msr2) FROM ", + String expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `name`, (xusersports.balias0) " + + "as `name`, (yusersports.balias0) as `name`, sum((basecube.msr2)) as `sum(msr2)` FROM ", " join " + getDbName() + "c1_usertable userdim_1 on basecube.userid = userdim_1.id " + " join (select user_interests_1.user_id as user_id, collect_set(usersports.name) as balias0 from " + getDbName() + "c1_user_interests_tbl user_interests_1 join " + getDbName() + "c1_sports_tbl usersports on " @@ -236,6 +291,27 @@ public void testBridgeTablesWithMultipleChains() throws Exception { TestCubeRewriter.compareQueries(hqlQuery, expected); // run with chain ref column query = "select sports, xsports, ysports, sum(msr2) from basecube where " + TWO_DAYS_RANGE; + expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `sports`, (xusersports.balias0) " + + "as `xsports`, (yusersports.balias0) as `ysports`, sum((basecube.msr2)) as `sum(msr2)` FROM ", + " join " + getDbName() + "c1_usertable userdim_1 on basecube.userid = userdim_1.id " + + " join (select user_interests_1.user_id as user_id, collect_set(usersports.name) as balias0 from " + + getDbName() + "c1_user_interests_tbl user_interests_1 join " + getDbName() + + "c1_sports_tbl usersports on " + + "user_interests_1.sport_id = usersports.id group by user_interests_1.user_id) " + + "usersports on userdim_1.id = usersports.user_id" + + " join " + getDbName() + "c1_usertable userdim_0 on basecube.yuserid = userdim_0.id " + + " join (select user_interests_0.user_id as user_id,collect_set(yusersports.name) as balias0 from " + + getDbName() + "c1_user_interests_tbl user_interests_0 join " + getDbName() + + "c1_sports_tbl yusersports on user_interests_0.sport_id = yusersports.id group by " + + "user_interests_0.user_id) yusersports on userdim_0.id =" + + " yusersports.user_id join " + getDbName() + "c1_usertable userdim on basecube.xuserid = userdim.id" + + " join (select user_interests.user_id as user_id,collect_set(xusersports.name) as balias0 from " + + getDbName() + "c1_user_interests_tbl user_interests join " + getDbName() + + "c1_sports_tbl xusersports on user_interests.sport_id = xusersports.id " + + "group by user_interests.user_id) xusersports on userdim.id = " + + " xusersports.user_id", + null, "group by usersports.balias0, xusersports.balias0, yusersports.balias0", null, + getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); hqlQuery = rewrite(query, hConf); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -247,8 +323,8 @@ public void testBridgeTablesWithMultipleChainsWithJoinType() throws Exception { String query = "select usersports.name, xusersports.name, yusersports.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE; String hqlQuery = rewrite(query, conf); - String expected = getExpectedQuery("basecube", "select usersports.balias0, xusersports.balias0, " - + "yusersports.balias0, sum(basecube.msr2) FROM ", + String expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `name`, (xusersports.balias0) " + + "as `name`, (yusersports.balias0) as `name`, sum((basecube.msr2)) as `sum(msr2)` FROM ", " left outer join " + getDbName() + "c1_usertable userdim_1 on basecube.userid = userdim_1.id " + " left outer join (select user_interests_1.user_id as user_id, collect_set(usersports.name) as balias0 from " + getDbName() + "c1_user_interests_tbl user_interests_1 join " + getDbName() + "c1_sports_tbl usersports on " @@ -268,6 +344,30 @@ public void testBridgeTablesWithMultipleChainsWithJoinType() throws Exception { TestCubeRewriter.compareQueries(hqlQuery, expected); // run with chain ref column query = "select sports, xsports, ysports, sum(msr2) from basecube where " + TWO_DAYS_RANGE; + expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `sports`, (xusersports.balias0) " + + "as `xsports`, (yusersports.balias0) as `ysports`, sum((basecube.msr2)) as `sum(msr2)` FROM ", + " left outer join " + getDbName() + "c1_usertable userdim_1 on basecube.userid = userdim_1.id " + + " left outer join (select user_interests_1.user_id as user_id, " + + "collect_set(usersports.name) as balias0 from " + + getDbName() + "c1_user_interests_tbl user_interests_1 join " + + getDbName() + "c1_sports_tbl usersports on " + + "user_interests_1.sport_id = usersports.id group by user_interests_1.user_id) " + + "usersports on userdim_1.id = usersports.user_id" + + " left outer join " + getDbName() + "c1_usertable userdim_0 on basecube.yuserid = userdim_0.id " + + " left outer join (select user_interests_0.user_id as user_id," + + "collect_set(yusersports.name) as balias0 from " + + getDbName() + "c1_user_interests_tbl user_interests_0 join " + getDbName() + + "c1_sports_tbl yusersports on " + + " user_interests_0.sport_id = yusersports.id group by user_interests_0.user_id) " + + "yusersports on userdim_0.id = yusersports.user_id left outer join " + getDbName() + + "c1_usertable userdim on basecube.xuserid = userdim.id" + + " left outer join (select user_interests.user_id as user_id," + + "collect_set(xusersports.name) as balias0 from " + getDbName() + + "c1_user_interests_tbl user_interests join " + getDbName() + "c1_sports_tbl xusersports" + + " on user_interests.sport_id = xusersports.id group by user_interests.user_id) " + + "xusersports on userdim.id = xusersports.user_id", null, + "group by usersports.balias0, xusersports.balias0, yusersports.balias0", null, + getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); hqlQuery = rewrite(query, conf); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -278,8 +378,9 @@ public void testBridgeTablesWithDimTablePartitioning() throws Exception { conf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); String query = "select usersports.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE; String hqlQuery = rewrite(query, conf); - String expected = getExpectedQuery("basecube", "select usersports.balias0, sum(basecube.msr2) FROM ", - " join " + getDbName() + "c2_usertable userdim ON basecube.userid = userdim.id and userdim.dt='latest' " + String expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `name`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", " join " + getDbName() + + "c2_usertable userdim ON basecube.userid = userdim.id and userdim.dt='latest' " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from " + getDbName() + "c2_user_interests_tbl user_interests" + " join " + getDbName() + "c2_sports_tbl usersports on user_interests.sport_id = usersports.id" @@ -291,6 +392,17 @@ public void testBridgeTablesWithDimTablePartitioning() throws Exception { TestCubeRewriter.compareQueries(hqlQuery, expected); // run with chain ref column query = "select sports, sum(msr2) from basecube where " + TWO_DAYS_RANGE; + expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `sports`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", " join " + getDbName() + + "c2_usertable userdim ON basecube.userid = userdim.id and userdim.dt='latest' " + + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + + " from " + getDbName() + "c2_user_interests_tbl user_interests" + + " join " + getDbName() + "c2_sports_tbl usersports on user_interests.sport_id = usersports.id" + + " and usersports.dt='latest and user_interests.dt='latest'" + + " group by user_interests.user_id) usersports" + + " on userdim.id = usersports.user_id ", + null, "group by usersports.balias0", null, + getWhereForDailyAndHourly2days("basecube", "c2_testfact1_base")); hqlQuery = rewrite(query, conf); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -300,8 +412,8 @@ public void testBridgeTablesWithNormalJoins() throws Exception { String query = "select usersports.name, cubestatecountry.name, cubecitystatecountry.name," + " sum(msr2) from basecube where " + TWO_DAYS_RANGE; String hqlQuery = rewrite(query, hConf); - String expected = getExpectedQuery("basecube", "select usersports.balias0, cubestatecountry.name, " - + "cubecitystatecountry.name, sum(basecube.msr2) FROM ", + String expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `name`, (cubestatecountry.name) " + + "as `name`, (cubecitystatecountry.name) as `name`, sum((basecube.msr2)) as `sum(msr2)` FROM ", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from " + getDbName() + "c1_user_interests_tbl user_interests" @@ -313,13 +425,33 @@ public void testBridgeTablesWithNormalJoins() throws Exception { + "c1_statetable statedim_0 on citydim.stateid=statedim_0.id and statedim_0.dt='latest'" + " join " + getDbName() + "c1_countrytable cubecitystatecountry on statedim_0.countryid=cubecitystatecountry.id" - + " join " + getDbName() + "c1_statetable statedim on basecube.stateid=statedim.id and (statedim.dt = 'latest')" + + " join " + getDbName() + "c1_statetable statedim on basecube.stateid=statedim.id " + + "and (statedim.dt = 'latest')" + " join " + getDbName() + "c1_countrytable cubestatecountry on statedim.countryid=cubestatecountry.id ", null, "group by usersports.balias0, cubestatecountry.name, cubecitystatecountry.name", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); TestCubeRewriter.compareQueries(hqlQuery, expected); // run with chain ref column query = "select sports, statecountry, citycountry, sum(msr2) from basecube where " + TWO_DAYS_RANGE; + expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `sports`, (cubestatecountry.name) " + + "as `statecountry`, (cubecitystatecountry.name) as `citycountry`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", + " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + + " from " + getDbName() + "c1_user_interests_tbl user_interests" + + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id" + + " group by user_interests.user_id) usersports" + + " on userdim.id = usersports.user_id " + + " join " + getDbName() + "c1_citytable citydim on basecube.cityid = citydim.id " + + "and (citydim.dt = 'latest') join " + getDbName() + + "c1_statetable statedim_0 on citydim.stateid=statedim_0.id and statedim_0.dt='latest'" + + " join " + getDbName() + + "c1_countrytable cubecitystatecountry on statedim_0.countryid=cubecitystatecountry.id" + + " join " + getDbName() + "c1_statetable statedim on basecube.stateid=statedim.id " + + "and (statedim.dt = 'latest')" + + " join " + getDbName() + "c1_countrytable cubestatecountry on statedim.countryid=cubestatecountry.id ", + null, "group by usersports.balias0, cubestatecountry.name, cubecitystatecountry.name", null, + getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); hqlQuery = rewrite(query, hConf); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -333,8 +465,9 @@ public void testBridgeTablesWithFilterBeforeFlattening() throws Exception { + " and not (some_filter(usersports.name, 'ASD') OR some_filter(usersports.name, 'ZXC'))" + " and myfunc(usersports.name) = 'CRT' and substr(usersports.name, 3) in ('CRI')"; String hqlQuery = rewrite(query, hConf); - String expected = getExpectedQuery("basecube", "select usersports.balias0, sum(basecube.msr2) FROM ", - " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + String expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `name`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", " join " + + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0," + " collect_set(myfunc(usersports.name)) as balias1, collect_set(substr(usersports.name, 3)) as balias2" + " from " + getDbName() + "c1_user_interests_tbl user_interests" @@ -357,6 +490,24 @@ public void testBridgeTablesWithFilterBeforeFlattening() throws Exception { + " and (some_filter(sports, 'CRICKET') OR some_filter(sports, 'FOOTBALL'))" + " and not (some_filter(sports, 'ASD') OR some_filter(sports, 'ZXC'))" + " and myfunc(sports) = 'CRT' and sports_abbr in ('CRI')"; + expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `sports`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", " join " + + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0," + + " collect_set(myfunc(usersports.name)) as balias1, collect_set(substr(usersports.name, 3)) as balias2" + + " from " + getDbName() + "c1_user_interests_tbl user_interests" + + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id " + + " group by user_interests.user_id) usersports" + + " on userdim.id = usersports.user_id ", + null, " and array_contains(usersports.balias0,'CRICKET') and (array_contains(usersports.balias0, 'BB')" + + " OR array_contains(usersports.balias0, 'FOOTBALL'))" + + " and not array_contains(usersports.balias0, 'RANDOM'))" + + " and not (array_contains(usersports.balias0, 'xyz') OR array_contains(usersports.balias0, 'ABC'))" + + " and (some_filter(usersports.name, 'CRICKET') OR some_filter(usersports.name, 'FOOTBALL'))" + + " and not (some_filter(usersports.name, 'ASD') OR some_filter(usersports.name, 'ZXC'))" + + " and (array_contains(usersports.balias1, 'CRT') AND array_contains(usersports.balias2, 'CRI'))" + + "group by usersports.balias0", null, + getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); hqlQuery = rewrite(query, hConf); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -364,22 +515,36 @@ public void testBridgeTablesWithFilterBeforeFlattening() throws Exception { @Test public void testBridgeTablesWithFilterAndOrderby() throws Exception { String query = "select usersports.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE - + " and usersports.name = 'CRICKET' order by usersports.name"; + + " and usersports.name = 'CRICKET' order by usersports.name"; String hqlQuery = rewrite(query, hConf); - String expected = getExpectedQuery("basecube", "select usersports.balias0, sum(basecube.msr2) FROM ", - " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " - + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" - + " from " + getDbName() + "c1_user_interests_tbl user_interests" - + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id " - + " group by user_interests.user_id) usersports" - + " on userdim.id = usersports.user_id ", - null, - " and array_contains(usersports.balias0, 'CRICKET') group by usersports.balias0 order by usersports.balias0 asc", - null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); + String expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `name`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", " join " + + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + + " from " + getDbName() + "c1_user_interests_tbl user_interests" + + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id " + + " group by user_interests.user_id) usersports" + + " on userdim.id = usersports.user_id ", + null, + " and array_contains(usersports.balias0, 'CRICKET') group by usersports.balias0 " + + "order by name asc", + null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); TestCubeRewriter.compareQueries(hqlQuery, expected); // run with chain ref column query = "select sports, sum(msr2) from basecube where " + TWO_DAYS_RANGE + " and sports = 'CRICKET' order by " - + "sports"; + + "sports"; + expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `sports`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", " join " + + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + + " from " + getDbName() + "c1_user_interests_tbl user_interests" + + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id " + + " group by user_interests.user_id) usersports" + + " on userdim.id = usersports.user_id ", + null, + " and array_contains(usersports.balias0, 'CRICKET') group by usersports.balias0 " + + "order by sports asc", + null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); hqlQuery = rewrite(query, hConf); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -391,7 +556,8 @@ public void testFlattenBridgeTablesWithCustomFilter() throws Exception { String query = "select usersports.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE + " and usersports.name in ('CRICKET','FOOTBALL')"; String hqlQuery = rewrite(query, conf); - String expected = getExpectedQuery("basecube", "select usersports.balias0, sum(basecube.msr2) FROM ", + String expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `name`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from " + getDbName() + "c1_user_interests_tbl user_interests" @@ -404,6 +570,17 @@ public void testFlattenBridgeTablesWithCustomFilter() throws Exception { TestCubeRewriter.compareQueries(hqlQuery, expected); // run with chain ref column query = "select sports, sum(msr2) from basecube where " + TWO_DAYS_RANGE + " and sports in ('CRICKET','FOOTBALL')"; + expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `sports`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", + " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + + " from " + getDbName() + "c1_user_interests_tbl user_interests" + + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id" + + " group by user_interests.user_id) usersports" + + " on userdim.id = usersports.user_id ", null, + " and (custom_filter(usersports.balias0, 'CRICKET') OR custom_filter(usersports.balias0, 'FOOTBALL'))" + + "group by usersports.balias0", + null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); hqlQuery = rewrite(query, conf); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -415,8 +592,9 @@ public void testBridgeTablesWithFilterAfterFlattening() throws Exception { String query = "select usersports.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE + " and usersports.name = 'CRICKET,FOOTBALL'"; String hqlQuery = rewrite(query, conf); - String expected = getExpectedQuery("basecube", "select usersports.name, sum(basecube.msr2) FROM ", - " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + String expected = getExpectedQuery("basecube", "SELECT (usersports.name) as `name`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", " join " + + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id" @@ -427,6 +605,16 @@ public void testBridgeTablesWithFilterAfterFlattening() throws Exception { TestCubeRewriter.compareQueries(hqlQuery, expected); // run with chain ref column query = "select sports, sum(msr2) from basecube where " + TWO_DAYS_RANGE + " and sports = 'CRICKET,FOOTBALL'"; + expected = getExpectedQuery("basecube", "SELECT (usersports.name) as `sports`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", " join " + + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + + " from " + getDbName() + "c1_user_interests_tbl user_interests" + + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id" + + " group by user_interests.user_id) usersports" + + " on userdim.id = usersports.user_id ", null, + " and usersports.name = 'CRICKET,FOOTBALL' group by usersports.name", null, + getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); hqlQuery = rewrite(query, conf); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -437,7 +625,8 @@ public void testBridgeTablesWithMultipleFactsWithFilterBeforeFlattening() throws + " and usersports.name = 'CRICKET'"; String hqlQuery = rewrite(query, hConf); String expected1 = getExpectedQuery("basecube", - "select usersports.balias0 as `name`, sum(basecube.msr2) as `msr2` FROM ", " join " + getDbName() + "SELECT (usersports.balias0) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 " + + "as `alias2` FROM ", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() @@ -446,7 +635,8 @@ public void testBridgeTablesWithMultipleFactsWithFilterBeforeFlattening() throws " and array_contains(usersports.balias0,'CRICKET') group by usersports.balias0", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); String expected2 = getExpectedQuery("basecube", - "select usersports.balias0 as `name`, sum(basecube.msr12) as `msr12` FROM ", " join " + getDbName() + "SELECT (usersports.balias0) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) " + + "as `alias2` FROM", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() @@ -457,16 +647,17 @@ public void testBridgeTablesWithMultipleFactsWithFilterBeforeFlattening() throws TestCubeRewriter.compareContains(expected1, hqlQuery); TestCubeRewriter.compareContains(expected2, hqlQuery); String lower = hqlQuery.toLowerCase(); - assertTrue(lower.startsWith("select coalesce(mq1.name, mq2.name) name, mq2.msr2 msr2, mq1.msr12 msr12 from ") - || lower.startsWith("select coalesce(mq1.name, mq2.name) name, mq1.msr2 msr2, mq2.msr12 msr12 from "), hqlQuery); + assertTrue(lower.startsWith("select (basecube.alias0) as `name`, sum((basecube.alias1)) as `msr2`, " + + "sum((basecube.alias2)) as `msr12` from"), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.name <=> mq2.name"), + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); // run with chain ref column query = "select sports, msr2, msr12 from basecube where " + TWO_DAYS_RANGE + " and sports = 'CRICKET'"; hqlQuery = rewrite(query, hConf); expected1 = getExpectedQuery("basecube", - "select usersports.balias0 as `sports`, sum(basecube.msr2) as `msr2` FROM ", " join " + getDbName() + "SELECT (usersports.balias0) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 " + + "as `alias2` FROM ", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() @@ -475,7 +666,8 @@ public void testBridgeTablesWithMultipleFactsWithFilterBeforeFlattening() throws "and array_contains(usersports.balias0,'CRICKET') group by usersports.balias0", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); expected2 = getExpectedQuery("basecube", - "select usersports.balias0 as `sports`, sum(basecube.msr12) as `msr12` FROM ", " join " + getDbName() + "SELECT (usersports.balias0) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) " + + "as `alias2` FROM ", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as balias0" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() @@ -486,11 +678,10 @@ public void testBridgeTablesWithMultipleFactsWithFilterBeforeFlattening() throws TestCubeRewriter.compareContains(expected1, hqlQuery); TestCubeRewriter.compareContains(expected2, hqlQuery); lower = hqlQuery.toLowerCase(); - assertTrue(lower.startsWith("select coalesce(mq1.sports, mq2.sports) sports, mq2.msr2 msr2, mq1.msr12 msr12 from ") - || lower.startsWith("select coalesce(mq1.sports, mq2.sports) sports, mq1.msr2 msr2, mq2.msr12 msr12 from "), - hqlQuery); + assertTrue(lower.startsWith("select (basecube.alias0) as `sports`, sum((basecube.alias1)) as `msr2`, " + + "sum((basecube.alias2)) as `msr12` from"), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.sports <=> mq2.sports"), + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @@ -502,8 +693,8 @@ public void testBridgeTablesWithMultipleFactsWithFilterAfterFlattening() throws + " and usersports.name = 'CRICKET,FOOTBALL'"; String hqlQuery = rewrite(query, conf); String expected1 = getExpectedQuery("basecube", - "select usersports.name as `name`, sum(basecube.msr2) as `msr2` FROM ", " join " + getDbName() - + "c1_usertable userdim ON basecube.userid = userdim.id " + "SELECT (usersports.name) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 as `alias2` FROM ", + " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id" @@ -511,7 +702,8 @@ public void testBridgeTablesWithMultipleFactsWithFilterAfterFlattening() throws " and usersports.name = 'CRICKET,FOOTBALL' group by usersports.name", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); String expected2 = getExpectedQuery("basecube", - "select usersports.name as `name`, sum(basecube.msr12) as `msr12` FROM ", " join " + getDbName() + "SELECT (usersports.name) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) as `alias2` FROM ", + " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() @@ -522,17 +714,18 @@ public void testBridgeTablesWithMultipleFactsWithFilterAfterFlattening() throws TestCubeRewriter.compareContains(expected1, hqlQuery); TestCubeRewriter.compareContains(expected2, hqlQuery); String lower = hqlQuery.toLowerCase(); - assertTrue(lower.startsWith("select coalesce(mq1.name, mq2.name) name, mq2.msr2 msr2, mq1.msr12 msr12 from ") - || lower.startsWith("select coalesce(mq1.name, mq2.name) name, mq1.msr2 msr2, mq2.msr12 msr12 from "), hqlQuery); + assertTrue(lower.startsWith("select (basecube.alias0) as `name`, sum((basecube.alias1)) as `msr2`, " + + "sum((basecube.alias2)) as `msr12` from"), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.name <=> mq2.name"), + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); // run with chain ref column query = "select sports, msr2, msr12 from basecube where " + TWO_DAYS_RANGE + " and sports = 'CRICKET,FOOTBALL'"; hqlQuery = rewrite(query, conf); expected1 = getExpectedQuery("basecube", - "select usersports.name as `sports`, sum(basecube.msr2) as `msr2` FROM ", " join " + getDbName() + "SELECT (usersports.name) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 " + + "as `alias2` FROM ", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() @@ -541,7 +734,8 @@ public void testBridgeTablesWithMultipleFactsWithFilterAfterFlattening() throws " and usersports.name = 'CRICKET,FOOTBALL' group by usersports.name", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); expected2 = getExpectedQuery("basecube", - "select usersports.name as `sports`, sum(basecube.msr12) as `msr12` FROM ", " join " + getDbName() + "SELECT (usersports.name) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) " + + "as `alias2` FROM ", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() @@ -552,11 +746,9 @@ public void testBridgeTablesWithMultipleFactsWithFilterAfterFlattening() throws TestCubeRewriter.compareContains(expected1, hqlQuery); TestCubeRewriter.compareContains(expected2, hqlQuery); lower = hqlQuery.toLowerCase(); - assertTrue(lower.startsWith("select coalesce(mq1.sports, mq2.sports) sports, mq2.msr2 msr2, mq1.msr12 msr12 from ") - || lower.startsWith("select coalesce(mq1.sports, mq2.sports) sports, mq1.msr2 msr2, mq2.msr12 msr12 from "), - hqlQuery); - - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.sports <=> mq2.sports"), + assertTrue(lower.startsWith("select (basecube.alias0) as `sports`, sum((basecube.alias1)) as `msr2`, " + + "sum((basecube.alias2)) as `msr12` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @@ -565,7 +757,8 @@ public void testBridgeTablesWithExpressionBeforeFlattening() throws Exception { String query = "select substr(usersports.name, 3), sum(msr2) from basecube where " + TWO_DAYS_RANGE + " and usersports.name = 'CRICKET'"; String hqlQuery = rewrite(query, hConf); - String expected = getExpectedQuery("basecube", "select usersports.balias0, sum(basecube.msr2) FROM ", + String expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `substr((usersports.name), 3)`, " + + "sum((basecube.msr2)) as `sum(msr2)` FROM ", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(substr(usersports.name, 3)) as balias0" + " collect_set(( usersports . name )) as balias1 from " + getDbName() + "c1_user_interests_tbl user_interests" @@ -577,6 +770,17 @@ public void testBridgeTablesWithExpressionBeforeFlattening() throws Exception { TestCubeRewriter.compareQueries(hqlQuery, expected); // run with chain ref column query = "select sports_abbr, sum(msr2) from basecube where " + TWO_DAYS_RANGE + " and sports = 'CRICKET'"; + expected = getExpectedQuery("basecube", "SELECT (usersports.balias0) as `sports_abbr`, " + + "sum((basecube.msr2)) as `sum(msr2)` FROM ", + " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + + " join (select user_interests.user_id as user_id,collect_set(substr(usersports.name, 3)) as balias0" + + " collect_set(( usersports . name )) as balias1 from " + getDbName() + + "c1_user_interests_tbl user_interests" + + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id " + + " group by user_interests.user_id) usersports" + + " on userdim.id = usersports.user_id ", + null, " and array_contains(usersports.balias1, 'CRICKET') group by usersports.balias0", null, + getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); hqlQuery = rewrite(query, hConf); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -588,8 +792,9 @@ public void testBridgeTablesWithExpressionAfterFlattening() throws Exception { String query = "select substr(usersports.name, 3), sum(msr2) from basecube where " + TWO_DAYS_RANGE + " and usersports.name = 'CRICKET,FOOTBALL'"; String hqlQuery = rewrite(query, conf); - String expected = getExpectedQuery("basecube", "select substr(usersports.name, 3), sum(basecube.msr2) FROM ", - " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + String expected = getExpectedQuery("basecube", "SELECT substr((usersports.name), 3) as " + + "`substr((usersports.name), 3)`, sum((basecube.msr2)) as `sum(msr2)` FROM ", " join " + + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id" @@ -599,7 +804,18 @@ public void testBridgeTablesWithExpressionAfterFlattening() throws Exception { getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); TestCubeRewriter.compareQueries(hqlQuery, expected); // run with chain ref column - query = "select sports_abbr, sum(msr2) from basecube where " + TWO_DAYS_RANGE + " and sports = 'CRICKET,FOOTBALL'"; + query = "select sports_abbr, sum(msr2) from basecube where " + TWO_DAYS_RANGE + + " and sports = 'CRICKET,FOOTBALL'"; + expected = getExpectedQuery("basecube", "SELECT substr((usersports.name), 3) as " + + "`sports_abbr`, sum((basecube.msr2)) as `sum(msr2)` FROM ", " join " + + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + + " from " + getDbName() + "c1_user_interests_tbl user_interests" + + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id" + + " group by user_interests.user_id) usersports" + + " on userdim.id = usersports.user_id ", null, + " and usersports.name = 'CRICKET,FOOTBALL' group by substr(usersports.name, 3)", null, + getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); hqlQuery = rewrite(query, conf); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -610,8 +826,8 @@ public void testBridgeTablesWithExpressionAndAliasesBeforeFlattening() throws Ex + " sum(msr2) from basecube where " + TWO_DAYS_RANGE + " and usersports.name = 'CRICKET' and substr(usersports.name, 3) = 'CRI' and (userid = 4 or userid = 5)"; String hqlQuery = rewrite(query, hConf); - String expected = getExpectedQuery("basecube", "select basecube.userid as `uid`, usersports.balias0 as `uname`, " - + " (usersports.balias1) as `sub user`, sum(basecube.msr2) FROM ", + String expected = getExpectedQuery("basecube", "SELECT (basecube.userid) as `uid`, (usersports.balias0) " + + "as `uname`, (usersports.balias1) as `sub user`, sum((basecube.msr2)) as `sum(msr2)` FROM ", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id, collect_set(usersports.name) as balias0, " + "collect_set(substr(usersports.name, 3)) as balias1" @@ -639,9 +855,9 @@ public void testBridgeTablesWithExpressionAndAliasesAfterFlattening() throws Exc + " sum(msr2) from basecube where " + TWO_DAYS_RANGE + " and usersports.name = 'CRICKET,FOOTBALL'"; String hqlQuery = rewrite(query, conf); - String expected = getExpectedQuery("basecube", "select usersports.name as `uname`, substr(usersports.name, 3) as " - + "`sub user`, sum(basecube.msr2) FROM ", - " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + String expected = getExpectedQuery("basecube", "SELECT (usersports.name) as `uname`, substr((usersports.name), 3) " + + "as `sub user`, sum((basecube.msr2)) as `sum(msr2)` FROM ", " join " + + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() + "c1_sports_tbl usersports on user_interests.sport_id = usersports.id" @@ -662,7 +878,8 @@ public void testBridgeTablesWithMultipleFactsWithExprBeforeFlattening() throws E + " and usersports.name in ('CRICKET', 'FOOTBALL')"; String hqlQuery = rewrite(query, hConf); String expected1 = getExpectedQuery("basecube", - "select usersports.balias0 as `expr1`, sum(basecube.msr2) as `msr2` FROM ", " join " + getDbName() + "SELECT (usersports.balias0) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 " + + "as `alias2` FROM ", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id, collect_set(substr(usersports.name, 3)) as balias0, " + " collect_set(usersports.name) as balias1 from" @@ -673,7 +890,8 @@ public void testBridgeTablesWithMultipleFactsWithExprBeforeFlattening() throws E + " group by usersports.balias0", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); String expected2 = getExpectedQuery("basecube", - "select usersports.balias0 as `expr1`, sum(basecube.msr12) as `msr12` FROM ", " join " + getDbName() + "SELECT (usersports.balias0) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) as `alias2` FROM " + , " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id, collect_set(substr(usersports.name, 3)) as balias0, " + " collect_set(usersports.name) as balias1 from" @@ -686,20 +904,19 @@ public void testBridgeTablesWithMultipleFactsWithExprBeforeFlattening() throws E TestCubeRewriter.compareContains(expected1, hqlQuery); TestCubeRewriter.compareContains(expected2, hqlQuery); String lower = hqlQuery.toLowerCase(); - assertTrue(lower.startsWith("select coalesce(mq1.expr1, mq2.expr1) `substr((usersports.name), 3)`," - + " mq2.msr2 msr2, mq1.msr12 msr12 from ") - || lower.startsWith("select coalesce(mq1.expr1, mq2.expr1) `substr((usersports.name), 3)`, mq1.msr2 msr2, " - + "mq2.msr12 msr12 from "), + assertTrue(lower.startsWith("select (basecube.alias0) as `substr((usersports.name), 3)`, " + + "sum((basecube.alias1)) as `msr2`, sum((basecube.alias2)) as `msr12` from"), hqlQuery); - - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.expr1 <=> mq2.expr1"), + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); + // run with chain ref column query = "select sports_abbr, msr2, msr12 from basecube where " + TWO_DAYS_RANGE + " and sports in " + "('CRICKET', 'FOOTBALL')"; hqlQuery = rewrite(query, hConf); expected1 = getExpectedQuery("basecube", - "select usersports.balias0 as `sports_abbr`, sum(basecube.msr2) as `msr2` FROM ", " join " + getDbName() + "SELECT (usersports.balias0) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 " + + "as `alias2` FROM ", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id, collect_set(substr((usersports.name), 3)) as balias0, " + " collect_set(usersports.name) as balias1 from" @@ -710,7 +927,8 @@ public void testBridgeTablesWithMultipleFactsWithExprBeforeFlattening() throws E + " group by usersports.balias0", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); expected2 = getExpectedQuery("basecube", - "select usersports.balias0 as `sports_abbr`, sum(basecube.msr12) as `msr12` FROM ", " join " + getDbName() + "SELECT (usersports.balias0) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) " + + "as `alias2` FROM ", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id, collect_set(substr((usersports.name), 3)) as balias0," + " collect_set(usersports.name) as balias1 from" @@ -724,13 +942,10 @@ public void testBridgeTablesWithMultipleFactsWithExprBeforeFlattening() throws E TestCubeRewriter.compareContains(expected2, hqlQuery); lower = hqlQuery.toLowerCase(); assertTrue(lower.startsWith( - "select coalesce(mq1.sports_abbr, mq2.sports_abbr) sports_abbr, mq2.msr2 msr2, mq1.msr12 msr12 from ") - || lower.startsWith( - "select coalesce(mq1.sports_abbr, mq2.sports_abbr) sports_abbr, mq1.msr2 msr2, mq2.msr12 msr12 from "), - hqlQuery); - - assertTrue(hqlQuery.contains("mq1 full outer join ") - && hqlQuery.endsWith("mq2 on mq1.sports_abbr <=> mq2.sports_abbr"), + "select (basecube.alias0) as `sports_abbr`, sum((basecube.alias1)) as `msr2`, " + + "sum((basecube.alias2)) as `msr12` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") + && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @@ -742,7 +957,8 @@ public void testBridgeTablesWithMultipleFactsWithExprAfterFlattening() throws Ex + " and usersports.name = 'CRICKET,FOOTBALL'"; String hqlQuery = rewrite(query, conf); String expected1 = getExpectedQuery("basecube", - "select substr(usersports.name, 3) as `expr1`, sum(basecube.msr2) as `msr2` FROM ", " join " + getDbName() + "SELECT substr((usersports.name), 3) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 " + + "as `alias2` FROM ", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() @@ -751,7 +967,8 @@ public void testBridgeTablesWithMultipleFactsWithExprAfterFlattening() throws Ex " and usersports.name = 'CRICKET,FOOTBALL' group by substr(usersports.name, 3)", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); String expected2 = getExpectedQuery("basecube", - "select substr(usersports.name, 3) as `expr1`, sum(basecube.msr12) as `msr12` FROM ", " join " + getDbName() + "SELECT substr((usersports.name), 3) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) " + + "as `alias2` FROM ", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() @@ -762,20 +979,19 @@ public void testBridgeTablesWithMultipleFactsWithExprAfterFlattening() throws Ex TestCubeRewriter.compareContains(expected1, hqlQuery); TestCubeRewriter.compareContains(expected2, hqlQuery); String lower = hqlQuery.toLowerCase(); - assertTrue(lower.startsWith("select coalesce(mq1.expr1, mq2.expr1) `substr((usersports.name), 3)`," - + " mq2.msr2 msr2, mq1.msr12 msr12 from ") - || lower.startsWith("select coalesce(mq1.expr1, mq2.expr1) `substr((usersports.name), 3)`, mq1.msr2 msr2," - + " mq2.msr12 msr12 from "), + assertTrue(lower.startsWith("select (basecube.alias0) as `substr((usersports.name), 3)`, " + + "sum((basecube.alias1)) as `msr2`, sum((basecube.alias2)) as `msr12` from"), hqlQuery); - assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.expr1 <=> mq2.expr1"), + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); // run with chain ref column query = "select sports_abbr, msr2, msr12 from basecube where " + TWO_DAYS_RANGE + " and sports = " + "'CRICKET,FOOTBALL'"; hqlQuery = rewrite(query, conf); expected1 = getExpectedQuery("basecube", - "select substr(usersports.name, 3) as `sports_abbr`, sum(basecube.msr2) as `msr2` FROM ", " join " + getDbName() + "SELECT substr((usersports.name), 3) as `alias0`, sum((basecube.msr2)) as `alias1`, 0.0 " + + "as `alias2` FROM ", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() @@ -784,7 +1000,8 @@ public void testBridgeTablesWithMultipleFactsWithExprAfterFlattening() throws Ex " and usersports.name = 'CRICKET,FOOTBALL' group by substr(usersports.name, 3)", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); expected2 = getExpectedQuery("basecube", - "select substr(usersports.name, 3) as `sports_abbr`, sum(basecube.msr12) as `msr12` FROM ", " join " + getDbName() + "SELECT substr((usersports.name), 3) as `alias0`, 0.0 as `alias1`, sum((basecube.msr12)) " + + "as `alias2` FROM ", " join " + getDbName() + "c1_usertable userdim ON basecube.userid = userdim.id " + " join (select user_interests.user_id as user_id,collect_set(usersports.name) as name" + " from " + getDbName() + "c1_user_interests_tbl user_interests" + " join " + getDbName() @@ -796,13 +1013,10 @@ public void testBridgeTablesWithMultipleFactsWithExprAfterFlattening() throws Ex TestCubeRewriter.compareContains(expected2, hqlQuery); lower = hqlQuery.toLowerCase(); assertTrue(lower.startsWith( - "select coalesce(mq1.sports_abbr, mq2.sports_abbr) sports_abbr, mq2.msr2 msr2, mq1.msr12 msr12 from ") - || lower.startsWith( - "select coalesce(mq1.sports_abbr, mq2.sports_abbr) sports_abbr, mq1.msr2 msr2, mq2.msr12 msr12 from "), - hqlQuery); - - assertTrue(hqlQuery.contains("mq1 full outer join ") - && hqlQuery.endsWith("mq2 on mq1.sports_abbr <=> mq2.sports_abbr"), + "select (basecube.alias0) as `sports_abbr`, sum((basecube.alias1)) as `msr2`, " + + "sum((basecube.alias2)) as `msr12` from"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") + && hqlQuery.endsWith("GROUP BY (basecube.alias0)"), hqlQuery); } @@ -848,7 +1062,8 @@ public void testBridgeTableQueryJoinColumns() throws Exception { " join " + getDbName() + "c1_usertable userchain ON basecube.userid = userchain.id " + " join ( select userinterestids.user_id as user_id, collect_set(userinterestids.sport_id) as balias0," + " collect_set(userinterestids.user_id) as balias1 from " + getDbName() + "c1_user_interests_tbl " - + " userinterestids group by userinterestids.user_id) userinterestids on userchain.id = userinterestids.user_id" + + " userinterestids group by userinterestids.user_id) userinterestids " + + "on userchain.id = userinterestids.user_id" + " join (select userinterestids.user_id as user_id, collect_set(usersports . id) as balias0 from" + getDbName() + " c1_user_interests_tbl userinterestids join " + getDbName() + "c1_sports_tbl" + " usersports on userinterestids.sport_id = usersports.id group by userinterestids.user_id) usersports" diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 7e06a5cbd..bab1080f0 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -35,8 +35,6 @@ import org.apache.lens.cube.error.NoCandidateDimAvailableException; import org.apache.lens.cube.error.NoCandidateFactAvailableException; import org.apache.lens.cube.metadata.*; -import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCause; -import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode; import org.apache.lens.server.api.LensServerAPITestUtil; import org.apache.lens.server.api.error.LensException; @@ -53,7 +51,6 @@ import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.common.base.Splitter; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import lombok.extern.slf4j.Slf4j; @@ -81,7 +78,7 @@ public Configuration getConf() { @Test public void testQueryWithNow() throws Exception { LensException e = getLensExceptionInRewrite( - "select SUM(msr2) from testCube where " + getTimeRangeString("NOW - 2DAYS", "NOW"), getConf()); + "select SUM(msr2) from testCube where " + getTimeRangeString("NOW - 2DAYS", "NOW"), getConf()); assertEquals(e.getErrorCode(), LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo().getErrorCode()); } @@ -90,7 +87,7 @@ public void testQueryWithContinuousUpdatePeriod() throws Exception { Configuration conf = getConf(); conf.set(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, "true"); conf.setClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, BetweenTimeRangeWriter.class, TimeRangeWriter.class); - + conf.setBoolean(REPLACE_TIMEDIM_WITH_PART_COL, false); DateFormat qFmt = new SimpleDateFormat("yyyy-MM-dd-HH:mm:ss"); String timeRangeString; timeRangeString = getTimeRangeString(DAILY, -2, 0, qFmt); @@ -99,15 +96,17 @@ public void testQueryWithContinuousUpdatePeriod() throws Exception { String to = getDateStringWithOffset(DAILY, 0, CONTINUOUS); String from = getDateStringWithOffset(DAILY, -2, CONTINUOUS); - String expected = "select SUM((testCube.msr15)) from TestQueryRewrite.c0_testFact_CONTINUOUS testcube" - + " WHERE ((( testcube . dt ) between '" + from + "' and '" + to + "' ))"; + String expected = "select SUM((testCube.msr15)) as `sum(msr15)` from " + + "TestQueryRewrite.c0_testFact_CONTINUOUS testcube" + + " WHERE ((( testcube . d_time ) between '" + from + "' and '" + to + "' ))"; System.out.println("rewrittenQuery.toHQL() " + rewrittenQuery.toHQL()); System.out.println("expected " + expected); compareQueries(rewrittenQuery.toHQL(), expected); //test with msr2 on different fact + conf.setBoolean(REPLACE_TIMEDIM_WITH_PART_COL, true); rewrittenQuery = rewriteCtx("select SUM(msr2) from testCube where " + timeRangeString, conf); - expected = "select SUM((testCube.msr2)) from TestQueryRewrite.c0_testFact testcube" + expected = "select SUM((testCube.msr2)) as `sum(msr2)` from TestQueryRewrite.c2_testfact testcube" + " WHERE ((( testcube . dt ) between '" + from + "' and '" + to + "' ))"; System.out.println("rewrittenQuery.toHQL() " + rewrittenQuery.toHQL()); System.out.println("expected " + expected); @@ -137,11 +136,12 @@ public void testCubeQuery() throws Exception { CubeQueryContext rewrittenQuery = rewriteCtx("select SUM(msr2) from testCube where " + TWO_DAYS_RANGE, getConfWithStorages("C2")); String expected = - getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ", null, null, + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); - compareQueries(rewrittenQuery.toHQL(), expected); + String hql = rewrittenQuery.toHQL(); + compareQueries(hql, expected); System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); - assertNotNull(rewrittenQuery.getNonExistingParts()); +// assertNotNull(rewrittenQuery.getNonExistingParts()); } @Test @@ -152,15 +152,9 @@ public void testMaxCoveringFact() throws Exception { conf.set(DRIVER_SUPPORTED_STORAGES, "C1,C2,C4"); CubeQueryContext cubeQueryContext = rewriteCtx("select SUM(msr2) from testCube where " + THIS_YEAR_RANGE, conf); - PruneCauses pruneCause = cubeQueryContext.getFactPruningMsgs(); - int lessDataCauses = 0; - for (Map.Entry> entry : pruneCause.entrySet()) { - for (CandidateTablePruneCause cause : entry.getValue()) { - if (cause.getCause().equals(LESS_DATA)) { - lessDataCauses++; - } - } - } + PruneCauses pruneCause = cubeQueryContext.getStoragePruningMsgs(); + long lessDataCauses = pruneCause.values().stream() + .flatMap(Collection::stream).map(CandidateTablePruneCause::getCause).filter(LESS_DATA::equals).count(); assertTrue(lessDataCauses > 0); } @@ -170,7 +164,7 @@ public void testLightestFactFirst() throws Exception { String hqlQuery = rewrite("select SUM(msr2) from testCube where " + TWO_DAYS_RANGE, getConfWithStorages( "C2")); String expected = - getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ", null, null, + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, expected); @@ -178,7 +172,7 @@ public void testLightestFactFirst() throws Exception { conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, true); hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ", null, null, + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, getWhereForHourly2days("c1_testfact2")); compareQueries(hqlQuery, expected); @@ -194,8 +188,8 @@ public void testLightestFactFirst() throws Exception { pruneCauses.getBrief().substring(0, endIndex), MISSING_PARTITIONS.errorFormat.substring(0, endIndex) ); - assertEquals(pruneCauses.getDetails().get("testfact").size(), 1); - assertEquals(pruneCauses.getDetails().get("testfact").iterator().next().getCause(), + assertEquals(pruneCauses.getDetails().get("c1_testfact").size(), 1); + assertEquals(pruneCauses.getDetails().get("c1_testfact").iterator().next().getCause(), MISSING_PARTITIONS); } @@ -204,11 +198,13 @@ public void testDerivedCube() throws ParseException, LensException, HiveExceptio CubeQueryContext rewrittenQuery = rewriteCtx("select SUM(msr2) from derivedCube where " + TWO_DAYS_RANGE, getConfWithStorages("C2")); String expected = - getExpectedQuery(DERIVED_CUBE_NAME, "select sum(derivedCube.msr2) FROM ", null, null, + getExpectedQuery(DERIVED_CUBE_NAME, "select sum(derivedCube.msr2) as `sum(msr2)` FROM ", null, null, getWhereForDailyAndHourly2days(DERIVED_CUBE_NAME, "C2_testfact")); compareQueries(rewrittenQuery.toHQL(), expected); - System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); - assertNotNull(rewrittenQuery.getNonExistingParts()); + + System.out.println("Non existing parts:" + ((StorageCandidate) rewrittenQuery.getCandidates().iterator().next()) + .getNonExistingPartitions()); + assertNotNull(((StorageCandidate) rewrittenQuery.getCandidates().iterator().next()).getNonExistingPartitions()); LensException th = getLensExceptionInRewrite( "select SUM(msr4) from derivedCube where " + TWO_DAYS_RANGE, getConf()); @@ -251,7 +247,7 @@ public void testCubeInsert() throws Exception { + TWO_DAYS_RANGE, conf); Map wh = getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact"); String expected = "insert overwrite directory 'target/test' " - + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ", null, null, wh); + + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, wh); compareQueries(hqlQuery, expected); hqlQuery = rewrite("insert overwrite directory" + " 'target/test' select SUM(msr2) from testCube where " @@ -262,18 +258,18 @@ public void testCubeInsert() throws Exception { + TWO_DAYS_RANGE, conf); wh = getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact"); expected = "insert overwrite local directory 'target/test' " - + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ", null, null, wh); + + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, wh); compareQueries(hqlQuery, expected); - hqlQuery = rewrite("insert overwrite local directory" + " 'target/test' select SUM(msr2) from testCube where " - + TWO_DAYS_RANGE, conf); + hqlQuery = rewrite("insert overwrite local directory" + " 'target/test' select SUM(msr2) as `sum(msr2)` " + + "from testCube where " + TWO_DAYS_RANGE, conf); compareQueries(hqlQuery, expected); hqlQuery = rewrite("insert overwrite table temp" + " select SUM(msr2) from testCube where " + TWO_DAYS_RANGE, conf); wh = getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact"); expected = "insert overwrite table temp " - + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ", null, null, wh); + + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, wh); compareQueries(hqlQuery, expected); hqlQuery = rewrite("insert overwrite table temp" + " select SUM(msr2) from testCube where " + TWO_DAYS_RANGE, @@ -315,7 +311,7 @@ public void testCubeWhereQuery() throws Exception { String hqlQuery, expected; hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, getConfWithStorages("C2")); expected = - getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ", null, null, + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, expected); @@ -324,7 +320,7 @@ public void testCubeWhereQuery() throws Exception { conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, true); hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ", null, null, + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)`FROM ", null, null, getWhereForHourly2days("c1_testfact2")); compareQueries(hqlQuery, expected); conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false); @@ -334,7 +330,7 @@ public void testCubeWhereQuery() throws Exception { conf.set(DRIVER_SUPPORTED_STORAGES, "C1"); hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ", null, null, + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C1_testfact")); compareQueries(hqlQuery, expected); @@ -342,7 +338,7 @@ public void testCubeWhereQuery() throws Exception { conf.set(CubeQueryConfUtil.getValidFactTablesKey(TEST_CUBE_NAME), "testFact"); hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ", null, null, + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, expected); @@ -350,7 +346,7 @@ public void testCubeWhereQuery() throws Exception { conf.set(CubeQueryConfUtil.getValidFactTablesKey(TEST_CUBE_NAME), "testFact2"); hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ", null, null, + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, getWhereForHourly2days("c1_testfact2")); compareQueries(hqlQuery, expected); @@ -359,7 +355,7 @@ public void testCubeWhereQuery() throws Exception { conf.set(getValidStorageTablesKey("testFact2"), "C1_testFact2"); hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ", null, null, + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, getWhereForHourly2days("c1_testfact2")); compareQueries(hqlQuery, expected); @@ -368,7 +364,7 @@ public void testCubeWhereQuery() throws Exception { conf.set(getValidUpdatePeriodsKey("testfact", "C1"), "HOURLY"); hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf); expected = getExpectedQuery(TEST_CUBE_NAME, - "select sum(testcube.msr2) FROM ", null, null, getWhereForHourly2days("c1_testfact")); + "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, getWhereForHourly2days("c1_testfact")); compareQueries(hqlQuery, expected); conf.set(DRIVER_SUPPORTED_STORAGES, "C2"); @@ -376,16 +372,16 @@ public void testCubeWhereQuery() throws Exception { conf.set(getValidUpdatePeriodsKey("testfact", "C2"), "HOURLY"); hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf); expected = getExpectedQuery(TEST_CUBE_NAME, - "select sum(testcube.msr2) FROM ", null, null, getWhereForHourly2days("c2_testfact")); + "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, getWhereForHourly2days("c2_testfact")); compareQueries(hqlQuery, expected); // max interval test conf = new Configuration(); conf.set(CubeQueryConfUtil.QUERY_MAX_INTERVAL, "HOURLY"); - conf.set(DRIVER_SUPPORTED_STORAGES, "C1,C2"); + conf.set(DRIVER_SUPPORTED_STORAGES, "C1"); hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf); expected = getExpectedQuery(TEST_CUBE_NAME, - "select sum(testcube.msr2) FROM ", null, null, getWhereForHourly2days("c1_testfact2")); + "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, getWhereForHourly2days("c1_testfact2")); compareQueries(hqlQuery, expected); } @@ -400,7 +396,8 @@ public void testPartColAsQueryColumn() throws Exception { "select cubecountry.name, msr2 from" + " testCube" + " where cubecountry.region = 'asia' and " + TWO_DAYS_RANGE, conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select cubecountry.name, sum(testcube.msr2)" + " FROM ", " JOIN " + getDbName() + getExpectedQuery(TEST_CUBE_NAME, "select cubecountry.name as `name`, sum(testcube.msr2) as `msr2` " + + " FROM ", " JOIN " + getDbName() + "c3_countrytable_partitioned cubecountry on testcube.countryid=cubecountry.id and cubecountry.dt='latest'", "cubecountry.region='asia'", " group by cubecountry.name ", null, @@ -410,7 +407,8 @@ public void testPartColAsQueryColumn() throws Exception { "select cubestate.name, cubestate.countryid, msr2 from" + " testCube" + " where cubestate.countryid = 5 and " + TWO_DAYS_RANGE, conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select cubestate.name, cubestate.countryid, sum(testcube.msr2)" + " FROM ", + getExpectedQuery(TEST_CUBE_NAME, "select cubestate.name as `name`, " + + "cubestate.countryid as `countryid`, sum(testcube.msr2) as `msr2`" + " FROM ", " JOIN " + getDbName() + "c3_statetable_partitioned cubestate ON" + " testCube.stateid = cubestate.id and cubestate.dt = 'latest'", "cubestate.countryid=5", @@ -428,7 +426,7 @@ public void testCubeJoinQuery() throws Exception { rewrite("select SUM(msr2) from testCube" + " join citydim on testCube.cityid = citydim.id" + " where " + TWO_DAYS_RANGE, conf); String expected = - getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2)" + " FROM ", " INNER JOIN " + getDbName() + getExpectedQuery(TEST_CUBE_NAME, "SELECT sum((testcube.msr2)) as `sum(msr2)` FROM ", " INNER JOIN " + getDbName() + "c2_citytable citydim ON" + " testCube.cityid = citydim.id", null, null, null, getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, expected); @@ -450,7 +448,7 @@ public void testCubeJoinQuery() throws Exception { + " right outer join zipdim on citydim.zipcode = zipdim.code" + " where " + TWO_DAYS_RANGE, getConf()); expected = getExpectedQuery(TEST_CUBE_NAME, - "select statedim.name," + " sum(testcube.msr2) FROM ", "INNER JOIN " + getDbName() + "select statedim.name as `name`," + " sum(testcube.msr2) as `SUM(msr2)` FROM ", "INNER JOIN " + getDbName() + "c1_citytable citydim ON testCube.cityid = citydim.id and citydim.dt='latest' LEFT OUTER JOIN " + getDbName() + "c1_statetable statedim" + " ON statedim.id = citydim.stateid AND " @@ -466,7 +464,8 @@ public void testCubeJoinQuery() throws Exception { + " left outer join statedim ST on ST.id = CT.stateid" + " right outer join zipdim ZT on CT.zipcode = ZT.code" + " where " + TWO_DAYS_RANGE, getConf()); expected = - getExpectedQuery("tc", "select st.name," + " sum(tc.msr2) FROM ", " INNER JOIN " + getDbName() + getExpectedQuery("tc", "select st.name as `name`," + " sum(tc.msr2) as `sum(msr2)` FROM ", + " INNER JOIN " + getDbName() + "c1_citytable ct ON" + " tc.cityid = ct.id and ct.dt='latest' LEFT OUTER JOIN " + getDbName() + "c1_statetable st" + " ON st.id = ct.stateid and (st.dt = 'latest') " + "RIGHT OUTER JOIN " + getDbName() + "c1_ziptable" @@ -480,7 +479,8 @@ public void testCubeJoinQuery() throws Exception { + " left outer join citydim on testCube.cityid = citydim.id" + " left outer join zipdim on citydim.zipcode = zipdim.code" + " where " + TWO_DAYS_RANGE, getConf()); expected = - getExpectedQuery(TEST_CUBE_NAME, "select citydim.name," + " sum(testcube.msr2) FROM ", " LEFT OUTER JOIN " + getExpectedQuery(TEST_CUBE_NAME, "select citydim.name as `name`," + " sum(testcube.msr2) as `sum(msr2)`FROM ", + " LEFT OUTER JOIN " + getDbName() + "c1_citytable citydim ON" + " testCube.cityid = citydim.id and (citydim.dt = 'latest') " + " LEFT OUTER JOIN " + getDbName() + "c1_ziptable" + " zipdim ON citydim.zipcode = zipdim.code AND " + "(zipdim.dt = 'latest')", null, " group by" + " citydim.name ", null, @@ -491,7 +491,7 @@ public void testCubeJoinQuery() throws Exception { rewrite("select SUM(msr2) from testCube" + " join countrydim on testCube.countryid = countrydim.id" + " where " + TWO_MONTHS_RANGE_UPTO_MONTH, getConf()); expected = - getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ", " INNER JOIN " + getDbName() + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", " INNER JOIN " + getDbName() + "c1_countrytable countrydim ON testCube.countryid = " + " countrydim.id", null, null, null, getWhereForMonthly2months("c2_testfactmonthly")); compareQueries(hqlQuery, expected); @@ -578,8 +578,9 @@ public void testCubeGroupbyWithConstantProjected() throws Exception { conf.set(DRIVER_SUPPORTED_STORAGES, "C2"); String hqlQuery1 = rewrite("select cityid, 99, \"placeHolder\", -1001, SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf); - String expected1 = getExpectedQuery(TEST_CUBE_NAME, "select testcube.cityid, 99, \"placeHolder\", -1001," - + " sum(testcube.msr2) FROM ", null, " group by testcube.cityid ", + String expected1 = getExpectedQuery(TEST_CUBE_NAME, "SELECT (testcube.cityid) as `cityid`, 99 as `99`, " + + "\"placeHolder\" as `\"placeHolder\"`, (-1001) as `(-1001)`, sum((testcube.msr2)) as `sum(msr2)` FROM ", + null, " group by testcube.cityid ", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery1, expected1); @@ -588,8 +589,9 @@ public void testCubeGroupbyWithConstantProjected() throws Exception { "select cityid, case when stateid = 'za' then \"Not Available\" end, 99, \"placeHolder\", -1001, " + "SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf); String expected2 = getExpectedQuery(TEST_CUBE_NAME, - "select testcube.cityid, case when testcube.stateid = 'za' then \"Not Available\" end, 99, \"placeHolder\"," - + " -1001, sum(testcube.msr2) FROM ", null, + "SELECT (testcube.cityid) as `cityid`, case when ((testcube.stateid) = 'za') then \"Not Available\" end " + + "as `case when (stateid = 'za') then \"Not Available\" end`, 99 as `99`, \"placeHolder\" " + + "as `\"placeHolder\"`, (-1001) as `(-1001)`, sum((testcube.msr2)) as `sum(msr2)` FROM ", null, " group by testcube.cityid, case when testcube.stateid = 'za' then \"Not Available\" end ", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery2, expected2); @@ -601,10 +603,14 @@ public void testCubeGroupbyWithConstantProjected() throws Exception { + "SUM(msr2), SUM(msr2 + 39), SUM(msr2) + 567 from testCube" + " where " + TWO_DAYS_RANGE, conf); String expected3 = getExpectedQuery( TEST_CUBE_NAME, - "select testcube.cityid, testcube.stateid + 99, 44 + testcube.stateid, testcube.stateid - 33," - + " 999 - testcube.stateid, TRUE, FALSE, round(123.4567,2), " - + "case when testcube.stateid='za' then 99 else -1001 end," - + " sum(testcube.msr2), sum(testcube.msr2 + 39), sum(testcube.msr2) + 567 FROM ", + "SELECT (testcube.cityid) as `cityid`, ((testcube.stateid) + 99) as `(stateid + 99)`, " + + "(44 + (testcube.stateid)) as `(44 + stateid)`, ((testcube.stateid) - 33) as `(stateid - 33)`, " + + "(999 - (testcube.stateid)) as `(999 - stateid)`, true as `true`, false " + + "as `false`, round(123.4567, 2) as `round(123.4567, 2)`, " + + "case when ((testcube.stateid) = 'za') then 99 else (-1001) end " + + "as `case when (stateid = 'za') then 99 else (-1001) end`, " + + "sum((testcube.msr2)) as `sum(msr2)`, sum(((testcube.msr2) + 39)) " + + "as `sum((msr2 + 39))`, (sum((testcube.msr2)) + 567) as `(sum(msr2) + 567)` FROM ", null, " group by testcube.cityid,testcube.stateid + 99, 44 + testcube.stateid, testcube.stateid - 33, " + "999 - testcube.stateid, " @@ -622,68 +628,74 @@ public void testCubeGroupbyQuery() throws Exception { rewrite("select name, SUM(msr2) from" + " testCube join citydim on testCube.cityid = citydim.id where " + TWO_DAYS_RANGE, conf); String expected = - getExpectedQuery(TEST_CUBE_NAME, "select citydim.name," + " sum(testcube.msr2) FROM ", "INNER JOIN " + getDbName() - + "c2_citytable citydim ON" + " testCube.cityid = citydim.id", null, " group by citydim.name ", + getExpectedQuery(TEST_CUBE_NAME, "select citydim.name as `name`, sum(testcube.msr2) as `sum(msr2)` FROM " + , "INNER JOIN " + getDbName() + "c2_citytable citydim ON" + " testCube.cityid = citydim.id", + null, " group by citydim.name ", null, getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, expected); hqlQuery = - rewrite("select SUM(msr2) from testCube" + " join citydim on testCube.cityid = citydim.id" + " where " + rewrite("select SUM(msr2) from testCube join citydim on testCube.cityid = citydim.id where " + TWO_DAYS_RANGE + " group by name", conf); compareQueries(hqlQuery, expected); - hqlQuery = rewrite("select cityid, SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf); + hqlQuery = rewrite("select cityid, sum(msr2) from testCube where " + TWO_DAYS_RANGE, conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select testcube.cityid," + " sum(testcube.msr2) FROM ", null, - " group by testcube.cityid ", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); + getExpectedQuery(TEST_CUBE_NAME, "select testcube.cityid as `cityid`, sum(testcube.msr2) as `sum(msr2)` from ", + null, " group by testcube.cityid ", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, expected); - hqlQuery = rewrite("select round(cityid), SUM(msr2) from" + " testCube where " + TWO_DAYS_RANGE, conf); + hqlQuery = rewrite("select round(cityid), sum(msr2) from" + " testCube where " + TWO_DAYS_RANGE, conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select round(testcube.cityid)," + " sum(testcube.msr2) FROM ", null, + getExpectedQuery(TEST_CUBE_NAME, "select round(testcube.cityid) as `round(cityid)`," + + " sum(testcube.msr2) as `sum(msr2)` FROM ", null, " group by round(testcube.cityid) ", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, expected); hqlQuery = - rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE + "group by round(zipcode)", conf); + rewrite("select sum(msr2) from testCube where " + TWO_DAYS_RANGE + "group by round(zipcode)", conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select round(testcube.zipcode)," + " sum(testcube.msr2) FROM ", null, + getExpectedQuery(TEST_CUBE_NAME, "select round(testcube.zipcode) as `round((testcube.zipcode))`," + + " sum(testcube.msr2) as `sum(msr2)` FROM ", null, " group by round(testcube.zipcode) ", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, expected); hqlQuery = - rewrite("select round(cityid), SUM(msr2) from" + " testCube where " + TWO_DAYS_RANGE + " group by zipcode", + rewrite("select round(cityid), sum(msr2) from" + " testCube where " + TWO_DAYS_RANGE + " group by zipcode", conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select " + " round(testcube.cityid), sum(testcube.msr2) FROM ", null, + getExpectedQuery(TEST_CUBE_NAME, "select round(testcube.cityid) as `round(cityid)`, " + + "sum(testcube.msr2) as `sum(msr2)` FROM ", null, " group by testcube.zipcode", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, expected); - hqlQuery = rewrite("select round(cityid), SUM(msr2) from" + " testCube where " + TWO_DAYS_RANGE, conf); + hqlQuery = rewrite("select round(cityid), sum(msr2) from testCube where " + TWO_DAYS_RANGE, conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select " + " round(testcube.cityid), sum(testcube.msr2) FROM ", null, + getExpectedQuery(TEST_CUBE_NAME, "select round(testcube.cityid) as `round(cityid)`, " + + "sum(testcube.msr2) as `sum(msr2)` FROM ", null, " group by round(testcube.cityid)", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, expected); hqlQuery = - rewrite("select cityid, SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE + " group by round(zipcode)", + rewrite("select cityid, sum(msr2) from testCube" + " where " + TWO_DAYS_RANGE + " group by round(zipcode)", conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select " + " testcube.cityid, sum(testcube.msr2) FROM ", null, - " group by round(testcube.zipcode)", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); + getExpectedQuery(TEST_CUBE_NAME, "select testcube.cityid as `cityid`, sum(testcube.msr2) as `sum(msr2)` FROM ", + null, " group by round(testcube.zipcode)", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, expected); hqlQuery = - rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE + " group by round(zipcode)", conf); + rewrite("select sum(msr2) from testCube where " + TWO_DAYS_RANGE + " group by round(zipcode)", conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select round(testcube.zipcode)," + " sum(testcube.msr2) FROM ", null, - " group by round(testcube.zipcode)", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); + getExpectedQuery(TEST_CUBE_NAME, "select round(testcube.zipcode) as `round(testcube.zipcode)`, " + + "sum(testcube.msr2) as `sum(msr2)` FROM ", null, " group by round(testcube.zipcode)", + getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, expected); hqlQuery = - rewrite("select cityid, msr2 from testCube" + " where " + TWO_DAYS_RANGE + " group by round(zipcode)", conf); + rewrite("select cityid, msr2 from testCube where " + TWO_DAYS_RANGE + " group by round(zipcode)", conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select " + " testcube.cityid, sum(testcube.msr2) FROM ", null, + getExpectedQuery(TEST_CUBE_NAME, "select testcube.cityid as `cityid`, sum(testcube.msr2) as `msr2` FROM ", null, " group by round(testcube.zipcode)", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, expected); @@ -691,24 +703,26 @@ public void testCubeGroupbyQuery() throws Exception { rewrite("select round(zipcode) rzc," + " msr2 from testCube where " + TWO_DAYS_RANGE + " group by zipcode" + " order by rzc", conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select round(testcube.zipcode) as `rzc`," + " sum(testcube.msr2) FROM ", null, - " group by testcube.zipcode order by rzc asc", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); + getExpectedQuery(TEST_CUBE_NAME, "select round(testcube.zipcode) as `rzc`, sum(testcube.msr2) as `msr2` FROM ", + null, " group by testcube.zipcode order by rzc asc", + getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, expected); //Dim attribute with aggregate function hqlQuery = - rewrite("select countofdistinctcityid, zipcode from" + " testCube where " + TWO_DAYS_RANGE, conf); + rewrite("select countofdistinctcityid, zipcode from testCube where " + TWO_DAYS_RANGE, conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select " + " count(distinct (testcube.cityid)), (testcube.zipcode) FROM ", - null, " group by (testcube.zipcode)", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); + getExpectedQuery(TEST_CUBE_NAME, "select count(distinct(testcube.cityid)) as `countofdistinctcityid`" + + ", (testcube.zipcode) as `zipcode` FROM ", null, " group by (testcube.zipcode)", + getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, expected); //Dim attribute with single row function hqlQuery = rewrite("select notnullcityid, zipcode from" + " testCube where " + TWO_DAYS_RANGE, conf); expected = - getExpectedQuery(TEST_CUBE_NAME, "select " + " distinct case when (testcube.cityid) is null then 0 " - + "else (testcube.cityid) end, (testcube.zipcode) FROM ", null, + getExpectedQuery(TEST_CUBE_NAME, "select distinct case when (testcube.cityid) is null then 0 " + + "else (testcube.cityid) end as `notnullcityid`, (testcube.zipcode) as `zipcode` FROM ", null, "", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, expected); @@ -894,14 +908,14 @@ public void testCubeQueryWithAilas() throws Exception { hqlQuery = rewrite("select SUM(msr2) from testCube mycube" + " where " + TWO_DAYS_RANGE, getConfWithStorages("C2")); expected = - getExpectedQuery("mycube", "select sum(mycube.msr2) FROM ", null, null, + getExpectedQuery("mycube", "select sum(mycube.msr2) as `sum(msr2)` FROM ", null, null, getWhereForDailyAndHourly2days("mycube", "C2_testfact")); compareQueries(hqlQuery, expected); hqlQuery = rewrite("select SUM(testCube.msr2) from testCube" + " where " + TWO_DAYS_RANGE, getConfWithStorages("C2")); expected = - getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ", null, null, + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(testCube.msr2)` FROM ", null, null, getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, expected); @@ -924,7 +938,7 @@ public void testCubeWhereQueryForMonth() throws Exception { String hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + TWO_MONTHS_RANGE_UPTO_HOURS, getConfWithStorages("C2")); String expected = - getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ", null, null, + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, getWhereForMonthlyDailyAndHourly2months("C2_testfact")); compareQueries(hqlQuery, expected); } @@ -938,27 +952,27 @@ public void testQueryWithMeasureWithDataCompletenessTagWithNoFailureOnPartialDat Configuration conf = getConf(); conf.setStrings(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL, "dt"); String hqlQuery = rewrite("select SUM(msr1) from basecube where " + TWO_DAYS_RANGE, conf); - String expected = getExpectedQuery("basecube", "select sum(basecube.msr1) FROM ", null, null, + String expected = getExpectedQuery("basecube", "select sum(basecube.msr1) as `sum(msr1)` FROM ", null, null, getWhereForHourly2days("basecube", "c1_testfact1_raw_base")); compareQueries(hqlQuery, expected); } @Test public void testQueryWithMeasureWithDataCompletenessPresentInMultipleFacts() throws ParseException, - LensException { + LensException { /*In this query a measure is used which is present in two facts with different %completeness. While resolving the facts, the fact with the higher dataCompletenessFactor gets picked up.*/ Configuration conf = getConf(); conf.setStrings(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL, "dt"); String hqlQuery = rewrite("select SUM(msr9) from basecube where " + TWO_DAYS_RANGE, conf); - String expected = getExpectedQuery("basecube", "select sum(basecube.msr9) FROM ", null, null, - getWhereForHourly2days("basecube", "c1_testfact5_raw_base")); + String expected = getExpectedQuery("basecube", "select sum(basecube.msr9) as `sum(msr9)` FROM ", null, null, + getWhereForHourly2days("basecube", "c1_testfact5_raw_base")); compareQueries(hqlQuery, expected); } @Test public void testCubeWhereQueryWithMeasureWithDataCompletenessAndFailIfPartialDataFlagSet() throws ParseException, - LensException { + LensException { /*In this query a measure is used for which dataCompletenessTag is set and the flag FAIL_QUERY_ON_PARTIAL_DATA is set. The partitions for the queried range are present but some of the them have incomplete data. So, the query throws NO_CANDIDATE_FACT_AVAILABLE Exception*/ @@ -966,13 +980,16 @@ public void testCubeWhereQueryWithMeasureWithDataCompletenessAndFailIfPartialDat conf.setStrings(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL, "dt"); conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, true); - LensException e = getLensExceptionInRewrite("select SUM(msr9) from basecube where " + TWO_DAYS_RANGE, conf); + LensException e = getLensExceptionInRewrite("select SUM(msr9) from basecube where " + + TWO_DAYS_RANGE, conf); assertEquals(e.getErrorCode(), LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo().getErrorCode()); NoCandidateFactAvailableException ne = (NoCandidateFactAvailableException) e; PruneCauses.BriefAndDetailedError pruneCauses = ne.getJsonMessage(); /*Since the Flag FAIL_QUERY_ON_PARTIAL_DATA is set, and the queried fact has incomplete data, hence, we expect the prune cause to be INCOMPLETE_PARTITION. The below check is to validate this.*/ - assertEquals(pruneCauses.getBrief(), String.format(INCOMPLETE_PARTITION.errorFormat, "[msr9]")); + for(String part: INCOMPLETE_PARTITION.errorFormat.split("%s")) { + assertTrue(pruneCauses.getBrief().contains(part), pruneCauses.getBrief()); + } } @Test @@ -990,31 +1007,27 @@ public void testCubeWhereQueryForMonthWithNoPartialData() throws Exception { pruneCauses.getBrief().substring(0, MISSING_PARTITIONS.errorFormat.length() - 3), MISSING_PARTITIONS.errorFormat.substring(0, MISSING_PARTITIONS.errorFormat.length() - 3), pruneCauses.getBrief()); - - Set expectedSet = - Sets.newTreeSet(Arrays.asList("summary1", "summary2", "testfact2_raw", "summary3", "testfact")); - boolean missingPartitionCause = false; - for (String key : pruneCauses.getDetails().keySet()) { - Set actualKeySet = Sets.newTreeSet(Splitter.on(',').split(key)); - if (expectedSet.equals(actualKeySet)) { - assertEquals(pruneCauses.getDetails().get(key).iterator() - .next().getCause(), MISSING_PARTITIONS); - missingPartitionCause = true; - } - } - assertTrue(missingPartitionCause, MISSING_PARTITIONS + " error does not occur for facttables set " + expectedSet - + " Details :" + pruneCauses.getDetails()); - assertEquals(pruneCauses.getDetails().get("testfactmonthly").iterator().next().getCause(), - NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE); - assertEquals(pruneCauses.getDetails().get("testfact2").iterator().next().getCause(), + List missingPartitionCauses = pruneCauses.enhanced().get( + Sets.newHashSet("c1_testfact2_raw", "c1_summary3", "c1_summary2", + "c1_summary1", "c2_testfact", "c1_testfact")); + assertEquals(missingPartitionCauses.size(), 1); + CandidateTablePruneCause missingPartitionCause = missingPartitionCauses.iterator().next(); + assertEquals(missingPartitionCause.getCause(), MISSING_PARTITIONS); + assertEquals(pruneCauses.getDetails().get("c1_testfact2").iterator().next().getCause(), MISSING_PARTITIONS); - assertEquals(pruneCauses.getDetails().get("cheapfact").iterator().next().getCause(), - NO_CANDIDATE_STORAGES); + /* + assertEquals(pruneCauses.getDetails().get("c4_testfact,c3_testfact,c3_testfact2_raw,c4_testfact2," + + "c99_cheapfact,c5_testfact").iterator().next().getCause(), + UNSUPPORTED_STORAGE); + CandidateTablePruneCause cheapFactPruneCauses = pruneCauses.getDetails().get("cheapfact").iterator().next(); - assertEquals(cheapFactPruneCauses.getStorageCauses().get("c0").getCause(), SkipStorageCode.RANGE_NOT_ANSWERABLE); - assertEquals(cheapFactPruneCauses.getStorageCauses().get("c99").getCause(), SkipStorageCode.UNSUPPORTED); + assertEquals(cheapFactPruneCauses.getDimStoragePruningCauses().get("c0"), + CandidateTablePruneCause.CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE); + assertEquals(cheapFactPruneCauses.getDimStoragePruningCauses().get("c99"), + CandidateTablePruneCause.CandidateTablePruneCode.UNSUPPORTED_STORAGE); assertEquals(pruneCauses.getDetails().get("summary4").iterator().next().getCause(), TIMEDIM_NOT_SUPPORTED); assertTrue(pruneCauses.getDetails().get("summary4").iterator().next().getUnsupportedTimeDims().contains("d_time")); + */ } @Test @@ -1023,26 +1036,28 @@ public void testCubeWhereQueryForMonthUptoMonths() throws Exception { String hqlQuery = rewrite("select cityid, SUM(msr2) from testCube" + " where " + TWO_MONTHS_RANGE_UPTO_MONTH, getConfWithStorages("C2")); String expected = - getExpectedQuery(TEST_CUBE_NAME, "select testcube.cityid," + " sum(testcube.msr2) FROM ", null, - "group by testcube.cityid", getWhereForMonthly2months("c2_testfact")); + getExpectedQuery(TEST_CUBE_NAME, "select testcube.cityid as `cityid`, sum(testcube.msr2) " + + "as `sum(msr2)` FROM ", null, "group by testcube.cityid", + getWhereForMonthly2months("c2_testfact")); compareQueries(hqlQuery, expected); } @Test public void testNoCandidateDimAvailableExceptionCompare() throws Exception { - //Max cause COLUMN_NOT_FOUND, Ordinal 9 - PruneCauses pr1 = new PruneCauses(); + //Max cause COLUMN_NOT_FOUND, Ordinal 2 + PruneCauses pr1 = new PruneCauses<>(); pr1.addPruningMsg(new CubeDimensionTable(new Table("test", "citydim")), - CandidateTablePruneCause.columnNotFound("test1", "test2", "test3")); + CandidateTablePruneCause.columnNotFound( + "test1", "test2", "test3")); NoCandidateDimAvailableException ne1 = new NoCandidateDimAvailableException(pr1); - //Max cause EXPRESSION_NOT_EVALUABLE, Ordinal 6 + //Max cause EXPRESSION_NOT_EVALUABLE, Ordinal 14 PruneCauses pr2 = new PruneCauses(); pr2.addPruningMsg(new CubeDimensionTable(new Table("test", "citydim")), CandidateTablePruneCause.expressionNotEvaluable("testexp1", "testexp2")); NoCandidateDimAvailableException ne2 = new NoCandidateDimAvailableException(pr2); - assertEquals(ne1.compareTo(ne2), 3); + assertEquals(ne1.compareTo(ne2), -7); } @Test @@ -1068,17 +1083,18 @@ public void testDimensionQueryWithMultipleStorages() throws Exception { NO_CANDIDATE_STORAGES.errorFormat, new HashMap>() { { - put("statetable", Arrays.asList(CandidateTablePruneCause.noCandidateStorages( - new HashMap() { + put("statetable", Arrays.asList(CandidateTablePruneCause.noCandidateStoragesForDimtable( + new HashMap() { { - put("c1_statetable", new SkipStorageCause(SkipStorageCode.NO_PARTITIONS)); + put("c1_statetable", CandidateTablePruneCause.CandidateTablePruneCode.NO_PARTITIONS); + put("c6_statetable", CandidateTablePruneCause.CandidateTablePruneCode.UNSUPPORTED_STORAGE); } })) ); - put("statetable_partitioned", Arrays.asList(CandidateTablePruneCause.noCandidateStorages( - new HashMap() { + put("statetable_partitioned", Arrays.asList(CandidateTablePruneCause.noCandidateStoragesForDimtable( + new HashMap() { { - put("C3_statetable_partitioned", new SkipStorageCause(SkipStorageCode.UNSUPPORTED)); + put("C3_statetable_partitioned", CandidateTablePruneCause.CandidateTablePruneCode.UNSUPPORTED_STORAGE); } })) ); @@ -1223,14 +1239,14 @@ public void testAliasReplacer() throws Exception { }; String[] expectedQueries = { - getExpectedQuery("t", "SELECT t.cityid, sum(t.msr2) FROM ", null, " group by t.cityid", + getExpectedQuery("t", "SELECT t.cityid as `cityid`, sum(t.msr2) as `msr2` FROM ", null, " group by t.cityid", getWhereForDailyAndHourly2days("t", "C2_testfact")), - getExpectedQuery(TEST_CUBE_NAME, "SELECT testCube.cityid, sum(testCube.msr2)" + " FROM ", + getExpectedQuery(TEST_CUBE_NAME, "SELECT testCube.cityid as `cityid`, sum(testCube.msr2) as `msr2`" + " FROM ", " testcube.cityid > 100 ", " group by testcube.cityid having" + " sum(testCube.msr2) < 1000", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")), - getExpectedQuery(TEST_CUBE_NAME, "SELECT testCube.cityid, sum(testCube.msr2)" + " FROM ", - " testcube.cityid > 100 ", " group by testcube.cityid having" - + " sum(testCube.msr2) < 1000 order by testCube.cityid asc", + getExpectedQuery(TEST_CUBE_NAME, "SELECT testCube.cityid as `cityid`, sum(testCube.msr2) as `msr2`" + " FROM ", + " testcube.cityid > 100 ", " group by testCube.cityid having" + + " sum(testCube.msr2) < 1000 order by cityid asc", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")), }; Configuration conf = getConf(); @@ -1246,23 +1262,26 @@ public void testFactsWithInvalidColumns() throws Exception { String hqlQuery = rewrite("select dim1, max(msr3)," + " msr2 from testCube" + " where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); String expected = - getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1, max(testcube.msr3), sum(testcube.msr2) FROM ", null, + getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1 as `dim1`, max(testcube.msr3) as `max(msr3)`" + + ", sum(testcube.msr2) as `msr2` FROM ", null, " group by testcube.dim1", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C1_summary1")); compareQueries(hqlQuery, expected); hqlQuery = rewrite("select dim1, dim2, COUNT(msr4)," + " SUM(msr2), msr3 from testCube" + " where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); expected = - getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1, testcube,dim2, count(testcube.msr4)," - + " sum(testcube.msr2), max(testcube.msr3) FROM ", null, " group by testcube.dim1, testcube.dim2", + getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1 as `dim1`, testcube,dim2 as `dim2`, " + + "count(testcube.msr4) as `count(msr4)`,sum(testcube.msr2) as `sum(msr2)`, " + + "max(testcube.msr3) as `msr3`FROM ", null, " group by testcube.dim1, testcube.dim2", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C1_summary2")); compareQueries(hqlQuery, expected); hqlQuery = rewrite("select dim1, dim2, cityid, msr4," + " SUM(msr2), msr3 from testCube" + " where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); expected = - getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1, testcube,dim2, testcube.cityid," - + " count(testcube.msr4), sum(testcube.msr2), max(testcube.msr3) FROM ", null, + getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1 as `dim1`, testcube,dim2 as `dim2`, " + + "testcube.cityid as `cityid`, count(testcube.msr4) as `msr4`, " + + "sum(testcube.msr2) as `sum(msr2)`, max(testcube.msr3) as `msr3` FROM ", null, " group by testcube.dim1, testcube.dim2, testcube.cityid", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C1_summary3")); compareQueries(hqlQuery, expected); @@ -1274,7 +1293,8 @@ public void testFactsWithTimedDimension() throws Exception { String hqlQuery = rewrite("select dim1, max(msr3)," + " msr2 from testCube" + " where " + TWO_DAYS_RANGE_IT, getConf()); String expected = - getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1, max(testcube.msr3), sum(testcube.msr2) FROM ", null, + getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1 as `dim1`, max(testcube.msr3) as `max(msr3)`, " + + "sum(testcube.msr2) as `msr2` FROM ", null, " group by testcube.dim1", getWhereForDailyAndHourly2daysWithTimeDim(TEST_CUBE_NAME, "it", "C2_summary1"), null); compareQueries(hqlQuery, expected); @@ -1282,8 +1302,9 @@ public void testFactsWithTimedDimension() throws Exception { rewrite("select dim1, dim2, COUNT(msr4)," + " SUM(msr2), msr3 from testCube" + " where " + TWO_DAYS_RANGE_IT, getConf()); expected = - getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1, testcube,dim2, count(testcube.msr4)," - + " sum(testcube.msr2), max(testcube.msr3) FROM ", null, " group by testcube.dim1, testcube.dim2", + getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1 as `dim1`, testcube,dim2 as `dim2`, " + + "count(testcube.msr4) as `count(msr4)`, sum(testcube.msr2) as `sum(msr2)`, " + + "max(testcube.msr3) as `msr3` FROM ", null, " group by testcube.dim1, testcube.dim2", getWhereForDailyAndHourly2daysWithTimeDim(TEST_CUBE_NAME, "it", "C2_summary2"), null); compareQueries(hqlQuery, expected); @@ -1291,8 +1312,9 @@ public void testFactsWithTimedDimension() throws Exception { rewrite("select dim1, dim2, cityid, count(msr4)," + " SUM(msr2), msr3 from testCube" + " where " + TWO_DAYS_RANGE_IT, getConf()); expected = - getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1, testcube,dim2, testcube.cityid," - + " count(testcube.msr4), sum(testcube.msr2), max(testcube.msr3) FROM ", null, + getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1 as `dim1`, testcube,dim2 as `dim2`, " + + "testcube.cityid as `cityid`, count(testcube.msr4) as `count(msr4)`, sum(testcube.msr2) " + + "as `sum(msr2)`, max(testcube.msr3) as `msr3`FROM ", null, " group by testcube.dim1, testcube.dim2, testcube.cityid", getWhereForDailyAndHourly2daysWithTimeDim(TEST_CUBE_NAME, "it", "C2_summary3"), null); @@ -1360,15 +1382,15 @@ public void testLookAhead() throws Exception { conf.setClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, AbridgedTimeRangeWriter.class, TimeRangeWriter.class); CubeQueryContext ctx = rewriteCtx("select dim1, max(msr3)," + " msr2 from testCube" + " where " + TWO_DAYS_RANGE_IT, conf); - assertEquals(ctx.candidateFacts.size(), 1); - CandidateFact candidateFact = ctx.candidateFacts.iterator().next(); - Set partsQueried = new TreeSet<>(candidateFact.getPartsQueried()); + assertEquals(ctx.getCandidates().size(), 1); + Candidate candidate = ctx.getCandidates().iterator().next(); + Set partsQueried = new TreeSet<>(((StorageCandidate)candidate).getParticipatingPartitions()); Date ceilDay = DAILY.getCeilDate(getDateWithOffset(DAILY, -2)); Date nextDay = DateUtils.addDays(ceilDay, 1); Date nextToNextDay = DateUtils.addDays(nextDay, 1); HashSet storageTables = Sets.newHashSet(); - for (String storageTable : candidateFact.getStorageTables()) { - storageTables.add(storageTable.split("\\.")[1]); + for (StorageCandidate sc : CandidateUtil.getStorageCandidates(candidate)) { + storageTables.add(sc.getStorageTable()); } TreeSet expectedPartsQueried = Sets.newTreeSet(); for (TimePartition p : Iterables.concat( @@ -1389,7 +1411,8 @@ public void testLookAhead() throws Exception { conf.setInt(CubeQueryConfUtil.LOOK_AHEAD_PT_PARTS_PFX, 3); ctx = rewriteCtx("select dim1, max(msr3)," + " msr2 from testCube" + " where " + TWO_DAYS_RANGE_IT, conf); - partsQueried = new TreeSet<>(ctx.candidateFacts.iterator().next().getPartsQueried()); + partsQueried = new TreeSet<>(((StorageCandidate)ctx.getCandidates().iterator().next()) + .getParticipatingPartitions()); // pt does not exist beyond 1 day. So in this test, max look ahead possible is 3 assertEquals(partsQueried, expectedPartsQueried); } @@ -1404,30 +1427,33 @@ public void testCubeQueryWithMultipleRanges() throws Exception { getWhereForDailyAndHourly2daysWithTimeDim(TEST_CUBE_NAME, "dt", TWODAYS_BACK, NOW) + " OR " + getWhereForDailyAndHourly2daysWithTimeDim(TEST_CUBE_NAME, "dt", BEFORE_6_DAYS, BEFORE_4_DAYS); - String expected = getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ", + String expected = getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, expectedRangeWhere, "c2_testfact"); compareQueries(hqlQuery, expected); hqlQuery = rewrite("select dim1, max(msr3)," + " msr2 from testCube" + " where " + TWO_DAYS_RANGE + " OR " + TWO_DAYS_RANGE_BEFORE_4_DAYS, getConfWithStorages("C1")); expected = - getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1, max(testcube.msr3), sum(testcube.msr2) FROM ", null, + getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1 as `dim1`, max(testcube.msr3) as `max(msr3)`" + + ", sum(testcube.msr2) as `msr2` FROM ", null, " group by testcube.dim1", expectedRangeWhere, "C1_summary1"); compareQueries(hqlQuery, expected); hqlQuery = rewrite("select dim1, dim2, COUNT(msr4)," + " SUM(msr2), msr3 from testCube" + " where " + TWO_DAYS_RANGE + " OR " + TWO_DAYS_RANGE_BEFORE_4_DAYS, getConfWithStorages("C1")); expected = - getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1, testcube,dim2, count(testcube.msr4)," - + " sum(testcube.msr2), max(testcube.msr3) FROM ", null, " group by testcube.dim1, testcube.dim2", + getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1 as `dim1`, testcube.dim2 as `dim2`, " + + "count(testcube.msr4) as `COUNT(msr4`, sum(testcube.msr2) as `sum(msr2)`, " + + "max(testcube.msr3) as `msr3` FROM ", null, " group by testcube.dim1, testcube.dim2", expectedRangeWhere, "C1_summary2"); compareQueries(hqlQuery, expected); hqlQuery = rewrite("select dim1, dim2, cityid, count(msr4)," + " SUM(msr2), msr3 from testCube" + " where " + TWO_DAYS_RANGE + " OR " + TWO_DAYS_RANGE_BEFORE_4_DAYS, getConfWithStorages("C1")); expected = - getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1, testcube,dim2, testcube.cityid," - + " count(testcube.msr4), sum(testcube.msr2), max(testcube.msr3) FROM ", null, + getExpectedQuery(TEST_CUBE_NAME, "select testcube.dim1 as `dim1`, testcube.dim2 as `dim2`, " + + "testcube.cityid as `cityid`, count(testcube.msr4) as `count(msr4)`, " + + "sum(testcube.msr2) as `sum(msr2)`, max(testcube.msr3) as `msr3` FROM ", null, " group by testcube.dim1, testcube.dim2, testcube.cityid", expectedRangeWhere, "C1_summary3"); compareQueries(hqlQuery, expected); } @@ -1468,7 +1494,8 @@ public void testJoinWithMultipleAliases() throws Exception { + db + "c1_citytable c2 ON (( s1 . countryid ) = ( c2 . id )) AND (c2.dt = 'latest')"; String expected = - getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2)" + " FROM ", expectedJoin, null, null, null, + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)`" + + " FROM ", expectedJoin, null, null, null, getWhereForHourly2days(TEST_CUBE_NAME, "C1_testfact2")); compareQueries(hqlQuery, expected); } diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java index 5505ed424..d97c0e652 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java @@ -20,8 +20,12 @@ package org.apache.lens.cube.parse; import static org.apache.lens.cube.metadata.DateFactory.*; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.columnNotFound; import static org.apache.lens.cube.parse.CubeTestSetup.*; +import static com.google.common.collect.Lists.newArrayList; +import static com.google.common.collect.Sets.newHashSet; + import java.util.*; import org.apache.lens.cube.error.NoCandidateFactAvailableException; @@ -36,8 +40,7 @@ import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.common.base.Splitter; -import com.google.common.collect.Sets; +import com.google.common.collect.Maps; public class TestDenormalizationResolver extends TestQueryRewrite { @@ -60,7 +63,8 @@ public void testDenormsAsDirectFields() throws ParseException, LensException, Hi String hqlQuery = rewrite("select dim2big1, max(msr3)," + " msr2 from testCube" + " where " + TWO_DAYS_RANGE_IT, conf); String expecteddim2big1 = - getExpectedQuery(cubeName, "select testcube.dim2big1," + " max(testcube.msr3), sum(testcube.msr2) FROM ", null, + getExpectedQuery(cubeName, "SELECT (testcube.dim2big1) as `dim2big1`, max((testcube.msr3)) as `max(msr3)`, " + + "sum((testcube.msr2)) as `msr2` FROM ", null, " group by testcube.dim2big1", getWhereForDailyAndHourly2daysWithTimeDim(cubeName, "it", "C2_summary4"), null); TestCubeRewriter.compareQueries(hqlQuery, expecteddim2big1); @@ -68,8 +72,9 @@ public void testDenormsAsDirectFields() throws ParseException, LensException, Hi hqlQuery = rewrite("select dim2big1, cubecity.name, max(msr3)," + " msr2 from testCube" + " where " + TWO_DAYS_RANGE_IT, conf); String expecteddim2big1WithAnotherTable = getExpectedQuery(cubeName, - "select testcube.dim2big1, cubecity.name, max(testcube.msr3), sum(testcube.msr2) FROM ", " JOIN " - + getDbName() + "c1_citytable cubecity " + "on testcube.cityid = cubecity.id and cubecity.dt = 'latest' ", null, + "SELECT (testcube.dim2big1) as `dim2big1`, (cubecity.name) as `name`, max((testcube.msr3)) as `max(msr3)`, " + + "sum((testcube.msr2)) as `msr2` FROM ", " JOIN " + getDbName() + "c1_citytable cubecity " + + "on testcube.cityid = cubecity.id and cubecity.dt = 'latest' ", null, " group by testcube.dim2big1, cubecity.name", null, getWhereForDailyAndHourly2daysWithTimeDim(cubeName, "it", "C2_summary4"), null); @@ -77,9 +82,9 @@ public void testDenormsAsDirectFields() throws ParseException, LensException, Hi hqlQuery = rewrite("select dim2big2, max(msr3)," + " msr2 from testCube" + " where " + TWO_DAYS_RANGE_IT, conf); String expecteddim2big2 = - getExpectedQuery(cubeName, "select testcube.dim2big2, max(testcube.msr3), sum(testcube.msr2) FROM ", null, - " group by testcube.dim2big2", getWhereForDailyAndHourly2daysWithTimeDim(cubeName, "it", "C2_summary4"), - null); + getExpectedQuery(cubeName, "SELECT (testcube.dim2big2) as `dim2big2`, max((testcube.msr3)) as `max(msr3)`, " + + "sum((testcube.msr2)) as `msr2` FROM ", null, " group by testcube.dim2big2", + getWhereForDailyAndHourly2daysWithTimeDim(cubeName, "it", "C2_summary4"), null); TestCubeRewriter.compareQueries(hqlQuery, expecteddim2big2); Configuration conf2 = new Configuration(conf); @@ -88,7 +93,8 @@ public void testDenormsAsDirectFields() throws ParseException, LensException, Hi + TWO_DAYS_RANGE_IT, conf2); String expected = getExpectedQuery(cubeName, - "select dim3chain.name, testcube.dim2big1, max(testcube.msr3), sum(testcube.msr2) FROM ", " JOIN " + "SELECT (dim3chain.name) as `name`, (testcube.dim2big1) as `dim2big1`, max((testcube.msr3)) as `max(msr3)`," + + " sum((testcube.msr2)) as `msr2` FROM ", " JOIN " + getDbName() + "c2_testdim2tbl3 testdim2 " + "on testcube.dim2big1 = testdim2.bigid1" + " join " + getDbName() + "c2_testdim3tbl dim3chain on " + "testdim2.testdim3id = dim3chain.id", null, " group by dim3chain.name, (testcube.dim2big1)", null, @@ -110,7 +116,8 @@ public void testDenormsWithJoins() throws Exception { String hqlQuery = rewrite("select dim2big1, max(msr3), msr2 from testCube where " + TWO_DAYS_RANGE, tconf); String expected = - getExpectedQuery(cubeName, "select dim2chain.bigid1, max(testcube.msr3), sum(testcube.msr2) FROM ", " JOIN " + getExpectedQuery(cubeName, "select (dim2chain.bigid1) as `dim2big1`, max((testcube.msr3)) " + + "as `max(msr3)`, sum((testcube.msr2)) as `msr2` FROM ", " JOIN " + getDbName() + "c1_testdim2tbl2 dim2chain ON testcube.dim2 = " + " dim2chain.id and (dim2chain.dt = 'latest') ", null, "group by (dim2chain.bigid1)", null, getWhereForDailyAndHourly2days(cubeName, "c1_summary2")); @@ -125,7 +132,8 @@ public void testDenormsWithJoinsWithChainFieldSelected() throws Exception { tconf); String expected = getExpectedQuery(cubeName, - "select dim2chain.name, dim2chain.bigid1, max(testcube.msr3), sum(testcube.msr2) FROM ", " JOIN " + "select (dim2chain.name) as `name`, (dim2chain.bigid1) as `dim2big1`, max((testcube.msr3)) as `max(msr3)`, " + + "sum((testcube.msr2)) as `msr2` FROM ", " JOIN " + getDbName() + "c1_testdim2tbl2 dim2chain ON testcube.dim2 = " + " dim2chain.id and (dim2chain.dt = 'latest') ", null, "group by dim2chain.name, dim2chain.bigid1", null, getWhereForDailyAndHourly2days(cubeName, "c1_summary2")); @@ -141,7 +149,8 @@ public void testDenormsWithJoinsWithChainFieldSelectedAndJoinTypeSpecified() thr tconf); String expected = getExpectedQuery(cubeName, - "select dim2chain.name, dim2chain.bigid1, max(testcube.msr3), sum(testcube.msr2) FROM ", " LEFT OUTER JOIN " + "select (dim2chain.name) as `name`, (dim2chain.bigid1) as `dim2big1`, max((testcube.msr3)) " + + "as `max(msr3)`, sum((testcube.msr2)) as `msr2` FROM ", " LEFT OUTER JOIN " + getDbName() + "c1_testdim2tbl2 dim2chain ON testcube.dim2 = " + " dim2chain.id and (dim2chain.dt = 'latest') ", null, "group by dim2chain.name, dim2chain.bigid1", null, getWhereForDailyAndHourly2days(cubeName, "c1_summary2")); @@ -167,7 +176,8 @@ public void testDenormsWithJoinsWithMergableChains() throws Exception { tconf); String expected = getExpectedQuery(cubeName, - "select dim3chain.name, dim2chain.bigid1, max(testcube.msr3), sum(testcube.msr2) FROM ", " JOIN " + " SELECT (dim3chain.name) as `name`, (dim2chain.bigid1) as `dim2big1`, max((testcube.msr3)) " + + "as `max(msr3)`, sum((testcube.msr2)) as `msr2` FROM ", " JOIN " + getDbName() + "c1_testdim2tbl3 dim2chain " + "on testcube.dim2 = dim2chain.id AND (dim2chain.dt = 'latest')" + " join " + getDbName() + "c1_testdim3tbl dim3chain on " + "dim2chain.testdim3id = dim3chain.id AND (dim3chain.dt = 'latest')", @@ -184,41 +194,23 @@ public void testDenormsWithJoinsWithNoCandidateStorages() throws Exception { "select dim2big2, max(msr3)," + " msr2 from testCube" + " where " + TWO_DAYS_RANGE, tconf); NoCandidateFactAvailableException ne = (NoCandidateFactAvailableException) e; PruneCauses.BriefAndDetailedError error = ne.getJsonMessage(); - Assert.assertEquals(error.getBrief(), CandidateTablePruneCode.NO_CANDIDATE_STORAGES.errorFormat); - - HashMap> details = error.getDetails(); - - for (Map.Entry> entry : details.entrySet()) { - if (entry.getValue().equals(Arrays.asList(CandidateTablePruneCause.columnNotFound("dim2big2")))) { - Set expectedKeySet = - Sets.newTreeSet(Splitter.on(',').split("summary1,cheapfact,testfactmonthly,testfact2,testfact")); - Assert.assertTrue(expectedKeySet.equals(Sets.newTreeSet(Splitter.on(',').split(entry.getKey())))); - } - - if (entry.getValue().equals( - Arrays.asList(new CandidateTablePruneCause(CandidateTablePruneCode.INVALID_DENORM_TABLE)))) { - Set expectedKeySet = - Sets.newTreeSet(Splitter.on(',').split("summary2,testfact2_raw,summary3")); - Assert.assertTrue(expectedKeySet.equals(Sets.newTreeSet(Splitter.on(',').split(entry.getKey())))); - } - - if (entry.getKey().equals("testfact_continuous")) { - Assert.assertTrue(entry.getValue().equals( - Arrays.asList(CandidateTablePruneCause.columnNotFound("msr2", "msr3"))) - || entry.getValue().equals(Arrays.asList(CandidateTablePruneCause.columnNotFound("msr3", "msr2")))); - } - - if (entry.getKey().equals("summary4")) { - List expectedPruneCauses = Arrays.asList(CandidateTablePruneCause.noCandidateStorages( - new HashMap() { - { - put("C2", new CandidateTablePruneCause.SkipStorageCause( - CandidateTablePruneCause.SkipStorageCode.UNSUPPORTED)); - } - })); - Assert.assertTrue(entry.getValue().equals(expectedPruneCauses)); - } - } + Assert.assertEquals(error.getBrief(), CandidateTablePruneCode.INVALID_DENORM_TABLE.errorFormat); + + Map, List> enhanced = error.enhanced(); + Map, List> expected = Maps.newHashMap(); + expected.put(newHashSet("c1_summary1", "c1_testfact", "c1_testfact2"), + newArrayList(columnNotFound("dim2big2"))); + expected.put(newHashSet("c2_summary2", "c2_summary3", "c1_testfact2_raw", "" + + "c3_testfact2_raw", "c1_summary3", "c1_summary2"), + newArrayList(new CandidateTablePruneCause(CandidateTablePruneCode.INVALID_DENORM_TABLE))); + expected.put(newHashSet("c0_testfact_continuous"), newArrayList(columnNotFound( + "msr2", "msr3"))); + expected.put(newHashSet("c2_summary2", "c2_summary3", "c2_summary4", "c4_testfact", "c2_summary1", + "c3_testfact", "c3_testfact2_raw", "c6_testfact", "c4_testfact2", "c5_testfact", "c99_cheapfact", + "c2_testfact", "c0_cheapfact", "c2_testfactmonthly", "c0_testfact"), + newArrayList(new CandidateTablePruneCause(CandidateTablePruneCode.UNSUPPORTED_STORAGE))); + + Assert.assertEquals(enhanced, expected); } @Test @@ -226,7 +218,8 @@ public void testCubeQueryWithExpressionHavingDenormColumnComingAsDirectColumn() String hqlQuery = rewrite("select substrdim2big1, max(msr3)," + " msr2 from testCube" + " where " + TWO_DAYS_RANGE_IT, conf); String expecteddim2big1 = - getExpectedQuery(cubeName, "select substr(testcube.dim2big1, 5), max(testcube.msr3), sum(testcube.msr2) FROM ", + getExpectedQuery(cubeName, "SELECT substr((testcube.dim2big1), 5) as `substrdim2big1`, max((testcube.msr3)) " + + "as `max(msr3)`, sum((testcube.msr2)) as `msr2` FROM ", null, " group by substr(testcube.dim2big1, 5)", getWhereForDailyAndHourly2daysWithTimeDim(cubeName, "it", "C2_summary4"), null); @@ -240,7 +233,8 @@ public void testCubeQueryWithExpressionHavingDenormColumnResultingJoin() throws String hqlQuery = rewrite("select substrdim2big1, max(msr3)," + " msr2 from testCube" + " where " + TWO_DAYS_RANGE, tconf); String expected = - getExpectedQuery(cubeName, "select substr(dim2chain.bigid1, 5), max(testcube.msr3), sum(testcube.msr2) FROM ", + getExpectedQuery(cubeName, "SELECT substr((dim2chain.bigid1), 5) as `substrdim2big1`, max((testcube.msr3)) " + + "as `max(msr3)`, sum((testcube.msr2)) as `msr2` FROM ", " JOIN " + getDbName() + "c1_testdim2tbl2 dim2chain ON testcube.dim2 = " + " dim2chain.id and (dim2chain.dt = 'latest') ", null, "group by substr(dim2chain.bigid1, 5)", null, getWhereForDailyAndHourly2days(cubeName, "c1_summary2")); @@ -274,15 +268,11 @@ public void testDimensionQuery() throws Exception { public void testCubeQueryWithTwoRefCols() throws Exception { Configuration tConf = new Configuration(conf); tConf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, ""); - CubeQueryContext cubeql = rewriteCtx("select dim2, test_time_dim2 from testcube where " + TWO_DAYS_RANGE, tConf); - Set candidateFacts = new HashSet(); - for (CandidateFact cfact : cubeql.getCandidateFacts()) { - candidateFacts.add(cfact.getName().toLowerCase()); - } - // testfact contains test_time_dim_day_id, but not dim2 - it should have been removed. - Assert.assertFalse(candidateFacts.contains("testfact")); - // summary2 contains dim2, but not test_time_dim2 - it should have been removed. - Assert.assertFalse(candidateFacts.contains("summary2")); + //test_time_dim2 and dim2 are not querable together + NoCandidateFactAvailableException e = (NoCandidateFactAvailableException)getLensExceptionInRewrite( + "select dim2, test_time_dim2 from testcube where " + TWO_DAYS_RANGE, tConf); + Assert.assertEquals(e.getJsonMessage().getBrief(), + "Range not answerable"); // getting storage update periods are not valid for given time range } @Test @@ -293,8 +283,9 @@ public void testCubeQueryWithHourDimJoin() throws Exception { tConf.set(CubeQueryConfUtil.getValidStorageTablesKey("testFact2"), "C1_testFact2"); String hqlQuery = rewrite("select test_time_dim2, msr2 from testcube where " + TWO_DAYS_RANGE, tConf); String expected = - getExpectedQuery(cubeName, "select timehourchain2.full_hour, sum(testcube.msr2) FROM ", " join " + getDbName() - + "c4_hourDimTbl timehourchain2 on testcube.test_time_dim_hour_id2 = timehourchain2.id", null, + getExpectedQuery(cubeName, "select timehourchain2.full_hour as `test_time_dim2`, sum(testcube.msr2) as `msr2` " + + "FROM ", " join " + getDbName() + + "c4_hourDimTbl timehourchain2 on testcube.test_time_dim_hour_id2 = timehourchain2.id", null, " group by timehourchain2 . full_hour ", null, getWhereForHourly2days("c1_testfact2")); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -308,9 +299,10 @@ public void testCubeQueryWithDayDimJoin() throws Exception { tConf.set(CubeQueryConfUtil.getValidStorageTablesKey("testFact"), "C1_testFact"); String hqlQuery = rewrite("select test_time_dim2, msr2 from testcube where " + TWO_DAYS_RANGE, tConf); String expected = - getExpectedQuery(cubeName, "select timedatechain2.full_date, sum(testcube.msr2) FROM ", " join " + getDbName() - + "c4_dayDimTbl timedatechain2 on testcube.test_time_dim_day_id2 = timedatechain2.id", null, - " group by timedatechain2 . full_date ", null, + getExpectedQuery(cubeName, "select timedatechain2.full_date as `test_time_dim2`, sum(testcube.msr2) as `msr2` " + + "FROM ", " join " + getDbName() + + "c4_dayDimTbl timedatechain2 on testcube.test_time_dim_day_id2 = timedatechain2.id", null, + " group by timedatechain2 . full_date ", null, getWhereForDailyAndHourly2days(cubeName, "c1_testfact")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -323,8 +315,8 @@ public void testCubeQueryWithOptionalDimsRemoved() throws Exception { + "c1_citytable citydim on basecube.cityid = citydim.id and (citydim.dt = 'latest') " + " join " + getDbName() + "c1_ziptable cityzip on citydim.zipcode = cityzip.code and (cityzip.dt = 'latest')"; String expected = - getExpectedQuery("basecube", "select cityzip.code, basecube.dim22, basecube.msr11 FROM ", - joinExpr, null, null, null, + getExpectedQuery("basecube", "SELECT (cityzip.code) as `code`, (basecube.dim22) as `dim22`, " + + "(basecube.msr11) as `msr11` FROM ", joinExpr, null, null, null, getWhereForHourly2days("basecube", "C1_testfact2_raw_base")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -338,7 +330,8 @@ public void testTwoFieldsFromDifferentChainButSameTable() throws Exception { + " join " + getDbName() + "c1_citytable cubecity2 on testcube.cityid2 = cubecity2.id and (cubecity2.dt = 'latest')"; String expected = - getExpectedQuery("testcube", "select cubecity1.name, cubecity2.name, sum(testcube.msr2) FROM ", + getExpectedQuery("testcube", "SELECT (cubecity1.name) as `name`, (cubecity2.name) as `name`, " + + "sum((testcube.msr2)) as `msr2` FROM ", joinExpr, null, " group by cubecity1.name, cubecity2.name", null, getWhereForHourly2days("testcube", "c1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -350,7 +343,7 @@ public void testDimensionQueryWithTwoRefCols() throws Exception { CubeQueryContext cubeql = rewriteCtx("select citydim.zipcode, citydim.statename from" + " citydim", tConf); Set candidateDims = new HashSet<>(); for (CandidateDim cdim : cubeql.getCandidateDims().get(cubeql.getMetastoreClient().getDimension("citydim"))) { - candidateDims.add(cdim.getName()); + candidateDims.add(cdim.getStorageTable()); } // city_table2 contains stateid, but not zipcode - it should have been removed. Assert.assertFalse(candidateDims.contains("city_table2")); @@ -386,7 +379,8 @@ public void testCubeQueryMultiChainRefCol() throws Exception { + " join " + getDbName() + "c1_countrytable cubecitystatecountry on statedim.countryid =" + " cubecitystatecountry.id"; String expected = - getExpectedQuery("basecube", "select cubecitystatecountry.capital, sum(basecube.msr12) FROM ", + getExpectedQuery("basecube", "SELECT (cubecitystatecountry.capital) as `cubecountrycapital`, " + + "sum((basecube.msr12)) as `msr12` FROM ", joinExpr, null, " group by cubecitystatecountry.capital ", null, getWhereForHourly2days("basecube", "C1_testfact2_raw_base")); TestCubeRewriter.compareQueries(hqlQuery, expected); diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java index 5d4e87f5a..1db371271 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java @@ -71,7 +71,7 @@ public void testCubeQueryExpressionSelection() throws Exception { // select with expression String hqlQuery = rewrite("select avgmsr from testCube where " + TWO_DAYS_RANGE, conf); String expected = - getExpectedQuery(cubeName, "select avg(testCube.msr1 + testCube.msr2) FROM ", null, null, + getExpectedQuery(cubeName, "select avg(testCube.msr1 + testCube.msr2) as `avgmsr` FROM ", null, null, getWhereForHourly2days("C1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -80,8 +80,8 @@ public void testCubeQueryExpressionSelection() throws Exception { public void testCubeQueryExpressionSelectionAlongWithColumn() throws Exception { String hqlQuery = rewrite("select dim1, roundedmsr2 from testCube" + " where " + TWO_DAYS_RANGE, conf); String expected = - getExpectedQuery(cubeName, "select testcube.dim1, round(sum(testcube.msr2)/1000) FROM ", null, - " group by testcube.dim1", getWhereForDailyAndHourly2days(cubeName, "c1_summary1")); + getExpectedQuery(cubeName, "select testcube.dim1 as `dim1`, round(sum(testcube.msr2)/1000) as `roundedmsr2` " + + "FROM ", null, " group by testcube.dim1", getWhereForDailyAndHourly2days(cubeName, "c1_summary1")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -91,7 +91,8 @@ public void testExpressionInWhereAfterTimerange() throws Exception { String hqlQuery = rewrite("select msr2 from testCube" + " where " + TWO_DAYS_RANGE + " and substrexpr != 'XYZ'", conf); String expected = - getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, " and substr(testCube.dim1, 3) != 'XYZ'", + getExpectedQuery(cubeName, "select sum(testcube.msr2) as `msr2` FROM ", null, + " and substr(testCube.dim1, 3) != 'XYZ'", getWhereForDailyAndHourly2days(cubeName, "c1_summary1")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -100,8 +101,8 @@ public void testExpressionInWhereBeforeTimerange() throws Exception { String hqlQuery = rewrite("select SUM(msr2) from testCube" + " where substrexpr != 'XYZ' and " + TWO_DAYS_RANGE, conf); String expected = - getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", "substr(testCube.dim1, 3) != 'XYZ'", null, - getWhereForDailyAndHourly2days(cubeName, "c1_summary1")); + getExpectedQuery(cubeName, "select sum(testcube.msr2) as `sum(msr2)` FROM ", "substr(testCube.dim1, 3) != 'XYZ'", + null, getWhereForDailyAndHourly2days(cubeName, "c1_summary1")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @Test @@ -110,7 +111,7 @@ public void testExpressionInSelectAndWhere() throws Exception { String hqlQuery = rewrite("select avgmsr from testCube" + " where " + TWO_DAYS_RANGE + " and substrexpr != 'XYZ'", conf); String expected = - getExpectedQuery(cubeName, "select avg(testCube.msr1 + testCube.msr2) FROM ", null, + getExpectedQuery(cubeName, "select avg(testCube.msr1 + testCube.msr2) as `avgmsr` FROM ", null, " and substr(testCube.dim1, 3) != 'XYZ'", getWhereForHourly2days("C1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -119,7 +120,7 @@ public void testNestedExpressionInWhere() throws Exception { String hqlQuery = rewrite("select avgmsr from testCube" + " where " + TWO_DAYS_RANGE + " and indiasubstr = true", conf); String expected = - getExpectedQuery(cubeName, "select avg(testCube.msr1 + testCube.msr2) FROM ", null, + getExpectedQuery(cubeName, "select avg(testCube.msr1 + testCube.msr2) as `avgmsr` FROM ", null, " and (substr(testCube.dim1, 3) = 'INDIA') = true", getWhereForHourly2days("C1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -130,8 +131,8 @@ public void testExpressionWithTableAlias() throws Exception { String hqlQuery = rewrite("select TC.avgmsr from testCube TC" + " where " + TWO_DAYS_RANGE + " and TC.substrexpr != 'XYZ'", conf); String expected = - getExpectedQuery("tc", "select avg(tc.msr1 + tc.msr2) FROM ", null, " and substr(tc.dim1, 3) != 'XYZ'", - getWhereForHourly2days("tc", "C1_testfact2_raw")); + getExpectedQuery("tc", "select avg(tc.msr1 + tc.msr2) as `avgmsr` FROM ", + null, " and substr(tc.dim1, 3) != 'XYZ'", getWhereForHourly2days("tc", "C1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -142,7 +143,7 @@ public void testCubeExpressionWithColumnAlias() throws Exception { rewrite("select TC.substrexpr as subdim1, TC.avgmsr from testCube TC" + " where " + TWO_DAYS_RANGE + " and subdim1 != 'XYZ'", conf); String expected = - getExpectedQuery("tc", "select substr(tc.dim1, 3) as `subdim1`, avg(tc.msr1 + tc.msr2) FROM ", null, + getExpectedQuery("tc", "select substr(tc.dim1, 3) as `subdim1`, avg(tc.msr1 + tc.msr2) as `avgmsr` FROM ", null, " and subdim1 != 'XYZ' group by substr(tc.dim1, 3)", getWhereForHourly2days("tc", "C1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -164,8 +165,8 @@ public void testExpressionInGroupbyToSelect() throws Exception { rewrite("select avgmsr from testCube" + " where " + TWO_DAYS_RANGE + " and substrexpr != 'XYZ' group by booleancut", conf); String expected = - getExpectedQuery(cubeName, "select testCube.dim1 != 'x' AND testCube.dim2 != 10 ," - + " avg(testCube.msr1 + testCube.msr2) FROM ", null, " and substr(testCube.dim1, 3) != 'XYZ'" + getExpectedQuery(cubeName, "SELECT (((testcube.dim1) != 'x') and ((testcube.dim2) != 10)) as `booleancut`, " + + "avg(((testcube.msr1) + (testcube.msr2))) as `avgmsr` FROM ", null, " and substr(testCube.dim1, 3) != 'XYZ'" + " group by testCube.dim1 != 'x' AND testCube.dim2 != 10", getWhereForHourly2days("C1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -176,8 +177,8 @@ public void testExpressionInSelectToGroupby() throws Exception { rewrite("select booleancut, avgmsr from testCube" + " where " + TWO_DAYS_RANGE + " and substrexpr != 'XYZ'", conf); String expected = - getExpectedQuery(cubeName, "select testCube.dim1 != 'x' AND testCube.dim2 != 10 ," - + " avg(testCube.msr1 + testCube.msr2) FROM ", null, " and substr(testCube.dim1, 3) != 'XYZ' " + getExpectedQuery(cubeName, "select testCube.dim1 != 'x' AND testCube.dim2 != 10 as `booleancut`," + + " avg(testCube.msr1 + testCube.msr2) as `avgmsr` FROM ", null, " and substr(testCube.dim1, 3) != 'XYZ' " + "group by testCube.dim1 != 'x' AND testCube.dim2 != 10", getWhereForHourly2days("C1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -189,8 +190,8 @@ public void testExpressionInSelectToGroupbyWithComplexExpression() throws Except rewrite("select booleancut, summsrs from testCube" + " where " + TWO_DAYS_RANGE + " and substrexpr != 'XYZ'", conf); String expected = - getExpectedQuery(cubeName, "select testCube.dim1 != 'x' AND testCube.dim2 != 10 ," - + " ((1000 + sum(testCube.msr1) + sum(testCube.msr2))/100) FROM ", null, + getExpectedQuery(cubeName, "select testCube.dim1 != 'x' AND testCube.dim2 != 10 as `booleancut`," + + " ((1000 + sum(testCube.msr1) + sum(testCube.msr2))/100) `summsrs` FROM ", null, " and substr(testCube.dim1, 3) != 'XYZ' group by testCube.dim1 != 'x' AND testCube.dim2 != 10", getWhereForHourly2days("C1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -210,9 +211,10 @@ public void testExpressionToJoin() throws Exception { + "c1_statetable cubestate on" + " testcube.stateid = cubestate.id and (cubestate.dt = 'latest')"; String expected = - getExpectedQuery(cubeName, "select concat(cubecity.name, \":\", cubestate.name)," - + " avg(testcube.msr1 + testcube.msr2) FROM ", join2 + join1, null, " and substr(testcube.dim1, 3) != 'XYZ'" - + " group by concat(cubecity.name, \":\", cubestate.name)", null, getWhereForHourly2days("C1_testfact2_raw")); + getExpectedQuery(cubeName, "select concat(cubecity.name, \":\", cubestate.name) as `cityandstate`," + + " avg(testcube.msr1 + testcube.msr2) as `avgmsr` FROM ", join2 + join1, null, + " and substr(testcube.dim1, 3) != 'XYZ' group by concat(cubecity.name, \":\", cubestate.name)", + null, getWhereForHourly2days("C1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -224,8 +226,9 @@ public void testExpressionToExcludeJoin() throws Exception { conf); String expected = - getExpectedQuery(cubeName, "select substr(testcube.concatedcitystate, 10)" - + " avg(testcube.msr1 + testcube.msr2) FROM ", null, null, " and substr(testcube.dim1, 3) != 'XYZ'" + getExpectedQuery(cubeName, "select substr(testcube.concatedcitystate, 10) AS `cityandstatenew`, " + + " avg(testcube.msr1 + testcube.msr2) AS `avgmsr` FROM ", null, null, + " and substr(testcube.dim1, 3) != 'XYZ'" + " group by substr(testcube.concatedcitystate, 10)", null, getWhereForHourly2days("C1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -250,8 +253,8 @@ public void testExpressionInHaving() throws Exception { rewrite("select booleancut, avgmsr from testCube" + " where " + TWO_DAYS_RANGE + " and substrexpr != 'XYZ'" + " having msr6 > 100.0", conf); String expected = - getExpectedQuery(cubeName, "select testCube.dim1 != 'x' AND testCube.dim2 != 10 ," - + " avg(testCube.msr1 + testCube.msr2) FROM ", null, " and substr(testCube.dim1, 3) != 'XYZ' " + getExpectedQuery(cubeName, "select testCube.dim1 != 'x' AND testCube.dim2 != 10 as `booleancut`," + + " avg(testCube.msr1 + testCube.msr2) as `avgmsr` FROM ", null, " and substr(testCube.dim1, 3) != 'XYZ' " + " group by testCube.dim1 != 'x' AND testCube.dim2 != 10" + " having (sum(testCube.msr2) + max(testCube.msr3))/ count(testcube.msr4) > 100.0", getWhereForHourly2days("C1_testfact2_raw")); @@ -265,11 +268,12 @@ public void testExpressionInOrderby() throws Exception { rewrite("select avgmsr from testCube " + " where " + TWO_DAYS_RANGE + " and substrexpr != 'XYZ'" + " group by booleancut having msr6 > 100.0 order by booleancut", conf); String expected = - getExpectedQuery(cubeName, "select testCube.dim1 != 'x' AND testCube.dim2 != 10 ," - + " avg(testCube.msr1 + testCube.msr2) FROM ", null, " and substr(testCube.dim1, 3) != 'XYZ' " + getExpectedQuery(cubeName, "SELECT (((testcube.dim1) != 'x') and ((testcube.dim2) != 10)) as `booleancut`, " + + "avg(((testcube.msr1) + (testcube.msr2))) as `avgmsr` FROM ", null, + " and substr(testCube.dim1, 3) != 'XYZ' " + " group by testCube.dim1 != 'x' AND testCube.dim2 != 10" + " having (sum(testCube.msr2) + max(testCube.msr3))/ count(testcube.msr4) > 100.0" - + " order by testCube.dim1 != 'x' AND testCube.dim2 != 10 asc", getWhereForHourly2days("C1_testfact2_raw")); + + " order by booleancut asc", getWhereForHourly2days("C1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @Test @@ -279,7 +283,7 @@ public void testExpressionWithAliasInOrderby() throws Exception { + " having msr6 > 100.0 order by bc", conf); String expected = getExpectedQuery(cubeName, "select testCube.dim1 != 'x' AND testCube.dim2 != 10 as `bc`," - + " sum(testCube.msr2) FROM ", null, " and substr(testCube.dim1, 3) != 'XYZ' " + + " sum(testCube.msr2) as `msr2` FROM ", null, " and substr(testCube.dim1, 3) != 'XYZ' " + " group by testCube.dim1 != 'x' AND testCube.dim2 != 10" + " having (sum(testCube.msr2) + max(testCube.msr3))/ count(testcube.msr4) > 100.0" + " order by bc asc", getWhereForDailyAndHourly2days(cubeName, "c1_summary2")); @@ -293,7 +297,7 @@ public void testMultipleExpressionsPickingFirstExpression() throws Exception { newConf.set(CubeQueryConfUtil.getValidFactTablesKey(cubeName), "testFact"); String hqlQuery = rewrite("select equalsums from testCube where " + TWO_DAYS_RANGE, newConf); String expected = - getExpectedQuery(cubeName, "select max(testcube.msr3) + count(testcube.msr4) FROM ", null, null, + getExpectedQuery(cubeName, "select max(testcube.msr3) + count(testcube.msr4) as `equalsums` FROM ", null, null, getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -301,8 +305,8 @@ public void testMultipleExpressionsPickingFirstExpression() throws Exception { @Test public void testMultipleExpressionsPickingSecondExpression() throws Exception { String hqlQuery = rewrite("select equalsums from testCube where " + TWO_DAYS_RANGE, conf); - String expected = getExpectedQuery(cubeName, "select (max(testCube.msr3) + sum(testCube.msr2))/100 FROM ", null, - null, getWhereForHourly2days(cubeName, "C1_testfact2")); + String expected = getExpectedQuery(cubeName, "select (max(testCube.msr3) + sum(testCube.msr2))/100 " + + "as `equalsums` FROM ", null, null, getWhereForHourly2days(cubeName, "C1_testfact2")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -310,8 +314,8 @@ public void testMultipleExpressionsPickingSecondExpression() throws Exception { public void testMaterializedExpressionPickingExpression() throws Exception { // select with expression String hqlQuery = rewrite("select msr5 from testCube where " + TWO_DAYS_RANGE, conf); - String expected = getExpectedQuery(cubeName, "select sum(testCube.msr2) + max(testCube.msr3) FROM ", null, null, - getWhereForHourly2days(cubeName, "C1_testfact2")); + String expected = getExpectedQuery(cubeName, "select (sum(testCube.msr2) + max(testCube.msr3)) as `msr5` FROM ", + null, null, getWhereForHourly2days(cubeName, "C1_testfact2")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -333,7 +337,7 @@ public void testMaterializedExpressionPickingMaterializedValue() throws Exceptio newConf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); newConf.set(CubeQueryConfUtil.getValidFactTablesKey(cubeName), "testFact"); String hqlQuery = rewrite("select msr5 from testCube where " + TWO_DAYS_RANGE, newConf); - String expected = getExpectedQuery(cubeName, "select testcube.msr5 FROM ", null, null, + String expected = getExpectedQuery(cubeName, "select testcube.msr5 as `msr5` FROM ", null, null, getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -342,8 +346,8 @@ public void testMaterializedExpressionPickingMaterializedValue() throws Exceptio public void testExprDimAttribute() throws Exception { // select with expression String hqlQuery = rewrite("select substrexpr from testCube where " + TWO_DAYS_RANGE, conf); - String expected = getExpectedQuery(cubeName, "select distinct substr(testCube.dim1, 3) FROM ", null, null, - getWhereForDailyAndHourly2days(cubeName, "c1_summary1")); + String expected = getExpectedQuery(cubeName, "select distinct substr(testCube.dim1, 3) as `substrexpr` " + + "FROM ", null, null, getWhereForDailyAndHourly2days(cubeName, "c1_summary1")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -462,7 +466,7 @@ public void testSingleColExpression() throws Exception { CubeQueryContext rewrittenQuery = rewriteCtx("select singlecolmsr2expr from testCube where " + TWO_DAYS_RANGE, tconf); String expected = - getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, + getExpectedQuery(cubeName, "select sum(testcube.msr2) as `singlecolmsr2expr` FROM ", null, null, getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); TestCubeRewriter.compareQueries(rewrittenQuery.toHQL(), expected); } @@ -474,7 +478,7 @@ public void testSingleDimColExpression() throws Exception { CubeQueryContext rewrittenQuery = rewriteCtx("select singlecoldim1expr from testCube where " + TWO_DAYS_RANGE, tconf); String expected = - getExpectedQuery(cubeName, "select distinct testcube.dim1 FROM ", null, null, + getExpectedQuery(cubeName, "select distinct testcube.dim1 as `singlecoldim1expr` FROM ", null, null, getWhereForDailyAndHourly2days(cubeName, "c1_summary1")); TestCubeRewriter.compareQueries(rewrittenQuery.toHQL(), expected); } @@ -498,7 +502,7 @@ public void testSingleDimColQualifiedExpression() throws Exception { CubeQueryContext rewrittenQuery = rewriteCtx("select singlecoldim1qualifiedexpr from testCube where " + TWO_DAYS_RANGE, tconf); String expected = - getExpectedQuery(cubeName, "select distinct testcube.dim1 FROM ", null, null, + getExpectedQuery(cubeName, "select distinct testcube.dim1 as `singlecoldim1qualifiedexpr` FROM ", null, null, getWhereForDailyAndHourly2days(cubeName, "C1_summary1")); TestCubeRewriter.compareQueries(rewrittenQuery.toHQL(), expected); } @@ -508,9 +512,10 @@ public void testSingleChainIdExpression() throws Exception { Configuration tconf = new Configuration(conf); tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); CubeQueryContext rewrittenQuery = - rewriteCtx("select singlecolchainid from testCube where " + TWO_DAYS_RANGE_IT, tconf); + rewriteCtx("select singlecolchainid from testCube where " + + TWO_DAYS_RANGE_IT, tconf); String expected = - getExpectedQuery(cubeName, "select distinct dim3chain.id FROM ", + getExpectedQuery(cubeName, "select distinct dim3chain.id as `singlecolchainid` FROM ", " join " + getDbName() + "c2_testdim3tbl dim3chain on testcube.testdim3id = dim3chain.id", null, null, null, getWhereForDailyAndHourly2daysWithTimeDim(cubeName, "it", "C2_summary1")); @@ -524,7 +529,7 @@ public void testSingleChainRefIdExpression() throws Exception { CubeQueryContext rewrittenQuery = rewriteCtx("select singlecolchainrefexpr from testCube where " + TWO_DAYS_RANGE_IT, tconf); String expected = - getExpectedQuery(cubeName, "select distinct testcube.testdim3id FROM ", null, null, + getExpectedQuery(cubeName, "select distinct testcube.testdim3id as `singlecolchainrefexpr` FROM ", null, null, getWhereForDailyAndHourly2daysWithTimeDim(cubeName, "it", "C2_summary1")); TestCubeRewriter.compareQueries(rewrittenQuery.toHQL(), expected); } @@ -536,7 +541,7 @@ public void testSingleChainRefColExpression() throws Exception { CubeQueryContext rewrittenQuery = rewriteCtx("select singlecolchainfield from testCube where " + TWO_DAYS_RANGE, tconf); String expected = - getExpectedQuery(cubeName, "select distinct cubecity.name FROM ", + getExpectedQuery(cubeName, "select distinct cubecity.name as `singlecolchainfield` FROM ", " join " + getDbName() + "c2_citytable cubecity ON testcube.cityid = cubecity.id", null, null, null, getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); TestCubeRewriter.compareQueries(rewrittenQuery.toHQL(), expected); @@ -574,7 +579,7 @@ public void testSingleColQualifiedExpression() throws Exception { CubeQueryContext rewrittenQuery = rewriteCtx("select singlecolmsr2qualifiedexpr from testCube where " + TWO_DAYS_RANGE, tconf); String expected = - getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, + getExpectedQuery(cubeName, "select sum(testcube.msr2) as `singlecolmsr2qualifiedexpr` FROM ", null, null, getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); TestCubeRewriter.compareQueries(rewrittenQuery.toHQL(), expected); } @@ -584,9 +589,10 @@ public void testSingleColQualifiedExpressionWithAlias() throws Exception { Configuration tconf = new Configuration(conf); tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); CubeQueryContext rewrittenQuery = - rewriteCtx("select singlecolmsr2qualifiedexpr from testCube tc where " + TWO_DAYS_RANGE, tconf); + rewriteCtx("select singlecolmsr2qualifiedexpr as `singlecolmsr2qualifiedexpr` from testCube tc where " + + TWO_DAYS_RANGE, tconf); String expected = - getExpectedQuery("tc", "select sum(tc.msr2) FROM ", null, null, + getExpectedQuery("tc", "select sum(tc.msr2) as `singlecolmsr2qualifiedexpr` FROM ", null, null, getWhereForDailyAndHourly2days("tc", "C2_testfact")); TestCubeRewriter.compareQueries(rewrittenQuery.toHQL(), expected); } diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java index 677d641fe..f5ddf7bba 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java @@ -19,16 +19,18 @@ package org.apache.lens.cube.parse; -import static org.apache.lens.cube.metadata.DateFactory.*; +import static org.apache.lens.cube.metadata.DateFactory.TWO_DAYS_RANGE; import static org.apache.lens.cube.parse.CubeTestSetup.*; import static org.apache.lens.cube.parse.TestCubeRewriter.compareQueries; -import static org.testng.Assert.*; - -import java.util.*; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import org.apache.lens.cube.error.LensCubeErrorCode; -import org.apache.lens.cube.metadata.*; +import org.apache.lens.cube.metadata.CubeMetastoreClient; +import org.apache.lens.cube.metadata.Dimension; import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.StringUtils; @@ -61,7 +63,15 @@ public void closeInstance() throws Exception { } private String getAutoResolvedFromString(CubeQueryContext query) throws LensException { - return query.getHqlContext().getFrom(); + String from = null; + if (query.getPickedCandidate() instanceof StorageCandidate) { + StorageCandidate sc = (StorageCandidate) query.getPickedCandidate(); + from = sc.getFromString(); + // Dim only query + } else if (query.getPickedCandidate() == null) { + from = query.getHqlContext().getFrom(); + } + return from; } @Test @@ -133,8 +143,9 @@ public void testJoinFilters() throws Exception { + " right outer join testDim4 on testdim3.testdim4id = testdim4.id and testDim4.name='TESTDIM4NAME'" + " WHERE " + TWO_DAYS_RANGE; String hqlQuery = rewrite(query, hconf); - String expected = getExpectedQuery("testcube", "select citydim.name, testDim4.name, sum(testcube.msr2) FROM ", - " left outer JOIN " + getDbName() + "c1_citytable citydim on testcube.cityid = citydim.id +" + String expected = getExpectedQuery("testcube", "SELECT (citydim.name) as `name`, (testdim4.name) as `name`, " + + "sum((testcube.msr2)) as `msr2` FROM ", + " left outer JOIN " + getDbName() + "c1_citytable citydim on testcube.cityid = citydim.id +" + " and (( citydim . name ) = 'FOOBAR' ) and (citydim.dt = 'latest')" + " right outer join " + getDbName() + "c1_testdim2tbl testdim2 on testcube.dim2 = testdim2.id and (testdim2.dt = 'latest')" @@ -175,8 +186,8 @@ public void testJoinTypeConf() throws Exception { String query = "select cubecity.name, msr2 FROM testCube WHERE " + TWO_DAYS_RANGE; String hqlQuery = rewrite(query, tConf); // Check that aliases are preserved in the join clause - String expected = getExpectedQuery("testcube", "select cubecity.name, sum(testcube.msr2) FROM ", - " left outer join " + getDbName() + String expected = getExpectedQuery("testcube", "SELECT (cubecity.name) as `name`, sum((testcube.msr2)) " + + "as `msr2` FROM ", " left outer join " + getDbName() + "c1_citytable cubecity ON testcube.cityid = cubecity.id and (cubecity.dt = 'latest')", null, " group by cubecity.name", null, getWhereForHourly2days("testcube", "c1_testfact2")); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -184,7 +195,7 @@ public void testJoinTypeConf() throws Exception { tConf.set(CubeQueryConfUtil.JOIN_TYPE_KEY, "FULLOUTER"); hqlQuery = rewrite(query, tConf); // Check that aliases are preserved in the join clause - expected = getExpectedQuery("testcube", "select cubecity.name, sum(testcube.msr2) FROM ", + expected = getExpectedQuery("testcube", "select cubecity.name as `name`, sum(testcube.msr2) as `msr2` FROM ", " full outer join " + getDbName() + "c1_citytable cubecity ON testcube.cityid = cubecity.id and (cubecity.dt = 'latest')", null, " group by cubecity.name", null, getWhereForHourly2days("testcube", "c1_testfact2")); @@ -193,7 +204,7 @@ public void testJoinTypeConf() throws Exception { tConf.set(CubeQueryConfUtil.JOIN_TYPE_KEY, "RIGHTOUTER"); hqlQuery = rewrite(query, tConf); // Check that aliases are preserved in the join clause - expected = getExpectedQuery("testcube", "select cubecity.name, sum(testcube.msr2) FROM ", + expected = getExpectedQuery("testcube", "select cubecity.name as `name`, sum(testcube.msr2) as `msr2` FROM ", " right outer join " + getDbName() + "c1_citytable cubecity ON testcube.cityid = cubecity.id", null, " and (cubecity.dt = 'latest') group by cubecity.name", null, @@ -209,7 +220,7 @@ public void testPreserveTableAliasWithFullJoin() throws Exception { String hqlQuery = rewrite(query, tConf); // Check that aliases are preserved in the join clause // Conf will be ignored in this case since user has specified the join condition - String expected = getExpectedQuery("t", "select c.name, sum(t.msr2) FROM ", + String expected = getExpectedQuery("t", "select c.name as `name`, sum(t.msr2) as `msr2` FROM ", " inner join " + getDbName() + "c1_citytable c ON t.cityid = c.id and c.dt = 'latest'", null, " group by c.name", null, getWhereForHourly2days("t", "c1_testfact2")); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -222,7 +233,7 @@ public void testPreserveTableAliasWithAutoJoin() throws Exception { String query = "select cubecity.name, t.msr2 FROM testCube t WHERE " + TWO_DAYS_RANGE; String hqlQuery = rewrite(query, tConf); // Check that aliases are preserved in the join clause - String expected = getExpectedQuery("t", "select cubecity.name, sum(t.msr2) FROM ", + String expected = getExpectedQuery("t", "select cubecity.name as `name`, sum(t.msr2) as `msr2` FROM ", " left outer join " + getDbName() + "c1_citytable cubecity ON t.cityid = cubecity.id and (cubecity.dt = 'latest')", null, " group by cubecity.name", null, getWhereForHourly2days("t", "c1_testfact2")); @@ -324,7 +335,8 @@ public void testJoinChains() throws ParseException, LensException, HiveException // Single joinchain with direct link query = "select cubestate.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE + " group by cubestate.name"; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("basecube", "select cubestate.name, sum(basecube.msr2) FROM ", + expected = getExpectedQuery("basecube", "SELECT (cubestate.name) as `name`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", " join " + getDbName() + "c1_statetable cubestate ON basecube.stateid=cubeState.id and cubeState.dt= 'latest'", null, "group by cubestate.name", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); @@ -333,8 +345,9 @@ public void testJoinChains() throws ParseException, LensException, HiveException // Single joinchain with two chains query = "select citystate.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE + " group by citystate.name"; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("basecube", "select citystate.name, sum(basecube.msr2) FROM ", - " join " + getDbName() + "c1_citytable citydim ON baseCube.cityid = citydim.id and citydim.dt = 'latest'" + expected = getExpectedQuery("basecube", "SELECT (citystate.name) as `name`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", + " join " + getDbName() + "c1_citytable citydim ON baseCube.cityid = citydim.id and citydim.dt = 'latest'" + " join " + getDbName() + "c1_statetable cityState ON citydim.stateid=cityState.id and cityState.dt= 'latest'", null, "group by citystate.name", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); @@ -343,9 +356,11 @@ public void testJoinChains() throws ParseException, LensException, HiveException // Single joinchain with two chains, accessed as refcolumn query = "select cityStateCapital, sum(msr2) from basecube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("basecube", "select citystate.capital, sum(basecube.msr2) FROM ", - " join " + getDbName() + "c1_citytable citydim ON baseCube.cityid = citydim.id and citydim.dt = 'latest'" - + " join " + getDbName() + "c1_statetable cityState ON citydim.stateid=cityState.id and cityState.dt= 'latest'", + expected = getExpectedQuery("basecube", "SELECT (citystate.capital) as `citystatecapital`, " + + "sum((basecube.msr2)) as `sum(msr2)` FROM ", + " join " + getDbName() + "c1_citytable citydim ON baseCube.cityid = citydim.id and citydim.dt = 'latest'" + + " join " + getDbName() + "c1_statetable cityState ON citydim.stateid=cityState.id " + + "and cityState.dt= 'latest'", null, "group by citystate.capital", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -358,18 +373,20 @@ public void testJoinChains() throws ParseException, LensException, HiveException // Adding Order by query = "select cityStateCapital, sum(msr2) from basecube where " + TWO_DAYS_RANGE + " order by cityStateCapital"; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("basecube", "select citystate.capital, sum(basecube.msr2) FROM ", - " join " + getDbName() + "c1_citytable citydim ON baseCube.cityid = citydim.id and citydim.dt = 'latest'" + expected = getExpectedQuery("basecube", "SELECT (citystate.capital) as `citystatecapital`, " + + "sum((basecube.msr2)) as `sum(msr2)` FROM ", " join " + + getDbName() + "c1_citytable citydim ON baseCube.cityid = citydim.id and citydim.dt = 'latest'" + " join " + getDbName() + "c1_statetable cityState ON citydim.stateid=cityState.id and cityState.dt= 'latest'", - null, "group by citystate.capital order by citystate.capital asc", + null, "group by citystate.capital order by citystatecapital", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); TestCubeRewriter.compareQueries(hqlQuery, expected); // Single joinchain, but one column accessed as refcol and another as chain.column query = "select citystate.name, cityStateCapital, sum(msr2) from basecube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("basecube", "select citystate.name, citystate.capital, sum(basecube.msr2) FROM ", - " join " + getDbName() + "c1_citytable citydim ON baseCube.cityid = citydim.id and citydim.dt = 'latest'" + expected = getExpectedQuery("basecube", "SELECT (citystate.name) as `name`, (citystate.capital) " + + "as `citystatecapital`, sum((basecube.msr2)) as `sum(msr2)` FROM ", " join " + + getDbName() + "c1_citytable citydim ON baseCube.cityid = citydim.id and citydim.dt = 'latest'" + " join " + getDbName() + "c1_statetable cityState ON citydim.stateid=cityState.id and cityState.dt= 'latest'", null, "group by citystate.name, citystate.capital", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); @@ -379,7 +396,7 @@ public void testJoinChains() throws ParseException, LensException, HiveException query = "select cubeState.name, cubecity.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); expected = getExpectedQuery("basecube", - "select cubestate.name, cubecity.name, sum(basecube.msr2) FROM ", + "SELECT (cubestate.name) as `name`, (cubecity.name) as `name`, sum((basecube.msr2)) as `sum(msr2)` FROM ", " join " + getDbName() + "c1_statetable cubestate on basecube.stateid = cubestate.id and cubestate.dt = 'latest'" + " join " + getDbName() + "c1_citytable cubecity on basecube.cityid = cubecity.id and cubecity.dt = 'latest'", null, "group by cubestate.name,cubecity.name", null, @@ -390,8 +407,9 @@ public void testJoinChains() throws ParseException, LensException, HiveException // Multiple join chains with same destination table query = "select cityState.name, cubeState.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("basecube", "select citystate.name, cubestate.name, sum(basecube.msr2) FROM ", - " join " + getDbName() + "c1_statetable cubestate on basecube.stateid=cubestate.id and cubestate.dt='latest'" + expected = getExpectedQuery("basecube", "SELECT (citystate.name) as `name`, (cubestate.name) " + + "as `name`, sum((basecube.msr2)) as `sum(msr2)` FROM ", " join " + getDbName() + + "c1_statetable cubestate on basecube.stateid=cubestate.id and cubestate.dt='latest'" + " join " + getDbName() + "c1_citytable citydim on basecube.cityid = citydim.id and " + "citydim.dt = 'latest'" + " join " + getDbName() + "c1_statetable citystate on citydim.stateid = citystate.id and " @@ -405,8 +423,9 @@ public void testJoinChains() throws ParseException, LensException, HiveException query = "select cubestate.name, cityStateCapital, sum(msr2) from basecube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); expected = getExpectedQuery("basecube", - "select cubestate.name, citystate.capital, sum(basecube.msr2) FROM ", - "" + "SELECT (cubestate.name) as `name`, (citystate.capital) as `citystatecapital`, " + + "sum((basecube.msr2)) as `sum(msr2)` FROM ", + "" + " join " + getDbName() + "c1_statetable cubestate on basecube.stateid=cubestate.id and cubestate.dt='latest'" + " join " + getDbName() + "c1_citytable citydim on basecube.cityid = citydim.id and citydim.dt = 'latest'" + " join " + getDbName() + "c1_statetable citystate on citydim.stateid=citystate.id and citystate.dt='latest'" @@ -420,7 +439,7 @@ public void testJoinChains() throws ParseException, LensException, HiveException query = "select cityState.name, cityZip.f1, sum(msr2) from basecube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); expected = getExpectedQuery("basecube", - "select citystate.name, cityzip.f1, sum(basecube.msr2) FROM ", + "SELECT (citystate.name) as `name`, (cityzip.f1) as `f1`, sum((basecube.msr2)) as `sum(msr2)` FROM ", " join " + getDbName() + "c1_citytable citydim on basecube.cityid = citydim.id and " + "citydim.dt = 'latest'" + " join " + getDbName() + "c1_statetable citystate on citydim.stateid = citystate.id and " @@ -437,8 +456,8 @@ public void testJoinChains() throws ParseException, LensException, HiveException query = "select cubeStateCountry.name, cubeCityStateCountry.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); expected = getExpectedQuery("basecube", - "select cubestatecountry.name, cubecitystatecountry.name, sum(basecube.msr2) FROM ", - "" + "SELECT (cubestatecountry.name) as `name`, (cubecitystatecountry.name) as `name`, sum((basecube.msr2)) " + + "as `sum(msr2)` FROM ", "" + " join " + getDbName() + "c1_citytable citydim on basecube.cityid = citydim.id and (citydim.dt = 'latest')" + " join " + getDbName() + "c1_statetable statedim_0 on citydim.stateid=statedim_0.id and statedim_0.dt='latest'" @@ -554,7 +573,8 @@ public void testMultiPaths() throws ParseException, LensException, HiveException query = "select dim3chain.name, sum(msr2) from testcube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("testcube", "select dim3chain.name, sum(testcube.msr2) FROM ", + expected = getExpectedQuery("testcube", "SELECT (dim3chain.name) as `name`, sum((testcube.msr2)) " + + "as `sum(msr2)` FROM ", " join " + getDbName() + "c1_testdim3tbl dim3chain ON testcube.testdim3id=dim3chain.id and dim3chain.dt='latest'", null, "group by dim3chain.name", null, getWhereForDailyAndHourly2days("testcube", "c1_summary1")); @@ -563,8 +583,9 @@ public void testMultiPaths() throws ParseException, LensException, HiveException // hit a fact where there is no direct path query = "select dim3chain.name, avg(msr2) from testcube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("testcube", "select dim3chain.name, avg(testcube.msr2) FROM ", - " join " + getDbName() + "c1_testdim2tbl testdim2 ON testcube.dim2 = testdim2.id and testdim2.dt = 'latest'" + expected = getExpectedQuery("testcube", "SELECT (dim3chain.name) as `name`, avg((testcube.msr2)) " + + "as `avg(msr2)` FROM ", " join " + + getDbName() + "c1_testdim2tbl testdim2 ON testcube.dim2 = testdim2.id and testdim2.dt = 'latest'" + " join " + getDbName() + "c1_testdim3tbl dim3chain " + "ON testdim2.testdim3id = dim3chain.id and dim3chain.dt = 'latest'", null, "group by dim3chain.name", @@ -574,8 +595,9 @@ public void testMultiPaths() throws ParseException, LensException, HiveException // resolve denorm variable through multi hop chain paths query = "select testdim3id, avg(msr2) from testcube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("testcube", "select dim3chain.id, avg(testcube.msr2) FROM ", - " join " + getDbName() + "c1_testdim2tbl testdim2 ON testcube.dim2 = testdim2.id and testdim2.dt = 'latest'" + expected = getExpectedQuery("testcube", "SELECT (dim3chain.id) as `testdim3id`, avg((testcube.msr2)) " + + "as `avg(msr2)` FROM", " join " + + getDbName() + "c1_testdim2tbl testdim2 ON testcube.dim2 = testdim2.id and testdim2.dt = 'latest'" + " join " + getDbName() + "c1_testdim3tbl dim3chain " + "ON testdim2.testdim3id = dim3chain.id and dim3chain.dt = 'latest'", null, "group by dim3chain.id", @@ -585,8 +607,9 @@ public void testMultiPaths() throws ParseException, LensException, HiveException // tests from multiple different chains query = "select dim4chain.name, testdim3id, avg(msr2) from testcube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("testcube", "select dim4chain.name, dim3chain.id, avg(testcube.msr2) FROM ", - " join " + getDbName() + "c1_testdim2tbl testdim2 ON testcube.dim2 = testdim2.id and testdim2.dt = 'latest'" + expected = getExpectedQuery("testcube", "select dim4chain.name as `name`, dim3chain.id as `testdim3id`, " + + "avg(testcube.msr2) as `avg(msr2)` FROM ", " join " + + getDbName() + "c1_testdim2tbl testdim2 ON testcube.dim2 = testdim2.id and testdim2.dt = 'latest'" + " join " + getDbName() + "c1_testdim3tbl dim3chain ON testdim2.testdim3id=dim3chain.id and dim3chain.dt='latest'" + " join " + getDbName() + "c1_testdim4tbl dim4chain ON dim3chain.testDim4id = dim4chain.id and" @@ -596,9 +619,10 @@ public void testMultiPaths() throws ParseException, LensException, HiveException query = "select cubecity.name, dim4chain.name, testdim3id, avg(msr2) from testcube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("testcube", "select cubecity.name, dim4chain.name, dim3chain.id, avg(testcube.msr2) " - + "FROM ", - " join " + getDbName() + "c1_testdim2tbl testdim2 ON testcube.dim2 = testdim2.id and testdim2.dt = 'latest'" + expected = getExpectedQuery("testcube", "select cubecity.name as `name`, dim4chain.name as `name`, " + + "dim3chain.id as `testdim3id`, avg(testcube.msr2) as `avg(msr2)`" + + "FROM ", " join " + + getDbName() + "c1_testdim2tbl testdim2 ON testcube.dim2 = testdim2.id and testdim2.dt = 'latest'" + " join " + getDbName() + "c1_testdim3tbl dim3chain ON testdim2.testdim3id=dim3chain.id and dim3chain.dt='latest'" + " join " + getDbName() + "c1_testdim4tbl dim4chain ON dim3chain.testDim4id = dim4chain.id and" @@ -611,8 +635,9 @@ public void testMultiPaths() throws ParseException, LensException, HiveException // test multi hops query = "select dim4chain.name, avg(msr2) from testcube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("testcube", "select dim4chain.name, avg(testcube.msr2) FROM ", - " join " + getDbName() + "c1_testdim2tbl testdim2 ON testcube.dim2 = testdim2.id and testdim2.dt = 'latest'" + expected = getExpectedQuery("testcube", "select dim4chain.name as `name`, avg(testcube.msr2) " + + "as `avg(msr2)` FROM ", " join " + + getDbName() + "c1_testdim2tbl testdim2 ON testcube.dim2 = testdim2.id and testdim2.dt = 'latest'" + " join " + getDbName() + "c1_testdim3tbl testdim3 ON testdim2.testdim3id=testdim3.id and testdim3.dt='latest'" + " join " + getDbName() + "c1_testdim4tbl dim4chain ON testdim3.testDim4id = dim4chain.id and" + " dim4chain.dt = 'latest'", null, "group by dim4chain.name", null, @@ -621,7 +646,7 @@ public void testMultiPaths() throws ParseException, LensException, HiveException query = "select dim4chain.name, sum(msr2) from testcube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("testcube", "select dim4chain.name, sum(testcube.msr2) FROM ", + expected = getExpectedQuery("testcube", "select dim4chain.name as `name`, sum(testcube.msr2) as `sum(msr2)` FROM ", " join " + getDbName() + "c1_testdim3tbl testdim3 ON testcube.testdim3id = testdim3.id and testdim3.dt = 'latest'" + " join " + getDbName() + "c1_testdim4tbl dim4chain ON testdim3.testDim4id = dim4chain.id and" + " dim4chain.dt = 'latest'", null, "group by dim4chain.name", null, diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java index 27a18f4af..3883beed7 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java @@ -21,8 +21,6 @@ import static org.apache.lens.cube.metadata.DateFactory.TWO_DAYS_RANGE; -import java.util.Arrays; - import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.metrics.LensMetricsRegistry; @@ -32,6 +30,7 @@ import org.testng.annotations.Test; import com.codahale.metrics.MetricRegistry; +import jersey.repackaged.com.google.common.collect.Sets; public class TestQueryMetrics extends TestQueryRewrite { @@ -44,28 +43,30 @@ public void testMethodGauges() throws Exception { rewriteCtx("select" + " SUM(msr2) from testCube where " + TWO_DAYS_RANGE, conf); MetricRegistry reg = LensMetricsRegistry.getStaticRegistry(); - Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList( - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.AggregateResolver-ITER-6", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.AliasReplacer-ITER-1", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.CandidateTableResolver-ITER-11", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.CandidateTableResolver-ITER-5", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.ColumnResolver-ITER-0", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.DenormalizationResolver-ITER-16", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.DenormalizationResolver-ITER-3", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.ExpressionResolver-ITER-17", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.ExpressionResolver-ITER-2", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.FieldValidator-ITER-8", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.GroupbyResolver-ITER-7", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.JoinResolver-ITER-9", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.LeastPartitionResolver-ITER-19", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.LightestDimensionResolver-ITER-20", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.LightestFactResolver-ITER-18", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.MaxCoveringFactResolver-ITER-14", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.StorageTableResolver-ITER-12", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.StorageTableResolver-ITER-13", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.StorageTableResolver-ITER-15", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.TimeRangeChecker-ITER-10", - "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.TimerangeResolver-ITER-4") - ), reg.getGauges().keySet().toString()); + Assert.assertEquals(reg.getGauges().keySet(), Sets.newHashSet( + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.ColumnResolver-ITER-0", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.AliasReplacer-ITER-1", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.ExpressionResolver-ITER-2", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.DenormalizationResolver-ITER-3", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.TimerangeResolver-ITER-4", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.CandidateTableResolver-ITER-5", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.AggregateResolver-ITER-6", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.GroupbyResolver-ITER-7", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.FieldValidator-ITER-8", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.JoinResolver-ITER-9", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.ColumnLifetimeChecker-ITER-10", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.StorageTableResolver-ITER-11", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.CandidateTableResolver-ITER-12", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse." + + "CandidateCoveringSetsResolver-ITER-13", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.StorageTableResolver-ITER-14", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.MaxCoveringFactResolver-ITER-15", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.StorageTableResolver-ITER-16", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.DenormalizationResolver-ITER-17", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.ExpressionResolver-ITER-18", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.LightestFactResolver-ITER-19", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.LeastPartitionResolver-ITER-20", + "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.LightestDimensionResolver-ITER-21" + )); } } diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java index 7f26b24f0..76ea77d52 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java @@ -59,8 +59,8 @@ public void testPlanExtractionForSimpleQuery() throws Exception { Assert.assertTrue(plan.getTablesQueried().contains("TestQueryRewrite.c2_testfact")); Assert.assertEquals(plan.getTableWeights().get("TestQueryRewrite.c2_testfact"), 1.0); Assert.assertFalse(plan.getPartitions().isEmpty()); - Assert.assertFalse(plan.getPartitions().get("testfact").isEmpty()); - Assert.assertTrue(plan.getPartitions().get("testfact").size() > 1); + Assert.assertFalse(plan.getPartitions().get("c2_testfact").isEmpty()); + Assert.assertTrue(plan.getPartitions().get("c2_testfact").size() > 1); } @Test @@ -79,8 +79,8 @@ public void testPlanExtractionForComplexQuery() throws Exception { Assert.assertEquals(plan.getTableWeights().get("TestQueryRewrite.c1_testfact2"), 1.0); Assert.assertEquals(plan.getTableWeights().get("TestQueryRewrite.c1_citytable"), 100.0); Assert.assertFalse(plan.getPartitions().isEmpty()); - Assert.assertFalse(plan.getPartitions().get("testfact2").isEmpty()); - Assert.assertTrue(plan.getPartitions().get("testfact2").size() > 1); + Assert.assertFalse(plan.getPartitions().get("c1_testfact2").isEmpty()); + Assert.assertTrue(plan.getPartitions().get("c1_testfact2").size() > 1); Assert.assertFalse(plan.getPartitions().get("citytable").isEmpty()); Assert.assertEquals(plan.getPartitions().get("citytable").size(), 1); } @@ -103,8 +103,8 @@ public void testPlanExtractionForMultipleQueries() throws Exception { Assert.assertEquals(plan.getTableWeights().get("TestQueryRewrite.c1_testfact2"), 1.0); Assert.assertEquals(plan.getTableWeights().get("TestQueryRewrite.c1_citytable"), 100.0); Assert.assertFalse(plan.getPartitions().isEmpty()); - Assert.assertFalse(plan.getPartitions().get("testfact2").isEmpty()); - Assert.assertTrue(plan.getPartitions().get("testfact2").size() > 1); + Assert.assertFalse(plan.getPartitions().get("c1_testfact2").isEmpty()); + Assert.assertTrue(plan.getPartitions().get("c1_testfact2").size() > 1); Assert.assertFalse(plan.getPartitions().get("citytable").isEmpty()); Assert.assertEquals(plan.getPartitions().get("citytable").size(), 1); } diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java index 280a8c441..d3938c130 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java @@ -20,19 +20,13 @@ package org.apache.lens.cube.parse; import static org.apache.lens.cube.metadata.DateFactory.*; -import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.COLUMN_NOT_FOUND; -import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.FACT_NOT_AVAILABLE_IN_RANGE; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.UNSUPPORTED_STORAGE; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; -import java.util.Calendar; -import java.util.GregorianCalendar; -import java.util.List; -import java.util.Set; +import java.util.*; -import org.apache.lens.cube.error.NoCandidateFactAvailableException; -import org.apache.lens.cube.metadata.TimeRange; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.error.LensException; @@ -44,7 +38,6 @@ import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.common.collect.Sets; public class TestTimeRangeResolver extends TestQueryRewrite { @@ -69,23 +62,9 @@ public Configuration getConf() { @Test public void testFactValidity() throws ParseException, LensException, HiveException, ClassNotFoundException { - LensException e = - getLensExceptionInRewrite("select msr2 from " + cubeName + " where " + LAST_YEAR_RANGE, - getConf()); - NoCandidateFactAvailableException ne = (NoCandidateFactAvailableException) e; - PruneCauses.BriefAndDetailedError causes = ne.getJsonMessage(); - assertTrue(causes.getBrief().contains("Columns [msr2] are not present in any table")); - assertEquals(causes.getDetails().size(), 2); - - Set expectedPruneCodes = Sets.newTreeSet(); - expectedPruneCodes.add(FACT_NOT_AVAILABLE_IN_RANGE); - expectedPruneCodes.add(COLUMN_NOT_FOUND); - Set actualPruneCodes = Sets.newTreeSet(); - for (List cause : causes.getDetails().values()) { - assertEquals(cause.size(), 1); - actualPruneCodes.add(cause.iterator().next().getCause()); - } - assertEquals(actualPruneCodes, expectedPruneCodes); + String query = "select msr2 from " + cubeName + " where " + LAST_YEAR_RANGE; + LensException e = getLensExceptionInRewrite(query, getConf()); + assertEquals(e.getErrorInfo().getErrorName(), "NO_UNION_CANDIDATE_AVAILABLE"); } @Test @@ -93,13 +72,28 @@ public void testAbsoluteValidity() throws ParseException, HiveException, LensExc CubeQueryContext ctx = rewriteCtx("select msr12 from basecube where " + TWO_DAYS_RANGE + " or " + TWO_DAYS_RANGE_BEFORE_4_DAYS, getConf()); - assertEquals(ctx.getFactPruningMsgs().get(ctx.getMetastoreClient().getCubeFact("testfact_deprecated")).size(), 1); - CandidateTablePruneCause pruningMsg = - ctx.getFactPruningMsgs().get(ctx.getMetastoreClient().getCubeFact("testfact_deprecated")).get(0); + List causes = findPruningMessagesForStorage("c3_testfact_deprecated", + ctx.getStoragePruningMsgs()); + assertEquals(causes.size(), 1); + assertEquals(causes.get(0).getCause(), UNSUPPORTED_STORAGE); + + causes = findPruningMessagesForStorage("c4_testfact_deprecated", + ctx.getStoragePruningMsgs()); + assertEquals(causes.size(), 1); + assertEquals(causes.get(0).getCause(), UNSUPPORTED_STORAGE); + // testfact_deprecated's validity should be in between of both ranges. So both ranges should be in the invalid list // That would prove that parsing of properties has gone through successfully - assertEquals(pruningMsg.getCause(), FACT_NOT_AVAILABLE_IN_RANGE); - assertTrue(pruningMsg.getInvalidRanges().containsAll(ctx.getTimeRanges())); + + causes = findPruningMessagesForStorage("c1_testfact_deprecated", + ctx.getStoragePruningMsgs()); + assertEquals(causes.size(), 1); + assertEquals(causes.get(0).getCause(), TIME_RANGE_NOT_ANSWERABLE); + + causes = findPruningMessagesForStorage("c2_testfact_deprecated", + ctx.getStoragePruningMsgs()); + assertEquals(causes.size(), 1); + assertEquals(causes.get(0).getCause(), TIME_RANGE_NOT_ANSWERABLE); } @Test @@ -107,12 +101,25 @@ public void testCustomNow() throws Exception { Configuration conf = getConf(); DateTime dt = new DateTime(1990, 3, 23, 12, 0, 0, 0); conf.setLong(LensConfConstants.QUERY_CURRENT_TIME_IN_MILLIS, dt.getMillis()); - CubeQueryContext ctx = rewriteCtx("select msr12 from basecube where time_range_in(d_time, 'now.day-275days','now')", - conf); - TimeRange timeRange = ctx.getTimeRanges().get(0); - // Month starts from zero. - Calendar from = new GregorianCalendar(1989, 5, 21, 0, 0, 0); - assertEquals(timeRange.getFromDate(), from.getTime()); - assertEquals(timeRange.getToDate(), dt.toDate()); + String query = "select msr12 from basecube where time_range_in(d_time, 'now.day-275days','now')"; + LensException e = getLensExceptionInRewrite(query, conf); + assertEquals(e.getMessage(), "NO_CANDIDATE_FACT_AVAILABLE[Range not answerable]"); } + + /** + * + * @param stoargeName storageName_factName + * @param allStoragePruningMsgs + * @return + */ + private static List findPruningMessagesForStorage(String stoargeName, + PruneCauses allStoragePruningMsgs) { + for (StorageCandidate sc : allStoragePruningMsgs.keySet()) { + if (sc.getStorageTable().equals(stoargeName)) { + return allStoragePruningMsgs.get(sc); + } + } + return new ArrayList(); + } + } diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java index 7010849e8..a8014bfe2 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java @@ -79,10 +79,11 @@ private Date getUptoHour(Date in) { return cal.getTime(); } - @Test + @Test(invocationCount = 1) public void testCubeQueryContinuousUpdatePeriod() throws Exception { LensException th = null; try { + conf.set("lens.cube.query.valid.testcube.facttables", "summary3"); rewrite("select" + " SUM(msr2) from testCube where " + TWO_DAYS_RANGE, conf); } catch (LensException e) { th = e; @@ -93,6 +94,7 @@ public void testCubeQueryContinuousUpdatePeriod() throws Exception { Assert .assertEquals(th.getErrorCode(), CANNOT_USE_TIMERANGE_WRITER.getLensErrorInfo().getErrorCode()); } + conf.unset("lens.cube.query.valid.testcube.facttables"); // hourly partitions for two days conf.setBoolean(FAIL_QUERY_ON_PARTIAL_DATA, true); DateFormat qFmt = new SimpleDateFormat("yyyy-MM-dd-HH:mm:ss"); @@ -101,10 +103,11 @@ public void testCubeQueryContinuousUpdatePeriod() throws Exception { String hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + twoDaysInRangeClause, conf); Map whereClauses = new HashMap(); whereClauses.put( - getDbName() + "c1_testfact", + getDbName() + "c2_testfact", TestBetweenTimeRangeWriter.getBetweenClause(cubeName, "dt", getDateWithOffset(DAILY, -2), getDateWithOffset(DAILY, 0), CONTINUOUS.format())); - String expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, whereClauses); + String expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) as `sum(msr2)` FROM ", + null, null, whereClauses); System.out.println("HQL:" + hqlQuery); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -118,13 +121,13 @@ public void testCubeQueryContinuousUpdatePeriod() throws Exception { whereClauses = new HashMap(); whereClauses.put( - getDbName() + "c1_testfact", + getDbName() + "c2_testfact", TestBetweenTimeRangeWriter.getBetweenClause(cubeName, "dt", getDateWithOffset(DAILY, -2), getDateWithOffset(DAILY, 0), CONTINUOUS.format()) + " OR" + TestBetweenTimeRangeWriter.getBetweenClause(cubeName, "dt", getDateWithOffset(DAILY, -6), getDateWithOffset(DAILY, 0), CONTINUOUS.format())); - expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, whereClauses); + expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, whereClauses); System.out.println("HQL:" + hqlQuery); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -132,10 +135,10 @@ public void testCubeQueryContinuousUpdatePeriod() throws Exception { conf.set(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT, "yyyy-MM-dd HH:mm:ss"); hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf); whereClauses = new HashMap(); - whereClauses.put(getDbName() + "c1_testfact", TestBetweenTimeRangeWriter.getBetweenClause(cubeName, + whereClauses.put(getDbName() + "c2_testfact", TestBetweenTimeRangeWriter.getBetweenClause(cubeName, "dt", getUptoHour(TWODAYS_BACK), getUptoHour(NOW), TestTimeRangeWriter.DB_FORMAT)); - expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, whereClauses); + expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, whereClauses); System.out.println("HQL:" + hqlQuery); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -159,7 +162,8 @@ public void testCubeQueryWithTimeDim() throws Exception { getUptoHour(getOneLess(NOW, UpdatePeriod.HOURLY.calendarField())), TestTimeRangeWriter.DB_FORMAT)); System.out.println("HQL:" + hqlQuery); String expected = - getExpectedQuery(cubeName, "select timehourchain1.full_hour, sum(testcube.msr2) FROM ", " join " + getDbName() + getExpectedQuery(cubeName, "select timehourchain1.full_hour as `test_time_dim`, sum(testcube.msr2) as `msr2`" + + " FROM ", " join " + getDbName() + "c4_hourDimTbl timehourchain1 on testcube.test_time_dim_hour_id = timehourchain1.id", null, " GROUP BY timehourchain1.full_hour", null, whereClauses); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -169,7 +173,7 @@ public void testCubeQueryWithTimeDim() throws Exception { hqlQuery = rewrite(query, tconf); System.out.println("HQL:" + hqlQuery); expected = - getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", " join " + getDbName() + getExpectedQuery(cubeName, "select sum(testcube.msr2) as `msr2` FROM ", " join " + getDbName() + "c4_hourDimTbl timehourchain1 on testcube.test_time_dim_hour_id = timehourchain1.id", null, null, null, whereClauses); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -179,7 +183,7 @@ public void testCubeQueryWithTimeDim() throws Exception { hqlQuery = rewrite(query, tconf); System.out.println("HQL:" + hqlQuery); expected = - getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", " join " + getDbName() + getExpectedQuery(cubeName, "select sum(testcube.msr2) as `msr2` FROM ", " join " + getDbName() + "c4_hourDimTbl timehourchain1 on testcube.test_time_dim_hour_id = timehourchain1.id", " testcube.cityid > 2 ", " and testcube.cityid != 5", null, whereClauses); @@ -202,7 +206,7 @@ public void testCubeQueryWithTimeDim() throws Exception { getUptoHour(getOneLess(BEFORE_4_DAYS, UpdatePeriod.HOURLY.calendarField())), TestTimeRangeWriter.DB_FORMAT)); expected = - getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", " join " + getDbName() + getExpectedQuery(cubeName, "select sum(testcube.msr2) as `sum(msr2)` FROM ", " join " + getDbName() + "c4_hourDimTbl timehourchain1 on testcube.test_time_dim_hour_id = timehourchain1.id", null, null, null, whereClauses); System.out.println("HQL:" + hqlQuery); @@ -214,7 +218,8 @@ public void testCubeQueryWithTimeDim() throws Exception { + " OR " + TWO_DAYS_RANGE_TTD_BEFORE_4_DAYS, tconf); expected = - getExpectedQuery(cubeName, "select to_date(timehourchain1.full_hour), sum(testcube.msr2) FROM ", " join " + getExpectedQuery(cubeName, "select to_date(timehourchain1.full_hour) as `to_date(test_time_dim)`, " + + "sum(testcube.msr2) as `sum(msr2)` FROM ", " join " + getDbName() + "c4_hourDimTbl timehourchain1 on testcube.test_time_dim_hour_id = timehourchain1.id", null, " group by to_date(timehourchain1.full_hour)", null, whereClauses); System.out.println("HQL:" + hqlQuery); @@ -240,7 +245,8 @@ public void testCubeQueryWithTimeDimThruChain() throws Exception { getUptoHour(getOneLess(NOW, UpdatePeriod.HOURLY.calendarField())), TestTimeRangeWriter.DB_FORMAT)); System.out.println("HQL:" + hqlQuery); String expected = - getExpectedQuery(cubeName, "select timehourchain2.full_hour, sum(testcube.msr2) FROM ", " join " + getDbName() + getExpectedQuery(cubeName, "select timehourchain2.full_hour as `test_time_dim2`, sum(testcube.msr2) as `msr2` " + + "FROM ", " join " + getDbName() + "c4_hourDimTbl timehourchain2 on testcube.test_time_dim_hour_id2 = timehourchain2.id", null, " GROUP BY timehourchain2.full_hour", null, whereClauses); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -250,7 +256,7 @@ public void testCubeQueryWithTimeDimThruChain() throws Exception { hqlQuery = rewrite(query, tconf); System.out.println("HQL:" + hqlQuery); expected = - getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", " join " + getDbName() + getExpectedQuery(cubeName, "select sum(testcube.msr2) as `msr2` FROM ", " join " + getDbName() + "c4_hourDimTbl timehourchain2 on testcube.test_time_dim_hour_id2 = timehourchain2.id", null, null, null, whereClauses); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -260,7 +266,7 @@ public void testCubeQueryWithTimeDimThruChain() throws Exception { hqlQuery = rewrite(query, tconf); System.out.println("HQL:" + hqlQuery); expected = - getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", " join " + getDbName() + getExpectedQuery(cubeName, "select sum(testcube.msr2) as `msr2` FROM ", " join " + getDbName() + "c4_hourDimTbl timehourchain2 on testcube.test_time_dim_hour_id2 = timehourchain2.id", " testcube.cityid > 2 ", " and testcube.cityid != 5", null, whereClauses); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -282,7 +288,7 @@ public void testCubeQueryWithTimeDimThruChain() throws Exception { getUptoHour(getOneLess(BEFORE_4_DAYS, UpdatePeriod.HOURLY.calendarField())), TestTimeRangeWriter.DB_FORMAT)); expected = - getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", " join " + getDbName() + getExpectedQuery(cubeName, "select sum(testcube.msr2) as `sum(msr2)`FROM ", " join " + getDbName() + "c4_hourDimTbl timehourchain2 on testcube.test_time_dim_hour_id2 = timehourchain2.id", null, null, null, whereClauses); System.out.println("HQL:" + hqlQuery); @@ -294,7 +300,8 @@ public void testCubeQueryWithTimeDimThruChain() throws Exception { + " OR " +TWO_DAYS_RANGE_TTD2_BEFORE_4_DAYS, tconf); expected = - getExpectedQuery(cubeName, "select to_date(timehourchain2.full_hour), sum(testcube.msr2) FROM ", " join " + getExpectedQuery(cubeName, "select to_date(timehourchain2.full_hour) as `to_date(test_time_dim2)`, " + + "sum(testcube.msr2) as `sum(msr2)` FROM ", " join " + getDbName() + "c4_hourDimTbl timehourchain2 on testcube.test_time_dim_hour_id2 = timehourchain2.id", null, " group by to_date(timehourchain2.full_hour)", null, whereClauses); System.out.println("HQL:" + hqlQuery); diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java new file mode 100644 index 000000000..0119f2e17 --- /dev/null +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java @@ -0,0 +1,169 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + +import static org.apache.lens.cube.metadata.DateFactory.TWO_MONTHS_RANGE_UPTO_DAYS; +import static org.apache.lens.cube.parse.CubeQueryConfUtil.*; +import static org.apache.lens.cube.parse.CubeTestSetup.*; +import static org.apache.lens.cube.parse.TestCubeRewriter.compareContains; + +import static org.testng.Assert.*; + +import org.apache.lens.server.api.LensServerAPITestUtil; +import org.apache.lens.server.api.error.LensException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.parse.ParseException; + +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import jodd.util.StringUtil; + +public class TestUnionAndJoinCandidates extends TestQueryRewrite { + + private Configuration conf; + + @BeforeTest + public void setupDriver() throws Exception { + conf = LensServerAPITestUtil.getConfigurationWithParams(getConf(), + //Supported storage + CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C1", + // Storage tables + getValidStorageTablesKey("union_join_ctx_fact1"), "C1_union_join_ctx_fact1", + getValidStorageTablesKey("union_join_ctx_fact2"), "C1_union_join_ctx_fact2", + getValidStorageTablesKey("union_join_ctx_fact3"), "C1_union_join_ctx_fact3", + // Update periods + getValidUpdatePeriodsKey("union_join_ctx_fact1", "C1"), "DAILY", + getValidUpdatePeriodsKey("union_join_ctx_fact2", "C1"), "DAILY", + getValidUpdatePeriodsKey("union_join_ctx_fact3", "C1"), "DAILY"); + conf.setBoolean(DISABLE_AUTO_JOINS, false); + conf.setBoolean(ENABLE_SELECT_TO_GROUPBY, true); + conf.setBoolean(ENABLE_GROUP_BY_TO_SELECT, true); + conf.setBoolean(DISABLE_AGGREGATE_RESOLVER, false); + conf.setBoolean(ENABLE_STORAGES_UNION, true); + } + + @Override + public Configuration getConf() { + return new Configuration(); + } + + @Test + public void testDuplicateProjectedFieldExclusion() throws ParseException, LensException { + String colsSelected = " union_join_ctx_cityid , union_join_ctx_msr1_greater_than_100, " + + " sum(union_join_ctx_msr1) "; + String whereCond = " union_join_ctx_zipcode = 'a' and union_join_ctx_cityid = 'b' and " + + "(" + TWO_MONTHS_RANGE_UPTO_DAYS + ")"; + String rewrittenQuery = rewrite("select " + colsSelected + " from basecube where " + whereCond, conf); + assertTrue(rewrittenQuery.contains("UNION ALL")); + assertEquals(StringUtil.count(rewrittenQuery, "sum((basecube.union_join_ctx_msr1))"), 2); + } + + @Test + public void testFinalCandidateRewrittenQuery() throws ParseException, LensException { + try { + // Query with non projected measure in having clause. + String colsSelected = "union_join_ctx_cityid, sum(union_join_ctx_msr2) "; + String having = " having sum(union_join_ctx_msr1) > 100"; + String whereCond = " union_join_ctx_zipcode = 'a' and union_join_ctx_cityid = 'b' and " + + "(" + TWO_MONTHS_RANGE_UPTO_DAYS + ")"; + String rewrittenQuery = rewrite("select " + colsSelected + " from basecube where " + whereCond + having, conf); + String expectedInnerSelect1 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, 0.0 as `alias1`, " + + "sum((basecube.union_join_ctx_msr1)) as `alias2` FROM TestQueryRewrite.c1_union_join_ctx_fact1 basecube "; + String expectedInnerSelect2 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, 0.0 as `alias1`, " + + "sum((basecube.union_join_ctx_msr1)) as `alias2` FROM TestQueryRewrite.c1_union_join_ctx_fact2 basecube "; + String expectedInnerSelect3 = " SELECT (basecube.union_join_ctx_cityid) as `alias0`, " + + "sum((basecube.union_join_ctx_msr2)) as `alias1`, 0.0 as `alias2` " + + "FROM TestQueryRewrite.c1_union_join_ctx_fact3 basecube "; + String outerHaving = "HAVING (sum((basecube.alias2)) > 100)"; + compareContains(expectedInnerSelect1, rewrittenQuery); + compareContains(expectedInnerSelect2, rewrittenQuery); + compareContains(expectedInnerSelect3, rewrittenQuery); + compareContains(outerHaving, rewrittenQuery); + + // Query with measure and dim only expression + colsSelected = " union_join_ctx_cityid , union_join_ctx_cityname , union_join_ctx_notnullcityid, " + + " sum(union_join_ctx_msr1), sum(union_join_ctx_msr2) "; + whereCond = " union_join_ctx_zipcode = 'a' and union_join_ctx_cityid = 'b' and " + + "(" + TWO_MONTHS_RANGE_UPTO_DAYS + ")"; + rewrittenQuery = rewrite("select " + colsSelected + " from basecube where " + whereCond, conf); + String outerSelect = "SELECT (basecube.alias0) as `union_join_ctx_cityid`, " + + "(basecube.alias1) as `union_join_ctx_cityname`, (basecube.alias2) as `union_join_ctx_notnullcityid`, " + + "sum((basecube.alias3)) as `sum(union_join_ctx_msr1)`, " + + "sum((basecube.alias4)) as `sum(union_join_ctx_msr2)` FROM "; + expectedInnerSelect1 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, (cubecityjoinunionctx.name) " + + "as `alias1`, case when (basecube.union_join_ctx_cityid) is null then 0 else " + + "(basecube.union_join_ctx_cityid) end as `alias2`, sum((basecube.union_join_ctx_msr1)) as `alias3`, " + + "0.0 as `alias4` FROM TestQueryRewrite.c1_union_join_ctx_fact1 basecube"; + expectedInnerSelect2 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, (cubecityjoinunionctx.name) " + + "as `alias1`, case when (basecube.union_join_ctx_cityid) is null then 0 else " + + "(basecube.union_join_ctx_cityid) end as `alias2`, sum((basecube.union_join_ctx_msr1)) as `alias3`, " + + "0.0 as `alias4` FROM TestQueryRewrite.c1_union_join_ctx_fact2"; + expectedInnerSelect3 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, (cubecityjoinunionctx.name) " + + "as `alias1`, case when (basecube.union_join_ctx_cityid) is null then 0 else " + + "(basecube.union_join_ctx_cityid) end as `alias2`, 0.0 as `alias3`, " + + "sum((basecube.union_join_ctx_msr2)) as `alias4` FROM TestQueryRewrite.c1_union_join_ctx_fact3"; + String outerGroupBy = "GROUP BY (basecube.alias0), (basecube.alias1), (basecube.alias2)"; + compareContains(outerSelect, rewrittenQuery); + compareContains(expectedInnerSelect1, rewrittenQuery); + compareContains(expectedInnerSelect2, rewrittenQuery); + compareContains(expectedInnerSelect3, rewrittenQuery); + compareContains(outerGroupBy, rewrittenQuery); + // Query with measure and measure expression eg. sum(case when....), case when sum(msr1)... + // and measure with constant sum(msr1) + 10 + colsSelected = " union_join_ctx_cityid as `city id`, union_join_ctx_cityname, sum(union_join_ctx_msr1), " + + "sum(union_join_ctx_msr2), union_join_ctx_non_zero_msr2_sum, union_join_ctx_msr1_greater_than_100, " + + "sum(union_join_ctx_msr1) + 10 "; + whereCond = " union_join_ctx_zipcode = 'a' and union_join_ctx_cityid = 'b' and " + + "(" + TWO_MONTHS_RANGE_UPTO_DAYS + ")"; + rewrittenQuery = rewrite("select " + colsSelected + " from basecube where " + whereCond, conf); + outerSelect = "SELECT (basecube.alias0) as `city id`, (basecube.alias1) as `union_join_ctx_cityname`, " + + "sum((basecube.alias2)) as `sum(union_join_ctx_msr1)`, sum((basecube.alias3)) " + + "as `sum(union_join_ctx_msr2)`, sum((basecube.alias4)) as `union_join_ctx_non_zero_msr2_sum`, " + + "case when (sum((basecube.alias2)) > 100) then \"high\" else \"low\" end as " + + "`union_join_ctx_msr1_greater_than_100`, (sum((basecube.alias2)) + 10) " + + "as `(sum(union_join_ctx_msr1) + 10)` FROM "; + expectedInnerSelect1 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, " + + "(cubecityjoinunionctx.name) as `alias1`, sum((basecube.union_join_ctx_msr1)) as `alias2`, " + + "0.0 as `alias3`, 0.0 as `alias4` FROM"; + expectedInnerSelect2 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, " + + "(cubecityjoinunionctx.name) as `alias1`, sum((basecube.union_join_ctx_msr1)) as `alias2`, " + + "0.0 as `alias3`, 0.0 as `alias4` FROM"; + expectedInnerSelect3 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, (cubecityjoinunionctx.name) " + + "as `alias1`, 0.0 as `alias2`, sum((basecube.union_join_ctx_msr2)) as `alias3`, " + + "sum(case when ((basecube.union_join_ctx_msr2) > 0) then (basecube.union_join_ctx_msr2) else 0 end) " + + "as `alias4` FROM"; + String innerGroupBy = "GROUP BY (basecube.union_join_ctx_cityid), (cubecityjoinunionctx.name)"; + outerGroupBy = "GROUP BY (basecube.alias0), (basecube.alias1)"; + + compareContains(outerSelect, rewrittenQuery); + compareContains(expectedInnerSelect1, rewrittenQuery); + compareContains(expectedInnerSelect2, rewrittenQuery); + compareContains(expectedInnerSelect3, rewrittenQuery); + compareContains(outerGroupBy, rewrittenQuery); + compareContains(innerGroupBy, rewrittenQuery); + + } finally { + getStorageToUpdatePeriodMap().clear(); + } + } + + +} diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java index d5bc81ccd..bed1e7aea 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java @@ -28,13 +28,13 @@ import static org.testng.Assert.*; import java.util.*; +import java.util.stream.Collectors; -import org.apache.lens.cube.error.LensCubeErrorCode; +import org.apache.lens.cube.error.NoCandidateFactAvailableException; import org.apache.lens.server.api.LensServerAPITestUtil; import org.apache.lens.server.api.error.LensException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.ql.parse.ParseException; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; @@ -50,12 +50,12 @@ public class TestUnionQueries extends TestQueryRewrite { @BeforeTest public void setupDriver() throws Exception { testConf = LensServerAPITestUtil.getConfiguration( - DRIVER_SUPPORTED_STORAGES, "C0,C1,C2", - DISABLE_AUTO_JOINS, false, - ENABLE_SELECT_TO_GROUPBY, true, - ENABLE_GROUP_BY_TO_SELECT, true, - DISABLE_AGGREGATE_RESOLVER, false, - ENABLE_STORAGES_UNION, true); + DRIVER_SUPPORTED_STORAGES, "C0,C1,C2", + DISABLE_AUTO_JOINS, false, + ENABLE_SELECT_TO_GROUPBY, true, + ENABLE_GROUP_BY_TO_SELECT, true, + DISABLE_AGGREGATE_RESOLVER, false, + ENABLE_STORAGES_UNION, true); } @Override @@ -65,202 +65,191 @@ public Configuration getConf() { @Test public void testUnionQueries() throws Exception { - Configuration conf = getConf(); - conf.set(getValidStorageTablesKey("testfact"), "C1_testFact,C2_testFact"); - conf.set(getValidUpdatePeriodsKey("testfact", "C1"), "DAILY,HOURLY"); - conf.set(getValidUpdatePeriodsKey("testfact2", "C1"), "YEARLY"); - conf.set(getValidUpdatePeriodsKey("testfact", "C2"), "MONTHLY,DAILY"); - conf.setBoolean(CubeQueryConfUtil.ENABLE_STORAGES_UNION, false); - ArrayList storages = Lists.newArrayList("c1_testfact", "c2_testfact"); + Configuration conf = LensServerAPITestUtil.getConfigurationWithParams(getConf(), + CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C6", + getValidFactTablesKey("basecube"), "testfact", + FAIL_QUERY_ON_PARTIAL_DATA, false); + ArrayList storages = Lists.newArrayList("daily_c6_testfact", "monthly_c6_testfact"); try { - getStorageToUpdatePeriodMap().put("c1_testfact", Lists.newArrayList(HOURLY, DAILY)); - getStorageToUpdatePeriodMap().put("c2_testfact", Lists.newArrayList(MONTHLY)); - // Union query String hqlQuery; String expected; StoragePartitionProvider provider = new StoragePartitionProvider() { @Override public Map providePartitionsForStorage(String storage) { - return getWhereForMonthlyDailyAndHourly2monthsUnionQuery(storage); + if (storage.contains("daily_c6_testfact")) { + return getWhereForDays(storage, ONE_MONTH_BACK_TRUNCATED, getTruncatedDateWithOffset(MONTHLY, 0)); + } else if (storage.contains("monthly_c6_testfact")) { + return getWhereForMonthly(storage, THREE_MONTHS_BACK_TRUNCATED, ONE_MONTH_BACK_TRUNCATED); + } + return null; } }; - try { - rewrite("select cityid as `City ID`, msr8, msr7 as `Third measure` " - + "from testCube where " + TWO_MONTHS_RANGE_UPTO_HOURS, conf); - fail("Union feature is disabled, should have failed"); - } catch (LensException e) { - assertEquals(e.getErrorCode(), LensCubeErrorCode.STORAGE_UNION_DISABLED.getLensErrorInfo().getErrorCode()); - } - conf.setBoolean(CubeQueryConfUtil.ENABLE_STORAGES_UNION, true); - - hqlQuery = rewrite("select ascii(cityname) as `City Name`, msr8, msr7 as `Third measure` " - + "from testCube where ascii(cityname) = 'c' and cityname = 'a' and zipcode = 'b' and " - + TWO_MONTHS_RANGE_UPTO_HOURS, conf); + hqlQuery = rewrite("select cityname1 as `City Name`, msr8, msr7 as `Third measure` " + + "from testCube where cityname1 = 'a' and zipcode = 'b' and " + + THREE_MONTHS_RANGE_UPTO_MONTH, conf); expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider, - "SELECT testcube.alias0 as `City Name`, sum(testcube.alias1) + max(testcube.alias2), " - + "case when sum(testcube.alias1) = 0 then 0 else sum(testcube.alias3)/sum(testcube.alias1) end " - + "as `Third Measure`", - null, "group by testcube.alias0", - "select ascii(cubecity.name) as `alias0`, sum(testcube.msr2) as `alias1`, " - + "max(testcube.msr3) as `alias2`, " - + "sum(case when testcube.cityid = 'x' then testcube.msr21 else testcube.msr22 end) as `alias3`", " join " - + getDbName() + "c1_citytable cubecity on testcube.cityid = cubecity.id and (cubecity.dt = 'latest')", - "ascii(cubecity.name) = 'c' and cubecity.name = 'a' and testcube.zipcode = 'b'", - "group by ascii(cubecity.name))"); - compareQueries(hqlQuery, expected); - hqlQuery = rewrite("select asciicity as `City Name`, msr8, msr7 as `Third measure` " - + "from testCube where asciicity = 'c' and cityname = 'a' and zipcode = 'b' and " - + TWO_MONTHS_RANGE_UPTO_HOURS, conf); + "SELECT (testcube.alias0) as `City Name`, (sum((testcube.alias1)) + max((testcube.alias2))) " + + "as `msr8`, case when (sum((testcube.alias1)) = 0) then 0 else (sum((testcube.alias4)) / " + + "sum((testcube.alias1))) end as `Third measure` ", + null, "group by testcube.alias0", + "SELECT (cubecity1.name) as `alias0`, sum((testcube.msr2)) as `alias1`, max((testcube.msr3)) " + + "as `alias2`, sum(case when ((testcube.cityid) = 'x') then (testcube.msr21) " + + "else (testcube.msr22) end) as `alias4` ", " join " + + getDbName() + "c6_citytable cubecity1 on testcube.cityid1 = cubecity1.id " + + "and (cubecity1.dt = 'latest') ", + "((cubecity1.name) = 'a') and ((testcube.zipcode) = 'b')", + "group by (cubecity1.name)"); compareQueries(hqlQuery, expected); hqlQuery = rewrite("select ascii(cityid) as `City ID`, msr8, msr7 as `Third measure` " - + "from testCube where ascii(cityid) = 'c' and cityid = 'a' and zipcode = 'b' and " - + TWO_MONTHS_RANGE_UPTO_HOURS, conf); + + "from testCube where ascii(cityid) = 'c' and cityid = 'a' and zipcode = 'b' and " + + THREE_MONTHS_RANGE_UPTO_MONTH, conf); expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider, - "SELECT testcube.alias0 as `City ID`, sum(testcube.alias1) + max(testcube.alias2), " - + "case when sum(testcube.alias1) = 0 then 0 else sum(testcube.alias3)/sum(testcube.alias1) end " - + "as `Third Measure`", - null, "group by testcube.alias0", - "select ascii(testcube.cityid) as `alias0`, sum(testcube.msr2) as `alias1`, " - + "max(testcube.msr3) as `alias2`, " - + "sum(case when testcube.cityid = 'x' then testcube.msr21 else testcube.msr22 end) as `alias3`", - "ascii(testcube.cityid) = 'c' and testcube.cityid = 'a' and testcube.zipcode = 'b'", - "group by ascii(testcube.cityid)"); + "SELECT (testcube.alias0) as `City ID`, (sum((testcube.alias1)) + max((testcube.alias2))) as `msr8`, " + + "case when (sum((testcube.alias1)) = 0) then 0 else (sum((testcube.alias4)) / sum((testcube.alias1))) " + + "end as `Third measure`", + null, "group by testcube.alias0", + "SELECT ascii((testcube.cityid)) as `alias0`, sum((testcube.msr2)) as `alias1`, max((testcube.msr3)) " + + "as `alias2`, sum(case when ((testcube.cityid) = 'x') then (testcube.msr21) " + + "else (testcube.msr22) end) as `alias4`", + "ascii(testcube.cityid) = 'c' and testcube.cityid = 'a' and testcube.zipcode = 'b'", + "group by ascii(testcube.cityid)"); compareQueries(hqlQuery, expected); hqlQuery = rewrite("select cityid as `City ID`, msr8, msr7 as `Third measure` " - + "from testCube where cityid = 'a' and zipcode = 'b' and " + TWO_MONTHS_RANGE_UPTO_HOURS, conf); + + "from testCube where cityid = 'a' and zipcode = 'b' and " + THREE_MONTHS_RANGE_UPTO_MONTH, conf); expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider, - "SELECT testcube.alias0 as `City ID`, sum(testcube.alias1) + max(testcube.alias2), " - + "case when sum(testcube.alias1) = 0 then 0 else sum(testcube.alias3)/sum(testcube.alias1) end " - + "as `Third Measure`", - null, "group by testcube.alias0", - "select testcube.cityid as `alias0`, sum(testcube.msr2) as `alias1`, " - + "max(testcube.msr3) as `alias2`, " - + "sum(case when testcube.cityid = 'x' then testcube.msr21 else testcube.msr22 end) as `alias3`", - "testcube.cityid = 'a' and testcube.zipcode = 'b'", "group by testcube.cityid"); + "SELECT (testcube.alias0) as `City ID`, (sum((testcube.alias1)) + max((testcube.alias2))) as `msr8`, " + + "case when (sum((testcube.alias1)) = 0) then 0 else (sum((testcube.alias4)) / sum((testcube.alias1)))" + + " end as `Third measure`", + null, "group by testcube.alias0", + "SELECT (testcube.cityid) as `alias0`, sum((testcube.msr2)) as `alias1`, max((testcube.msr3)) as `alias2`, " + + "sum(case when ((testcube.cityid) = 'x') then (testcube.msr21) else (testcube.msr22) end) as `alias4`", + "testcube.cityid = 'a' and testcube.zipcode = 'b'", "group by testcube.cityid"); compareQueries(hqlQuery, expected); hqlQuery = rewrite("select cityid as `City ID`, msr3 as `Third measure` from testCube where " - + TWO_MONTHS_RANGE_UPTO_HOURS + " having msr7 > 10", conf); + + THREE_MONTHS_RANGE_UPTO_MONTH + " having msr7 > 10", conf); expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider, - "SELECT testcube.alias0 as `City ID`, max(testcube.alias1) as `Third measure`", - null, "group by testcube.alias0 having " - + "(case when sum(testcube.alias2)=0 then 0 else sum(testcube.alias3)/sum(testcube.alias2) end > 10 )", - "SELECT testcube.cityid as `alias0`, max(testcube.msr3) as `alias1`, " - + "sum(testcube.msr2) as `alias2`, " - + "sum(case when testcube.cityid='x' then testcube.msr21 else testcube.msr22 end) as `alias3`", - null, "group by testcube.cityid"); + "SELECT testcube.alias0 as `City ID`, max(testcube.alias1) as `Third measure`", + null, "group by testcube.alias0 having " + + "(case when sum(testcube.alias2)=0 then 0 else sum(testcube.alias3)/sum(testcube.alias2) end > 10 )", + "SELECT testcube.cityid as `alias0`, max(testcube.msr3) as `alias1`, " + + "sum(testcube.msr2) as `alias2`, " + + "sum(case when testcube.cityid='x' then testcube.msr21 else testcube.msr22 end) as `alias3`", + null, "group by testcube.cityid"); compareQueries(hqlQuery, expected); hqlQuery = rewrite("select cityid as `City ID`, msr3 as `Third measure` from testCube where " - + TWO_MONTHS_RANGE_UPTO_HOURS + " having msr8 > 10", conf); + + THREE_MONTHS_RANGE_UPTO_MONTH + " having msr8 > 10", conf); expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider, - "SELECT testcube.alias0 as `City ID`, max(testcube.alias1) as `Third measure`", - null, "GROUP BY testcube.alias0 " - + "HAVING (sum(testcube.alias2) + max(testcube.alias1)) > 10 ", - "SELECT testcube.cityid as `alias0`, max(testcube.msr3) as `alias1`, " - + "sum(testcube.msr2)as `alias2`", null, "group by testcube.cityid"); + "SELECT testcube.alias0 as `City ID`, max(testcube.alias1) as `Third measure`", + null, "GROUP BY testcube.alias0 " + + "HAVING (sum(testcube.alias2) + max(testcube.alias1)) > 10 ", + "SELECT testcube.cityid as `alias0`, max(testcube.msr3) as `alias1`, " + + "sum(testcube.msr2)as `alias2`", null, "group by testcube.cityid"); compareQueries(hqlQuery, expected); hqlQuery = rewrite("select msr3 as `Measure 3` from testCube where " - + TWO_MONTHS_RANGE_UPTO_HOURS + " having msr2 > 10 and msr2 < 100", conf); + + THREE_MONTHS_RANGE_UPTO_MONTH + " having msr2 > 10 and msr2 < 100", conf); expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider, - "SELECT max(testcube.alias0) as `Measure 3` ", - null, " HAVING sum(testcube.alias1) > 10 and sum(testcube.alias1) < 100", - "SELECT max(testcube.msr3) as `alias0`, sum(testcube.msr2) as `alias1`", null, null); + "SELECT max(testcube.alias0) as `Measure 3` ", + null, " HAVING sum(testcube.alias1) > 10 and sum(testcube.alias1) < 100", + "SELECT max(testcube.msr3) as `alias0`, sum(testcube.msr2) as `alias1`", null, null); compareQueries(hqlQuery, expected); hqlQuery = rewrite("select zipcode, cityid as `City ID`, msr3 as `Measure 3`, msr4, " - + "SUM(msr2) as `Measure 2` from testCube where " - + TWO_MONTHS_RANGE_UPTO_HOURS + " having msr4 > 10 order by cityid desc limit 5", conf); + + "SUM(msr2) as `Measure 2` from testCube where " + + THREE_MONTHS_RANGE_UPTO_MONTH + " having msr4 > 10 order by cityid desc limit 5", conf); expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider, - "SELECT testcube.alias0, testcube.alias1 as `City ID`, max(testcube.alias2) as `Measure 3`, " - + "count(testcube.alias3), sum(testcube.alias4) as `Measure 2`", - null, "group by testcube.alias0, testcube.alias1 " - + " having count(testcube.alias3) > 10 order by testcube.alias1 desc limit 5", - "select testcube.zipcode as `alias0`, testcube.cityid as `alias1`, " - + "max(testcube.msr3) as `alias2`,count(testcube.msr4) as `alias3`, sum(testcube.msr2) as `alias4`", - null, "group by testcube.zipcode, testcube.cityid "); - compareQueries(hqlQuery, expected); - - conf.setBoolean(CubeQueryConfUtil.ENABLE_GROUP_BY_TO_SELECT, false); - conf.setBoolean(ENABLE_SELECT_TO_GROUPBY, false); - hqlQuery = rewrite("select cityid as `City ID`, msr3 as `Measure 3`, " - + "SUM(msr2) as `Measure 2` from testCube" + " where " - + TWO_MONTHS_RANGE_UPTO_HOURS + " group by zipcode having msr4 > 10 order by cityid desc limit 5", conf); - - expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider, - "SELECT testcube.alias0 as `City ID`,max(testcube.alias1) as `Measure 3`,sum(testcube.alias2) as `Measure 2` ", - null, "group by testcube.alias3 having count(testcube.alias4) > 10 order by testcube.alias0 desc limit 5", - "SELECT testcube.cityid as `alias0`, max(testcube.msr3) as `alias1`, " - + "sum(testcube.msr2) as `alias2`, testcube.zipcode as `alias3`, count(testcube .msr4) as `alias4` FROM ", - null, "GROUP BY testcube.zipcode"); + "SELECT (testcube.alias0) as `zipcode`, (testcube.alias1) as `City ID`, max((testcube.alias2)) " + + "as `Measure 3`, count((testcube.alias3)) as `msr4`, sum((testcube.alias4)) as `Measure 2`", + null, "group by testcube.alias0, testcube.alias1 " + + " having count(testcube.alias3) > 10 order by testcube.alias1 desc limit 5", + "SELECT (testcube.zipcode) as `alias0`, (testcube.cityid) as `alias1`, max((testcube.msr3)) as `alias2`, " + + "count((testcube.msr4)) as `alias3`, sum((testcube.msr2)) as `alias4`", + null, "group by testcube.zipcode, testcube.cityid "); compareQueries(hqlQuery, expected); } finally { getStorageToUpdatePeriodMap().clear(); } } - @Test public void testDimAttrExpressionQuery() throws Exception { - Configuration conf = getConf(); - conf.set(getValidStorageTablesKey("testfact"), "C1_testFact,C2_testFact"); - conf.set(getValidUpdatePeriodsKey("testfact", "C1"), "DAILY,HOURLY"); - conf.set(getValidUpdatePeriodsKey("testfact2", "C1"), "YEARLY"); - conf.set(getValidUpdatePeriodsKey("testfact", "C2"), "MONTHLY,DAILY"); + Configuration conf = LensServerAPITestUtil.getConfigurationWithParams(getConf(), + CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C6", + getValidFactTablesKey("testcube"), "testfact", + FAIL_QUERY_ON_PARTIAL_DATA, false); + ArrayList storages = Lists.newArrayList("daily_c6_testfact", "monthly_c6_testfact"); + StoragePartitionProvider provider = new StoragePartitionProvider() { + @Override + public Map providePartitionsForStorage(String storage) { + if (storage.contains("daily_c6_testfact")) { + return getWhereForDays(storage, ONE_MONTH_BACK_TRUNCATED, getTruncatedDateWithOffset(MONTHLY, 0)); + } else if (storage.contains("monthly_c6_testfact")) { + return getWhereForMonthly(storage, THREE_MONTHS_BACK_TRUNCATED, ONE_MONTH_BACK_TRUNCATED); + } + return null; + } + }; + // exception in following line String hqlQuery = rewrite("select asciicity as `City Name`, cityAndState as citystate, isIndia as isIndia," - + " msr8, msr7 as `Third measure` " - + "from testCube where asciicity = 'c' and cityname = 'a' and zipcode = 'b' and " - + TWO_MONTHS_RANGE_UPTO_HOURS, conf); + + " msr8, msr7 as `Third measure` " + + "from testCube where asciicity = 'c' and cityname = 'a' and zipcode = 'b' and " + + THREE_MONTHS_RANGE_UPTO_MONTH, conf); String joinExpr1 = " join " - + getDbName() + "c1_statetable cubestate on testcube.stateid = cubestate.id and (cubestate.dt = 'latest') join" - + getDbName() + "c1_citytable cubecity on testcube.cityid = cubecity.id and (cubecity.dt = 'latest')"; + + getDbName() + "c6_statetable cubestate on testcube.stateid = cubestate.id and (cubestate.dt = 'latest') join" + + getDbName() + "c6_citytable cubecity on testcube.cityid = cubecity.id and (cubecity.dt = 'latest')"; String joinExpr2 = " join " - + getDbName() + "c1_citytable cubecity on testcube.cityid = cubecity.id and (cubecity.dt = 'latest') join" - + getDbName() + "c1_statetable cubestate on testcube.stateid = cubestate.id and (cubestate.dt = 'latest')"; + + getDbName() + "c6_citytable cubecity on testcube.cityid = cubecity.id and (cubecity.dt = 'latest') join" + + getDbName() + "c6_statetable cubestate on testcube.stateid = cubestate.id and (cubestate.dt = 'latest')"; String expected1 = getExpectedQueryForDimAttrExpressionQuery(joinExpr1); String expected2 = getExpectedQueryForDimAttrExpressionQuery(joinExpr2); assertTrue(new TestQuery(hqlQuery).equals(new TestQuery(expected1)) - || new TestQuery(hqlQuery).equals(new TestQuery(expected2)), - "Actual :" + hqlQuery + " Expected1:" + expected1 + " Expected2 : "+ expected2); + || new TestQuery(hqlQuery).equals(new TestQuery(expected2)), + "Actual :" + hqlQuery + " Expected1:" + expected1 + " Expected2 : "+ expected2); } private String getExpectedQueryForDimAttrExpressionQuery(String joinExpr) { try { - ArrayList storages = Lists.newArrayList("c1_testfact", "c2_testfact"); - getStorageToUpdatePeriodMap().put("c1_testfact", Lists.newArrayList(HOURLY, DAILY)); - getStorageToUpdatePeriodMap().put("c2_testfact", Lists.newArrayList(MONTHLY)); + ArrayList storages = Lists.newArrayList("daily_c6_testfact", "monthly_c6_testfact"); + StoragePartitionProvider provider = new StoragePartitionProvider() { @Override public Map providePartitionsForStorage(String storage) { - return getWhereForMonthlyDailyAndHourly2monthsUnionQuery(storage); + if (storage.contains("daily_c6_testfact")) { + return getWhereForDays(storage, ONE_MONTH_BACK_TRUNCATED, getTruncatedDateWithOffset(MONTHLY, 0)); + } else if (storage.contains("monthly_c6_testfact")) { + return getWhereForMonthly(storage, THREE_MONTHS_BACK_TRUNCATED, ONE_MONTH_BACK_TRUNCATED); + } + return null; } }; return getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider, - "SELECT testcube.alias0 as `City Name`, testcube.alias1 as citystate, testcube.alias2 as isIndia, " - + "sum(testcube.alias3) + max(testcube.alias4), " - + "case when sum(testcube.alias3) = 0 then 0 else sum(testcube.alias5)/sum(testcube.alias3) end " - + "as `Third Measure`", - null, " group by testcube.alias0, testcube.alias1, testcube.alias2", - "select ascii(cubecity.name) as `alias0`, concat(cubecity.name, \":\", cubestate.name) as alias1," - + "cubecity.name == 'DELHI' OR cubestate.name == 'KARNATAKA' OR cubestate.name == 'MAHARASHTRA' as alias2," - + "sum(testcube.msr2) as `alias3`, max(testcube.msr3) as `alias4`, " - + "sum(case when testcube.cityid = 'x' then testcube.msr21 else testcube.msr22 end) as `alias5`", joinExpr, - "ascii(cubecity.name) = 'c' and cubecity.name = 'a' and testcube.zipcode = 'b'", - " group by ascii(cubecity.name)), concat(cubecity.name, \":\", cubestate.name)," - + "cubecity.name == 'DELHI' OR cubestate.name == 'KARNATAKA' OR cubestate.name == 'MAHARASHTRA'"); + "SELECT (testcube.alias0) as `City Name`, (testcube.alias1) as `citystate`, (testcube.alias2) as `isIndia`, " + + "(sum((testcube.alias3)) + max((testcube.alias4))) as `msr8`, case when (sum((testcube.alias3)) = 0) " + + "then 0 else (sum((testcube.alias6)) / sum((testcube.alias3))) end as `Third measure` ", + null, " group by testcube.alias0, testcube.alias1, testcube.alias2", + "SELECT ascii((cubecity.name)) as `alias0`, concat((cubecity.name), \":\", (cubestate.name)) as `alias1`, " + + "(((cubecity.name) == 'DELHI') or ((cubestate.name) == 'KARNATAKA') or ((cubestate.name) " + + "== 'MAHARASHTRA')) as `alias2`, sum((testcube.msr2)) as `alias3`, max((testcube.msr3)) as `alias4`, " + + "sum(case when ((testcube.cityid) = 'x') then (testcube.msr21) else (testcube.msr22) end) " + + "as `alias6` ", joinExpr, + "ascii(cubecity.name) = 'c' and cubecity.name = 'a' and testcube.zipcode = 'b'", + " GROUP BY ascii((cubecity.name)), concat((cubecity.name), \":\", (cubestate.name)), " + + "(((cubecity.name) == 'DELHI') or ((cubestate.name) == 'KARNATAKA') " + + "or ((cubestate.name) == 'MAHARASHTRA'))"); } finally { getStorageToUpdatePeriodMap().clear(); } @@ -268,134 +257,34 @@ public Map providePartitionsForStorage(String storage) { @Test public void testNonAggregateOverAggregateFunction() throws Exception { try { - Configuration conf = getConf(); - conf.set(getValidStorageTablesKey("testfact"), "C1_testFact,C2_testFact"); - conf.set(getValidUpdatePeriodsKey("testfact", "C1"), "DAILY,HOURLY"); - conf.set(getValidUpdatePeriodsKey("testfact2", "C1"), "YEARLY"); - conf.set(getValidUpdatePeriodsKey("testfact", "C2"), "MONTHLY,DAILY"); - ArrayList storages = Lists.newArrayList("c1_testfact", "c2_testfact"); - getStorageToUpdatePeriodMap().put("c1_testfact", Lists.newArrayList(HOURLY, DAILY)); - getStorageToUpdatePeriodMap().put("c2_testfact", Lists.newArrayList(MONTHLY)); + Configuration conf = LensServerAPITestUtil.getConfigurationWithParams(getConf(), + CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C6", + getValidFactTablesKey("testcube"), "testfact", + FAIL_QUERY_ON_PARTIAL_DATA, false); + ArrayList storages = Lists.newArrayList("daily_c6_testfact", "monthly_c6_testfact"); + StoragePartitionProvider provider = new StoragePartitionProvider() { @Override public Map providePartitionsForStorage(String storage) { - return getWhereForMonthlyDailyAndHourly2monthsUnionQuery(storage); + if (storage.contains("daily_c6_testfact")) { + return getWhereForDays(storage, ONE_MONTH_BACK_TRUNCATED, getTruncatedDateWithOffset(MONTHLY, 0)); + } else if (storage.contains("monthly_c6_testfact")) { + return getWhereForMonthly(storage, THREE_MONTHS_BACK_TRUNCATED, ONE_MONTH_BACK_TRUNCATED); + } + return null; } }; - String hqlQuery = rewrite("select cityid as `City ID`, msr3 as `Measure 3`, " - + "round(SUM(msr2)) as `Measure 2` from testCube" + " where " - + TWO_MONTHS_RANGE_UPTO_HOURS + " group by zipcode having msr4 > 10 order by cityid desc, stateid asc, zipcode " - + "asc limit 5", - conf); - String expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider, - "SELECT testcube.alias0 as `City ID`,max(testcube.alias1) as `Measure 3`,round(sum(testcube.alias2)) as " - + "`Measure 2` ", null, "group by testcube.alias3 having count(testcube.alias4) > 10 " - + "order by testcube.alias0 desc, testcube.alias5 asc, testcube.alias3 asc limit 5", - "SELECT testcube.cityid as `alias0`, max(testcube.msr3) as `alias1`, sum(testcube.msr2) as `alias2`, " - + "testcube.zipcode as `alias3`, count(testcube .msr4) as `alias4`, (testcube.stateid) as `alias5` FROM ", - null, "GROUP BY testcube.zipcode"); - compareQueries(hqlQuery, expected); - } finally { - getStorageToUpdatePeriodMap().clear(); - } - } - @Test - public void testMultiFactMultiStorage() throws ParseException, LensException { - try { - Configuration conf = LensServerAPITestUtil.getConfigurationWithParams(getConf(), - CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C1,C2", - getValidStorageTablesKey("testfact"), "C1_testFact,C2_testFact", - getValidUpdatePeriodsKey("testfact", "C1"), "HOURLY", - getValidUpdatePeriodsKey("testfact", "C2"), "DAILY", - getValidUpdatePeriodsKey("testfact2_raw", "C1"), "YEARLY", - getValidUpdatePeriodsKey("testfact2_raw", "C2"), "YEARLY"); - getStorageToUpdatePeriodMap().put("c1_testfact", Lists.newArrayList(HOURLY)); - getStorageToUpdatePeriodMap().put("c2_testfact", Lists.newArrayList(DAILY)); - String whereCond = "zipcode = 'a' and cityid = 'b' and (" + TWO_DAYS_RANGE_SPLIT_OVER_UPDATE_PERIODS + ")"; - String hqlQuery = rewrite("select zipcode, count(msr4), sum(msr15) from testCube where " + whereCond, conf); - System.out.println(hqlQuery); - String possibleStart1 = "SELECT COALESCE(mq1.zipcode, mq2.zipcode) zipcode, mq1.expr2 `count(msr4)`," - + " mq2.expr3 `sum(msr15)` FROM "; - String possibleStart2 = "SELECT COALESCE(mq1.zipcode, mq2.zipcode) zipcode, mq2.expr2 `count(msr4)`," - + " mq1.expr3 `sum(msr15)` FROM "; - - assertTrue(hqlQuery.startsWith(possibleStart1) || hqlQuery.startsWith(possibleStart2)); - compareContains(rewrite("select zipcode as `zipcode`, sum(msr15) as `expr3` from testcube where " + whereCond, - conf), hqlQuery); - compareContains(rewrite("select zipcode as `zipcode`, count(msr4) as `expr2` from testcube where " + whereCond, - conf), hqlQuery); - assertTrue(hqlQuery.endsWith("on mq1.zipcode <=> mq2.zipcode")); - // No time_range_in should be remaining - assertFalse(hqlQuery.contains("time_range_in")); - //TODO: handle having after LENS-813, also handle for order by and limit - } finally { - getStorageToUpdatePeriodMap().clear(); - } - } - - @Test - public void testCubeWhereQueryWithMultipleTables() throws Exception { - Configuration conf = getConf(); - conf.setBoolean(CubeQueryConfUtil.ENABLE_STORAGES_UNION, true); - conf.set(getValidStorageTablesKey("testfact"), "C1_testFact,C2_testFact"); - conf.set(getValidUpdatePeriodsKey("testfact", "C1"), "DAILY"); - conf.set(getValidUpdatePeriodsKey("testfact2", "C1"), "YEARLY"); - conf.set(getValidUpdatePeriodsKey("testfact", "C2"), "HOURLY"); - - getStorageToUpdatePeriodMap().put("c1_testfact", Lists.newArrayList(DAILY)); - getStorageToUpdatePeriodMap().put("c2_testfact", Lists.newArrayList(HOURLY)); - StoragePartitionProvider provider = new StoragePartitionProvider() { - @Override - public Map providePartitionsForStorage(String storage) { - return getWhereForDailyAndHourly2days(TEST_CUBE_NAME, storage); - } - }; - try { - // Union query - String hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf); - System.out.println("HQL:" + hqlQuery); - - String expected = getExpectedUnionQuery(TEST_CUBE_NAME, - Lists.newArrayList("c1_testfact", "c2_testfact"), provider, - "select sum(testcube.alias0) ", null, null, - "select sum(testcube.msr2) as `alias0` from ", null, null - ); - compareQueries(hqlQuery, expected); - } finally { - getStorageToUpdatePeriodMap().clear(); - } - } - - @Test - public void testCubeWhereQueryWithMultipleTablesForMonth() throws Exception { - Configuration conf = getConf(); - conf.set(DRIVER_SUPPORTED_STORAGES, "C1,C2,C3"); - conf.set(getValidStorageTablesKey("testfact"), ""); - conf.set(getValidUpdatePeriodsKey("testfact", "C1"), "HOURLY"); - conf.set(getValidUpdatePeriodsKey("testfact2", "C1"), "YEARLY"); - conf.set(getValidUpdatePeriodsKey("testfact2_raw", "C3"), "YEARLY"); - conf.set(getValidUpdatePeriodsKey("testfact", "C2"), "DAILY"); - conf.set(getValidUpdatePeriodsKey("testfact", "C3"), "MONTHLY"); - - getStorageToUpdatePeriodMap().put("c1_testfact", Lists.newArrayList(HOURLY)); - getStorageToUpdatePeriodMap().put("c2_testfact", Lists.newArrayList(DAILY)); - getStorageToUpdatePeriodMap().put("c3_testfact", Lists.newArrayList(MONTHLY)); - StoragePartitionProvider provider = new StoragePartitionProvider() { - @Override - public Map providePartitionsForStorage(String storage) { - return getWhereForMonthlyDailyAndHourly2monthsUnionQuery(storage); - } - }; - try { - // Union query - String hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + TWO_MONTHS_RANGE_UPTO_HOURS, conf); - System.out.println("HQL:" + hqlQuery); - ArrayList storages = Lists.newArrayList("c1_testfact", "c2_testfact", "c3_testfact"); + String hqlQuery = rewrite("select cityid as `City ID`, msr3 as `Measure 3`, " + + "round(SUM(msr2)) as `Measure 2` from testCube" + " where " + + THREE_MONTHS_RANGE_UPTO_MONTH + " group by cityid having msr3 > 10 order by cityid desc limit 5", conf); String expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider, - "select sum(testcube.alias0)", null, null, - "select sum(testcube.msr2) as `alias0` from ", null, null - ); + "SELECT (testcube.alias0) as `City ID`, max((testcube.alias1)) as `Measure 3`, round(sum((testcube.alias2))) " + + "as `Measure 2` ", null, "GROUP BY (testcube.alias0) HAVING (max((testcube.alias1)) > 10) " + + "ORDER BY testcube.alias0 desc LIMIT 5", + "SELECT (testcube.cityid) as `alias0`, max((testcube.msr3)) as `alias1`, " + + "sum((testcube.msr2)) as `alias2` FROM ", + null, "GROUP BY testcube.cityid"); compareQueries(hqlQuery, expected); } finally { getStorageToUpdatePeriodMap().clear(); @@ -434,33 +323,77 @@ public void testConvertDimFilterToFactFilterForMultiFact() throws Exception { @Test public void testSingleFactMultiStorage() throws Exception { Configuration conf = LensServerAPITestUtil.getConfigurationWithParams(getConf(), - CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C3,C5", - getValidFactTablesKey("testcube"), "testfact", - getValidUpdatePeriodsKey("testfact", "C3"), "DAILY", - getValidUpdatePeriodsKey("testfact", "C5"), "DAILY", - FAIL_QUERY_ON_PARTIAL_DATA, false); + CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C6", + getValidFactTablesKey("testcube"), "testfact", + FAIL_QUERY_ON_PARTIAL_DATA, false); - String hqlQuery = rewrite("select count(msr4) from testCube where " + TWO_MONTHS_RANGE_UPTO_DAYS, conf); + String hqlQuery = rewrite("select count(msr4) from testCube where " + THREE_MONTHS_RANGE_UPTO_MONTH, conf); System.out.println(hqlQuery); // No time_range_in should be remaining assertFalse(hqlQuery.contains("time_range_in")); - ArrayList storages = Lists.newArrayList("c3_testfact", "c5_testfact"); + ArrayList storages = Lists.newArrayList("daily_c6_testfact", "monthly_c6_testfact"); StoragePartitionProvider provider = new StoragePartitionProvider() { @Override public Map providePartitionsForStorage(String storage) { - if (storage.contains("c3")) { - return getWhereForDays(storage, TWO_MONTHS_BACK, getDateWithOffset(DAILY, -10)); - } else if (storage.contains("c5")) { - return getWhereForDays(storage, getDateWithOffset(DAILY, -10), NOW); + if (storage.contains("daily_c6_testfact")) { + return getWhereForDays(storage, ONE_MONTH_BACK_TRUNCATED, getTruncatedDateWithOffset(MONTHLY, 0)); + } else if (storage.contains("monthly_c6_testfact")) { + return getWhereForMonthly(storage, THREE_MONTHS_BACK_TRUNCATED, ONE_MONTH_BACK_TRUNCATED); } return null; } }; String expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider, - "select count(testcube.alias0)", null, null, - "select count(testcube.msr4) as `alias0` from ", null, null + "SELECT count((testcube.alias0)) as `count(msr4)`", null, null, + "select count(testcube.msr4) as `alias0` from ", null, null ); compareQueries(hqlQuery, expected); } + + + @Test + public void testSingleFactSingleStorageWithMultipleTableDescriptions() throws Exception { + Configuration conf = LensServerAPITestUtil.getConfigurationWithParams(getConf(), + CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C6", + getValidFactTablesKey("testcube"), "testfact", + FAIL_QUERY_ON_PARTIAL_DATA, false); + + //If not beginning of month. Expecting this to pass at beginning of every month (example April 01 00:00) + if (!THREE_MONTHS_RANGE_UPTO_DAYS.equals(THREE_MONTHS_RANGE_UPTO_MONTH)) { + LensException e = getLensExceptionInRewrite("select count(msr4) from testCube where " + + THREE_MONTHS_RANGE_UPTO_DAYS, conf); + assertTrue(e instanceof NoCandidateFactAvailableException); + Set>> causes = + ((NoCandidateFactAvailableException) e).getBriefAndDetailedError() + .entrySet().stream().filter(x -> x.getKey().getStorageTable() + .equalsIgnoreCase("c6_testfact")).collect(Collectors.toSet()); + assertEquals(causes.size(), 1); + List pruneCauses = causes.iterator().next().getValue(); + assertEquals(pruneCauses.size(), 1); + assertEquals(pruneCauses.get(0).getCause(), CandidateTablePruneCause. + CandidateTablePruneCode.STORAGE_NOT_AVAILABLE_IN_RANGE); + } + + String hqlQuery2 = rewrite("select count(msr4) from testCube where " + THREE_MONTHS_RANGE_UPTO_MONTH, conf); + System.out.println(hqlQuery2); + + ArrayList storages = Lists.newArrayList("daily_c6_testfact", "monthly_c6_testfact"); + StoragePartitionProvider provider = new StoragePartitionProvider() { + @Override + public Map providePartitionsForStorage(String storage) { + if (storage.contains("daily_c6_testfact")) { + return getWhereForDays(storage, ONE_MONTH_BACK_TRUNCATED, getTruncatedDateWithOffset(MONTHLY, 0)); + } else if (storage.contains("monthly_c6_testfact")) { + return getWhereForMonthly(storage, THREE_MONTHS_BACK_TRUNCATED, ONE_MONTH_BACK_TRUNCATED); + } + return null; + } + }; + String expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider, + "select count(testcube.alias0) AS `count(msr4)`", null, null, + "select count((testcube.msr4)) AS `alias0` from ", null, null + ); + compareQueries(hqlQuery2, expected); + } } diff --git a/lens-cube/src/test/resources/schema/cubes/base/basecube.xml b/lens-cube/src/test/resources/schema/cubes/base/basecube.xml new file mode 100644 index 000000000..22703ab59 --- /dev/null +++ b/lens-cube/src/test/resources/schema/cubes/base/basecube.xml @@ -0,0 +1,972 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + APAC + EMEA + USA + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml new file mode 100644 index 000000000..2edf419b9 --- /dev/null +++ b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml @@ -0,0 +1,663 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + APAC + EMEA + USA + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/cubes/derived/der1.xml b/lens-cube/src/test/resources/schema/cubes/derived/der1.xml new file mode 100644 index 000000000..cf36cc73d --- /dev/null +++ b/lens-cube/src/test/resources/schema/cubes/derived/der1.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + msr1 + msr11 + msr9 + + + dim1 + dim11 + d_time + + diff --git a/lens-cube/src/test/resources/schema/cubes/derived/der2.xml b/lens-cube/src/test/resources/schema/cubes/derived/der2.xml new file mode 100644 index 000000000..337e7f450 --- /dev/null +++ b/lens-cube/src/test/resources/schema/cubes/derived/der2.xml @@ -0,0 +1,63 @@ + + + + + + + + + + + + + + directmsr + msr2 + msr12 + msr14 + msr13 + + + test_time_dim_hour_id + test_time_dim_hour_id2 + yuserid + test_time_dim_day_id + dim2 + dim2bignew + dim1 + stateid + test_time_dim2 + xuserid + dim2big1 + cityid + userid + dim2big2 + test_time_dim + test_time_dim_day_id2 + dim13 + dim11 + dim22 + dim12 + d_time + + diff --git a/lens-cube/src/test/resources/schema/cubes/derived/der3.xml b/lens-cube/src/test/resources/schema/cubes/derived/der3.xml new file mode 100644 index 000000000..ce9a511b1 --- /dev/null +++ b/lens-cube/src/test/resources/schema/cubes/derived/der3.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + msr3 + msr13 + + + test_time_dim_hour_id + test_time_dim_hour_id2 + test_time_dim_day_id + dim1 + stateid + test_time_dim2 + cityid + countryid + zipcode + test_time_dim + test_time_dim_day_id2 + regionname + d_time + + diff --git a/lens-cube/src/test/resources/schema/cubes/derived/derivedcube.xml b/lens-cube/src/test/resources/schema/cubes/derived/derivedcube.xml new file mode 100644 index 000000000..4245498a2 --- /dev/null +++ b/lens-cube/src/test/resources/schema/cubes/derived/derivedcube.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + msr3 + msr2 + msr1 + msr9 + + + dim2 + dim2bignew + dim1 + dim2big1 + dim2big2 + + diff --git a/lens-cube/src/test/resources/schema/cubes/derived/union_join_ctx_der1.xml b/lens-cube/src/test/resources/schema/cubes/derived/union_join_ctx_der1.xml new file mode 100644 index 000000000..6237d8741 --- /dev/null +++ b/lens-cube/src/test/resources/schema/cubes/derived/union_join_ctx_der1.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + union_join_ctx_msr2 + union_join_ctx_msr1 + union_join_ctx_msr3 + + + union_join_ctx_zipcode + union_join_ctx_cityname + d_time + union_join_ctx_cityid + + diff --git a/lens-cube/src/test/resources/schema/dimensions/citydim.xml b/lens-cube/src/test/resources/schema/dimensions/citydim.xml new file mode 100644 index 000000000..fd57fcd8f --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimensions/citydim.xml @@ -0,0 +1,122 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/dimensions/countrydim.xml b/lens-cube/src/test/resources/schema/dimensions/countrydim.xml new file mode 100644 index 000000000..bd52a9af5 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimensions/countrydim.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/dimensions/cycledim1.xml b/lens-cube/src/test/resources/schema/dimensions/cycledim1.xml new file mode 100644 index 000000000..7ffb276a7 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimensions/cycledim1.xml @@ -0,0 +1,71 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/dimensions/cycledim2.xml b/lens-cube/src/test/resources/schema/dimensions/cycledim2.xml new file mode 100644 index 000000000..b721c4705 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimensions/cycledim2.xml @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/dimensions/daydim.xml b/lens-cube/src/test/resources/schema/dimensions/daydim.xml new file mode 100644 index 000000000..89f2a6b8f --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimensions/daydim.xml @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/dimensions/hourdim.xml b/lens-cube/src/test/resources/schema/dimensions/hourdim.xml new file mode 100644 index 000000000..0a2ea1cd3 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimensions/hourdim.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/dimensions/sports.xml b/lens-cube/src/test/resources/schema/dimensions/sports.xml new file mode 100644 index 000000000..3a0623f4a --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimensions/sports.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/dimensions/statedim.xml b/lens-cube/src/test/resources/schema/dimensions/statedim.xml new file mode 100644 index 000000000..5676348dc --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimensions/statedim.xml @@ -0,0 +1,73 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/dimensions/testdim2.xml b/lens-cube/src/test/resources/schema/dimensions/testdim2.xml new file mode 100644 index 000000000..f40e8e2bf --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimensions/testdim2.xml @@ -0,0 +1,223 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/dimensions/testdim3.xml b/lens-cube/src/test/resources/schema/dimensions/testdim3.xml new file mode 100644 index 000000000..6f7d29582 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimensions/testdim3.xml @@ -0,0 +1,68 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/dimensions/testdim4.xml b/lens-cube/src/test/resources/schema/dimensions/testdim4.xml new file mode 100644 index 000000000..ca615bb9c --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimensions/testdim4.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/dimensions/unreachabledim.xml b/lens-cube/src/test/resources/schema/dimensions/unreachabledim.xml new file mode 100644 index 000000000..290448d57 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimensions/unreachabledim.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/dimensions/user_interests.xml b/lens-cube/src/test/resources/schema/dimensions/user_interests.xml new file mode 100644 index 000000000..4a677a31c --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimensions/user_interests.xml @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/dimensions/userdim.xml b/lens-cube/src/test/resources/schema/dimensions/userdim.xml new file mode 100644 index 000000000..af5c7202c --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimensions/userdim.xml @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/dimensions/zipdim.xml b/lens-cube/src/test/resources/schema/dimensions/zipdim.xml new file mode 100644 index 000000000..29f3af6f8 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimensions/zipdim.xml @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/dimtables/citytable.xml b/lens-cube/src/test/resources/schema/dimtables/citytable.xml new file mode 100644 index 000000000..41ec206b7 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/citytable.xml @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + + + + + + + HOURLY + + C1 + + + + + + + + + + + + + + + C2 + + + + + + + + + + + + + + HOURLY + + C6 + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/citytable2.xml b/lens-cube/src/test/resources/schema/dimtables/citytable2.xml new file mode 100644 index 000000000..0ea091177 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/citytable2.xml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + C4 + + + + + + + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/citytable3.xml b/lens-cube/src/test/resources/schema/dimtables/citytable3.xml new file mode 100644 index 000000000..2b895aadc --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/citytable3.xml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + C4 + + + + + + + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/citytable4.xml b/lens-cube/src/test/resources/schema/dimtables/citytable4.xml new file mode 100644 index 000000000..43881bf30 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/citytable4.xml @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + C4 + + + + + + + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/countrytable.xml b/lens-cube/src/test/resources/schema/dimtables/countrytable.xml new file mode 100644 index 000000000..4a7b3f51d --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/countrytable.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + C1 + + + + + + + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/countrytable_partitioned.xml b/lens-cube/src/test/resources/schema/dimtables/countrytable_partitioned.xml new file mode 100644 index 000000000..2c3de87db --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/countrytable_partitioned.xml @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + HOURLY + + C3 + + + + + + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/cycledim1tbl.xml b/lens-cube/src/test/resources/schema/dimtables/cycledim1tbl.xml new file mode 100644 index 000000000..5c1467ce0 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/cycledim1tbl.xml @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + HOURLY + + C1 + + + + + + + + + + + dt + + + + + C2 + + + + + + + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/cycledim2tbl.xml b/lens-cube/src/test/resources/schema/dimtables/cycledim2tbl.xml new file mode 100644 index 000000000..aaac96ec5 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/cycledim2tbl.xml @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + HOURLY + + C1 + + + + + + + + + + + dt + + + + + C2 + + + + + + + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/daydimtbl.xml b/lens-cube/src/test/resources/schema/dimtables/daydimtbl.xml new file mode 100644 index 000000000..39ef456b3 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/daydimtbl.xml @@ -0,0 +1,68 @@ + + + + + + + + + + + + + + + + + + HOURLY + + C3 + + + + + + + + + + + dt + + + + + C4 + + + + + + + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/hourdimtbl.xml b/lens-cube/src/test/resources/schema/dimtables/hourdimtbl.xml new file mode 100644 index 000000000..86298d1e7 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/hourdimtbl.xml @@ -0,0 +1,68 @@ + + + + + + + + + + + + + + + + + + HOURLY + + C3 + + + + + + + + + + + dt + + + + + C4 + + + + + + + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/sports_tbl.xml b/lens-cube/src/test/resources/schema/dimtables/sports_tbl.xml new file mode 100644 index 000000000..9dca80afe --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/sports_tbl.xml @@ -0,0 +1,68 @@ + + + + + + + + + + + + + + + + + + C1 + + + + + + + + + + + + + + HOURLY + + C2 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/statetable.xml b/lens-cube/src/test/resources/schema/dimtables/statetable.xml new file mode 100644 index 000000000..58eb94825 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/statetable.xml @@ -0,0 +1,74 @@ + + + + + + + + + + + + + + + + + + + + HOURLY + + C1 + + + + + + + + + + + dt + + + + + HOURLY + + C6 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/statetable_partitioned.xml b/lens-cube/src/test/resources/schema/dimtables/statetable_partitioned.xml new file mode 100644 index 000000000..826d8dcd0 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/statetable_partitioned.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + HOURLY + + C3 + + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/testdim2tbl.xml b/lens-cube/src/test/resources/schema/dimtables/testdim2tbl.xml new file mode 100644 index 000000000..d04cb5c99 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/testdim2tbl.xml @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + + + HOURLY + + C1 + + + + + + + + + + + dt + + + + + C2 + + + + + + + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/testdim2tbl2.xml b/lens-cube/src/test/resources/schema/dimtables/testdim2tbl2.xml new file mode 100644 index 000000000..efc0969f2 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/testdim2tbl2.xml @@ -0,0 +1,89 @@ + + + + + + + + + + + + + + + + + + + + + HOURLY + + C3 + + + + + + + + + + + dt + + + + + HOURLY + + C1 + + + + + + + + + + + dt + + + + + C2 + + + + + + + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/testdim2tbl3.xml b/lens-cube/src/test/resources/schema/dimtables/testdim2tbl3.xml new file mode 100644 index 000000000..d9b39c79d --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/testdim2tbl3.xml @@ -0,0 +1,89 @@ + + + + + + + + + + + + + + + + + + + + + HOURLY + + C3 + + + + + + + + + + + dt + + + + + HOURLY + + C1 + + + + + + + + + + + dt + + + + + C2 + + + + + + + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/testdim3tbl.xml b/lens-cube/src/test/resources/schema/dimtables/testdim3tbl.xml new file mode 100644 index 000000000..ffd09c034 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/testdim3tbl.xml @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + HOURLY + + C1 + + + + + + + + + + + dt + + + + + C2 + + + + + + + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/testdim4tbl.xml b/lens-cube/src/test/resources/schema/dimtables/testdim4tbl.xml new file mode 100644 index 000000000..d81daa733 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/testdim4tbl.xml @@ -0,0 +1,68 @@ + + + + + + + + + + + + + + + + + + HOURLY + + C1 + + + + + + + + + + + dt + + + + + C2 + + + + + + + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/unreachabledimtable.xml b/lens-cube/src/test/resources/schema/dimtables/unreachabledimtable.xml new file mode 100644 index 000000000..e940cf712 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/unreachabledimtable.xml @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + HOURLY + + C1 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/user_interests_tbl.xml b/lens-cube/src/test/resources/schema/dimtables/user_interests_tbl.xml new file mode 100644 index 000000000..cec16a50d --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/user_interests_tbl.xml @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + + C1 + + + + + + + + + + + + + + HOURLY + + C2 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/usertable.xml b/lens-cube/src/test/resources/schema/dimtables/usertable.xml new file mode 100644 index 000000000..de2083d88 --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/usertable.xml @@ -0,0 +1,72 @@ + + + + + + + + + + + + + + + + + + + + + + C1 + + + + + + + + + + + + + + HOURLY + + C2 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/dimtables/ziptable.xml b/lens-cube/src/test/resources/schema/dimtables/ziptable.xml new file mode 100644 index 000000000..7287886fe --- /dev/null +++ b/lens-cube/src/test/resources/schema/dimtables/ziptable.xml @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + HOURLY + + C1 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/cheapfact.xml b/lens-cube/src/test/resources/schema/facts/cheapfact.xml new file mode 100644 index 000000000..2b11c5b1c --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/cheapfact.xml @@ -0,0 +1,101 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C99 + + + + + + + + + + + + ttd + ttd2 + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C0 + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/summary1.xml b/lens-cube/src/test/resources/schema/facts/summary1.xml new file mode 100644 index 000000000..44756c99c --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/summary1.xml @@ -0,0 +1,101 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MINUTELY + HOURLY + DAILY + + C1 + + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + + C2 + + + + + + + + + + + + + pt + it + et + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/summary2.xml b/lens-cube/src/test/resources/schema/facts/summary2.xml new file mode 100644 index 000000000..380801fe4 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/summary2.xml @@ -0,0 +1,103 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MINUTELY + HOURLY + DAILY + + C1 + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + + C2 + + + + + + + + + + + + + pt + it + et + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/summary3.xml b/lens-cube/src/test/resources/schema/facts/summary3.xml new file mode 100644 index 000000000..77e296c32 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/summary3.xml @@ -0,0 +1,104 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MINUTELY + HOURLY + DAILY + + C1 + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + + C2 + + + + + + + + + + + + + pt + it + et + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/summary4.xml b/lens-cube/src/test/resources/schema/facts/summary4.xml new file mode 100644 index 000000000..e26ef6839 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/summary4.xml @@ -0,0 +1,85 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MINUTELY + HOURLY + DAILY + + C2 + + + + + + + + + + + + + pt + it + et + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/testfact.xml b/lens-cube/src/test/resources/schema/facts/testfact.xml new file mode 100644 index 000000000..aa9322aba --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact.xml @@ -0,0 +1,251 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + CONTINUOUS + + C3 + + + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + CONTINUOUS + + C4 + + + + + + + + + + + + ttd + ttd2 + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + CONTINUOUS + + C5 + + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + CONTINUOUS + + C0 + + + + + + + + + + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + CONTINUOUS + + C1 + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + CONTINUOUS + + C2 + + + + + + + + + + + dt + + + + C6 + + + DAILY + + + + + + + + + + + + + dt + + + + MONTHLY + + + + + + + + + + + + + dt + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/testfact1_base.xml b/lens-cube/src/test/resources/schema/facts/testfact1_base.xml new file mode 100644 index 000000000..81aaddb0f --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact1_base.xml @@ -0,0 +1,155 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C3 + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C4 + + + + + + + + + + + + ttd + ttd2 + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C1 + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C2 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/testfact1_raw_base.xml b/lens-cube/src/test/resources/schema/facts/testfact1_raw_base.xml new file mode 100644 index 000000000..65f1dc3bf --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact1_raw_base.xml @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + HOURLY + + C3 + + + + + + + + + + + dt + + + + + HOURLY + + C1 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/testfact2.xml b/lens-cube/src/test/resources/schema/facts/testfact2.xml new file mode 100644 index 000000000..41d204500 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact2.xml @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + HOURLY + + C4 + + + + + + + + + + + + + + ttd + ttd2 + + + + + HOURLY + + C1 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/testfact2_base.xml b/lens-cube/src/test/resources/schema/facts/testfact2_base.xml new file mode 100644 index 000000000..f113ff5e2 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact2_base.xml @@ -0,0 +1,140 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C3 + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C4 + + + + + + + + + + + + ttd + ttd2 + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C1 + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C2 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/testfact2_raw.xml b/lens-cube/src/test/resources/schema/facts/testfact2_raw.xml new file mode 100644 index 000000000..9c3675338 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact2_raw.xml @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + HOURLY + + C3 + + + + + + + + + + + dt + + + + + HOURLY + + C1 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/testfact2_raw_base.xml b/lens-cube/src/test/resources/schema/facts/testfact2_raw_base.xml new file mode 100644 index 000000000..0bab461a3 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact2_raw_base.xml @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + HOURLY + + C1 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/testfact3_base.xml b/lens-cube/src/test/resources/schema/facts/testfact3_base.xml new file mode 100644 index 000000000..e27664fad --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact3_base.xml @@ -0,0 +1,137 @@ + + + + + + + + + + + + + + + + + + + + + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C3 + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C4 + + + + + + + + + + + + ttd + ttd2 + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C1 + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C2 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/testfact3_raw_base.xml b/lens-cube/src/test/resources/schema/facts/testfact3_raw_base.xml new file mode 100644 index 000000000..6becd8eb3 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact3_raw_base.xml @@ -0,0 +1,64 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + HOURLY + + C1 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/testfact4_raw_base.xml b/lens-cube/src/test/resources/schema/facts/testfact4_raw_base.xml new file mode 100644 index 000000000..5d1136bee --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact4_raw_base.xml @@ -0,0 +1,65 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + HOURLY + + C1 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/testfact5_base.xml b/lens-cube/src/test/resources/schema/facts/testfact5_base.xml new file mode 100644 index 000000000..d302c487b --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact5_base.xml @@ -0,0 +1,148 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C3 + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C4 + + + + + + + + + + + + ttd + ttd2 + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C1 + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C2 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/testfact5_raw_base.xml b/lens-cube/src/test/resources/schema/facts/testfact5_raw_base.xml new file mode 100644 index 000000000..dbfa9068c --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact5_raw_base.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + HOURLY + + C1 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/testfact6_base.xml b/lens-cube/src/test/resources/schema/facts/testfact6_base.xml new file mode 100644 index 000000000..b62a1a140 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact6_base.xml @@ -0,0 +1,137 @@ + + + + + + + + + + + + + + + + + + + + + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C3 + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C4 + + + + + + + + + + + + ttd + ttd2 + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C1 + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C2 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/testfact_continuous.xml b/lens-cube/src/test/resources/schema/facts/testfact_continuous.xml new file mode 100644 index 000000000..6723c986a --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact_continuous.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + + + CONTINUOUS + + C0 + + + + + + + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/testfact_deprecated.xml b/lens-cube/src/test/resources/schema/facts/testfact_deprecated.xml new file mode 100644 index 000000000..ce3b54a91 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact_deprecated.xml @@ -0,0 +1,146 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C3 + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C4 + + + + + + + + + + + + ttd + ttd2 + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C1 + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + MONTHLY + QUARTERLY + YEARLY + + C2 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/testfactmonthly.xml b/lens-cube/src/test/resources/schema/facts/testfactmonthly.xml new file mode 100644 index 000000000..6f0f6877a --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfactmonthly.xml @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MONTHLY + + C2 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact1.xml b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact1.xml new file mode 100644 index 000000000..fcef02d62 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact1.xml @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + DAILY + + C1 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact2.xml b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact2.xml new file mode 100644 index 000000000..c883640d5 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact2.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + DAILY + + C1 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact3.xml b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact3.xml new file mode 100644 index 000000000..27e859e3d --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact3.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + DAILY + + C1 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact5.xml b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact5.xml new file mode 100644 index 000000000..28871944e --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact5.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + DAILY + + C1 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact6.xml b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact6.xml new file mode 100644 index 000000000..22e3ea83f --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/union_join_ctx_fact6.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + DAILY + + C1 + + + + + + + + + + + dt + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/segmentations/seg1.xml b/lens-cube/src/test/resources/schema/segmentations/seg1.xml new file mode 100644 index 000000000..7ed48a1ca --- /dev/null +++ b/lens-cube/src/test/resources/schema/segmentations/seg1.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/storages/c0.xml b/lens-cube/src/test/resources/schema/storages/c0.xml new file mode 100644 index 000000000..d27f7d39d --- /dev/null +++ b/lens-cube/src/test/resources/schema/storages/c0.xml @@ -0,0 +1,26 @@ + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/storages/c1.xml b/lens-cube/src/test/resources/schema/storages/c1.xml new file mode 100644 index 000000000..3233e4b81 --- /dev/null +++ b/lens-cube/src/test/resources/schema/storages/c1.xml @@ -0,0 +1,26 @@ + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/storages/c2.xml b/lens-cube/src/test/resources/schema/storages/c2.xml new file mode 100644 index 000000000..0078ed0fe --- /dev/null +++ b/lens-cube/src/test/resources/schema/storages/c2.xml @@ -0,0 +1,26 @@ + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/storages/c3.xml b/lens-cube/src/test/resources/schema/storages/c3.xml new file mode 100644 index 000000000..5af207473 --- /dev/null +++ b/lens-cube/src/test/resources/schema/storages/c3.xml @@ -0,0 +1,26 @@ + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/storages/c4.xml b/lens-cube/src/test/resources/schema/storages/c4.xml new file mode 100644 index 000000000..037ab8603 --- /dev/null +++ b/lens-cube/src/test/resources/schema/storages/c4.xml @@ -0,0 +1,26 @@ + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/storages/c5.xml b/lens-cube/src/test/resources/schema/storages/c5.xml new file mode 100644 index 000000000..65e7b4a82 --- /dev/null +++ b/lens-cube/src/test/resources/schema/storages/c5.xml @@ -0,0 +1,26 @@ + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/resources/schema/storages/c99.xml b/lens-cube/src/test/resources/schema/storages/c99.xml new file mode 100644 index 000000000..2e9355ff3 --- /dev/null +++ b/lens-cube/src/test/resources/schema/storages/c99.xml @@ -0,0 +1,26 @@ + + + + + + + \ No newline at end of file diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index 24660e193..194b380cc 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -18,7 +18,7 @@ */ package org.apache.lens.server.metastore; -import static org.apache.lens.server.metastore.JAXBUtils.*; +import static org.apache.lens.cube.metadata.JAXBUtils.*; import java.util.*; import java.util.Date; @@ -168,10 +168,7 @@ public List getAllCubeNames(LensSessionHandle sessionid) throws LensExce @Override public void createCube(LensSessionHandle sessionid, XCube cube) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - CubeMetastoreClient msClient = getClient(sessionid); - Cube parent = cube instanceof XDerivedCube ? (Cube) msClient.getCube( - ((XDerivedCube) cube).getParent()) : null; - msClient.createCube(JAXBUtils.hiveCubeFromXCube(cube, parent)); + getClient(sessionid).createCube(cube); log.info("Created cube " + cube.getName()); } } @@ -200,7 +197,7 @@ public XCube getCube(LensSessionHandle sessionid, String cubeName) throws LensEx * @param cubeName cube name */ public void dropCube(LensSessionHandle sessionid, String cubeName) throws LensException { - try(SessionContext ignored = new SessionContext(sessionid)) { + try (SessionContext ignored = new SessionContext(sessionid)) { getClient(sessionid).dropCube(cubeName); } } @@ -214,10 +211,7 @@ public void dropCube(LensSessionHandle sessionid, String cubeName) throws LensEx @Override public void updateCube(LensSessionHandle sessionid, XCube cube) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - CubeMetastoreClient msClient = getClient(sessionid); - Cube parent = cube instanceof XDerivedCube ? (Cube) msClient.getCube( - ((XDerivedCube) cube).getParent()) : null; - msClient.alterCube(cube.getName(), JAXBUtils.hiveCubeFromXCube(cube, parent)); + getClient(sessionid).alterCube(cube); log.info("Cube updated " + cube.getName()); } catch (HiveException e) { throw new LensException(e); @@ -232,24 +226,8 @@ public void updateCube(LensSessionHandle sessionid, XCube cube) throws LensExcep */ @Override public void createDimensionTable(LensSessionHandle sessionid, XDimensionTable xDimTable) throws LensException { - String dimTblName = xDimTable.getTableName(); - List columns = JAXBUtils.fieldSchemaListFromColumns(xDimTable.getColumns()); - Map updatePeriodMap = - JAXBUtils.dumpPeriodsFromStorageTables(xDimTable.getStorageTables()); - - Map properties = JAXBUtils.mapFromXProperties(xDimTable.getProperties()); - Map storageDesc = JAXBUtils.tableDescPrefixMapFromXStorageTables( - xDimTable.getStorageTables()); - try (SessionContext ignored = new SessionContext(sessionid)){ - log.info("# Columns: " + columns); - getClient(sessionid).createCubeDimensionTable(xDimTable.getDimensionName(), - dimTblName, - columns, - xDimTable.getWeight(), - updatePeriodMap, - properties, - storageDesc); + getClient(sessionid).createCubeDimensionTable(xDimTable); log.info("Dimension Table created " + xDimTable.getTableName()); } } @@ -265,31 +243,14 @@ public void dropDimensionTable(LensSessionHandle sessionid, String dimTblName, b @Override public XDimensionTable getDimensionTable(LensSessionHandle sessionid, String dimTblName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - CubeMetastoreClient msClient = getClient(sessionid); - CubeDimensionTable dimTable = msClient.getDimensionTable(dimTblName); - XDimensionTable dt = JAXBUtils.dimTableFromCubeDimTable(dimTable); - if (dimTable.getStorages() != null && !dimTable.getStorages().isEmpty()) { - for (String storageName : dimTable.getStorages()) { - XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( - msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, storageName))); - tblElement.setStorageName(storageName); - UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); - if (p != null) { - tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); - } - dt.getStorageTables().getStorageTable().add(tblElement); - } - } - return dt; + return getClient(sessionid).getXDimensionTable(dimTblName); } } @Override public void updateDimensionTable(LensSessionHandle sessionid, XDimensionTable dimensionTable) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).alterCubeDimensionTable(dimensionTable.getTableName(), - JAXBUtils.cubeDimTableFromDimTable(dimensionTable), - JAXBUtils.tableDescPrefixMapFromXStorageTables(dimensionTable.getStorageTables())); + getClient(sessionid).alterCubeDimensionTable(dimensionTable); log.info("Updated dimension table " + dimensionTable.getTableName()); } catch (HiveException exc) { throw new LensException(exc); @@ -395,43 +356,7 @@ public void dropStorageOfDimTable(LensSessionHandle sessionid, String dimTblName @Override public XFactTable getFactTable(LensSessionHandle sessionid, String fact) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - CubeMetastoreClient msClient = getClient(sessionid); - CubeFactTable cft = msClient.getFactTable(fact); - XFactTable factTable = JAXBUtils.factTableFromCubeFactTable(cft); - Map> storageMap = cft.getStoragePrefixUpdatePeriodMap(); - for (String storageName : cft.getStorages()) { - Set updatePeriods = cft.getUpdatePeriods().get(storageName); - // This map tells if there are different tables for different update period. - Map updatePeriodToTableMap = storageMap.get(storageName); - Set tableNames = new HashSet<>(); - for (UpdatePeriod updatePeriod : updatePeriods) { - tableNames.add(updatePeriodToTableMap.get(updatePeriod)); - } - if (tableNames.size() <= 1) { - XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( - msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(fact, storageName))); - tblElement.setStorageName(storageName); - for (UpdatePeriod p : updatePeriods) { - tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); - } - factTable.getStorageTables().getStorageTable().add(tblElement); - } else { - // Multiple storage tables. - XStorageTableElement tblElement = new XStorageTableElement(); - tblElement.setStorageName(storageName); - XUpdatePeriods xUpdatePeriods = new XUpdatePeriods(); - tblElement.setUpdatePeriods(xUpdatePeriods); - for (Map.Entry entry : updatePeriodToTableMap.entrySet()) { - XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor(); - updatePeriodTableDescriptor.setTableDesc(getStorageTableDescFromHiveTable( - msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(fact, (String) entry.getValue())))); - updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod)entry.getKey()).name())); - xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor); - } - factTable.getStorageTables().getStorageTable().add(tblElement); - } - } - return factTable; + return getClient(sessionid).getXFactTable(fact); } } @@ -448,54 +373,34 @@ public XSegmentation getSegmentation(LensSessionHandle sessionid, String cubeSeg @Override public void createFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).createCubeFactTable(fact.getCubeName(), - fact.getName(), - JAXBUtils.fieldSchemaListFromColumns(fact.getColumns()), - JAXBUtils.getFactUpdatePeriodsFromStorageTables(fact.getStorageTables()), - fact.getWeight(), - addFactColStartTimePropertyToFactProperties(fact), - JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()), - JAXBUtils.storageTablePrefixMapOfStorage(fact.getStorageTables())); + getClient(sessionid).createCubeFactTable(fact); log.info("Created fact table " + fact.getName()); } } - public Map addFactColStartTimePropertyToFactProperties(XFactTable fact) { - Map props = new HashMap(); - props.putAll(JAXBUtils.mapFromXProperties(fact.getProperties())); - props.putAll(JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns())); - return props; - } @Override - public void createSegmentation(LensSessionHandle sessionid, XSegmentation cubeSeg) throws LensException { + public void updateFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).createSegmentation( - cubeSeg.getCubeName(), - cubeSeg.getName(), - JAXBUtils.segmentsFromXSegments(cubeSeg.getSegements()), - cubeSeg.getWeight(), - JAXBUtils.mapFromXProperties(cubeSeg.getProperties())); - log.info("Created segmentation " + cubeSeg.getName()); + getClient(sessionid).alterCubeFactTable(fact); + log.info("Updated fact table " + fact.getName()); + } catch (HiveException e) { + throw new LensException(e); } } @Override - public void updateFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException { + public void createSegmentation(LensSessionHandle sessionid, XSegmentation cubeSeg) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).alterCubeFactTable(fact.getName(), JAXBUtils.cubeFactFromFactTable(fact), - JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()), - JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns())); - log.info("Updated fact table " + fact.getName()); - } catch (HiveException e) { - throw new LensException(e); + getClient(sessionid).createSegmentation(cubeSeg); + log.info("Created segmentation " + cubeSeg.getName()); } } @Override public void updateSegmentation(LensSessionHandle sessionid, XSegmentation cubeSeg) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).alterSegmentation(cubeSeg.getName(), segmentationFromXSegmentation(cubeSeg)); + getClient(sessionid).alterSegmentation(cubeSeg); log.info("Updated segmentation " + cubeSeg.getName()); } catch (HiveException e) { throw new LensException(e); @@ -906,7 +811,7 @@ public void dropPartitionFromStorageByFilter(LensSessionHandle sessionid, String public void createStorage(LensSessionHandle sessionid, XStorage storage) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).createStorage(JAXBUtils.storageFromXStorage(storage)); + getClient(sessionid).createStorage(storage); log.info("Created storage " + storage.getName()); } @@ -925,8 +830,7 @@ public void dropStorage(LensSessionHandle sessionid, String storageName) public void alterStorage(LensSessionHandle sessionid, String storageName, XStorage storage) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).alterStorage(storageName, - JAXBUtils.storageFromXStorage(storage)); + getClient(sessionid).alterStorage(storage); log.info("Altered storage " + storageName); } catch (HiveException e) { throw new LensException(e); @@ -1015,7 +919,7 @@ public List getAllQueryableCubeNames(LensSessionHandle sessionid) public void createDimension(LensSessionHandle sessionid, XDimension dimension) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).createDimension(JAXBUtils.dimensionFromXDimension(dimension)); + getClient(sessionid).createDimension(dimension); log.info("Created dimension " + dimension.getName()); } } @@ -1041,8 +945,7 @@ public void dropDimension(LensSessionHandle sessionid, String dimName) public void updateDimension(LensSessionHandle sessionid, String dimName, XDimension dimension) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).alterDimension(dimName, - JAXBUtils.dimensionFromXDimension(dimension)); + getClient(sessionid).alterDimension(dimension); log.info("Altered dimension " + dimName); } catch (HiveException e) { throw new LensException(e); diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java index 9d823da84..9ba1eb1cb 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java @@ -1,4 +1,4 @@ -/** + /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java b/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java index 02e2f8b17..df4e07a00 100644 --- a/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java +++ b/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java @@ -39,6 +39,7 @@ import org.apache.lens.api.metastore.ObjectFactory; import org.apache.lens.api.metastore.XCube; import org.apache.lens.api.metastore.XFactTable; +import org.apache.lens.api.metastore.XStorage; import org.apache.lens.api.query.*; import org.apache.lens.api.result.LensAPIResult; @@ -166,6 +167,16 @@ public static void createFactFailFast(final WebTarget target, final LensSessionH checkResponse(result); } + public static void createStorageFailFast(final WebTarget target, final LensSessionHandle sessionId, + final XStorage storage, MediaType mt) { + APIResult result = target.path("metastore").path("storages").queryParam("sessionid", sessionId) + .request(mt).post(Entity.entity( + new GenericEntity>(cubeObjectFactory.createXStorage(storage)) { + }, mt), + APIResult.class); + checkResponse(result); + } + public static APIResult setCurrentDatabase(final WebTarget target, final LensSessionHandle sessionId, final String dbName, MediaType mt) { diff --git a/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java b/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java index 040051955..fbfd898ae 100644 --- a/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java +++ b/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java @@ -50,6 +50,10 @@ public static String getRandomDbName() { return "db" + getRandomName(); } + public static String getRandomStorageName() { + return "storage" + getRandomName(); + } + public static String getRandomCubeName() { return "cube" + getRandomName(); } diff --git a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java index 567c92934..064da0129 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java @@ -47,9 +47,11 @@ import org.apache.lens.api.result.LensErrorTO; import org.apache.lens.api.util.MoxyJsonConfigurationContextResolver; import org.apache.lens.cube.error.ColUnAvailableInTimeRange; +import org.apache.lens.cube.metadata.HDFSStorage; import org.apache.lens.server.LensJerseyTest; import org.apache.lens.server.LensRequestLoggingFilter; import org.apache.lens.server.common.ErrorResponseExpectedData; +import org.apache.lens.server.common.RestAPITestUtil; import org.apache.lens.server.error.GenericExceptionMapper; import org.apache.lens.server.error.LensJAXBValidationExceptionMapper; import org.apache.lens.server.metastore.MetastoreResource; @@ -255,6 +257,7 @@ private void testColUnAvailableInTimeRange(@NonNull final Optional col final String testCube = getRandomCubeName(); final String testDimensionField = getRandomDimensionField(); final String testFact = getRandomFactName(); + final String testStorage = getRandomStorageName(); /* Setup: Begin */ LensSessionHandle sessionId = openSession(target, "foo", "bar", new LensConf(), mt); @@ -268,9 +271,21 @@ private void testColUnAvailableInTimeRange(@NonNull final Optional col XCube xcube = createXCubeWithDummyMeasure(testCube, Optional.of("dt"), testXDim); createCubeFailFast(target, sessionId, xcube, mt); + /* Create Storage */ + XStorage xs = new XStorage(); + xs.setClassname(HDFSStorage.class.getCanonicalName()); + xs.setName(testStorage); + RestAPITestUtil.createStorageFailFast(target, sessionId, xs, mt); + /* Create a fact with test dimension field */ XColumn xColumn = createXColumn(testDimensionField); XFactTable xFactTable = createXFactTableWithColumns(testFact, testCube, xColumn); + + //Create a StorageTable + XStorageTables tables = new XStorageTables(); + tables.getStorageTable().add(createStorageTblElement(testStorage,"DAILY")); + xFactTable.setStorageTables(tables); + createFactFailFast(target, sessionId, xFactTable, mt); /* Setup: End */ @@ -343,6 +358,18 @@ public void testExplainWebappException(MediaType mt) throws InterruptedException } finally { closeSessionFailFast(target(), sessionId, mt); } + } + private XStorageTableElement createStorageTblElement(String storageName, String... updatePeriod) { + XStorageTableElement tbl = new XStorageTableElement(); + tbl.setUpdatePeriods(new XUpdatePeriods()); + tbl.setStorageName(storageName); + if (updatePeriod != null) { + for (String p : updatePeriod) { + tbl.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p)); + } + } + tbl.setTableDesc(new XStorageTableDesc()); + return tbl; } }