From e2cf15bc9eefc169063d7fdf12b8e2d4a7df4b7b Mon Sep 17 00:00:00 2001 From: Rajashekhar Date: Fri, 2 Aug 2019 16:19:26 +0530 Subject: [PATCH 01/16] fixed the memory leak and bug in prepred query --- .../src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java | 2 ++ .../apache/lens/server/api/query/PreparedQueryContext.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java index 62f615b93..34e3ebf85 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java @@ -556,6 +556,8 @@ public String getPlan() { public QueryCost estimate(AbstractQueryContext qctx) throws LensException { MethodMetricsContext validateGauge = MethodMetricsFactory.createMethodGauge(qctx.getDriverConf(this), true, VALIDATE_GAUGE); + String rewrittenQuery = rewriteQuery(qctx); + qctx.setSelectedDriverQuery(rewrittenQuery); validate(qctx); validateGauge.markSuccess(); return calculateQueryCost(qctx); diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java index 69ee7c4fe..1834c8b0e 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java @@ -77,7 +77,7 @@ public class PreparedQueryContext extends AbstractQueryContext implements Delaye /** * The millis in week. */ - private static long millisInWeek = 7 * 24 * 60 * 60 * 1000; + private static long millisInTenMinutes = 10 * 60 * 1000; /** * Instantiates a new prepared query context. @@ -129,7 +129,7 @@ public long getDelay(TimeUnit units) { if (this.prepareStartTime != null) { Date now = new Date(); long elapsedMills = now.getTime() - this.prepareStartTime.getTime(); - delayMillis = millisInWeek - elapsedMills; + delayMillis = millisInTenMinutes - elapsedMills; return units.convert(delayMillis, TimeUnit.MILLISECONDS); } else { return Integer.MAX_VALUE; From bfac71316992cae07a1614efec178157d1659703 Mon Sep 17 00:00:00 2001 From: Rajashekhar Date: Tue, 6 Aug 2019 19:43:30 +0530 Subject: [PATCH 02/16] fix for the RNA-14041 --- .../apache/lens/server/query/LensServerDAO.java | 16 +++++++++++++++- .../server/query/QueryExecutionServiceImpl.java | 8 +++++++- .../apache/lens/server/query/TestLensDAO.java | 5 ++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java index 28b678b3b..bc0b2d7dc 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java @@ -35,11 +35,11 @@ import org.apache.lens.api.query.QueryStatus; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.FinishedLensQuery; +import org.apache.lens.server.api.query.PreparedLensQuery; import org.apache.lens.server.api.query.PreparedQueryContext; import org.apache.lens.server.api.query.QueryContext; import org.apache.lens.server.session.LensSessionImpl; import org.apache.lens.server.util.UtilityMethods; - import org.apache.commons.codec.binary.Base64; import org.apache.commons.dbutils.*; import org.apache.commons.dbutils.handlers.BeanHandler; @@ -868,4 +868,18 @@ public void insertPreparedQuery(PreparedQueryContext preparedQueryContext) throw DbUtils.closeQuietly(conn); } } + + public PreparedLensQuery getPreparedQuery(String handle) { + ResultSetHandler rsh = new BeanHandler<>(PreparedLensQuery.class, + new BasicRowProcessor(new FinishedLensQueryBeanProcessor())); + String sql = "select * from prepared_queries where handle=?"; + QueryRunner runner = new QueryRunner(ds); + try { + PreparedLensQuery preparedLensQuery = runner.query(sql, rsh, handle); + return preparedLensQuery; + } catch (SQLException e) { + log.error("SQL exception while executing query.", e); + } + return null; + } } diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index 18bd2d06f..7121c7a6a 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -2461,7 +2461,13 @@ private PreparedQueryContext getPreparedQueryContext(LensSessionHandle sessionHa acquire(sessionHandle); PreparedQueryContext ctx = preparedQueries.get(prepareHandle); if (ctx == null) { - throw new NotFoundException("Prepared query not found " + prepareHandle); + PreparedLensQuery preparedLensQuery = lensServerDao.getPreparedQuery(prepareHandle.getQueryHandleString()); + if (preparedLensQuery == null) { + throw new NotFoundException("Prepared query not found " + prepareHandle); + } + ctx = new PreparedQueryContext(preparedLensQuery.getUserquery(), preparedLensQuery.getSubmitter(), conf, + drivers.values()); + ctx.setSelectedDriverQuery(preparedLensQuery.getDriverquery()); } return ctx; } finally { diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java index 23b9813a5..3d820bb85 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java @@ -260,7 +260,10 @@ public void testPreparedQueryDAO() throws Exception { PreparedQueryContext preparedQueryContext = new PreparedQueryContext("query", "user1", driverConf, createDriver(driverConf)); preparedQueryContext.setPrepareEndTime(new Date()); - service.lensServerDao.insertPreparedQuery(preparedQueryContext); + service.lensServerDao.insertPreparedQuery(preparedQueryContext); + PreparedLensQuery preparedLensQuery = service.lensServerDao.getPreparedQuery(preparedQueryContext.getQueryHandleString()); + Assert.assertEquals(preparedLensQuery.getHandle(), preparedQueryContext.getQueryHandleString()); + Assert.assertEquals(preparedLensQuery.getSubmitter(), "user1"); } catch (Exception e) { Assert.fail("it shouldn't be coming in this catch block"); } From 410bfc033039812b42eba7d048012500c1746702 Mon Sep 17 00:00:00 2001 From: Rajashekhar Date: Thu, 8 Aug 2019 12:25:59 +0530 Subject: [PATCH 03/16] fixed the code review comments --- .../java/org/apache/lens/server/query/LensServerDAO.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java index bc0b2d7dc..69ed0109b 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java @@ -869,9 +869,9 @@ public void insertPreparedQuery(PreparedQueryContext preparedQueryContext) throw } } - public PreparedLensQuery getPreparedQuery(String handle) { + public PreparedLensQuery getPreparedQuery(String handle) throws LensException { ResultSetHandler rsh = new BeanHandler<>(PreparedLensQuery.class, - new BasicRowProcessor(new FinishedLensQueryBeanProcessor())); + new BasicRowProcessor(new BeanProcessor())); String sql = "select * from prepared_queries where handle=?"; QueryRunner runner = new QueryRunner(ds); try { @@ -879,7 +879,7 @@ public PreparedLensQuery getPreparedQuery(String handle) { return preparedLensQuery; } catch (SQLException e) { log.error("SQL exception while executing query.", e); + throw new LensException(e); } - return null; } } From a56794bc45bb015338efb3850a9369aea28d89ad Mon Sep 17 00:00:00 2001 From: Rajashekhar Date: Thu, 8 Aug 2019 12:42:23 +0530 Subject: [PATCH 04/16] added class for the Prepared Bean --- .../server/api/query/PreparedLensQuery.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedLensQuery.java diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedLensQuery.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedLensQuery.java new file mode 100644 index 000000000..8729a639a --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedLensQuery.java @@ -0,0 +1,42 @@ +package org.apache.lens.server.api.query; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@ToString +public class PreparedLensQuery { + + @Getter + @Setter + private String handle; + + @Getter + @Setter + private String userquery; + + @Getter + @Setter + private String submitter; + + @Getter + @Setter + private long timetaken; + + @Getter + @Setter + private String queryname; + + @Getter + @Setter + + private String drivername; + @Getter + @Setter + + private String driverquery; + + @Getter + @Setter + private long starttime; +} From b7d4f2a5aa162f166046c83d29cd1c9209880c61 Mon Sep 17 00:00:00 2001 From: Rajashekhar Date: Tue, 13 Aug 2019 11:51:04 +0530 Subject: [PATCH 05/16] Revert "fixed the memory leak and bug in prepred query" This reverts commit e2cf15bc9eefc169063d7fdf12b8e2d4a7df4b7b. --- .../src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java | 2 -- .../apache/lens/server/api/query/PreparedQueryContext.java | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java index 34e3ebf85..62f615b93 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java @@ -556,8 +556,6 @@ public String getPlan() { public QueryCost estimate(AbstractQueryContext qctx) throws LensException { MethodMetricsContext validateGauge = MethodMetricsFactory.createMethodGauge(qctx.getDriverConf(this), true, VALIDATE_GAUGE); - String rewrittenQuery = rewriteQuery(qctx); - qctx.setSelectedDriverQuery(rewrittenQuery); validate(qctx); validateGauge.markSuccess(); return calculateQueryCost(qctx); diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java index 1834c8b0e..69ee7c4fe 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java @@ -77,7 +77,7 @@ public class PreparedQueryContext extends AbstractQueryContext implements Delaye /** * The millis in week. */ - private static long millisInTenMinutes = 10 * 60 * 1000; + private static long millisInWeek = 7 * 24 * 60 * 60 * 1000; /** * Instantiates a new prepared query context. @@ -129,7 +129,7 @@ public long getDelay(TimeUnit units) { if (this.prepareStartTime != null) { Date now = new Date(); long elapsedMills = now.getTime() - this.prepareStartTime.getTime(); - delayMillis = millisInTenMinutes - elapsedMills; + delayMillis = millisInWeek - elapsedMills; return units.convert(delayMillis, TimeUnit.MILLISECONDS); } else { return Integer.MAX_VALUE; From 055576176cc08a246535cbfde66e4a44b7c0c9f6 Mon Sep 17 00:00:00 2001 From: Rajashekhar Date: Tue, 13 Aug 2019 11:53:16 +0530 Subject: [PATCH 06/16] Revert "Revert "fixed the memory leak and bug in prepred query"" This reverts commit b7d4f2a5aa162f166046c83d29cd1c9209880c61. --- .../src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java | 2 ++ .../apache/lens/server/api/query/PreparedQueryContext.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java index 62f615b93..34e3ebf85 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java @@ -556,6 +556,8 @@ public String getPlan() { public QueryCost estimate(AbstractQueryContext qctx) throws LensException { MethodMetricsContext validateGauge = MethodMetricsFactory.createMethodGauge(qctx.getDriverConf(this), true, VALIDATE_GAUGE); + String rewrittenQuery = rewriteQuery(qctx); + qctx.setSelectedDriverQuery(rewrittenQuery); validate(qctx); validateGauge.markSuccess(); return calculateQueryCost(qctx); diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java index 69ee7c4fe..1834c8b0e 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java @@ -77,7 +77,7 @@ public class PreparedQueryContext extends AbstractQueryContext implements Delaye /** * The millis in week. */ - private static long millisInWeek = 7 * 24 * 60 * 60 * 1000; + private static long millisInTenMinutes = 10 * 60 * 1000; /** * Instantiates a new prepared query context. @@ -129,7 +129,7 @@ public long getDelay(TimeUnit units) { if (this.prepareStartTime != null) { Date now = new Date(); long elapsedMills = now.getTime() - this.prepareStartTime.getTime(); - delayMillis = millisInWeek - elapsedMills; + delayMillis = millisInTenMinutes - elapsedMills; return units.convert(delayMillis, TimeUnit.MILLISECONDS); } else { return Integer.MAX_VALUE; From 49c9dcb08ef6c6483a1c07cf3a0aaff4206a993a Mon Sep 17 00:00:00 2001 From: Rajashekhar Date: Tue, 13 Aug 2019 11:53:26 +0530 Subject: [PATCH 07/16] Revert "added class for the Prepared Bean" This reverts commit a56794bc45bb015338efb3850a9369aea28d89ad. --- .../server/api/query/PreparedLensQuery.java | 42 ------------------- 1 file changed, 42 deletions(-) delete mode 100644 lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedLensQuery.java diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedLensQuery.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedLensQuery.java deleted file mode 100644 index 8729a639a..000000000 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedLensQuery.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.apache.lens.server.api.query; - -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; - -@ToString -public class PreparedLensQuery { - - @Getter - @Setter - private String handle; - - @Getter - @Setter - private String userquery; - - @Getter - @Setter - private String submitter; - - @Getter - @Setter - private long timetaken; - - @Getter - @Setter - private String queryname; - - @Getter - @Setter - - private String drivername; - @Getter - @Setter - - private String driverquery; - - @Getter - @Setter - private long starttime; -} From 23125c264c0bb3dac449b3de0f53f3a274a23311 Mon Sep 17 00:00:00 2001 From: Rajashekhar Date: Tue, 13 Aug 2019 11:53:34 +0530 Subject: [PATCH 08/16] Revert "fixed the code review comments" This reverts commit 410bfc033039812b42eba7d048012500c1746702. --- .../java/org/apache/lens/server/query/LensServerDAO.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java index 69ed0109b..bc0b2d7dc 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java @@ -869,9 +869,9 @@ public void insertPreparedQuery(PreparedQueryContext preparedQueryContext) throw } } - public PreparedLensQuery getPreparedQuery(String handle) throws LensException { + public PreparedLensQuery getPreparedQuery(String handle) { ResultSetHandler rsh = new BeanHandler<>(PreparedLensQuery.class, - new BasicRowProcessor(new BeanProcessor())); + new BasicRowProcessor(new FinishedLensQueryBeanProcessor())); String sql = "select * from prepared_queries where handle=?"; QueryRunner runner = new QueryRunner(ds); try { @@ -879,7 +879,7 @@ public PreparedLensQuery getPreparedQuery(String handle) throws LensException { return preparedLensQuery; } catch (SQLException e) { log.error("SQL exception while executing query.", e); - throw new LensException(e); } + return null; } } From 4aa21244593c836314d4a6437c749e6ee00c7892 Mon Sep 17 00:00:00 2001 From: Rajashekhar Date: Tue, 13 Aug 2019 11:53:48 +0530 Subject: [PATCH 09/16] Revert "fix for the RNA-14041" This reverts commit bfac71316992cae07a1614efec178157d1659703. --- .../apache/lens/server/query/LensServerDAO.java | 16 +--------------- .../server/query/QueryExecutionServiceImpl.java | 8 +------- .../apache/lens/server/query/TestLensDAO.java | 5 +---- 3 files changed, 3 insertions(+), 26 deletions(-) diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java index bc0b2d7dc..28b678b3b 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java @@ -35,11 +35,11 @@ import org.apache.lens.api.query.QueryStatus; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.FinishedLensQuery; -import org.apache.lens.server.api.query.PreparedLensQuery; import org.apache.lens.server.api.query.PreparedQueryContext; import org.apache.lens.server.api.query.QueryContext; import org.apache.lens.server.session.LensSessionImpl; import org.apache.lens.server.util.UtilityMethods; + import org.apache.commons.codec.binary.Base64; import org.apache.commons.dbutils.*; import org.apache.commons.dbutils.handlers.BeanHandler; @@ -868,18 +868,4 @@ public void insertPreparedQuery(PreparedQueryContext preparedQueryContext) throw DbUtils.closeQuietly(conn); } } - - public PreparedLensQuery getPreparedQuery(String handle) { - ResultSetHandler rsh = new BeanHandler<>(PreparedLensQuery.class, - new BasicRowProcessor(new FinishedLensQueryBeanProcessor())); - String sql = "select * from prepared_queries where handle=?"; - QueryRunner runner = new QueryRunner(ds); - try { - PreparedLensQuery preparedLensQuery = runner.query(sql, rsh, handle); - return preparedLensQuery; - } catch (SQLException e) { - log.error("SQL exception while executing query.", e); - } - return null; - } } diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index 7121c7a6a..18bd2d06f 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -2461,13 +2461,7 @@ private PreparedQueryContext getPreparedQueryContext(LensSessionHandle sessionHa acquire(sessionHandle); PreparedQueryContext ctx = preparedQueries.get(prepareHandle); if (ctx == null) { - PreparedLensQuery preparedLensQuery = lensServerDao.getPreparedQuery(prepareHandle.getQueryHandleString()); - if (preparedLensQuery == null) { - throw new NotFoundException("Prepared query not found " + prepareHandle); - } - ctx = new PreparedQueryContext(preparedLensQuery.getUserquery(), preparedLensQuery.getSubmitter(), conf, - drivers.values()); - ctx.setSelectedDriverQuery(preparedLensQuery.getDriverquery()); + throw new NotFoundException("Prepared query not found " + prepareHandle); } return ctx; } finally { diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java index 3d820bb85..23b9813a5 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java @@ -260,10 +260,7 @@ public void testPreparedQueryDAO() throws Exception { PreparedQueryContext preparedQueryContext = new PreparedQueryContext("query", "user1", driverConf, createDriver(driverConf)); preparedQueryContext.setPrepareEndTime(new Date()); - service.lensServerDao.insertPreparedQuery(preparedQueryContext); - PreparedLensQuery preparedLensQuery = service.lensServerDao.getPreparedQuery(preparedQueryContext.getQueryHandleString()); - Assert.assertEquals(preparedLensQuery.getHandle(), preparedQueryContext.getQueryHandleString()); - Assert.assertEquals(preparedLensQuery.getSubmitter(), "user1"); + service.lensServerDao.insertPreparedQuery(preparedQueryContext); } catch (Exception e) { Assert.fail("it shouldn't be coming in this catch block"); } From 570b07025ca3fcac28aff8858dcadbd52f90cb0e Mon Sep 17 00:00:00 2001 From: Rajashekhar Date: Tue, 13 Aug 2019 11:53:57 +0530 Subject: [PATCH 10/16] Revert "fixed the memory leak and bug in prepred query" This reverts commit e2cf15bc9eefc169063d7fdf12b8e2d4a7df4b7b. --- .../src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java | 2 -- .../apache/lens/server/api/query/PreparedQueryContext.java | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java index 34e3ebf85..62f615b93 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java @@ -556,8 +556,6 @@ public String getPlan() { public QueryCost estimate(AbstractQueryContext qctx) throws LensException { MethodMetricsContext validateGauge = MethodMetricsFactory.createMethodGauge(qctx.getDriverConf(this), true, VALIDATE_GAUGE); - String rewrittenQuery = rewriteQuery(qctx); - qctx.setSelectedDriverQuery(rewrittenQuery); validate(qctx); validateGauge.markSuccess(); return calculateQueryCost(qctx); diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java index 1834c8b0e..69ee7c4fe 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java @@ -77,7 +77,7 @@ public class PreparedQueryContext extends AbstractQueryContext implements Delaye /** * The millis in week. */ - private static long millisInTenMinutes = 10 * 60 * 1000; + private static long millisInWeek = 7 * 24 * 60 * 60 * 1000; /** * Instantiates a new prepared query context. @@ -129,7 +129,7 @@ public long getDelay(TimeUnit units) { if (this.prepareStartTime != null) { Date now = new Date(); long elapsedMills = now.getTime() - this.prepareStartTime.getTime(); - delayMillis = millisInTenMinutes - elapsedMills; + delayMillis = millisInWeek - elapsedMills; return units.convert(delayMillis, TimeUnit.MILLISECONDS); } else { return Integer.MAX_VALUE; From 37fdb922d239d6eba182653526f8bf72b8ab7e60 Mon Sep 17 00:00:00 2001 From: Rajashekhar Date: Tue, 13 Aug 2019 11:55:41 +0530 Subject: [PATCH 11/16] Changes in JDBCDriver.java was supposed to happen in #34, --- .../src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java index 62f615b93..34e3ebf85 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java @@ -556,6 +556,8 @@ public String getPlan() { public QueryCost estimate(AbstractQueryContext qctx) throws LensException { MethodMetricsContext validateGauge = MethodMetricsFactory.createMethodGauge(qctx.getDriverConf(this), true, VALIDATE_GAUGE); + String rewrittenQuery = rewriteQuery(qctx); + qctx.setSelectedDriverQuery(rewrittenQuery); validate(qctx); validateGauge.markSuccess(); return calculateQueryCost(qctx); From 5482db464f219991db4ef3a37f96a86629397723 Mon Sep 17 00:00:00 2001 From: Rajashekhar Date: Thu, 22 Aug 2019 17:12:25 +0530 Subject: [PATCH 12/16] added this line back, because if multiple jdbc drivers(more than one) are present, then selected driver's query should be finally set --- .../org/apache/lens/server/query/QueryExecutionServiceImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index 18bd2d06f..f240f8878 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -2057,6 +2057,7 @@ public QueryPrepareHandle prepare(LensSessionHandle sessionHandle, String query, acquire(sessionHandle); prepared = prepareQuery(sessionHandle, query, lensConf, SubmitOp.PREPARE); prepared.setQueryName(queryName); + prepared.getSelectedDriver().prepare(prepared); lensServerDao.insertPreparedQuery(prepared); return prepared.getPrepareHandle(); } catch (LensException e) { From f13dee25e7cd6b14e247294703bd2b92f1bdea87 Mon Sep 17 00:00:00 2001 From: Rajashekhar Date: Thu, 22 Aug 2019 17:14:13 +0530 Subject: [PATCH 13/16] added jdbc driver changes, removed the code which even makes calls to the driver --- .../apache/lens/driver/jdbc/JDBCDriver.java | 49 +++---------------- .../lens/driver/jdbc/TestJdbcDriver.java | 42 +++------------- 2 files changed, 12 insertions(+), 79 deletions(-) diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java index 34e3ebf85..3a439aeaa 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java @@ -556,8 +556,6 @@ public String getPlan() { public QueryCost estimate(AbstractQueryContext qctx) throws LensException { MethodMetricsContext validateGauge = MethodMetricsFactory.createMethodGauge(qctx.getDriverConf(this), true, VALIDATE_GAUGE); - String rewrittenQuery = rewriteQuery(qctx); - qctx.setSelectedDriverQuery(rewrittenQuery); validate(qctx); validateGauge.markSuccess(); return calculateQueryCost(qctx); @@ -635,14 +633,7 @@ public void validate(AbstractQueryContext pContext) throws LensException { PreparedStatement stmt; // Estimate queries need to get connection from estimate pool to make sure // we are not blocked by data queries. - stmt = prepareInternal(pContext, true, true, "validate-"); - if (stmt != null) { - try { - stmt.close(); - } catch (SQLException e) { - throw new LensException(); - } - } + prepareInternal(pContext, true, true, "validate-"); } } @@ -709,12 +700,12 @@ protected final ConnectionProvider getConnectionProvider() { * @return prepared statement of the query * @throws LensException */ - private PreparedStatement prepareInternal(AbstractQueryContext pContext) throws LensException { + private void prepareInternal(AbstractQueryContext pContext) throws LensException { if (pContext.getDriverQuery(this) == null) { throw new NullPointerException("Null driver query for " + pContext.getUserQuery()); } checkConfigured(); - return prepareInternal(pContext, false, false, "prepare-"); + prepareInternal(pContext, false, false, "prepare-"); } @@ -727,7 +718,7 @@ private PreparedStatement prepareInternal(AbstractQueryContext pContext) throws * @return prepared statement * @throws LensException */ - private PreparedStatement prepareInternal(AbstractQueryContext pContext, + private void prepareInternal(AbstractQueryContext pContext, boolean calledForEstimate, boolean checkConfigured, String metricCallStack) throws LensException { @@ -744,33 +735,8 @@ private PreparedStatement prepareInternal(AbstractQueryContext pContext, MethodMetricsContext sqlRewriteGauge = MethodMetricsFactory.createMethodGauge(pContext.getDriverConf(this), true, metricCallStack + COLUMNAR_SQL_REWRITE_GAUGE); String rewrittenQuery = rewriteQuery(pContext); + pContext.setSelectedDriverQuery(rewrittenQuery); sqlRewriteGauge.markSuccess(); - MethodMetricsContext jdbcPrepareGauge = MethodMetricsFactory.createMethodGauge(pContext.getDriverConf(this), true, - metricCallStack + JDBC_PREPARE_GAUGE); - - PreparedStatement stmt = null; - Connection conn = null; - try { - conn = calledForEstimate ? getEstimateConnection() : getConnection(); - stmt = conn.prepareStatement(rewrittenQuery); - if (!pContext.getDriverConf(this).getBoolean(JDBC_VALIDATE_SKIP_WARNINGS, - DEFAULT_JDBC_VALIDATE_SKIP_WARNINGS) && stmt.getWarnings() != null) { - throw new LensException(stmt.getWarnings()); - } - } catch (SQLException sql) { - handleJDBCSQLException(sql); - } finally { - if (conn != null) { - try { - conn.close(); - } catch (SQLException e) { - log.error("Error closing connection: {}", rewrittenQuery, e); - } - } - jdbcPrepareGauge.markSuccess(); - } - log.info("Prepared: {}", rewrittenQuery); - return stmt; } /** @@ -809,10 +775,7 @@ public void prepare(PreparedQueryContext pContext) throws LensException { // already prepared return; } - PreparedStatement stmt = prepareInternal(pContext); - if (stmt != null) { - preparedQueries.put(pContext.getPrepareHandle(), stmt); - } + prepareInternal(pContext); } /** diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java index c08b04a27..c1023b828 100644 --- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java +++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java @@ -294,23 +294,7 @@ public void testEstimate() throws Exception { } /** - * Test estimate failing - * - * @throws Exception the exception - */ - @Test - public void testEstimateFailing() throws Exception { - String query2 = "SELECT * FROM estimate_test2"; // Select query against non existing table - try { - driver.estimate(createExplainContext(query2, baseConf)); - Assert.fail("Running estimate on a non existing table."); - } catch (LensException ex) { - Assert.assertEquals(LensUtil.getCauseMessage(ex), "user lacks privilege or object not found: ESTIMATE_TEST2"); - } - } - - /** - * Test estimate failing + * Test estimate guages * * @throws Exception the exception */ @@ -326,7 +310,6 @@ public void testEstimateGauges() throws Exception { String driverQualifiledName = driver.getFullyQualifiedName(); Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList( "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-validate-columnar-sql-rewrite", - "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-validate-jdbc-prepare-statement", "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-validate-thru-prepare", "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-jdbc-check-allowed-query"))); } @@ -359,7 +342,11 @@ public void testMetricsEnabled() throws Exception { cost = driver.estimate(pContext2); Assert.assertEquals(cost, JDBC_COST); driver.prepare(pContext2); - driver.explainAndPrepare(pContext2); + + PreparedQueryContext pContext3 = new PreparedQueryContext(query1, "SA", metricConf, drivers); + cost = driver.estimate(pContext3); + Assert.assertEquals(cost, JDBC_COST); + driver.explainAndPrepare(pContext3); } /** @@ -653,23 +640,6 @@ public void testPrepare() throws Exception { driver.prepare(pContext); } - /** - * Test prepare failing - * - * @throws Exception the exception - */ - @Test - public void testPrepareFailing() throws Exception { - String query = "SELECT * FROM prepare_test2"; // Select query against non existing table - try { - PreparedQueryContext pContext = new PreparedQueryContext(query, "SA", baseConf, drivers); - driver.prepare(pContext); - Assert.fail("Running prepare on a non existing table."); - } catch (LensException ex) { - Assert.assertEquals(LensUtil.getCauseMessage(ex), "user lacks privilege or object not found: PREPARE_TEST2"); - } - } - /** * Test prepare skip warnings * From ee4e06675a45713093f5981700b66644971428b4 Mon Sep 17 00:00:00 2001 From: Rajashekhar Date: Thu, 22 Aug 2019 17:15:53 +0530 Subject: [PATCH 14/16] made 10 minutes back --- .../apache/lens/server/api/query/PreparedQueryContext.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java index 69ee7c4fe..1834c8b0e 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java @@ -77,7 +77,7 @@ public class PreparedQueryContext extends AbstractQueryContext implements Delaye /** * The millis in week. */ - private static long millisInWeek = 7 * 24 * 60 * 60 * 1000; + private static long millisInTenMinutes = 10 * 60 * 1000; /** * Instantiates a new prepared query context. @@ -129,7 +129,7 @@ public long getDelay(TimeUnit units) { if (this.prepareStartTime != null) { Date now = new Date(); long elapsedMills = now.getTime() - this.prepareStartTime.getTime(); - delayMillis = millisInWeek - elapsedMills; + delayMillis = millisInTenMinutes - elapsedMills; return units.convert(delayMillis, TimeUnit.MILLISECONDS); } else { return Integer.MAX_VALUE; From 40d4fe1529cd4f855d54b5a9f522d9aeea95502b Mon Sep 17 00:00:00 2001 From: Rajashekhar Date: Thu, 29 Aug 2019 11:29:59 +0530 Subject: [PATCH 15/16] formatting --- .../test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java index c1023b828..377b2dfb8 100644 --- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java +++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java @@ -44,7 +44,6 @@ import org.apache.lens.server.api.query.QueryContext; import org.apache.lens.server.api.query.cost.QueryCost; import org.apache.lens.server.api.query.cost.StaticQueryCost; -import org.apache.lens.server.api.util.LensUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; @@ -342,7 +341,7 @@ public void testMetricsEnabled() throws Exception { cost = driver.estimate(pContext2); Assert.assertEquals(cost, JDBC_COST); driver.prepare(pContext2); - + PreparedQueryContext pContext3 = new PreparedQueryContext(query1, "SA", metricConf, drivers); cost = driver.estimate(pContext3); Assert.assertEquals(cost, JDBC_COST); From cb704e3f05e4e01ab372295cc9f9d0e02646d7df Mon Sep 17 00:00:00 2001 From: Rajashekhar Date: Thu, 29 Aug 2019 11:39:09 +0530 Subject: [PATCH 16/16] formatting --- .../java/org/apache/lens/server/query/LensServerDAO.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java index 28b678b3b..bba6caca0 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java @@ -120,7 +120,7 @@ public void createPreparedQueriesTable() throws Exception { log.warn("Unable to create prepared_queries queries table", e); } } - + public void createFailedAttemptsTable() throws Exception { String sql = "CREATE TABLE if not exists failed_attempts (handle varchar(255) not null," + "attempt_number int, drivername varchar(10000), progress float, progressmessage varchar(10000), " @@ -845,8 +845,9 @@ public boolean deleteActiveSession(LensSessionHandle sessionId) throws LensExcep * @throws SQLException the exception */ public void insertPreparedQuery(PreparedQueryContext preparedQueryContext) throws LensException { - String sql = "insert into prepared_queries (handle, userquery, submitter, timetaken, queryname, drivername, " - + "driverquery, starttime)" + " values (?,?,?,?,?,?,?,?)"; + String sql = + "insert into prepared_queries (handle, userquery, submitter, timetaken, queryname, drivername, " + + "driverquery, starttime)" + " values (?,?,?,?,?,?,?,?)"; Connection conn = null; try { conn = getConnection(); @@ -864,7 +865,7 @@ public void insertPreparedQuery(PreparedQueryContext preparedQueryContext) throw } catch (SQLException e) { log.error("Failed to insert prepared query into database with error, " + e); throw new LensException(e); - } finally { + } finally { DbUtils.closeQuietly(conn); } }